diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 116d72ee7b..012736ba13 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15831,6 +15831,7 @@ dependencies = [ "url", "urlencoding", "uuid", + "windmill-api-auth", "windmill-audit", "windmill-autoscaling", "windmill-common", @@ -15844,9 +15845,32 @@ dependencies = [ "windmill-parser-sql", "windmill-parser-ts", "windmill-queue", + "windmill-triggers", "windmill-worker", ] +[[package]] +name = "windmill-api-auth" +version = "1.628.2" +dependencies = [ + "axum 0.7.9", + "chrono", + "http 1.4.0", + "itertools 0.14.0", + "lazy_static", + "quick_cache", + "serde", + "serde_json", + "sqlx", + "time", + "tokio", + "tower-cookies", + "tracing", + "uuid", + "windmill-audit", + "windmill-common", +] + [[package]] name = "windmill-api-client" version = "1.628.2" @@ -16337,6 +16361,97 @@ dependencies = [ "windmill-parser-sql", ] +[[package]] +name = "windmill-triggers" +version = "1.628.2" +dependencies = [ + "anyhow", + "async-nats", + "async-recursion", + "async-stream", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sqs", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-types", + "axum 0.7.9", + "backon", + "base64 0.22.1", + "byteorder", + "bytes", + "chrono", + "constant_time_eq 0.3.1", + "dashmap 6.1.0", + "datafusion", + "futures", + "google-cloud-googleapis", + "google-cloud-pubsub", + "hex", + "hmac", + "http 1.4.0", + "hyper 1.8.1", + "itertools 0.14.0", + "jsonwebtoken 8.3.0", + "lazy_static", + "mail-parser", + "matchit 0.7.3", + "native-tls", + "nkeys", + "object_store", + "openssl", + "pg_escape", + "pin-project", + "postgres-native-tls 0.5.0", + "postgres-native-tls 0.5.1", + "prometheus", + "quick_cache", + "rand 0.9.0", + "rdkafka", + "rdkafka-sys", + "regex", + "reqwest 0.13.1", + "rumqttc", + "rust_decimal", + "serde", + "serde_json", + "serde_urlencoded", + "serde_yml", + "sha1", + "sha2 0.10.9", + "sql-builder", + "sqlx", + "strum 0.27.2", + "thiserror 2.0.18", + "tokio", + "tokio-native-tls", + "tokio-postgres 0.7.11", + "tokio-postgres 0.7.13", + "tokio-stream", + "tokio-tungstenite 0.24.0", + "tokio-util", + "tonic", + "tower 0.4.13", + "tower-cookies", + "tower-http", + "tracing", + "ulid", + "url", + "urlencoding", + "uuid", + "windmill-api-auth", + "windmill-audit", + "windmill-common", + "windmill-git-sync", + "windmill-oauth", + "windmill-parser", + "windmill-parser-py", + "windmill-parser-ts", + "windmill-queue", +] + [[package]] name = "windmill-worker" version = "1.628.2" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 899eb1d611..3e1e2931bb 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -8,6 +8,8 @@ edition.workspace = true resolver = "2" members = [ "./windmill-api", + "./windmill-api-auth", + "./windmill-triggers", "./windmill-queue", "./windmill-worker", "./windmill-common", @@ -217,6 +219,8 @@ tempfile.workspace = true [workspace.dependencies] windmill-api = { path = "./windmill-api", default-features = false } +windmill-api-auth = { path = "./windmill-api-auth" } +windmill-triggers = { path = "./windmill-triggers" } windmill-queue = { path = "./windmill-queue" } windmill-worker = { path = "./windmill-worker" } windmill-common = { path = "./windmill-common", default-features = false } diff --git a/backend/windmill-api-auth/Cargo.toml b/backend/windmill-api-auth/Cargo.toml new file mode 100644 index 0000000000..6dfedf6e30 --- /dev/null +++ b/backend/windmill-api-auth/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "windmill-api-auth" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_api_auth" +path = "src/lib.rs" + +[features] +default = [] +enterprise = ["windmill-common/enterprise"] +no_auth = [] +private = ["windmill-common/private"] +cloud = ["windmill-common/cloud"] + +[dependencies] +windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true + +axum.workspace = true +chrono.workspace = true +http.workspace = true +itertools.workspace = true +lazy_static.workspace = true +quick_cache.workspace = true +serde.workspace = true +serde_json.workspace = true +sqlx.workspace = true +time.workspace = true +tokio.workspace = true +tower-cookies.workspace = true +tracing.workspace = true +uuid.workspace = true diff --git a/backend/windmill-api-auth/src/auth.rs b/backend/windmill-api-auth/src/auth.rs new file mode 100644 index 0000000000..581c4a8762 --- /dev/null +++ b/backend/windmill-api-auth/src/auth.rs @@ -0,0 +1,868 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{ + async_trait, + extract::{FromRequestParts, OriginalUri, Query}, + Extension, Json, +}; +use chrono::TimeZone; +use http::{request::Parts, StatusCode}; +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; +use tower_cookies::Cookies; +use tracing::Span; + +use crate::{scopes, ApiAuthed, OptJobAuthed}; +use std::{ + str::FromStr, + sync::{ + atomic::{AtomicI64, AtomicU64, Ordering}, + Arc, + }, +}; + +use windmill_common::{ + auth::{ + fetch_authed_from_permissioned_as, get_folders_for_user, get_groups_for_user, + JWTAuthClaims, TOKEN_PREFIX_LEN, + }, + db::DB, + error::{Error, JsonResult}, + jwt, + users::{username_to_permissioned_as, COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, +}; + +lazy_static::lazy_static! { + // Global auth cache accessible from main.rs for direct invalidation + pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); +} + +// Global function to invalidate a specific token from cache +pub fn invalidate_token_from_cache(token: &str) { + // Remove all cache entries for this token (across all workspaces) + AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); + tracing::info!( + "Invalidated token from auth cache: {}...", + &token[..token.len().min(8)] + ); +} + +#[derive(Clone)] +pub struct ExpiringAuthCache { + pub authed: ApiAuthed, + pub expiry: chrono::DateTime, + pub job_id: Option, +} + +/// Callback trait for enterprise JWT external auth. +/// windmill-api provides the implementation; windmill-api-auth calls it through this trait. +#[async_trait] +pub trait JwtExtAuthBackend: Send + Sync + 'static { + async fn jwt_ext_auth( + &self, + w_id: Option<&String>, + token: &str, + db: &DB, + ) -> Option<(ApiAuthed, usize, Option)>; +} + +/// A no-op backend used when enterprise JWT ext auth is not available. +pub struct NoopJwtExtAuth; + +#[async_trait] +impl JwtExtAuthBackend for NoopJwtExtAuth { + async fn jwt_ext_auth( + &self, + _w_id: Option<&String>, + _token: &str, + _db: &DB, + ) -> Option<(ApiAuthed, usize, Option)> { + None + } +} + +pub struct AuthCache { + db: DB, + superadmin_secret: Option, + jwt_ext_auth: Arc, +} + +impl AuthCache { + pub fn new( + db: DB, + superadmin_secret: Option, + jwt_ext_auth: Arc, + ) -> Self { + AuthCache { + db, + superadmin_secret, + jwt_ext_auth, + } + } + + pub async fn invalidate(&self, w_id: &str, token: String) { + AUTH_CACHE.remove(&(w_id.to_string(), token)); + } + + pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { + Some(self.get_opt_job_authed(w_id, token).await?.authed) + } + + pub async fn get_opt_job_authed( + &self, + w_id: Option, + token: &str, + ) -> Option { + let key = ( + w_id.as_ref().unwrap_or(&"".to_string()).to_string(), + token.to_string(), + ); + let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); + match s { + Some(ExpiringAuthCache { + authed, expiry, job_id, + }) if expiry > chrono::Utc::now() => Some(OptJobAuthed { authed, job_id }), + _ if token.starts_with("jwt_ext_") => { + let authed_and_exp = self + .jwt_ext_auth + .jwt_ext_auth( + w_id.as_ref(), + token.trim_start_matches("jwt_ext_"), + &self.db, + ) + .await; + + if let Some((authed, exp, job_id)) = authed_and_exp { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } else { + None + } + } + _ if token.starts_with("jwt_") => { + let jwt_token = token.trim_start_matches("jwt_"); + + let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; + + match jwt_result { + Ok(claims) => { + if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { + tracing::error!("JWT auth error: workspace_id mismatch"); + return None; + } + let username_override = username_override_from_label(claims.label); + + let authed = ApiAuthed { + email: claims.email, + username: claims.username, + is_admin: claims.is_admin, + is_operator: claims.is_operator, + groups: claims.groups, + folders: claims.folders, + scopes: None, + username_override, + token_prefix: claims.audit_span, + }; + let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc + .timestamp_nanos(claims.exp as i64 * 1_000_000_000), + job_id, + }, + ); + + Some(OptJobAuthed { authed, job_id }) + } + Err(err) => { + tracing::error!("JWT auth error: {:?}", err); + None + } + } + } + _ => { + let user_o = sqlx::query!( + "UPDATE token SET last_used_at = now() WHERE + token = $1 + AND (expiration > NOW() OR expiration IS NULL) + AND (workspace_id IS NULL OR workspace_id = $2) + RETURNING owner, email, super_admin, scopes, label", + token, + w_id.as_ref(), + ) + .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) + .fetch_optional(&self.db) + .await + .ok() + .flatten(); + + if let Some(user) = user_o { + let authed_o = { + match user { + (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { + let username_override = username_override_from_label(label); + if let Some((prefix, name)) = owner.split_once('/') { + if prefix == "u" { + let (is_admin, is_operator) = if super_admin { + (true, false) + } else { + let r = sqlx::query!( + "SELECT is_admin, operator FROM usr where username = $1 AND \ + workspace_id = $2 AND disabled = false", + name, + &w_id.as_ref().unwrap() + ) + .fetch_one(&self.db) + .await + .ok(); + if let Some(r) = r { + (r.is_admin, r.operator) + } else { + (false, true) + } + }; + + let w_id = &w_id.unwrap(); + let groups = + get_groups_for_user(w_id, &name, &email, &self.db) + .await + .ok() + .unwrap_or_default(); + + let folders = + get_folders_for_user(w_id, &name, &groups, &self.db) + .await + .ok() + .unwrap_or_default(); + + Some(ApiAuthed { + email: email, + username: name.to_string(), + is_admin, + is_operator, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } else { + let groups = vec![name.to_string()]; + let folders = get_folders_for_user( + &w_id.unwrap(), + "", + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email: email, + username: format!("group-{name}"), + is_admin: false, + groups, + is_operator: false, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + } else { + let groups = vec![]; + let folders = vec![]; + Some(ApiAuthed { + email: email, + username: owner, + is_admin: super_admin, + is_operator: true, + groups, + folders, + scopes: None, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + } + (_, Some(email), super_admin, scopes, label) => { + let username_override = username_override_from_label(label); + if w_id.is_some() { + let row_o = sqlx::query!( + "SELECT username, is_admin, operator FROM usr WHERE + email = $1 AND workspace_id = $2 AND disabled = false", + &email, + w_id.as_ref().unwrap() + ) + .map(|x| (x.username, x.is_admin, x.operator)) + .fetch_optional(&self.db) + .await + .unwrap_or(Some(("error".to_string(), false, false))); + + match row_o { + Some((username, is_admin, is_operator)) => { + let groups = get_groups_for_user( + &w_id.as_ref().unwrap(), + &username, + &email, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + + let folders = get_folders_for_user( + &w_id.unwrap(), + &username, + &groups, + &self.db, + ) + .await + .ok() + .unwrap_or_default(); + Some(ApiAuthed { + email, + username, + is_admin: is_admin || super_admin, + is_operator, + groups, + folders, + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + None if super_admin => Some(ApiAuthed { + email: email.clone(), + username: email, + is_admin: super_admin, + is_operator: false, + groups: vec![], + folders: vec![], + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }), + None => None, + } + } else { + Some(ApiAuthed { + email: email.to_string(), + username: email, + is_admin: super_admin, + is_operator: true, + groups: Vec::new(), + folders: Vec::new(), + scopes, + username_override, + token_prefix: Some( + token[0..TOKEN_PREFIX_LEN].to_string(), + ), + }) + } + } + _ => None, + } + }; + if let Some(authed) = authed_o.as_ref() { + AUTH_CACHE.insert( + key, + ExpiringAuthCache { + authed: authed.clone(), + expiry: chrono::Utc::now() + + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + } + authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) + } else if self + .superadmin_secret + .as_ref() + .map(|x| x == token) + .unwrap_or(false) + { + let authed = ApiAuthed { + email: SUPERADMIN_SECRET_EMAIL.to_string(), + username: "superadmin_secret".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), + }; + Some(OptJobAuthed { authed, job_id: None }) + } else { + None + } + } + } + } +} + +pub async fn extract_token(parts: &mut Parts, state: &S) -> Option { + let auth_header = parts + .headers + .get(http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|s| s.strip_prefix("Bearer ")); + + let from_cookie = match auth_header { + Some(x) => Some(x.to_owned()), + None => Extension::::from_request_parts(parts, state) + .await + .ok() + .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), + }; + + #[derive(Deserialize)] + struct Token { + token: Option, + } + match from_cookie { + Some(token) => Some(token), + None => Query::::from_request_parts(parts, state) + .await + .ok() + .and_then(|token| token.token.clone()), + } +} + +#[derive(Clone, Debug)] +pub struct Tokened { + pub token: String, +} + +#[derive(Clone, Debug)] +pub struct OptTokened { + #[allow(dead_code)] + pub token: Option, +} + +struct BruteForceCounter { + counter: AtomicU64, + last_reset: AtomicI64, +} + +lazy_static::lazy_static! { + static ref BRUTE_FORCE_COUNTER: BruteForceCounter = + BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; +} + +impl BruteForceCounter { + async fn increment(&self) { + let now = time::OffsetDateTime::now_utc().unix_timestamp(); + if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { + tracing::error!( + "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" + ); + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + if now - self.last_reset.load(Ordering::Relaxed) > 60 { + self.counter.store(0, Ordering::Relaxed); + self.last_reset.store(now, Ordering::Relaxed); + } + } +} + +#[async_trait] +impl FromRequestParts for Tokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(Tokened { + token: "".to_string(), + }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(tokened.clone()) + } else { + let token_o = extract_token(parts, state).await; + if let Some(token) = token_o { + let tokened = Self { token }; + parts.extensions.insert(tokened.clone()); + Ok(tokened) + } else { + BRUTE_FORCE_COUNTER.increment().await; + Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) + } + } + } +} + +#[async_trait] +impl FromRequestParts for OptTokened +where + S: Send + Sync, +{ + type Rejection = (StatusCode, String); + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(OptTokened { token: None }); + }; + let already_tokened = parts.extensions.get::(); + if let Some(tokened) = already_tokened { + Ok(OptTokened { + token: Some(tokened.token.clone()), + }) + } else { + let token_o = extract_token(parts, state).await; + Ok(OptTokened { token: token_o }) + } + } +} + +pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { + if let Some(scopes) = scopes { + for scope in scopes.iter_mut() { + if scope.starts_with("run:") { + let (_, part_scope) = scope.split_once(":").unwrap(); + + if let Some((kind, path)) = part_scope.split_once("/") { + //appending a 's' as runnable kind is singular while new scope format expect it to be plural + *scope = format!("jobs:run:{}s:{}", kind, path); + } + } else if scope.starts_with("jobs:") { + // Map old jobs scopes to new format + let new_scope = match scope.as_str() { + "jobs:listjobs" => "jobs:read", + "jobs:runscript" => "jobs:run:scripts", + "jobs:runflow" => "jobs:run:flows", + "jobs:resumeflow" => "jobs:run:flows", + "jobs:deletejob" => "jobs:write", + _ => continue, + }; + + *scope = new_scope.to_string(); + } + } + } +} + +fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { + let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { + Some(path_vec[3].to_owned()) + } else if path_vec.len() >= 5 + && path_vec[0] == "" + && path_vec[1] == "api" + && path_vec[2] == "mcp" + && path_vec[3] == "w" + { + Some(path_vec[4].to_owned()) + } else { + if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" + { + Some(path_vec[4].to_owned()) + } else { + None + } + }; + + workspace_id +} + +#[async_trait] +impl FromRequestParts for ApiAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + let opt_job_authed = OptJobAuthed::from_request_parts(parts, state).await?; + Ok(opt_job_authed.authed) + } +} + +#[async_trait] +impl FromRequestParts for OptJobAuthed +where + S: Send + Sync, +{ + type Rejection = Error; + + async fn from_request_parts( + parts: &mut Parts, + state: &S, + ) -> std::result::Result { + if parts.method == http::Method::OPTIONS { + return Ok(OptJobAuthed::default()); + }; + + #[cfg(feature = "no_auth")] + { + let authed = ApiAuthed { + email: "admin@windmill.dev".to_string(), + username: "admin".to_string(), + is_admin: true, + is_operator: false, + groups: Vec::new(), + folders: Vec::new(), + scopes: None, + username_override: None, + token_prefix: None, + }; + return Ok(OptJobAuthed { + authed, + job_id: None, + }); + } + + let already_authed = parts.extensions.get::(); + + if let Some(authed) = already_authed { + return Ok(authed.clone()); + } + + let already_tokened = parts.extensions.get::(); + let token_o = if let Some(token) = already_tokened { + Some(token.token.clone()) + } else { + extract_token(parts, state).await + }; + if let Some(token) = token_o { + if let Ok(Extension(cache)) = + Extension::>::from_request_parts(parts, state).await + { + let original_uri = OriginalUri::from_request_parts(parts, state) + .await + .ok() + .map(|x| x.0) + .unwrap_or_default(); + let path_vec: Vec<&str> = original_uri.path().split("/").collect(); + let workspace_id = maybe_get_workspace_id_from_path(&path_vec); + + if let Some(mut opt_job_authed) = + cache.get_opt_job_authed(workspace_id.clone(), &token).await + { + let authed = &mut opt_job_authed.authed; + if authed.scopes.is_some() { + transform_old_scope_to_new_scope(authed.scopes.as_mut()); + + let path = original_uri.path(); + let method = parts.method.as_str(); + + if let Err(err) = scopes::check_scopes_for_route( + authed.scopes.as_deref(), + path, + method, + ) { + BRUTE_FORCE_COUNTER.increment().await; + return Err(err); + } + } + parts.extensions.insert(authed.clone()); + + Span::current().record("username", &authed.username.as_str()); + Span::current().record("email", &authed.email); + + if let Some(workspace_id) = workspace_id { + Span::current().record("workspace_id", &workspace_id); + } + return Ok(opt_job_authed); + } + } + } + BRUTE_FORCE_COUNTER.increment().await; + Err(Error::NotAuthorized("Unauthorized".to_string())) + } +} + +fn username_override_from_label(label: Option) -> Option { + match label { + Some(label) + if label.starts_with("webhook-") + || label.starts_with("http-") + || label.starts_with("email-") + || label.starts_with("ws-") => + { + Some(label) + } + Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( + label + .trim_start_matches("ephemeral-script-end-user-") + .to_string(), + ), + Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), + Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { + Some(format!("label-{label}")) + } + _ => None, + } +} + +#[derive(FromRow, Serialize)] +pub struct TruncatedTokenWithEmail { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, + pub email: Option, +} + +pub async fn list_tokens_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult> { + let tokens = if is_flow { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:flows:' || $2]::text[] + OR scopes @> ARRAY['run:flow/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + } else { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] + OR scopes @> ARRAY['run:script/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + }; + + Ok(Json(tokens)) +} + +lazy_static::lazy_static! { + pub static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300); +} + +pub async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> windmill_common::error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await +} + +#[allow(unused)] +pub async fn fetch_api_authed_from_permissioned_as( + permissioned_as: String, + email: String, + w_id: &str, + db: &DB, + username_override: Option, +) -> windmill_common::error::Result { + let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); + + let mut api_authed = match API_AUTHED_CACHE.get(&key) { + Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { + tracing::debug!("API authed cache hit for user {}", email); + expiring_authed.authed + } + _ => { + tracing::debug!("API authed cache miss for user {}", email); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + + let api_authed = ApiAuthed { + username: authed.username, + email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: None, + token_prefix: authed.token_prefix, + }; + + API_AUTHED_CACHE.insert( + key, + ExpiringAuthCache { + authed: api_authed.clone(), + expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), + job_id: None, + }, + ); + + api_authed + } + }; + + api_authed.username_override = username_override; + Ok(api_authed) +} diff --git a/backend/windmill-api-auth/src/lib.rs b/backend/windmill-api-auth/src/lib.rs new file mode 100644 index 0000000000..afb27d5c32 --- /dev/null +++ b/backend/windmill-api-auth/src/lib.rs @@ -0,0 +1,14 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +mod auth; +pub mod scopes; +mod types; + +pub use auth::*; +pub use types::*; diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs new file mode 100644 index 0000000000..5a8cc8b148 --- /dev/null +++ b/backend/windmill-api-auth/src/scopes.rs @@ -0,0 +1,959 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use windmill_common::error::{Error, Result}; + +use crate::ApiAuthed; + +/// Comprehensive scope system for JWT token authorization +/// +/// Scopes follow the format: {domain}:{action}[:{resource}] +/// Examples: +/// - "jobs:read" - Read access to jobs +/// - "scripts:write:f/folder/*" - Write access to scripts in a folder +/// - "*" - Full access (superuser) + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScopeDefinition { + pub domain: String, + pub action: String, + pub kind: Option, // For jobs:run:kind (optional) + pub resource: Option>, +} + +impl ScopeDefinition { + pub fn new( + domain: &str, + action: &str, + kind: Option<&str>, + resource: Option>, + ) -> Self { + Self { + domain: domain.to_string(), + action: action.to_string(), + kind: kind.map(|s| s.to_string()), + resource: resource, + } + } + + pub fn from_scope_string(scope: &str) -> Result { + let parts: Vec<&str> = scope.split(':').collect(); + + let into_owned_vec = |resources: &str| -> Vec { + let resources = resources + .split(",") + .collect_vec() + .into_iter() + .map(ToOwned::to_owned) + .collect_vec(); + + resources + }; + + match parts.len() { + 2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action + 3 => { + if parts[0] == "jobs" && parts[1] == "run" { + Ok(Self::new(parts[0], parts[1], Some(parts[2]), None)) + } else { + Ok(Self::new( + parts[0], + parts[1], + None, + Some(into_owned_vec(parts[2])), + )) + } + } + 4 => { + if parts[0] == "jobs" && parts[1] == "run" { + Ok(Self::new( + parts[0], + parts[1], + Some(parts[2]), + Some(into_owned_vec(parts[3])), + )) + } else { + Err(Error::BadRequest(format!( + "Invalid 4-part scope: {}", + scope + ))) + } + } + _ => Err(Error::BadRequest(format!( + "Invalid scope format: {}", + scope + ))), + } + } + + pub fn as_string(&self) -> String { + match (&self.kind, &self.resource) { + (Some(kind), Some(resource)) => { + format!( + "{}:{}:{}:{}", + self.domain, + self.action, + kind, + resource.join(",") + ) + } + (Some(kind), None) => { + format!("{}:{}:{}", self.domain, self.action, kind) + } + (None, Some(resource)) => { + format!("{}:{}:{}", self.domain, self.action, resource.join(",")) + } + (None, None) => format!("{}:{}", self.domain, self.action), + } + } + + pub fn includes(&self, other: &ScopeDefinition) -> bool { + if self.domain != other.domain { + return false; + } + + match (self.action.as_str(), other.action.as_str()) { + (a, b) if (a == "write" && b == "read") || (a == b) => {} + _ => return false, + } + + if self.domain == "jobs" && self.action == "run" { + match (&self.kind, &other.kind) { + (Some(self_kind), Some(other_kind)) => { + if self_kind != other_kind { + return false; + } + } + (Some(_), None) => { + return false; + } + (None, _) => { + return true; + } + } + } + + match (&self.resource, &other.resource) { + (Some(self_resources), Some(other_resources)) => { + resources_match(self_resources, other_resources) + } + (Some(_), None) => false, + (None, _) => true, + } + } +} + +fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool { + if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) { + return true; + } + + if scope_resources.len() <= 4 && accepted_resources.len() <= 4 { + return resources_match_small(scope_resources, accepted_resources); + } + + resources_match_large(scope_resources, accepted_resources) +} + +fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool { + for required in accepted_resources { + for scope_resource in scope_resources { + if resource_matches_pattern(scope_resource, required) { + return true; + } + } + } + false +} + +fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool { + let mut exact_matches = HashSet::new(); + let mut patterns = Vec::new(); + + for scope_resource in scope_resources { + if scope_resource.contains('*') { + patterns.push(scope_resource); + } else { + exact_matches.insert(scope_resource); + } + } + + for accepted_resource in accepted_resources { + if exact_matches.contains(accepted_resource) { + return true; + } + + for pattern in &patterns { + if resource_matches_pattern(pattern, accepted_resource) { + return true; + } + } + } + + false +} + +fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool { + if scope_resource == accepted_resource { + return true; + } + + let matches_wildcard = |pattern: &str, resource: &str| -> bool { + if !pattern.ends_with("/*") { + return false; + } + + let prefix = &pattern[..pattern.len() - 2]; + + if !resource.starts_with(prefix) { + return false; + } + + // If the resource is exactly the prefix, it matches + if resource.len() == prefix.len() { + return true; + } + + // If the resource is longer, the next character must be '/' for a valid match + // This prevents "u/user" from matching "u/use/*" + resource.chars().nth(prefix.len()) == Some('/') + }; + + // Check if either resource is a wildcard pattern and matches the other + matches_wildcard(scope_resource, accepted_resource) + || matches_wildcard(accepted_resource, scope_resource) +} + +/// Available scope domains (top-level API categories) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ScopeDomain { + // Core resource domains + Jobs, + Scripts, + Flows, + Apps, + Variables, + Resources, + Schedules, + Folders, + Users, + Groups, + Workspaces, + + // Trigger domains + HttpTriggers, + WebsocketTriggers, + KafkaTriggers, + NatsTriggers, + MqttTriggers, + SqsTriggers, + GcpTriggers, + PostgresTriggers, + EmailTriggers, + + // Native trigger domains + NativeTriggers, + + // System domains + Audit, + Settings, + Workers, + ServiceLogs, + Configs, + OAuth, + AI, + + Indexer, + Teams, // Microsoft Teams integration + GitSync, // Git synchronization + + // Special domains + Capture, // Webhook capture + Drafts, // Draft resources + Favorites, // User favorites + Inputs, // Input templates + JobHelpers, // Job helper functions + ConcurrencyGroups, // Concurrency groups + Oidc, // OpenID Connect + Openapi, // OpenAPI generation + + // Additional domains + Acls, // Granular access control lists + RawApps, // Raw application data + AgentWorkers, // Agent workers management + Mcp, // MCP +} + +impl ScopeDomain { + pub fn as_str(&self) -> &'static str { + match self { + Self::Jobs => "jobs", + Self::Scripts => "scripts", + Self::Flows => "flows", + Self::Apps => "apps", + Self::Variables => "variables", + Self::Resources => "resources", + Self::Schedules => "schedules", + Self::Folders => "folders", + Self::Users => "users", + Self::Groups => "groups", + Self::Workspaces => "workspaces", + Self::HttpTriggers => "http_triggers", + Self::WebsocketTriggers => "websocket_triggers", + Self::KafkaTriggers => "kafka_triggers", + Self::NatsTriggers => "nats_triggers", + Self::MqttTriggers => "mqtt_triggers", + Self::SqsTriggers => "sqs_triggers", + Self::GcpTriggers => "gcp_triggers", + Self::PostgresTriggers => "postgres_triggers", + Self::EmailTriggers => "email_triggers", + Self::NativeTriggers => "native_triggers", + Self::Audit => "audit", + Self::Settings => "settings", + Self::Workers => "workers", + Self::ServiceLogs => "service_logs", + Self::Configs => "configs", + Self::OAuth => "oauth", + Self::AI => "ai", + Self::Capture => "capture", + Self::Drafts => "drafts", + Self::Favorites => "favorites", + Self::Inputs => "inputs", + Self::JobHelpers => "job_helpers", + Self::ConcurrencyGroups => "concurrency_groups", + Self::Oidc => "oidc", + Self::Openapi => "openapi", + Self::Acls => "acls", + Self::RawApps => "raw_apps", + Self::AgentWorkers => "agent_workers", + Self::Indexer => "indexer", + Self::Teams => "teams", + Self::GitSync => "git_sync", + Self::Mcp => "mcp", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "jobs" | "jobs_u" => Some(Self::Jobs), + "scripts" => Some(Self::Scripts), + "flows" => Some(Self::Flows), + "apps" | "apps_u" => Some(Self::Apps), + "variables" => Some(Self::Variables), + "resources" => Some(Self::Resources), + "schedules" => Some(Self::Schedules), + "folders" => Some(Self::Folders), + "users" => Some(Self::Users), + "groups" => Some(Self::Groups), + "workspaces" => Some(Self::Workspaces), + "http_triggers" => Some(Self::HttpTriggers), + "websocket_triggers" => Some(Self::WebsocketTriggers), + "kafka_triggers" => Some(Self::KafkaTriggers), + "nats_triggers" => Some(Self::NatsTriggers), + "mqtt_triggers" => Some(Self::MqttTriggers), + "sqs_triggers" => Some(Self::SqsTriggers), + "gcp_triggers" => Some(Self::GcpTriggers), + "postgres_triggers" => Some(Self::PostgresTriggers), + "email_triggers" => Some(Self::EmailTriggers), + "audit" => Some(Self::Audit), + "settings" => Some(Self::Settings), + "workers" => Some(Self::Workers), + "service_logs" => Some(Self::ServiceLogs), + "configs" => Some(Self::Configs), + "oauth" => Some(Self::OAuth), + "ai" => Some(Self::AI), + "indexer" | "srch" => Some(Self::Indexer), + "teams" => Some(Self::Teams), + "native_triggers" => Some(Self::NativeTriggers), + "git_sync" | "github_app" => Some(Self::GitSync), + "capture" => Some(Self::Capture), + "drafts" => Some(Self::Drafts), + "favorites" => Some(Self::Favorites), + "inputs" => Some(Self::Inputs), + "job_helpers" => Some(Self::JobHelpers), + "concurrency_groups" => Some(Self::ConcurrencyGroups), + "oidc" => Some(Self::Oidc), + "openapi" => Some(Self::Openapi), + "acls" => Some(Self::Acls), + "raw_apps" => Some(Self::RawApps), + "agent_workers" => Some(Self::AgentWorkers), + "mcp" => Some(Self::Mcp), + _ => None, + } + } +} + +/// Available scope actions +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ScopeAction { + Read, // GET operations, list, view + Write, // POST, PUT, PATCH, DELETE operations, create, update, delete + Run, // Special action for running (scripts, flows, etc.) +} + +impl ScopeAction { + pub fn as_str(&self) -> &'static str { + match self { + Self::Read => "read", + Self::Write => "write", + Self::Run => "run", + } + } + + pub fn from_str(s: &str) -> Option { + match s { + "read" => Some(Self::Read), + "write" => Some(Self::Write), + "delete" => Some(Self::Write), + "run" => Some(Self::Run), + _ => None, + } + } + + /// Check if this action includes another action + /// Write includes Read + pub fn includes(&self, other: &ScopeAction) -> bool { + match (self, other) { + (ScopeAction::Write, ScopeAction::Read) => true, + (ScopeAction::Run, ScopeAction::Read) => true, + (a, b) => a == b, + } + } +} + +pub fn check_route_access( + token_scopes: &[String], + route_path: &str, + http_method: &str, +) -> Result<()> { + // Map HTTP method to scope action (considering route context) + let required_action = map_http_method_to_action(http_method, route_path); + + // Find the domain and kind for this route + let (required_domain, required_kind, route_suffix) = extract_domain_from_route(route_path)?; + + // Backward compatibility: MCP handlers expect unusual scope actions: all, favorites, hub. + if required_domain == ScopeDomain::Mcp { + return Ok(()); + } + + let mut is_scoped_token = false; + // Check if any token scope grants the required access + for scope_str in token_scopes { + if !scope_str.starts_with("if_jobs:filter_tags:") { + if let Ok(scope) = ScopeDefinition::from_scope_string(scope_str) { + if scope_grants_access( + &scope, + required_domain, + required_action, + required_kind.as_deref(), + route_suffix.as_deref(), + )? { + return Ok(()); + } + } + if !is_scoped_token { + is_scoped_token = true; + } + } + } + + //Edge case for backward compatibility, if only scopes defined was filter tag then don't treat this we don't treat the token + //as a restricted token + if !is_scoped_token { + return Ok(()); + } + let scope_display = if let Some(kind) = required_kind { + format!( + "{}:{}:{}", + required_domain.as_str(), + required_action.as_str(), + kind + ) + } else { + format!("{}:{}", required_domain.as_str(), required_action.as_str()) + }; + + Err(Error::NotAuthorized(format!( + "Access denied. Required scope: {}", + scope_display + ))) +} + +const SCRIPT_JOBS: [&'static str; 8] = [ + "jobs/run/p", + "jobs/run/h", + "jobs/run_wait_result/p", + "jobs/run_wait_result/h", + "jobs/run/preview_bundle", + "jobs/run/preview", + "jobs/run_and_stream/p", + "jobs/run_and_stream/h", +]; + +const FLOW_JOBS: [&'static str; 6] = [ + "jobs/run/f", + "jobs/run_wait_result/f", + "jobs/run/preview_flow", + "jobs/restart/f", + "jobs/flow/resume", + "jobs/run_and_stream/f", +]; + +lazy_static::lazy_static! { + static ref RUN_PATH_ACTIONS: Vec<&'static str> = { + let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component"]; + + v.extend(SCRIPT_JOBS); + v.extend(FLOW_JOBS); + v + }; +} + +fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { + if RUN_PATH_ACTIONS + .iter() + .any(|run_path| route_path.contains(run_path)) + { + return ScopeAction::Run; + } + + match method.to_uppercase().as_str() { + "GET" | "HEAD" | "OPTIONS" => ScopeAction::Read, + "POST" | "PUT" | "PATCH" | "DELETE" => ScopeAction::Write, + _ => ScopeAction::Read, + } +} + +/// Checks the route path to determine the runnable kind (either "flows" or "scripts"). +/// +/// The order of checks is important: +/// - Flow-related paths are checked first to avoid false positives, as some flow paths +/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). +/// +/// Returns `"flows"` or `"scripts"` based on the match, or `None` if no match is found. +fn determine_kind_from_route(route_path: &str) -> Option { + if route_path.starts_with("jobs") { + if FLOW_JOBS.iter().any(|path| route_path.starts_with(path)) { + return Some("flows".to_string()); + } else if SCRIPT_JOBS.iter().any(|path| route_path.starts_with(path)) { + return Some("scripts".to_string()); + } + } + None +} + +fn extract_domain_from_route( + route_path: &str, +) -> Result<(ScopeDomain, Option, Option)> { + let parts: Vec<&str> = route_path.split('/').collect(); + + let (domain, kind, route_suffix) = if parts.len() >= 5 && parts[1] == "api" && parts[2] == "w" { + let domain_part = parts[4]; + let route_suffix = &parts[4..].join("/"); + + let domain = ScopeDomain::from_str(domain_part); + + let kind = determine_kind_from_route(&route_suffix); + + (domain, kind, Some(route_suffix.to_owned())) + } else if parts.len() >= 3 && parts[1] == "api" { + ( + ScopeDomain::from_str(parts[2]), + None, + Some(parts[2..].join("/")), + ) + } else { + (None, None, None) + }; + + if let Some(domain) = domain { + return Ok((domain, kind, route_suffix)); + } + + Err(Error::BadRequest(format!( + "Could not extract domain from route: {}", + route_path + ))) +} + +const RUN_WHITELISTED_GET_PATHS: [&'static str; 19] = [ + "jobs_u/get_flow/", + "jobs_u/get_root_job_id/", + "jobs_u/get/", + "jobs_u/get_logs/", + "jobs_u/get_args/", + "jobs_u/get_flow_debug_info/", + "jobs_u/completed/get/", + "jobs_u/completed/get_result/", + "jobs_u/completed/get_result_maybe/", + "jobs_u/getupdate/", + "jobs_u/getupdate_sse/", + "jobs_u/get_log_file/", + "jobs/result_by_id/", + "jobs/resume_urls/", + "jobs/flow/user_states/", + "jobs/job_signature/", + "jobs/completed/get/", + "jobs/completed/get_result/", + "jobs/completed/get_result_maybe/", +]; + +fn scope_grants_access( + scope: &ScopeDefinition, + required_domain: ScopeDomain, + required_action: ScopeAction, + required_kind: Option<&str>, + route_path: Option<&str>, +) -> Result { + // Check domain match + let scope_domain = ScopeDomain::from_str(&scope.domain) + .ok_or_else(|| Error::BadRequest(format!("Invalid scope domain: {}", scope.domain)))?; + + if scope_domain != required_domain { + return Ok(false); + } + + // Check action match (with hierarchical permissions) + let scope_action = ScopeAction::from_str(&scope.action) + .ok_or_else(|| Error::BadRequest(format!("Invalid scope action: {}", scope.action)))?; + + if !scope_action.includes(&required_action) + && !(scope_domain == ScopeDomain::Jobs + && required_action == ScopeAction::Read + && route_path.is_some_and(|p| { + RUN_WHITELISTED_GET_PATHS + .iter() + .any(|path| p.starts_with(path)) + })) + { + return Ok(false); + } + + if scope_domain == ScopeDomain::Jobs && required_action == ScopeAction::Run { + match (&scope.kind, required_kind) { + (Some(scope_kind), Some(req_kind)) => { + if scope_kind != req_kind { + return Ok(false); + } + } + (None, _) => {} + (Some(_), None) => { + return Ok(false); + } + } + } + + // No resource specified means access to entire domain + Ok(true) +} + +/// Helper function to check if scopes allow access to a route +pub fn check_scopes_for_route( + token_scopes: Option<&[String]>, + route_path: &str, + http_method: &str, +) -> Result<()> { + // If no scopes defined, allow access (backward compatibility) + let scopes = match token_scopes { + Some(s) if !s.is_empty() => s, + _ => return Ok(()), + }; + + check_route_access(scopes, route_path, http_method) +} + +/// Check if an authed user's scopes allow a specific action +pub fn check_scopes(authed: &ApiAuthed, required: F) -> Result<()> +where + F: FnOnce() -> String, +{ + if let Some(scopes) = authed.scopes.as_ref() { + let mut is_scoped_token = false; + let required_scope = ScopeDefinition::from_scope_string(&required())?; + for scope in scopes { + if !scope.starts_with("if_jobs:filter_tags:") { + if !is_scoped_token { + is_scoped_token = true; + } + + match ScopeDefinition::from_scope_string(scope) { + Ok(scope) if scope.includes(&required_scope) => return Ok(()), + _ => {} + } + } + } + + if is_scoped_token { + return Err(Error::NotAuthorized(format!( + "Required scope: {}", + required_scope.as_string() + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_scope_definition_parsing() { + let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "read"); + assert_eq!(scope.kind, None); + assert_eq!(scope.resource, None); + + let scope = ScopeDefinition::from_scope_string("jobs:run:scripts:f/folder/*").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("scripts".to_string())); + assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); + + // Test jobs:run:kind parsing + let scope = ScopeDefinition::from_scope_string("jobs:run:scripts").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("scripts".to_string())); + assert_eq!(scope.resource, None); + + // Test jobs:run:kind:resource parsing + let scope = ScopeDefinition::from_scope_string("jobs:run:flows:f/folder/*").unwrap(); + assert_eq!(scope.domain, "jobs"); + assert_eq!(scope.action, "run"); + assert_eq!(scope.kind, Some("flows".to_string())); + assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); + + // Test comma-separated resources parsing + let scope = + ScopeDefinition::from_scope_string("scripts:read:path1,path2,f/folder/*").unwrap(); + assert_eq!(scope.domain, "scripts"); + assert_eq!(scope.action, "read"); + assert_eq!(scope.kind, None); + assert_eq!( + scope.resource, + Some(vec![ + "path1".to_string(), + "path2".to_string(), + "f/folder/*".to_string() + ]) + ); + } + + #[test] + fn test_scope_action_hierarchy() { + assert!(ScopeAction::Write.includes(&ScopeAction::Read)); + assert!(!ScopeAction::Read.includes(&ScopeAction::Write)); + assert!(ScopeAction::Run.includes(&ScopeAction::Read)); + assert!(!ScopeAction::Run.includes(&ScopeAction::Write)); + } + + #[test] + fn test_route_domain_extraction() { + let (domain, kind, route_suffix) = + extract_domain_from_route("/api/w/test_workspace/jobs/123").unwrap(); + assert_eq!(domain, ScopeDomain::Jobs); + assert_eq!(kind, None); + assert_eq!(route_suffix, Some("jobs/123".to_string())); + + let (domain, kind, route_suffix) = + extract_domain_from_route("/api/w/test_workspace/scripts/test_script").unwrap(); + assert_eq!(domain, ScopeDomain::Scripts); + assert_eq!(kind, None); + assert_eq!(route_suffix, Some("scripts/test_script".to_string())); + } + + #[test] + fn test_specific_scope_access() { + let scopes = vec!["jobs:read".to_string()]; + + assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "GET").is_ok()); + + // DELETE now requires write permission, so it should still fail with read-only scope + assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "DELETE").is_err()); + } + + #[test] + fn test_new_domain_parsing() { + // Test that new domains are properly parsed + assert_eq!(ScopeDomain::from_str("acls"), Some(ScopeDomain::Acls)); + assert_eq!( + ScopeDomain::from_str("raw_apps"), + Some(ScopeDomain::RawApps) + ); + assert_eq!( + ScopeDomain::from_str("agent_workers"), + Some(ScopeDomain::AgentWorkers) + ); + + // Test that string conversion works both ways + assert_eq!(ScopeDomain::Acls.as_str(), "acls"); + assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); + assert_eq!(ScopeDomain::AgentWorkers.as_str(), "agent_workers"); + } + + #[test] + fn test_resource_array_matching() { + // Test wildcard access + let scope_all = ScopeDefinition::new("scripts", "read", None, Some(vec!["*".to_string()])); + let required = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(scope_all.includes(&required)); + + // Test exact matches + let scope_exact = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + let required_subset = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + assert!(scope_exact.includes(&required_subset)); + + // Test partial match - should grant access if ANY required resource matches + let scope_limited = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + let required_partial = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(scope_limited.includes(&required_partial)); // path1 matches, so access granted + + // Test no match - scope doesn't cover any of the required resources + let scope_different = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path3".to_string()])); + let required_no_match = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + assert!(!scope_different.includes(&required_no_match)); + + // Test pattern matching + let scope_pattern = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/*".to_string()]), + ); + let required_in_folder = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/script1".to_string()]), + ); + assert!(scope_pattern.includes(&required_in_folder)); + + // Test mixed patterns and exact matches + let scope_mixed = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["exact_path".to_string(), "f/folder/*".to_string()]), + ); + let required_mixed1 = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["exact_path".to_string()]), + ); + let required_mixed2 = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["f/folder/script2".to_string()]), + ); + assert!(scope_mixed.includes(&required_mixed1)); + assert!(scope_mixed.includes(&required_mixed2)); + } + + #[test] + fn test_efficiency_small_vs_large_arrays() { + // Test small array optimization path + let scope_small = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["path1".to_string(), "path2".to_string()]), + ); + let required_small = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); + assert!(scope_small.includes(&required_small)); + + // Test large array optimization path + let large_scope_vec: Vec = (0..10).map(|i| format!("path{}", i)).collect(); + let scope_large = ScopeDefinition::new("scripts", "read", None, Some(large_scope_vec)); + let required_large = + ScopeDefinition::new("scripts", "read", None, Some(vec!["path5".to_string()])); + assert!(scope_large.includes(&required_large)); + } + + #[test] + fn test_user_example_case() { + // User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"] + // Should grant access because scope "u/dieri/*" falls under required pattern "u/*" + let user_scope = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()])); + let required_mixed = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]), + ); + assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/* + + // Another example: scope covers one but not both paths + let scope_specific = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["folder/file1".to_string()]), + ); + let required_multi = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["folder/file1".to_string(), "other/file2".to_string()]), + ); + assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly + + // Test bidirectional pattern matching more explicitly + let scope_broad = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); + let required_specific = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dieri/script.py".to_string()]), + ); + assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py + + let scope_specific_path = ScopeDefinition::new( + "scripts", + "read", + None, + Some(vec!["u/dieri/script.py".to_string()]), + ); + let required_broad = + ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); + assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/* + } +} diff --git a/backend/windmill-api-auth/src/types.rs b/backend/windmill-api-auth/src/types.rs new file mode 100644 index 0000000000..1cafefaecb --- /dev/null +++ b/backend/windmill-api-auth/src/types.rs @@ -0,0 +1,125 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_audit::audit_oss::AuditAuthorable; +use windmill_common::db::{Authable, Authed, AuthedRef}; + +#[derive(Default, Clone, Debug)] +pub struct OptJobAuthed { + pub job_id: Option, + pub authed: ApiAuthed, +} + +#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] +pub struct ApiAuthed { + pub email: String, + pub username: String, + pub is_admin: bool, + pub is_operator: bool, + pub groups: Vec, + // (folder name, can write, is owner) + pub folders: Vec<(String, bool, bool)>, + pub scopes: Option>, + pub username_override: Option, + pub token_prefix: Option, +} + +impl ApiAuthed { + pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { + AuthedRef { + email: &self.email, + username: &self.username, + is_admin: &self.is_admin, + is_operator: &self.is_operator, + groups: &self.groups, + folders: &self.folders, + scopes: &self.scopes, + token_prefix: &self.token_prefix, + } + } + + pub fn display_username(&self) -> &str { + self.username_override.as_ref().unwrap_or(&self.username) + } +} + +impl From for Authed { + fn from(value: ApiAuthed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + token_prefix: value.token_prefix, + } + } +} + +impl From for ApiAuthed { + fn from(value: Authed) -> Self { + Self { + email: value.email, + username: value.username, + is_admin: value.is_admin, + is_operator: value.is_operator, + groups: value.groups, + folders: value.folders, + scopes: value.scopes, + username_override: None, // Authed doesn't have this field, so default to None + token_prefix: value.token_prefix, + } + } +} + +impl AuditAuthorable for ApiAuthed { + fn username(&self) -> &str { + self.username.as_str() + } + fn email(&self) -> &str { + self.email.as_str() + } + fn username_override(&self) -> Option<&str> { + self.username_override.as_deref() + } + fn token_prefix(&self) -> Option<&str> { + self.token_prefix.as_deref() + } +} + +impl Authable for ApiAuthed { + fn is_admin(&self) -> bool { + self.is_admin + } + + fn is_operator(&self) -> bool { + self.is_operator + } + + fn groups(&self) -> &[String] { + &self.groups + } + + fn folders(&self) -> &[(String, bool, bool)] { + &self.folders + } + + fn scopes(&self) -> Option<&[std::string::String]> { + self.scopes.as_ref().map(|x| x.as_slice()) + } + + fn email(&self) -> &str { + &self.email + } + + fn username(&self) -> &str { + &self.username + } +} diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index b33326a015..7f7d390117 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -10,41 +10,43 @@ path = "src/lib.rs" [features] default = [] -private = ["windmill-audit/private", "windmill-common/private"] -enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"] -stripe = [] +private = ["windmill-audit/private", "windmill-common/private", "windmill-triggers/private"] +enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise", "windmill-api-auth/enterprise", "windmill-triggers/enterprise"] +stripe = ["windmill-triggers/stripe"] agent_worker_server = [] enterprise_saml = ["dep:samael", "dep:libxml"] benchmark = [] embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn"] -parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet"] -prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet", "windmill-worker/parquet", "windmill-triggers/parquet"] +prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus", "windmill-triggers/prometheus"] +openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-triggers/openidconnect"] tantivy = ["dep:windmill-indexer"] -kafka = ["dep:rdkafka", "dep:rdkafka-sys"] -kafka-gssapi = ["kafka", "rdkafka/gssapi"] -nats = ["dep:async-nats", "dep:nkeys"] -websocket = ["dep:tokio-tungstenite"] -smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"] -license = ["dep:rsa"] +kafka = ["dep:rdkafka", "dep:rdkafka-sys", "windmill-triggers/kafka"] +kafka-gssapi = ["kafka", "rdkafka/gssapi", "windmill-triggers/kafka-gssapi"] +nats = ["dep:async-nats", "dep:nkeys", "windmill-triggers/nats"] +websocket = ["dep:tokio-tungstenite", "windmill-triggers/websocket"] +smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp", "windmill-triggers/smtp"] +license = ["dep:rsa", "windmill-triggers/license"] zip = ["dep:async_zip"] oauth2 = ["dep:windmill-oauth"] -http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq"] +http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq", "windmill-triggers/http_trigger"] static_frontend = ["dep:rust-embed"] -postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"] -mqtt_trigger = ["dep:thiserror", "dep:rumqttc"] -native_trigger = ["dep:strum", "dep:backon", "oauth2"] -sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config"] +postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls", "windmill-triggers/postgres_trigger"] +mqtt_trigger = ["dep:thiserror", "dep:rumqttc", "windmill-triggers/mqtt_trigger"] +native_trigger = ["dep:strum", "dep:backon", "oauth2", "windmill-triggers/native_trigger"] +sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config", "windmill-triggers/sqs_trigger"] deno_core = ["dep:deno_core", "dep:deno_error"] -gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] -cloud = ["windmill-common/cloud"] +gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic", "windmill-triggers/gcp_trigger"] +cloud = ["windmill-common/cloud", "windmill-triggers/cloud"] mcp = ["dep:windmill-mcp", "windmill-mcp/server", "windmill-mcp/auth"] bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "windmill-common/bedrock", "dep:aws-config"] python = [] -no_auth = [] +no_auth = ["windmill-api-auth/no_auth", "windmill-triggers/no_auth"] [dependencies] windmill-mcp = { workspace = true, optional = true } +windmill-api-auth.workspace = true +windmill-triggers.workspace = true windmill-queue.workspace = true windmill-common = { workspace = true, default-features = false } windmill-audit.workspace = true diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index 9383063919..ff8e3b5959 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -1,766 +1,8 @@ -#[cfg(feature = "enterprise")] -use crate::ee_oss::ExternalJwks; -use axum::{ - async_trait, - extract::{FromRequestParts, OriginalUri, Query}, - Extension, Json, +// Re-export all auth types from windmill-api-auth +pub use windmill_api_auth::{ + extract_token, invalidate_token_from_cache, list_tokens_internal, transform_old_scope_to_new_scope, + AuthCache, ExpiringAuthCache, JwtExtAuthBackend, NoopJwtExtAuth, + OptTokened, Tokened, TruncatedTokenWithEmail, + AUTH_CACHE, API_AUTHED_CACHE, + fetch_api_authed, fetch_api_authed_from_permissioned_as, }; -use chrono::TimeZone; -use http::{request::Parts, StatusCode}; -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use tower_cookies::Cookies; -use tracing::Span; - -use crate::db::{ApiAuthed, OptJobAuthed, DB}; -use std::{ - str::FromStr, - sync::{ - atomic::{AtomicI64, AtomicU64, Ordering}, - Arc, - }, -}; -#[cfg(feature = "enterprise")] -use tokio::sync::RwLock; - -use windmill_common::{ - auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, - error::{Error, JsonResult}, - jwt, - users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, -}; - -lazy_static::lazy_static! { - // Global auth cache accessible from main.rs for direct invalidation - pub static ref AUTH_CACHE: Cache<(String, String), ExpiringAuthCache> = Cache::new(300); - -} -// Global function to invalidate a specific token from cache -pub fn invalidate_token_from_cache(token: &str) { - // Remove all cache entries for this token (across all workspaces) - AUTH_CACHE.retain(|(_workspace_id, cached_token), _cached_value| cached_token != token); - tracing::info!( - "Invalidated token from auth cache: {}...", - &token[..token.len().min(8)] - ); -} - -#[derive(Clone)] -pub struct ExpiringAuthCache { - pub authed: ApiAuthed, - pub expiry: chrono::DateTime, - pub job_id: Option, -} - -pub struct AuthCache { - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] - ext_jwks: Option>>, -} - -impl AuthCache { - pub fn new( - db: DB, - superadmin_secret: Option, - #[cfg(feature = "enterprise")] ext_jwks: Option>>, - ) -> Self { - AuthCache { - db, - superadmin_secret, - #[cfg(feature = "enterprise")] - ext_jwks, - } - } - - pub async fn invalidate(&self, w_id: &str, token: String) { - AUTH_CACHE.remove(&(w_id.to_string(), token)); - } - - pub async fn get_authed(&self, w_id: Option, token: &str) -> Option { - Some(self.get_opt_job_authed(w_id, token).await?.authed) - } - - pub async fn get_opt_job_authed( - &self, - w_id: Option, - token: &str, - ) -> Option { - let key = ( - w_id.as_ref().unwrap_or(&"".to_string()).to_string(), - token.to_string(), - ); - let s = AUTH_CACHE.get(&key).map(|c| c.to_owned()); - match s { - Some(ExpiringAuthCache { authed, expiry, job_id }) if expiry > chrono::Utc::now() => { - Some(OptJobAuthed { authed, job_id }) - } - #[cfg(feature = "enterprise")] - _ if token.starts_with("jwt_ext_") => { - let authed_and_exp = match crate::ee_oss::jwt_ext_auth( - w_id.as_ref(), - token.trim_start_matches("jwt_ext_"), - self.ext_jwks.clone(), - &self.db, - ) - .await - { - Ok(r) => Some(r), - Err(e) => { - tracing::error!("JWT_EXT auth error: {:?}", e); - None - } - }; - - if let Some((authed, exp, job_id)) = authed_and_exp.clone() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc.timestamp_nanos(exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } else { - None - } - } - _ if token.starts_with("jwt_") => { - let jwt_token = token.trim_start_matches("jwt_"); - - let jwt_result = jwt::decode_with_internal_secret::(jwt_token).await; - - match jwt_result { - Ok(claims) => { - if w_id.is_some_and(|w_id| !claims.allowed_in_workspace(&w_id)) { - tracing::error!("JWT auth error: workspace_id mismatch"); - return None; - } - let username_override = username_override_from_label(claims.label); - - let authed = crate::db::ApiAuthed { - email: claims.email, - username: claims.username, - is_admin: claims.is_admin, - is_operator: claims.is_operator, - groups: claims.groups, - folders: claims.folders, - scopes: None, - username_override, - token_prefix: claims.audit_span, - }; - let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok()); - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc - .timestamp_nanos(claims.exp as i64 * 1_000_000_000), - job_id, - }, - ); - - Some(OptJobAuthed { authed, job_id }) - } - Err(err) => { - tracing::error!("JWT auth error: {:?}", err); - None - } - } - } - _ => { - let user_o = sqlx::query!( - "UPDATE token SET last_used_at = now() WHERE - token = $1 - AND (expiration > NOW() OR expiration IS NULL) - AND (workspace_id IS NULL OR workspace_id = $2) - RETURNING owner, email, super_admin, scopes, label", - token, - w_id.as_ref(), - ) - .map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label)) - .fetch_optional(&self.db) - .await - .ok() - .flatten(); - - if let Some(user) = user_o { - let authed_o = { - match user { - (Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => { - let username_override = username_override_from_label(label); - if let Some((prefix, name)) = owner.split_once('/') { - if prefix == "u" { - let (is_admin, is_operator) = if super_admin { - (true, false) - } else { - let r = sqlx::query!( - "SELECT is_admin, operator FROM usr where username = $1 AND \ - workspace_id = $2 AND disabled = false", - name, - &w_id.as_ref().unwrap() - ) - .fetch_one(&self.db) - .await - .ok(); - if let Some(r) = r { - (r.is_admin, r.operator) - } else { - (false, true) - } - }; - - let w_id = &w_id.unwrap(); - let groups = - get_groups_for_user(w_id, &name, &email, &self.db) - .await - .ok() - .unwrap_or_default(); - - let folders = - get_folders_for_user(w_id, &name, &groups, &self.db) - .await - .ok() - .unwrap_or_default(); - - Some(ApiAuthed { - email: email, - username: name.to_string(), - is_admin, - is_operator, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } else { - let groups = vec![name.to_string()]; - let folders = get_folders_for_user( - &w_id.unwrap(), - "", - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email: email, - username: format!("group-{name}"), - is_admin: false, - groups, - is_operator: false, - folders, - scopes: None, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - } else { - let groups = vec![]; - let folders = vec![]; - Some(ApiAuthed { - email: email, - username: owner, - is_admin: super_admin, - is_operator: true, - groups, - folders, - scopes: None, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - (_, Some(email), super_admin, scopes, label) => { - let username_override = username_override_from_label(label); - if w_id.is_some() { - let row_o = sqlx::query!( - "SELECT username, is_admin, operator FROM usr WHERE - email = $1 AND workspace_id = $2 AND disabled = false", - &email, - w_id.as_ref().unwrap() - ) - .map(|x| (x.username, x.is_admin, x.operator)) - .fetch_optional(&self.db) - .await - .unwrap_or(Some(("error".to_string(), false, false))); - - match row_o { - Some((username, is_admin, is_operator)) => { - let groups = get_groups_for_user( - &w_id.as_ref().unwrap(), - &username, - &email, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - - let folders = get_folders_for_user( - &w_id.unwrap(), - &username, - &groups, - &self.db, - ) - .await - .ok() - .unwrap_or_default(); - Some(ApiAuthed { - email, - username, - is_admin: is_admin || super_admin, - is_operator, - groups, - folders, - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }) - } - None if super_admin => Some(ApiAuthed { - email: email.clone(), - username: email, - is_admin: super_admin, - is_operator: false, - groups: vec![], - folders: vec![], - scopes, - username_override, - token_prefix: Some( - token[0..TOKEN_PREFIX_LEN].to_string(), - ), - }), - None => None, - } - } else { - Some(ApiAuthed { - email: email.to_string(), - username: email, - is_admin: super_admin, - is_operator: true, - groups: Vec::new(), - folders: Vec::new(), - scopes, - username_override, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }) - } - } - _ => None, - } - }; - if let Some(authed) = authed_o.as_ref() { - AUTH_CACHE.insert( - key, - ExpiringAuthCache { - authed: authed.clone(), - expiry: chrono::Utc::now() - + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - } - authed_o.map(|authed| OptJobAuthed { authed, job_id: None }) - } else if self - .superadmin_secret - .as_ref() - .map(|x| x == token) - .unwrap_or(false) - { - let authed = ApiAuthed { - email: SUPERADMIN_SECRET_EMAIL.to_string(), - username: "superadmin_secret".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: Some(token[0..TOKEN_PREFIX_LEN].to_string()), - }; - Some(OptJobAuthed { authed, job_id: None }) - } else { - None - } - } - } - } -} - -async fn extract_token(parts: &mut Parts, state: &S) -> Option { - let auth_header = parts - .headers - .get(http::header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|s| s.strip_prefix("Bearer ")); - - let from_cookie = match auth_header { - Some(x) => Some(x.to_owned()), - None => Extension::::from_request_parts(parts, state) - .await - .ok() - .and_then(|cookies| cookies.get(COOKIE_NAME).map(|c| c.value().to_owned())), - }; - - #[derive(Deserialize)] - struct Token { - token: Option, - } - match from_cookie { - Some(token) => Some(token), - None => Query::::from_request_parts(parts, state) - .await - .ok() - .and_then(|token| token.token.clone()), - } -} - -#[derive(Clone, Debug)] -pub struct Tokened { - pub token: String, -} - -#[derive(Clone, Debug)] -pub struct OptTokened { - #[allow(dead_code)] - pub token: Option, -} - -struct BruteForceCounter { - counter: AtomicU64, - last_reset: AtomicI64, -} - -lazy_static::lazy_static! { - static ref BRUTE_FORCE_COUNTER: BruteForceCounter = - BruteForceCounter { last_reset: AtomicI64::new(0), counter: AtomicU64::new(0) }; -} - -impl BruteForceCounter { - async fn increment(&self) { - let now = time::OffsetDateTime::now_utc().unix_timestamp(); - if self.counter.fetch_add(1, Ordering::Relaxed) > 10000 { - tracing::error!( - "Brute force attack to find valid token detected, sleeping unauthorized response for 2 seconds" - ); - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - if now - self.last_reset.load(Ordering::Relaxed) > 60 { - self.counter.store(0, Ordering::Relaxed); - self.last_reset.store(now, Ordering::Relaxed); - } - } -} - -#[async_trait] -impl FromRequestParts for Tokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(Tokened { token: "".to_string() }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(tokened.clone()) - } else { - let token_o = extract_token(parts, state).await; - if let Some(token) = token_o { - let tokened = Self { token }; - parts.extensions.insert(tokened.clone()); - Ok(tokened) - } else { - BRUTE_FORCE_COUNTER.increment().await; - Err((StatusCode::UNAUTHORIZED, "Unauthorized".to_owned())) - } - } - } -} - -#[async_trait] -impl FromRequestParts for OptTokened -where - S: Send + Sync, -{ - type Rejection = (StatusCode, String); - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptTokened { token: None }); - }; - let already_tokened = parts.extensions.get::(); - if let Some(tokened) = already_tokened { - Ok(OptTokened { token: Some(tokened.token.clone()) }) - } else { - let token_o = extract_token(parts, state).await; - Ok(OptTokened { token: token_o }) - } - } -} - -pub fn transform_old_scope_to_new_scope(scopes: Option<&mut Vec>) { - if let Some(scopes) = scopes { - for scope in scopes.iter_mut() { - if scope.starts_with("run:") { - let (_, part_scope) = scope.split_once(":").unwrap(); - - if let Some((kind, path)) = part_scope.split_once("/") { - //appending a 's' as runnable kind is singular while new scope format expect it to be plural - *scope = format!("jobs:run:{}s:{}", kind, path); - } - } else if scope.starts_with("jobs:") { - // Map old jobs scopes to new format - let new_scope = match scope.as_str() { - "jobs:listjobs" => "jobs:read", - "jobs:runscript" => "jobs:run:scripts", - "jobs:runflow" => "jobs:run:flows", - "jobs:resumeflow" => "jobs:run:flows", - "jobs:deletejob" => "jobs:write", - _ => continue, - }; - - *scope = new_scope.to_string(); - } - } - } -} - -fn maybe_get_workspace_id_from_path(path_vec: &[&str]) -> Option { - let workspace_id = if path_vec.len() >= 4 && path_vec[0] == "" && path_vec[2] == "w" { - Some(path_vec[3].to_owned()) - } else if path_vec.len() >= 5 - && path_vec[0] == "" - && path_vec[1] == "api" - && path_vec[2] == "mcp" - && path_vec[3] == "w" - { - Some(path_vec[4].to_owned()) - } else { - if path_vec.len() >= 5 && path_vec[0] == "" && path_vec[2] == "srch" && path_vec[3] == "w" { - Some(path_vec[4].to_owned()) - } else { - None - } - }; - - workspace_id -} - -#[async_trait] -impl FromRequestParts for ApiAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - let opt_job_authed = OptJobAuthed::from_request_parts(parts, state).await?; - Ok(opt_job_authed.authed) - } -} - -#[async_trait] -impl FromRequestParts for OptJobAuthed -where - S: Send + Sync, -{ - type Rejection = Error; - - async fn from_request_parts( - parts: &mut Parts, - state: &S, - ) -> std::result::Result { - if parts.method == http::Method::OPTIONS { - return Ok(OptJobAuthed::default()); - }; - - #[cfg(feature = "no_auth")] - { - let authed = ApiAuthed { - email: "admin@windmill.dev".to_string(), - username: "admin".to_string(), - is_admin: true, - is_operator: false, - groups: Vec::new(), - folders: Vec::new(), - scopes: None, - username_override: None, - token_prefix: None, - }; - return Ok(OptJobAuthed { authed, job_id: None }); - } - - let already_authed = parts.extensions.get::(); - - if let Some(authed) = already_authed { - return Ok(authed.clone()); - } - - let already_tokened = parts.extensions.get::(); - let token_o = if let Some(token) = already_tokened { - Some(token.token.clone()) - } else { - extract_token(parts, state).await - }; - if let Some(token) = token_o { - if let Ok(Extension(cache)) = - Extension::>::from_request_parts(parts, state).await - { - let original_uri = OriginalUri::from_request_parts(parts, state) - .await - .ok() - .map(|x| x.0) - .unwrap_or_default(); - let path_vec: Vec<&str> = original_uri.path().split("/").collect(); - let workspace_id = maybe_get_workspace_id_from_path(&path_vec); - - if let Some(mut opt_job_authed) = - cache.get_opt_job_authed(workspace_id.clone(), &token).await - { - let authed = &mut opt_job_authed.authed; - if authed.scopes.is_some() { - transform_old_scope_to_new_scope(authed.scopes.as_mut()); - - let path = original_uri.path(); - let method = parts.method.as_str(); - - if let Err(err) = crate::scopes::check_scopes_for_route( - authed.scopes.as_deref(), - path, - method, - ) { - BRUTE_FORCE_COUNTER.increment().await; - return Err(err); - } - } - parts.extensions.insert(authed.clone()); - - Span::current().record("username", &authed.username.as_str()); - Span::current().record("email", &authed.email); - - if let Some(workspace_id) = workspace_id { - Span::current().record("workspace_id", &workspace_id); - } - return Ok(opt_job_authed); - } - } - } - BRUTE_FORCE_COUNTER.increment().await; - Err(Error::NotAuthorized("Unauthorized".to_string())) - } -} - -fn username_override_from_label(label: Option) -> Option { - match label { - Some(label) - if label.starts_with("webhook-") - || label.starts_with("http-") - || label.starts_with("email-") - || label.starts_with("ws-") => - { - Some(label) - } - Some(label) if label.starts_with("ephemeral-script-end-user-") => Some( - label - .trim_start_matches("ephemeral-script-end-user-") - .to_string(), - ), - Some(label) if label == "Ephemeral lsp token" => Some("lsp".to_string()), - Some(label) if label != "ephemeral-script" && label != "session" && !label.is_empty() => { - Some(format!("label-{label}")) - } - _ => None, - } -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedTokenWithEmail { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, - pub scopes: Option>, - pub email: Option, -} - -pub async fn list_tokens_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult> { - let tokens = if is_flow { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:flows:' || $2]::text[] - OR scopes @> ARRAY['run:flow/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - } else { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] - OR scopes @> ARRAY['run:script/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - }; - - Ok(Json(tokens)) -} diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index 9157c3cb01..5368f76d38 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -14,13 +14,9 @@ use sqlx::{ }; use tokio::task::JoinHandle; -use windmill_audit::audit_oss::AuditAuthorable; +pub use windmill_api_auth::{ApiAuthed, OptJobAuthed}; pub use windmill_common::db::DB; -use windmill_common::{ - db::{Authable, Authed, AuthedRef}, - error::Error, - utils::generate_lock_id, -}; +use windmill_common::{error::Error, utils::generate_lock_id}; async fn current_database(conn: &mut PgConnection) -> Result { // language=SQL @@ -292,119 +288,3 @@ pub async fn migrate( Ok(None) } -#[derive(Default, Clone, Debug)] -pub struct OptJobAuthed { - pub job_id: Option, - pub authed: ApiAuthed, -} - -#[derive(Clone, Debug, Default, Hash, Eq, PartialEq)] -pub struct ApiAuthed { - pub email: String, - pub username: String, - pub is_admin: bool, - pub is_operator: bool, - pub groups: Vec, - // (folder name, can write, is owner) - pub folders: Vec<(String, bool, bool)>, - pub scopes: Option>, - pub username_override: Option, - pub token_prefix: Option, -} - -impl ApiAuthed { - pub fn to_authed_ref<'e>(&'e self) -> AuthedRef<'e> { - AuthedRef { - email: &self.email, - username: &self.username, - is_admin: &self.is_admin, - is_operator: &self.is_operator, - groups: &self.groups, - folders: &self.folders, - scopes: &self.scopes, - token_prefix: &self.token_prefix, - } - } -} - -impl From for Authed { - fn from(value: ApiAuthed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - token_prefix: value.token_prefix, - } - } -} - -impl From for ApiAuthed { - fn from(value: Authed) -> Self { - Self { - email: value.email, - username: value.username, - is_admin: value.is_admin, - is_operator: value.is_operator, - groups: value.groups, - folders: value.folders, - scopes: value.scopes, - username_override: None, // Authed doesn't have this field, so default to None - token_prefix: value.token_prefix, - } - } -} - -impl ApiAuthed { - pub fn display_username(&self) -> &str { - self.username_override.as_ref().unwrap_or(&self.username) - } -} - -impl AuditAuthorable for ApiAuthed { - fn username(&self) -> &str { - self.username.as_str() - } - fn email(&self) -> &str { - self.email.as_str() - } - fn username_override(&self) -> Option<&str> { - self.username_override.as_deref() - } - fn token_prefix(&self) -> Option<&str> { - self.token_prefix.as_deref() - } -} - -impl Authable for ApiAuthed { - fn is_admin(&self) -> bool { - self.is_admin - } - - fn is_operator(&self) -> bool { - self.is_operator - } - - fn groups(&self) -> &[String] { - &self.groups - } - - fn folders(&self) -> &[(String, bool, bool)] { - &self.folders - } - - fn scopes(&self) -> Option<&[std::string::String]> { - self.scopes.as_ref().map(|x| x.as_slice()) - } - - fn email(&self) -> &str { - &self.email - } - - fn username(&self) -> &str { - &self.username - } -} diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index 6e54b45dc5..e0a004563d 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -44,6 +44,30 @@ impl ExternalJwks { } } +#[cfg(all(feature = "enterprise", not(feature = "private")))] +pub struct ExternalJwksAuthBackend { + pub ext_jwks: Option>>, +} + +#[cfg(all(feature = "enterprise", not(feature = "private")))] +#[axum::async_trait] +impl windmill_api_auth::JwtExtAuthBackend for ExternalJwksAuthBackend { + async fn jwt_ext_auth( + &self, + w_id: Option<&String>, + token: &str, + db: &windmill_common::DB, + ) -> Option<(crate::db::ApiAuthed, usize, Option)> { + match jwt_ext_auth(w_id, token, self.ext_jwks.clone(), db).await { + Ok(r) => Some(r), + Err(e) => { + tracing::error!("JWT_EXT auth error: {:?}", e); + None + } + } + } +} + #[cfg(all( feature = "enterprise", any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), diff --git a/backend/windmill-api/src/job_ops_impl.rs b/backend/windmill-api/src/job_ops_impl.rs new file mode 100644 index 0000000000..3979a59229 --- /dev/null +++ b/backend/windmill-api/src/job_ops_impl.rs @@ -0,0 +1,300 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Implementation of `windmill_triggers::jobs_ext::JobOps` trait. +//! This bridges windmill-triggers back to windmill-api internals. + +use axum::response::Response; +use serde_json::value::RawValue; +use sqlx::Postgres; +#[cfg(feature = "parquet")] +use std::sync::Arc; +use uuid::Uuid; +use windmill_api_auth::{ApiAuthed, OptTokened}; +use windmill_common::{ + db::UserDB, + error, + triggers::TriggerMetadata, + utils::StripPath, + DB, +}; +use windmill_queue::PushArgsOwned; +use windmill_triggers::jobs_ext::{JobOps, JobUpdateSSEStream, RunJobQuery}; + +pub struct JobOpsImpl; + +#[axum::async_trait] +impl JobOps for JobOpsImpl { + async fn push_script_job_by_path_into_queue( + &self, + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, + ) -> error::Result<( + Uuid, + Option, + Option>, + )> { + // Convert RunJobQuery from windmill-triggers to the windmill-api version + let api_run_query = crate::jobs::RunJobQuery { + scheduled_for: run_query.scheduled_for, + scheduled_in_secs: run_query.scheduled_in_secs, + parent_job: run_query.parent_job, + root_job: run_query.root_job, + invisible_to_owner: run_query.invisible_to_owner, + queue_limit: run_query.queue_limit, + payload: run_query.payload, + job_id: run_query.job_id, + tag: run_query.tag, + timeout: run_query.timeout, + cache_ttl: run_query.cache_ttl, + cache_ignore_s3_path: run_query.cache_ignore_s3_path, + skip_preprocessor: run_query.skip_preprocessor, + poll_delay_ms: run_query.poll_delay_ms, + memory_id: run_query.memory_id, + trigger_external_id: run_query.trigger_external_id, + service_name: run_query.service_name, + suspended_mode: run_query.suspended_mode, + }; + crate::jobs::push_script_job_by_path_into_queue( + authed, db, tx_o, user_db, w_id, script_path, api_run_query, args, trigger, + ) + .await + } + + async fn push_flow_job_by_path_into_queue( + &self, + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, + ) -> error::Result<( + Uuid, + Option, + Option>, + )> { + let api_run_query = crate::jobs::RunJobQuery { + scheduled_for: run_query.scheduled_for, + scheduled_in_secs: run_query.scheduled_in_secs, + parent_job: run_query.parent_job, + root_job: run_query.root_job, + invisible_to_owner: run_query.invisible_to_owner, + queue_limit: run_query.queue_limit, + payload: run_query.payload, + job_id: run_query.job_id, + tag: run_query.tag, + timeout: run_query.timeout, + cache_ttl: run_query.cache_ttl, + cache_ignore_s3_path: run_query.cache_ignore_s3_path, + skip_preprocessor: run_query.skip_preprocessor, + poll_delay_ms: run_query.poll_delay_ms, + memory_id: run_query.memory_id, + trigger_external_id: run_query.trigger_external_id, + service_name: run_query.service_name, + suspended_mode: run_query.suspended_mode, + }; + crate::jobs::push_flow_job_by_path_into_queue( + authed, db, tx_o, user_db, w_id, flow_path, api_run_query, args, trigger, + ) + .await + } + + async fn run_wait_result_internal( + &self, + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, + ) -> error::Result<(Box, bool)> { + crate::jobs::run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username) + .await + } + + fn result_to_response( + &self, + result: Box, + success: bool, + ) -> error::Result { + crate::jobs::result_to_response(result, success) + } + + async fn cancel_jobs( + &self, + jobs: Vec, + db: &DB, + username: &str, + w_id: &str, + force_cancel: bool, + ) -> error::JsonResult> { + crate::jobs::cancel_jobs(jobs, db, username, w_id, force_cancel).await + } + + fn start_job_update_sse_stream( + &self, + opt_authed: Option, + opt_tokened: OptTokened, + db: DB, + w_id: String, + job_id: Uuid, + initial_log_offset: Option, + initial_stream_offset: Option, + get_progress: Option, + running: Option, + only_result: Option, + fast: Option, + no_logs: Option, + is_flow: Option, + tx: tokio::sync::mpsc::Sender, + poll_delay_ms: Option, + ) { + // Convert between windmill-api's SSE type and windmill-triggers' SSE type + let (bridge_tx, mut bridge_rx) = + tokio::sync::mpsc::channel::(32); + + crate::jobs::start_job_update_sse_stream( + opt_authed, + opt_tokened, + db, + w_id, + job_id, + initial_log_offset, + initial_stream_offset, + get_progress, + running, + only_result, + fast, + no_logs, + is_flow, + bridge_tx, + poll_delay_ms, + ); + + // Spawn a task to convert between the two SSE stream types + tokio::spawn(async move { + while let Some(msg) = bridge_rx.recv().await { + let converted = match msg { + crate::jobs::JobUpdateSSEStream::Update(update) => { + JobUpdateSSEStream::Update( + serde_json::to_value(&update).unwrap_or_default(), + ) + } + crate::jobs::JobUpdateSSEStream::Error { error } => { + JobUpdateSSEStream::Error { error } + } + crate::jobs::JobUpdateSSEStream::NotFound => JobUpdateSSEStream::NotFound, + crate::jobs::JobUpdateSSEStream::Timeout => JobUpdateSSEStream::Timeout, + crate::jobs::JobUpdateSSEStream::Ping => JobUpdateSSEStream::Ping, + }; + if tx.send(converted).await.is_err() { + break; + } + } + }); + } + + async fn try_get_resource_from_db( + &self, + authed: &ApiAuthed, + user_db: Option, + db: &DB, + resource_path: &str, + w_id: &str, + ) -> error::Result { + use windmill_common::db::DbWithOptAuthed; + + let db_with_authed = DbWithOptAuthed::from_authed(authed, db.clone(), user_db); + let resource = crate::resources::get_resource_value_interpolated_internal( + &db_with_authed, + w_id, + resource_path, + None, + None, + false, + ) + .await?; + + match resource { + Some(v) => Ok(v), + None => Err(error::Error::NotFound(format!( + "resource at path :{} does not exist", + resource_path + ))), + } + } + + async fn interpolate( + &self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + s: String, + ) -> Result { + #[cfg(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), + ))] + { + crate::ee_oss::interpolate(authed, db, w_id, s).await + } + #[cfg(not(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), + )))] + { + let _ = (authed, db, w_id); + Ok(s) + } + } + + #[cfg(feature = "parquet")] + fn get_random_file_name(&self, ext: Option) -> String { + crate::job_helpers_oss::get_random_file_name(ext) + } + + #[cfg(feature = "parquet")] + async fn upload_file_internal( + &self, + s3_client: Arc, + file_key: &str, + bytes_stream: std::pin::Pin< + Box> + Send>, + >, + options: object_store::PutMultipartOpts, + ) -> error::Result<()> { + crate::job_helpers_oss::upload_file_internal(s3_client, file_key, bytes_stream, options) + .await + .map(|_| ()) + } + + #[cfg(feature = "parquet")] + async fn get_workspace_s3_resource( + &self, + authed: &ApiAuthed, + db: &DB, + user_db: Option, + w_id: &str, + storage: Option, + ) -> error::Result<( + Option, + Option, + )> { + crate::job_helpers_oss::get_workspace_s3_resource(authed, db, user_db, w_id, storage).await + } +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index a78b6fa2a3..9fe5297045 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -132,7 +132,13 @@ pub mod job_helpers_ee; #[cfg(feature = "parquet")] mod job_helpers_oss; pub mod job_metrics; +pub mod job_ops_impl; pub mod jobs; + +// Re-export windmill-triggers extension modules so EE files that use +// `crate::jobs_ext`, `crate::resource_ext` etc. work in both crates. +pub use windmill_triggers::jobs_ext; +pub use windmill_triggers::resource_ext; pub mod jobs_export; #[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; @@ -299,19 +305,30 @@ pub async fn run_server( .expect("could not create initial server dir"); } - #[cfg(feature = "enterprise")] - let ext_jwks = ExternalJwks::load().await; + let jwt_ext_auth: Arc = { + #[cfg(feature = "enterprise")] + { + let ext_jwks = ExternalJwks::load().await; + Arc::new(crate::ee_oss::ExternalJwksAuthBackend { ext_jwks }) + } + #[cfg(not(feature = "enterprise"))] + { + Arc::new(windmill_api_auth::NoopJwtExtAuth) + } + }; let auth_cache = Arc::new(crate::auth::AuthCache::new( db.clone(), std::env::var("SUPERADMIN_SECRET").ok(), - #[cfg(feature = "enterprise")] - ext_jwks, + jwt_ext_auth, )); let argon2 = Arc::new(Argon2::default()); // Initialize debug signing key for debugger authentication debug::init_debug_signing_key().await; + // Initialize windmill-triggers JobOps with the real implementation + windmill_triggers::jobs_ext::set_ops(Arc::new(job_ops_impl::JobOpsImpl)); + let disable_response_logs = std::env::var("DISABLE_RESPONSE_LOGS") .ok() .map(|x| x == "true") diff --git a/backend/windmill-api/src/scopes.rs b/backend/windmill-api/src/scopes.rs index 1c1bdd4f26..a852fefcb2 100644 --- a/backend/windmill-api/src/scopes.rs +++ b/backend/windmill-api/src/scopes.rs @@ -1,934 +1,3 @@ -/* - * Author: Windmill Labs, Inc - * Copyright: Windmill Labs, Inc 2024 - * This file and its contents are licensed under the AGPLv3 License. - * Please see the included NOTICE for copyright information and - * LICENSE-AGPL for a copy of the license. - */ - -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use std::collections::HashSet; -use windmill_common::error::{Error, Result}; - -/// Comprehensive scope system for JWT token authorization -/// -/// Scopes follow the format: {domain}:{action}[:{resource}] -/// Examples: -/// - "jobs:read" - Read access to jobs -/// - "scripts:write:f/folder/*" - Write access to scripts in a folder -/// - "*" - Full access (superuser) - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ScopeDefinition { - pub domain: String, - pub action: String, - pub kind: Option, // For jobs:run:kind (optional) - pub resource: Option>, -} - -impl ScopeDefinition { - pub fn new( - domain: &str, - action: &str, - kind: Option<&str>, - resource: Option>, - ) -> Self { - Self { - domain: domain.to_string(), - action: action.to_string(), - kind: kind.map(|s| s.to_string()), - resource: resource, - } - } - - pub fn from_scope_string(scope: &str) -> Result { - let parts: Vec<&str> = scope.split(':').collect(); - - let into_owned_vec = |resources: &str| -> Vec { - let resources = resources - .split(",") - .collect_vec() - .into_iter() - .map(ToOwned::to_owned) - .collect_vec(); - - resources - }; - - match parts.len() { - 2 => Ok(Self::new(parts[0], parts[1], None, None)), // domain:action - 3 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new(parts[0], parts[1], Some(parts[2]), None)) - } else { - Ok(Self::new( - parts[0], - parts[1], - None, - Some(into_owned_vec(parts[2])), - )) - } - } - 4 => { - if parts[0] == "jobs" && parts[1] == "run" { - Ok(Self::new( - parts[0], - parts[1], - Some(parts[2]), - Some(into_owned_vec(parts[3])), - )) - } else { - Err(Error::BadRequest(format!( - "Invalid 4-part scope: {}", - scope - ))) - } - } - _ => Err(Error::BadRequest(format!( - "Invalid scope format: {}", - scope - ))), - } - } - - pub fn as_string(&self) -> String { - match (&self.kind, &self.resource) { - (Some(kind), Some(resource)) => { - format!( - "{}:{}:{}:{}", - self.domain, - self.action, - kind, - resource.join(",") - ) - } - (Some(kind), None) => { - format!("{}:{}:{}", self.domain, self.action, kind) - } - (None, Some(resource)) => { - format!("{}:{}:{}", self.domain, self.action, resource.join(",")) - } - (None, None) => format!("{}:{}", self.domain, self.action), - } - } - - pub fn includes(&self, other: &ScopeDefinition) -> bool { - if self.domain != other.domain { - return false; - } - - match (self.action.as_str(), other.action.as_str()) { - (a, b) if (a == "write" && b == "read") || (a == b) => {} - _ => return false, - } - - if self.domain == "jobs" && self.action == "run" { - match (&self.kind, &other.kind) { - (Some(self_kind), Some(other_kind)) => { - if self_kind != other_kind { - return false; - } - } - (Some(_), None) => { - return false; - } - (None, _) => { - return true; - } - } - } - - match (&self.resource, &other.resource) { - (Some(self_resources), Some(other_resources)) => { - resources_match(self_resources, other_resources) - } - (Some(_), None) => false, - (None, _) => true, - } - } -} - -fn resources_match(scope_resources: &[String], accepted_resources: &[String]) -> bool { - if scope_resources.contains(&"*".to_string()) || accepted_resources.contains(&"*".to_string()) { - return true; - } - - if scope_resources.len() <= 4 && accepted_resources.len() <= 4 { - return resources_match_small(scope_resources, accepted_resources); - } - - resources_match_large(scope_resources, accepted_resources) -} - -fn resources_match_small(scope_resources: &[String], accepted_resources: &[String]) -> bool { - for required in accepted_resources { - for scope_resource in scope_resources { - if resource_matches_pattern(scope_resource, required) { - return true; - } - } - } - false -} - -fn resources_match_large(scope_resources: &[String], accepted_resources: &[String]) -> bool { - let mut exact_matches = HashSet::new(); - let mut patterns = Vec::new(); - - for scope_resource in scope_resources { - if scope_resource.contains('*') { - patterns.push(scope_resource); - } else { - exact_matches.insert(scope_resource); - } - } - - for accepted_resource in accepted_resources { - if exact_matches.contains(accepted_resource) { - return true; - } - - for pattern in &patterns { - if resource_matches_pattern(pattern, accepted_resource) { - return true; - } - } - } - - false -} - -fn resource_matches_pattern(scope_resource: &str, accepted_resource: &str) -> bool { - if scope_resource == accepted_resource { - return true; - } - - let matches_wildcard = |pattern: &str, resource: &str| -> bool { - if !pattern.ends_with("/*") { - return false; - } - - let prefix = &pattern[..pattern.len() - 2]; - - if !resource.starts_with(prefix) { - return false; - } - - // If the resource is exactly the prefix, it matches - if resource.len() == prefix.len() { - return true; - } - - // If the resource is longer, the next character must be '/' for a valid match - // This prevents "u/user" from matching "u/use/*" - resource.chars().nth(prefix.len()) == Some('/') - }; - - // Check if either resource is a wildcard pattern and matches the other - matches_wildcard(scope_resource, accepted_resource) - || matches_wildcard(accepted_resource, scope_resource) -} - -/// Available scope domains (top-level API categories) -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeDomain { - // Core resource domains - Jobs, - Scripts, - Flows, - Apps, - Variables, - Resources, - Schedules, - Folders, - Users, - Groups, - Workspaces, - - // Trigger domains - HttpTriggers, - WebsocketTriggers, - KafkaTriggers, - NatsTriggers, - MqttTriggers, - SqsTriggers, - GcpTriggers, - PostgresTriggers, - EmailTriggers, - - // Native trigger domains - NativeTriggers, - - // System domains - Audit, - Settings, - Workers, - ServiceLogs, - Configs, - OAuth, - AI, - - Indexer, - Teams, // Microsoft Teams integration - GitSync, // Git synchronization - - // Special domains - Capture, // Webhook capture - Drafts, // Draft resources - Favorites, // User favorites - Inputs, // Input templates - JobHelpers, // Job helper functions - ConcurrencyGroups, // Concurrency groups - Oidc, // OpenID Connect - Openapi, // OpenAPI generation - - // Additional domains - Acls, // Granular access control lists - RawApps, // Raw application data - AgentWorkers, // Agent workers management - Mcp, // MCP -} - -impl ScopeDomain { - pub fn as_str(&self) -> &'static str { - match self { - Self::Jobs => "jobs", - Self::Scripts => "scripts", - Self::Flows => "flows", - Self::Apps => "apps", - Self::Variables => "variables", - Self::Resources => "resources", - Self::Schedules => "schedules", - Self::Folders => "folders", - Self::Users => "users", - Self::Groups => "groups", - Self::Workspaces => "workspaces", - Self::HttpTriggers => "http_triggers", - Self::WebsocketTriggers => "websocket_triggers", - Self::KafkaTriggers => "kafka_triggers", - Self::NatsTriggers => "nats_triggers", - Self::MqttTriggers => "mqtt_triggers", - Self::SqsTriggers => "sqs_triggers", - Self::GcpTriggers => "gcp_triggers", - Self::PostgresTriggers => "postgres_triggers", - Self::EmailTriggers => "email_triggers", - Self::NativeTriggers => "native_triggers", - Self::Audit => "audit", - Self::Settings => "settings", - Self::Workers => "workers", - Self::ServiceLogs => "service_logs", - Self::Configs => "configs", - Self::OAuth => "oauth", - Self::AI => "ai", - Self::Capture => "capture", - Self::Drafts => "drafts", - Self::Favorites => "favorites", - Self::Inputs => "inputs", - Self::JobHelpers => "job_helpers", - Self::ConcurrencyGroups => "concurrency_groups", - Self::Oidc => "oidc", - Self::Openapi => "openapi", - Self::Acls => "acls", - Self::RawApps => "raw_apps", - Self::AgentWorkers => "agent_workers", - Self::Indexer => "indexer", - Self::Teams => "teams", - Self::GitSync => "git_sync", - Self::Mcp => "mcp", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "jobs" | "jobs_u" => Some(Self::Jobs), - "scripts" => Some(Self::Scripts), - "flows" => Some(Self::Flows), - "apps" | "apps_u" => Some(Self::Apps), - "variables" => Some(Self::Variables), - "resources" => Some(Self::Resources), - "schedules" => Some(Self::Schedules), - "folders" => Some(Self::Folders), - "users" => Some(Self::Users), - "groups" => Some(Self::Groups), - "workspaces" => Some(Self::Workspaces), - "http_triggers" => Some(Self::HttpTriggers), - "websocket_triggers" => Some(Self::WebsocketTriggers), - "kafka_triggers" => Some(Self::KafkaTriggers), - "nats_triggers" => Some(Self::NatsTriggers), - "mqtt_triggers" => Some(Self::MqttTriggers), - "sqs_triggers" => Some(Self::SqsTriggers), - "gcp_triggers" => Some(Self::GcpTriggers), - "postgres_triggers" => Some(Self::PostgresTriggers), - "email_triggers" => Some(Self::EmailTriggers), - "audit" => Some(Self::Audit), - "settings" => Some(Self::Settings), - "workers" => Some(Self::Workers), - "service_logs" => Some(Self::ServiceLogs), - "configs" => Some(Self::Configs), - "oauth" => Some(Self::OAuth), - "ai" => Some(Self::AI), - "indexer" | "srch" => Some(Self::Indexer), - "teams" => Some(Self::Teams), - "native_triggers" => Some(Self::NativeTriggers), - "git_sync" | "github_app" => Some(Self::GitSync), - "capture" => Some(Self::Capture), - "drafts" => Some(Self::Drafts), - "favorites" => Some(Self::Favorites), - "inputs" => Some(Self::Inputs), - "job_helpers" => Some(Self::JobHelpers), - "concurrency_groups" => Some(Self::ConcurrencyGroups), - "oidc" => Some(Self::Oidc), - "openapi" => Some(Self::Openapi), - "acls" => Some(Self::Acls), - "raw_apps" => Some(Self::RawApps), - "agent_workers" => Some(Self::AgentWorkers), - "mcp" => Some(Self::Mcp), - _ => None, - } - } -} - -/// Available scope actions -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ScopeAction { - Read, // GET operations, list, view - Write, // POST, PUT, PATCH, DELETE operations, create, update, delete - Run, // Special action for running (scripts, flows, etc.) -} - -impl ScopeAction { - pub fn as_str(&self) -> &'static str { - match self { - Self::Read => "read", - Self::Write => "write", - Self::Run => "run", - } - } - - pub fn from_str(s: &str) -> Option { - match s { - "read" => Some(Self::Read), - "write" => Some(Self::Write), - "delete" => Some(Self::Write), - "run" => Some(Self::Run), - _ => None, - } - } - - /// Check if this action includes another action - /// Write includes Read - pub fn includes(&self, other: &ScopeAction) -> bool { - match (self, other) { - (ScopeAction::Write, ScopeAction::Read) => true, - (ScopeAction::Run, ScopeAction::Read) => true, - (a, b) => a == b, - } - } -} - -pub fn check_route_access( - token_scopes: &[String], - route_path: &str, - http_method: &str, -) -> Result<()> { - // Map HTTP method to scope action (considering route context) - let required_action = map_http_method_to_action(http_method, route_path); - - // Find the domain and kind for this route - let (required_domain, required_kind, route_suffix) = extract_domain_from_route(route_path)?; - - // Backward compatibility: MCP handlers expect unusual scope actions: all, favorites, hub. - if required_domain == ScopeDomain::Mcp { - return Ok(()); - } - - // tracing::error!("Checking route access {:?} {:?} {:?} {:?}", required_action, required_domain, required_kind, route_suffix); - let mut is_scoped_token = false; - // Check if any token scope grants the required access - for scope_str in token_scopes { - if !scope_str.starts_with("if_jobs:filter_tags:") { - if let Ok(scope) = ScopeDefinition::from_scope_string(scope_str) { - // tracing::error!("Checking scope {:?} for required domain {:?} and action {:?} and kind {:?} and route suffix {:?}", scope, required_domain, required_action, required_kind, route_suffix); - if scope_grants_access( - &scope, - required_domain, - required_action, - required_kind.as_deref(), - route_suffix.as_deref(), - )? { - // tracing::error!("Scope grants access: {:?}", scope); - return Ok(()); - } - } - if !is_scoped_token { - is_scoped_token = true; - } - } - } - - //Edge case for backward compatibility, if only scopes defined was filter tag then don't treat this we don't treat the token - //as a restricted token - if !is_scoped_token { - return Ok(()); - } - let scope_display = if let Some(kind) = required_kind { - format!( - "{}:{}:{}", - required_domain.as_str(), - required_action.as_str(), - kind - ) - } else { - format!("{}:{}", required_domain.as_str(), required_action.as_str()) - }; - - Err(Error::NotAuthorized(format!( - "Access denied. Required scope: {}", - scope_display - ))) -} - -const SCRIPT_JOBS: [&'static str; 8] = [ - "jobs/run/p", - "jobs/run/h", - "jobs/run_wait_result/p", - "jobs/run_wait_result/h", - "jobs/run/preview_bundle", - "jobs/run/preview", - "jobs/run_and_stream/p", - "jobs/run_and_stream/h", -]; - -const FLOW_JOBS: [&'static str; 6] = [ - "jobs/run/f", - "jobs/run_wait_result/f", - "jobs/run/preview_flow", - "jobs/restart/f", - "jobs/flow/resume", - "jobs/run_and_stream/f", -]; - -lazy_static::lazy_static! { - static ref RUN_PATH_ACTIONS: Vec<&'static str> = { - let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component"]; - - v.extend(SCRIPT_JOBS); - v.extend(FLOW_JOBS); - v - }; -} - -fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { - if RUN_PATH_ACTIONS - .iter() - .any(|run_path| route_path.contains(run_path)) - { - return ScopeAction::Run; - } - - match method.to_uppercase().as_str() { - "GET" | "HEAD" | "OPTIONS" => ScopeAction::Read, - "POST" | "PUT" | "PATCH" | "DELETE" => ScopeAction::Write, - _ => ScopeAction::Read, - } -} - -/// Checks the route path to determine the runnable kind (either "flows" or "scripts"). -/// -/// The order of checks is important: -/// - Flow-related paths are checked first to avoid false positives, as some flow paths -/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). -/// -/// Returns `"flows"` or `"scripts"` based on the match, or `None` if no match is found. -fn determine_kind_from_route(route_path: &str) -> Option { - if route_path.starts_with("jobs") { - if FLOW_JOBS.iter().any(|path| route_path.starts_with(path)) { - return Some("flows".to_string()); - } else if SCRIPT_JOBS.iter().any(|path| route_path.starts_with(path)) { - return Some("scripts".to_string()); - } - } - None -} - -fn extract_domain_from_route( - route_path: &str, -) -> Result<(ScopeDomain, Option, Option)> { - // Examples: - // - /api/w/workspace/jobs/123 -> jobs domain (workspaced) - // - /api/teams/sync -> teams domain (global) - // - /api/srch/index/search -> indexer domain (global) - let parts: Vec<&str> = route_path.split('/').collect(); - - let (domain, kind, route_suffix) = if parts.len() >= 5 && parts[1] == "api" && parts[2] == "w" { - let domain_part = parts[4]; - let route_suffix = &parts[4..].join("/"); - - let domain = ScopeDomain::from_str(domain_part); - - let kind = determine_kind_from_route(&route_suffix); - - (domain, kind, Some(route_suffix.to_owned())) - } else if parts.len() >= 3 && parts[1] == "api" { - ( - ScopeDomain::from_str(parts[2]), - None, - Some(parts[2..].join("/")), - ) - } else { - (None, None, None) - }; - - if let Some(domain) = domain { - // tracing::error!("Extracted domain {:?} from route {:?} with kind {:?} and route suffix {:?}", domain, route_path, kind, route_suffix); - return Ok((domain, kind, route_suffix)); - } - - Err(Error::BadRequest(format!( - "Could not extract domain from route: {}", - route_path - ))) -} - -const RUN_WHITELISTED_GET_PATHS: [&'static str; 19] = [ - "jobs_u/get_flow/", - "jobs_u/get_root_job_id/", - "jobs_u/get/", - "jobs_u/get_logs/", - "jobs_u/get_args/", - "jobs_u/get_flow_debug_info/", - "jobs_u/completed/get/", - "jobs_u/completed/get_result/", - "jobs_u/completed/get_result_maybe/", - "jobs_u/getupdate/", - "jobs_u/getupdate_sse/", - "jobs_u/get_log_file/", - "jobs/result_by_id/", - "jobs/resume_urls/", - "jobs/flow/user_states/", - "jobs/job_signature/", - "jobs/completed/get/", - "jobs/completed/get_result/", - "jobs/completed/get_result_maybe/", -]; - -fn scope_grants_access( - scope: &ScopeDefinition, - required_domain: ScopeDomain, - required_action: ScopeAction, - required_kind: Option<&str>, - route_path: Option<&str>, -) -> Result { - // Check domain match - let scope_domain = ScopeDomain::from_str(&scope.domain) - .ok_or_else(|| Error::BadRequest(format!("Invalid scope domain: {}", scope.domain)))?; - - if scope_domain != required_domain { - return Ok(false); - } - - // Check action match (with hierarchical permissions) - let scope_action = ScopeAction::from_str(&scope.action) - .ok_or_else(|| Error::BadRequest(format!("Invalid scope action: {}", scope.action)))?; - - if !scope_action.includes(&required_action) - && !(scope_domain == ScopeDomain::Jobs - && required_action == ScopeAction::Read - && route_path.is_some_and(|p| { - RUN_WHITELISTED_GET_PATHS - .iter() - .any(|path| p.starts_with(path)) - })) - { - return Ok(false); - } - - if scope_domain == ScopeDomain::Jobs && required_action == ScopeAction::Run { - match (&scope.kind, required_kind) { - (Some(scope_kind), Some(req_kind)) => { - if scope_kind != req_kind { - return Ok(false); - } - } - (None, _) => {} - (Some(_), None) => { - return Ok(false); - } - } - } - - // No resource specified means access to entire domain - Ok(true) -} - -/// Helper function to check if scopes allow access to a route -pub fn check_scopes_for_route( - token_scopes: Option<&[String]>, - route_path: &str, - http_method: &str, -) -> Result<()> { - // If no scopes defined, allow access (backward compatibility) - let scopes = match token_scopes { - Some(s) if !s.is_empty() => s, - _ => return Ok(()), - }; - - check_route_access(scopes, route_path, http_method) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_scope_definition_parsing() { - let scope = ScopeDefinition::from_scope_string("jobs:read").unwrap(); - assert_eq!(scope.domain, "jobs"); - assert_eq!(scope.action, "read"); - assert_eq!(scope.kind, None); - assert_eq!(scope.resource, None); - - let scope = ScopeDefinition::from_scope_string("jobs:run:scripts:f/folder/*").unwrap(); - assert_eq!(scope.domain, "jobs"); - assert_eq!(scope.action, "run"); - assert_eq!(scope.kind, Some("scripts".to_string())); - assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); - - // Test jobs:run:kind parsing - let scope = ScopeDefinition::from_scope_string("jobs:run:scripts").unwrap(); - assert_eq!(scope.domain, "jobs"); - assert_eq!(scope.action, "run"); - assert_eq!(scope.kind, Some("scripts".to_string())); - assert_eq!(scope.resource, None); - - // Test jobs:run:kind:resource parsing - let scope = ScopeDefinition::from_scope_string("jobs:run:flows:f/folder/*").unwrap(); - assert_eq!(scope.domain, "jobs"); - assert_eq!(scope.action, "run"); - assert_eq!(scope.kind, Some("flows".to_string())); - assert_eq!(scope.resource, Some(vec!["f/folder/*".to_string()])); - - // Test comma-separated resources parsing - let scope = - ScopeDefinition::from_scope_string("scripts:read:path1,path2,f/folder/*").unwrap(); - assert_eq!(scope.domain, "scripts"); - assert_eq!(scope.action, "read"); - assert_eq!(scope.kind, None); - assert_eq!( - scope.resource, - Some(vec![ - "path1".to_string(), - "path2".to_string(), - "f/folder/*".to_string() - ]) - ); - } - - #[test] - fn test_scope_action_hierarchy() { - assert!(ScopeAction::Write.includes(&ScopeAction::Read)); - assert!(!ScopeAction::Read.includes(&ScopeAction::Write)); - assert!(ScopeAction::Run.includes(&ScopeAction::Read)); - assert!(!ScopeAction::Run.includes(&ScopeAction::Write)); - } - - #[test] - fn test_route_domain_extraction() { - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/jobs/123").unwrap(); - assert_eq!(domain, ScopeDomain::Jobs); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("jobs/123".to_string())); - - let (domain, kind, route_suffix) = - extract_domain_from_route("/api/w/test_workspace/scripts/test_script").unwrap(); - assert_eq!(domain, ScopeDomain::Scripts); - assert_eq!(kind, None); - assert_eq!(route_suffix, Some("scripts/test_script".to_string())); - } - - #[test] - fn test_specific_scope_access() { - let scopes = vec!["jobs:read".to_string()]; - - assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "GET").is_ok()); - - // DELETE now requires write permission, so it should still fail with read-only scope - assert!(check_route_access(&scopes, "/api/w/test_workspace/jobs/123", "DELETE").is_err()); - } - - #[test] - fn test_new_domain_parsing() { - // Test that new domains are properly parsed - assert_eq!(ScopeDomain::from_str("acls"), Some(ScopeDomain::Acls)); - assert_eq!( - ScopeDomain::from_str("raw_apps"), - Some(ScopeDomain::RawApps) - ); - assert_eq!( - ScopeDomain::from_str("agent_workers"), - Some(ScopeDomain::AgentWorkers) - ); - - // Test that string conversion works both ways - assert_eq!(ScopeDomain::Acls.as_str(), "acls"); - assert_eq!(ScopeDomain::RawApps.as_str(), "raw_apps"); - assert_eq!(ScopeDomain::AgentWorkers.as_str(), "agent_workers"); - } - - #[test] - fn test_resource_array_matching() { - // Test wildcard access - let scope_all = ScopeDefinition::new("scripts", "read", None, Some(vec!["*".to_string()])); - let required = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["path1".to_string(), "path2".to_string()]), - ); - assert!(scope_all.includes(&required)); - - // Test exact matches - let scope_exact = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["path1".to_string(), "path2".to_string()]), - ); - let required_subset = - ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); - assert!(scope_exact.includes(&required_subset)); - - // Test partial match - should grant access if ANY required resource matches - let scope_limited = - ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); - let required_partial = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["path1".to_string(), "path2".to_string()]), - ); - assert!(scope_limited.includes(&required_partial)); // path1 matches, so access granted - - // Test no match - scope doesn't cover any of the required resources - let scope_different = - ScopeDefinition::new("scripts", "read", None, Some(vec!["path3".to_string()])); - let required_no_match = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["path1".to_string(), "path2".to_string()]), - ); - assert!(!scope_different.includes(&required_no_match)); - - // Test pattern matching - let scope_pattern = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["f/folder/*".to_string()]), - ); - let required_in_folder = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["f/folder/script1".to_string()]), - ); - assert!(scope_pattern.includes(&required_in_folder)); - - // Test mixed patterns and exact matches - let scope_mixed = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["exact_path".to_string(), "f/folder/*".to_string()]), - ); - let required_mixed1 = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["exact_path".to_string()]), - ); - let required_mixed2 = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["f/folder/script2".to_string()]), - ); - assert!(scope_mixed.includes(&required_mixed1)); - assert!(scope_mixed.includes(&required_mixed2)); - } - - #[test] - fn test_efficiency_small_vs_large_arrays() { - // Test small array optimization path - let scope_small = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["path1".to_string(), "path2".to_string()]), - ); - let required_small = - ScopeDefinition::new("scripts", "read", None, Some(vec!["path1".to_string()])); - assert!(scope_small.includes(&required_small)); - - // Test large array optimization path - let large_scope_vec: Vec = (0..10).map(|i| format!("path{}", i)).collect(); - let scope_large = ScopeDefinition::new("scripts", "read", None, Some(large_scope_vec)); - let required_large = - ScopeDefinition::new("scripts", "read", None, Some(vec!["path5".to_string()])); - assert!(scope_large.includes(&required_large)); - } - - #[test] - fn test_user_example_case() { - // User's example: scope has "u/dieri/*", required has ["u/dadad/wqdq", "u/*"] - // Should grant access because scope "u/dieri/*" falls under required pattern "u/*" - let user_scope = - ScopeDefinition::new("scripts", "read", None, Some(vec!["u/dieri/*".to_string()])); - let required_mixed = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["u/dadad/wqdq".to_string(), "u/*".to_string()]), - ); - assert!(user_scope.includes(&required_mixed)); // Should match because u/dieri/* falls under u/* - - // Another example: scope covers one but not both paths - let scope_specific = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["folder/file1".to_string()]), - ); - let required_multi = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["folder/file1".to_string(), "other/file2".to_string()]), - ); - assert!(scope_specific.includes(&required_multi)); // Should match because folder/file1 matches exactly - - // Test bidirectional pattern matching more explicitly - let scope_broad = - ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); - let required_specific = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["u/dieri/script.py".to_string()]), - ); - assert!(scope_broad.includes(&required_specific)); // u/* covers u/dieri/script.py - - let scope_specific_path = ScopeDefinition::new( - "scripts", - "read", - None, - Some(vec!["u/dieri/script.py".to_string()]), - ); - let required_broad = - ScopeDefinition::new("scripts", "read", None, Some(vec!["u/*".to_string()])); - assert!(scope_specific_path.includes(&required_broad)); // u/dieri/script.py satisfies u/* - } -} +// Re-export all scope types from windmill-api-auth +#[allow(unused_imports)] +pub use windmill_api_auth::scopes::*; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 2e28999779..0b9aa5ce24 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -8,7 +8,6 @@ #![allow(non_snake_case)] -use quick_cache::sync::Cache; use sqlx::{PgConnection, Postgres, Transaction}; use std::sync::atomic::AtomicBool; @@ -24,7 +23,7 @@ use crate::utils::{ generate_instance_wide_unique_username, get_instance_username_or_create_pending, }; use crate::{ - auth::ExpiringAuthCache, db::DB, utils::require_super_admin, webhook_util::WebhookShared, + db::DB, utils::require_super_admin, webhook_util::WebhookShared, COOKIE_DOMAIN, IS_SECURE, }; use argon2::{Argon2, PasswordHash, PasswordVerifier}; @@ -47,11 +46,11 @@ use tracing::Instrument; use windmill_audit::audit_oss::audit_log; use windmill_audit::ActionKind; use windmill_common::audit::AuditAuthor; -use windmill_common::auth::{fetch_authed_from_permissioned_as, TOKEN_PREFIX_LEN}; +use windmill_common::auth::TOKEN_PREFIX_LEN; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::oauth2::InstanceEvent; use windmill_common::users::COOKIE_NAME; -use windmill_common::users::{truncate_token, username_to_permissioned_as}; +use windmill_common::users::truncate_token; use windmill_common::utils::paginate; use windmill_common::worker::CLOUD_HOSTED; use windmill_common::BASE_URL; @@ -197,70 +196,9 @@ where } } +pub use windmill_api_auth::fetch_api_authed; #[allow(unused)] -pub async fn fetch_api_authed( - username: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let permissioned_as = username_to_permissioned_as(username.as_str()); - fetch_api_authed_from_permissioned_as(permissioned_as, email, w_id, db, username_override).await -} - -lazy_static::lazy_static! { - static ref API_AUTHED_CACHE: Cache<(String,String,String), ExpiringAuthCache> = Cache::new(300); -} - -#[allow(unused)] -pub async fn fetch_api_authed_from_permissioned_as( - permissioned_as: String, - email: String, - w_id: &str, - db: &DB, - username_override: Option, -) -> error::Result { - let key = (w_id.to_string(), permissioned_as.clone(), email.clone()); - - let mut api_authed = match API_AUTHED_CACHE.get(&key) { - Some(expiring_authed) if expiring_authed.expiry > chrono::Utc::now() => { - tracing::debug!("API authed cache hit for user {}", email); - expiring_authed.authed - } - _ => { - tracing::debug!("API authed cache miss for user {}", email); - let authed = - fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; - - let api_authed = ApiAuthed { - username: authed.username, - email, - is_admin: authed.is_admin, - is_operator: authed.is_operator, - groups: authed.groups, - folders: authed.folders, - scopes: authed.scopes, - username_override: None, - token_prefix: authed.token_prefix, - }; - - API_AUTHED_CACHE.insert( - key, - ExpiringAuthCache { - authed: api_authed.clone(), - expiry: chrono::Utc::now() + chrono::Duration::try_seconds(120).unwrap(), - job_id: None, - }, - ); - - api_authed - } - }; - - api_authed.username_override = username_override; - Ok(api_authed) -} +pub use windmill_api_auth::fetch_api_authed_from_permissioned_as; #[derive(FromRow, Serialize)] pub struct User { diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index ea31585cba..0f05e816e8 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -18,7 +18,7 @@ use windmill_common::{ DB, }; -use crate::{db::ApiAuthed, scopes::ScopeDefinition}; +pub use windmill_api_auth::scopes::check_scopes; #[cfg(feature = "enterprise")] use windmill_common::error::JsonResult; @@ -49,36 +49,6 @@ pub async fn require_super_admin(db: &DB, email: &str) -> error::Result<()> { } } -pub fn check_scopes(authed: &ApiAuthed, required: F) -> error::Result<()> -where - F: FnOnce() -> String, -{ - if let Some(scopes) = authed.scopes.as_ref() { - let mut is_scoped_token = false; - let required_scope = ScopeDefinition::from_scope_string(&required())?; - for scope in scopes { - if !scope.starts_with("if_jobs:filter_tags:") { - if !is_scoped_token { - is_scoped_token = true; - } - - match ScopeDefinition::from_scope_string(scope) { - Ok(scope) if scope.includes(&required_scope) => return Ok(()), - _ => {} - } - } - } - - if is_scoped_token { - return Err(Error::NotAuthorized(format!( - "Required scope: {}", - required_scope.as_string() - ))); - } - } - Ok(()) -} - pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> { let is_devops = is_devops_email(db, email).await?; diff --git a/backend/windmill-triggers/Cargo.toml b/backend/windmill-triggers/Cargo.toml new file mode 100644 index 0000000000..cd008e51cb --- /dev/null +++ b/backend/windmill-triggers/Cargo.toml @@ -0,0 +1,133 @@ +[package] +name = "windmill-triggers" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +name = "windmill_triggers" +path = "src/lib.rs" + +[features] +default = [] +private = ["windmill-api-auth/private", "windmill-common/private"] +enterprise = ["windmill-api-auth/enterprise", "windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"] +cloud = ["windmill-api-auth/cloud", "windmill-common/cloud"] +license = [] +no_auth = ["windmill-api-auth/no_auth"] +kafka = ["dep:rdkafka", "dep:rdkafka-sys"] +kafka-gssapi = ["kafka", "rdkafka/gssapi"] +nats = ["dep:async-nats", "dep:nkeys"] +websocket = ["dep:tokio-tungstenite"] +smtp = ["dep:mail-parser", "dep:openssl", "windmill-common/smtp"] +http_trigger = ["dep:matchit", "dep:thiserror", "dep:sha1", "dep:constant_time_eq"] +postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal", "dep:rust-postgres-native-tls"] +mqtt_trigger = ["dep:thiserror", "dep:rumqttc"] +native_trigger = ["dep:strum", "dep:backon", "oauth2"] +sqs_trigger = ["dep:aws-sdk-sqs", "dep:aws-sdk-sts", "dep:aws-sdk-sso", "dep:aws-sdk-ssooidc", "dep:thiserror", "dep:backon", "dep:aws-config", "windmill-common/aws_auth"] +gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic", "dep:tokio-util"] +parquet = ["dep:datafusion", "dep:object_store", "windmill-common/parquet"] +oauth2 = ["dep:windmill-oauth"] +openidconnect = ["windmill-common/openidconnect"] +benchmark = [] +stripe = [] +prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"] + +[dependencies] +windmill-api-auth.workspace = true +windmill-common = { workspace = true, default-features = false } +windmill-queue.workspace = true +windmill-audit.workspace = true +windmill-git-sync.workspace = true +windmill-oauth = { workspace = true, optional = true } +windmill-parser.workspace = true +windmill-parser-ts.workspace = true +windmill-parser-py.workspace = true + +# Core framework +tokio.workspace = true +tokio-stream.workspace = true +axum.workspace = true +tower.workspace = true +tower-http.workspace = true +tower-cookies.workspace = true +hyper.workspace = true +http.workspace = true + +# Data handling +serde.workspace = true +serde_json.workspace = true +serde_urlencoded.workspace = true +serde_yml.workspace = true + +# Database +sqlx.workspace = true +sql-builder.workspace = true + +# HTTP +reqwest.workspace = true + +# Utilities +chrono.workspace = true +uuid.workspace = true +anyhow.workspace = true +itertools.workspace = true +futures.workspace = true +rand.workspace = true +bytes.workspace = true +base64.workspace = true +regex.workspace = true +ulid.workspace = true +async-trait.workspace = true +quick_cache.workspace = true +lazy_static.workspace = true +jsonwebtoken.workspace = true +hmac.workspace = true +sha2.workspace = true +url.workspace = true +async-recursion.workspace = true +async-stream.workspace = true +pin-project.workspace = true + +# Optional dependencies +tokio-tungstenite = { workspace = true, optional = true } +rust-postgres = { workspace = true, optional = true } +rust-postgres-native-tls = { workspace = true, optional = true } +pg_escape = { workspace = true, optional = true } +byteorder = { workspace = true, optional = true } +native-tls.workspace = true +tokio-native-tls.workspace = true +tokio-postgres.workspace = true +postgres-native-tls.workspace = true +rumqttc = { workspace = true, optional = true } +rdkafka = { workspace = true, optional = true } +rdkafka-sys = { workspace = true, optional = true } +async-nats = { workspace = true, optional = true } +nkeys = { workspace = true, optional = true } +aws-sdk-sqs = { workspace = true, optional = true } +aws-sdk-sts = { workspace = true, optional = true } +aws-sdk-sso = { workspace = true, optional = true } +aws-sdk-ssooidc = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-credential-types.workspace = true +aws-smithy-types.workspace = true +google-cloud-pubsub = { workspace = true, optional = true } +google-cloud-googleapis = { workspace = true, optional = true } +tonic = { workspace = true, optional = true } +tokio-util = { workspace = true, optional = true } +matchit = { workspace = true, optional = true } +sha1 = { workspace = true, optional = true } +constant_time_eq = { workspace = true, optional = true } +thiserror = { workspace = true, optional = true } +rust_decimal = { workspace = true, optional = true } +mail-parser = { workspace = true, features = ["serde_support"], optional = true } +openssl = { workspace = true, optional = true } +strum = { workspace = true, optional = true } +backon = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } +datafusion = { workspace = true, optional = true } +object_store = { workspace = true, optional = true } +tracing.workspace = true +dashmap.workspace = true +hex.workspace = true +urlencoding.workspace = true diff --git a/backend/windmill-triggers/src/args_ext.rs b/backend/windmill-triggers/src/args_ext.rs new file mode 100644 index 0000000000..c53c85cb50 --- /dev/null +++ b/backend/windmill-triggers/src/args_ext.rs @@ -0,0 +1,684 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +// Re-exported from windmill-api/src/args.rs +// This module handles webhook argument processing for triggers. + +use std::collections::HashMap; + +use axum::{ + extract::{FromRequest, Multipart, Query, Request}, + http::HeaderMap, + response::{IntoResponse, Response}, +}; +use bytes::Bytes; +use http::{header::CONTENT_TYPE, StatusCode}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::types::JsonRawValue; +use windmill_common::{ + error::Error, + triggers::{RunnableFormat, RunnableFormatVersion, TriggerKind}, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +use crate::triggers::trigger_helpers::{get_runnable_format, RunnableId}; +use windmill_api_auth::ApiAuthed; + +#[derive(Debug)] +pub enum RawBody { + Json(String), + CEJson(String), + Text(String), + Xml(String), + UrlEncoded(Bytes), + Multipart(Multipart), + Empty, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum Body { + HashMap(HashMap>), + NoHashMap(Box), +} + +#[derive(Debug, Clone, Default)] +pub struct WebhookArgsMetadata { + pub raw_string: Option, + pub headers: HeaderMap, + pub query: Option, + pub method: http::Method, + pub query_wrap_body: bool, + pub query_use_raw: bool, + pub query_include_header: Option, + pub query_include_query: Option, +} + +pub struct RawWebhookArgs { + pub body: RawBody, + pub metadata: WebhookArgsMetadata, +} + +#[derive(Debug, Clone)] +pub struct WebhookArgs { + pub body: Body, + pub metadata: WebhookArgsMetadata, +} + +impl RawWebhookArgs { + #[cfg(not(feature = "parquet"))] + pub async fn process_multipart( + _multipart: Multipart, + _authed: &ApiAuthed, + _db: &DB, + _w_id: &str, + ) -> Result>, Error> { + return Err(Error::BadRequest(format!( + "multipart/form-data requires the parquet feature" + ))); + } + + #[cfg(feature = "parquet")] + async fn process_multipart( + mut multipart: Multipart, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + ) -> Result>, Error> { + use object_store::{Attribute, Attributes}; + use windmill_common::s3_helpers::build_object_store_client; + + let job_ops = crate::jobs_ext::get_ops(); + let (_, s3_resource) = job_ops.get_workspace_s3_resource(authed, db, None, w_id, None).await?; + + if let Some(s3_resource) = s3_resource { + let s3_client = build_object_store_client(&s3_resource).await?; + + let mut body = HashMap::new(); + let mut files = HashMap::new(); + + while let Some(field) = multipart.next_field().await.map_err(|e| { + Error::BadRequest(format!("Error reading multipart field: {}", e.body_text())) + })? { + if let Some(name) = field.name().map(|x| x.to_string()) { + if let Some(content_type) = field.content_type() { + let ext = field + .file_name() + .map(|x| x.split('.').last()) + .flatten() + .map(|x| x.to_string()); + + let file_key = job_ops.get_random_file_name(ext); + + let options = Attributes::from_iter(vec![ + (Attribute::ContentType, content_type.to_string()), + ( + Attribute::ContentDisposition, + if let Some(filename) = field.file_name() { + format!("inline; filename=\"{}\"", filename) + } else { + "inline".to_string() + }, + ), + ]) + .into(); + + let field_bytes = field.bytes().await.map_err(|e| { + Error::BadRequest(format!("Error reading multipart field bytes: {}", e.body_text())) + })?; + let bytes_stream = futures::stream::once(async move { + Ok::(field_bytes) + }); + + job_ops + .upload_file_internal(s3_client.clone(), &file_key, Box::pin(bytes_stream), options) + .await?; + + files.entry(name).or_insert(vec![]).push(serde_json::json!({ + "s3": &file_key + })); + } else { + body.insert(name, to_raw_value(&field.text().await.unwrap_or_default())); + } + } + } + + for (k, v) in files { + body.insert(k, to_raw_value(&v)); + } + + Ok(body) + } else { + Err(Error::BadRequest(format!( + "You need to connect your workspace to an S3 bucket to use multipart/form-data" + ))) + } + } + + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + force_use_raw: Option, + ) -> Result { + let use_raw = force_use_raw.unwrap_or(self.metadata.query_use_raw); + + match self.body { + RawBody::Multipart(multipart) => { + let body = Self::process_multipart(multipart, authed, db, w_id).await?; + Ok(WebhookArgs { body: Body::HashMap(body), metadata: self.metadata }) + } + RawBody::Empty => { + let mut metadata = self.metadata; + if use_raw { + metadata.raw_string = Some("".to_string()); + } + Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata }) + } + RawBody::Text(s) | RawBody::Xml(s) => Ok(WebhookArgs { + body: Body::HashMap(HashMap::new()), + metadata: WebhookArgsMetadata { raw_string: Some(s), ..self.metadata }, + }), + RawBody::UrlEncoded(bytes) => { + let mut metadata = self.metadata; + if use_raw { + let raw_string = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)))?; + metadata.raw_string = Some(raw_string); + } + let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) + .map_err(|e| Error::BadRequest(format!("invalid urlencoded data: {}", e)))?; + let payload = payload + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect::>(); + + Ok(WebhookArgs { body: Body::HashMap(payload), metadata }) + } + RawBody::Json(s) => WebhookArgs::from_json(self.metadata, use_raw, s).await, + RawBody::CEJson(s) => WebhookArgs::from_ce_json(self.metadata, use_raw, s).await, + } + } + + pub async fn to_main_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + ) -> Result { + let args = self.process_args(authed, db, w_id, None).await?; + args.to_main_args() + } + + pub async fn to_args_from_runnable( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + let args = self.process_args(authed, db, w_id, None).await?; + args.to_args_from_runnable(db, w_id, runnable_id, skip_preprocessor) + .await + } +} + +#[derive(Serialize)] +struct WebhookPreprocessorEvent { + kind: String, + body: Box, + raw_string: Option, + headers: HashMap>, + query: HashMap>, +} + +impl WebhookArgs { + pub fn to_main_args(self) -> Result { + self.to_args_from_format(RunnableFormat { + has_preprocessor: false, + version: RunnableFormatVersion::V2, + }) + } + + pub async fn to_args_from_runnable( + self, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + if skip_preprocessor.unwrap_or(false) { + self.to_main_args() + } else { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; + + self.to_args_from_format(runnable_format) + } + } + + pub fn to_args_from_format( + self, + runnable_format: RunnableFormat, + ) -> Result { + let headers = build_headers( + &self.metadata.headers, + self.metadata.query_include_header, + runnable_format.has_preprocessor, + ); + + let query = build_query( + self.metadata.query.as_deref(), + self.metadata.query_include_query, + runnable_format.has_preprocessor, + ); + + match runnable_format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + let mut args = HashMap::new(); + + args.insert( + "event".to_string(), + to_raw_value(&WebhookPreprocessorEvent { + kind: "webhook".to_string(), + body: to_raw_value(&self.body), + raw_string: self.metadata.raw_string, + headers, + query, + }), + ); + + Ok(PushArgsOwned { args, extra: None }) + } + RunnableFormat { has_preprocessor, .. } => { + let mut extra = HashMap::new(); + + let WebhookArgsMetadata { query_wrap_body, raw_string, .. } = self.metadata; + + for (k, v) in headers { + extra.insert(k, v); + } + + for (k, v) in query { + extra.insert(k, v); + } + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + if has_preprocessor { + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })), + ); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.body { + Body::HashMap(mut body) => { + if query_wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); + } + Ok(PushArgsOwned { args: body, extra }) + } + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) + } + } + } + } + } +} + +#[derive(Deserialize)] +pub struct RequestQuery { + pub raw: Option, + pub wrap_body: Option, + pub include_header: Option, + pub include_query: Option, +} + +async fn req_to_string( + req: Request, + _state: &S, +) -> Result { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response()) +} + +pub async fn try_from_request_body( + request: Request, + _state: &S, + is_http_trigger: bool, +) -> Result +where + S: Send + Sync, +{ + let (content_type, metadata) = { + let headers_map = request.headers(); + let content_type_header = headers_map.get(CONTENT_TYPE); + let content_type = content_type_header.and_then(|value| value.to_str().ok()); + let uri = request.uri(); + let request_query = Query::::try_from_uri(uri).unwrap().0; + + let query = uri.query().map(|s| s.to_owned()); + let raw = !is_http_trigger && request_query.raw.unwrap_or(false); + let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false); + ( + content_type, + WebhookArgsMetadata { + headers: headers_map.clone(), + query, + method: request.method().clone(), + raw_string: None, + query_wrap_body: wrap_body, + query_use_raw: raw, + query_include_header: request_query.include_header, + query_include_query: request_query.include_query, + }, + ) + }; + + let no_content_type = content_type.is_none(); + if no_content_type || content_type.unwrap().starts_with("application/json") { + let bytes = Bytes::from_request(request, _state) + .await + .map_err(IntoResponse::into_response)?; + if no_content_type && bytes.is_empty() { + Ok(RawWebhookArgs { body: RawBody::Empty, metadata }) + } else { + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + Ok(RawWebhookArgs { body: RawBody::Json(str), metadata }) + } + } else if content_type + .unwrap() + .starts_with("application/cloudevents+json") + { + let str = req_to_string(request, _state).await?; + + Ok(RawWebhookArgs { body: RawBody::CEJson(str), metadata }) + } else if content_type + .unwrap() + .starts_with("application/cloudevents-batch+json") + { + Err( + Error::BadRequest(format!("Cloud events batching is not supported yet")) + .into_response(), + ) + } else if content_type.unwrap().starts_with("text/plain") { + let str = req_to_string(request, _state).await?; + Ok(RawWebhookArgs { body: RawBody::Text(str), metadata }) + } else if content_type + .unwrap() + .starts_with("application/x-www-form-urlencoded") + { + let bytes = Bytes::from_request(request, _state) + .await + .map_err(IntoResponse::into_response)?; + + Ok(RawWebhookArgs { body: RawBody::UrlEncoded(bytes), metadata }) + } else if content_type.unwrap().starts_with("application/xml") + || content_type.unwrap().starts_with("text/xml") + { + let str = req_to_string(request, _state).await?; + Ok(RawWebhookArgs { body: RawBody::Xml(str), metadata }) + } else if content_type.unwrap().starts_with("multipart/form-data") { + let multipart = Multipart::from_request(request, _state) + .await + .map_err(IntoResponse::into_response)?; + + Ok(RawWebhookArgs { body: RawBody::Multipart(multipart), metadata }) + } else { + Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) + } +} + +#[axum::async_trait] +impl FromRequest for RawWebhookArgs +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let args = try_from_request_body(request, _state, false).await?; + + Ok(args) + } +} + +lazy_static::lazy_static! { + static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") + .ok().map(|x| x + .split(',') + .map(|s| s.to_string()) + .collect()).unwrap_or_default(); +} + +pub fn build_headers( + headers: &HeaderMap, + include_header: Option, + include_all_headers: bool, +) -> HashMap> { + let mut selected_headers = HashMap::new(); + + if include_all_headers { + for (k, v) in headers.iter() { + selected_headers.insert( + k.to_string(), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + } else { + let whitelist = include_header + .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) + .unwrap_or_default(); + whitelist + .iter() + .chain(INCLUDE_HEADERS.iter()) + .for_each(|h| { + if let Some(v) = headers.get(h) { + selected_headers.insert( + h.to_string().to_lowercase().replace('-', "_"), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + }); + } + selected_headers +} + +pub fn build_query( + query: Option<&str>, + include_query: Option, + include_all_query: bool, +) -> HashMap> { + let Some(query) = query else { + return HashMap::new(); + }; + + if include_all_query { + let queries = + serde_urlencoded::from_str::>(&query).unwrap_or_default(); + queries + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect() + } else { + let parse_query_args = include_query + .map(|s| s.split(",").map(|p| p.to_string()).collect::>()) + .unwrap_or_default(); + let mut args = HashMap::new(); + if !parse_query_args.is_empty() { + let queries = + serde_urlencoded::from_str::>(&query).unwrap_or_default(); + parse_query_args.iter().for_each(|h| { + if let Some(v) = queries.get(h) { + args.insert(h.to_string(), to_raw_value(v)); + } + }); + } + args + } +} + +fn restructure_cloudevents_metadata( + mut p: HashMap>, +) -> Result>, Error> { + let data = p + .remove("data") + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let str = data.to_string(); + + let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{'; + + if wrap_body { + let args = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); + Ok(hm) + } else { + let mut hm = serde_json::from_str::>>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new); + hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); + Ok(hm) + } +} + +impl WebhookArgs { + async fn from_json( + mut metadata: WebhookArgsMetadata, + use_raw: bool, + str: String, + ) -> Result { + if use_raw { + metadata.raw_string = Some(str.clone()); + } + + let no_hashmap = str.len() > 0 && str.chars().next().unwrap() != '{'; + + if no_hashmap { + let args = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + + Ok(Self { body: Body::NoHashMap(args), metadata }) + } else { + let hm = serde_json::from_str::>>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new); + Ok(Self { body: Body::HashMap(hm), metadata }) + } + } + + async fn from_ce_json( + mut metadata: WebhookArgsMetadata, + use_raw: bool, + str: String, + ) -> Result { + if use_raw { + metadata.raw_string = Some(str.clone()); + } + + let hm = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid cloudevents+json: {}", e)))?; + let hm = restructure_cloudevents_metadata(hm)?; + Ok(Self { body: Body::HashMap(hm), metadata }) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[tokio::test] + async fn test_cloudevents_json_payload() { + let r1 = r#" + { + "specversion" : "1.0", + "type" : "com.example.someevent", + "source" : "/mycontext", + "subject": null, + "id" : "C234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : { + "appinfoA" : "abc", + "appinfoB" : 123, + "appinfoC" : true + } + } + "#; + let r2 = r#" + { + "specversion" : "1.0", + "type" : "com.example.someevent", + "source" : "/mycontext", + "subject": null, + "id" : "C234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : 1.5 + } + "#; + let metadata = WebhookArgsMetadata::default(); + + let a1 = WebhookArgs::from_ce_json(metadata.clone(), false, r1.to_string()) + .await + .expect("Failed to parse the cloudevent"); + let a2 = WebhookArgs::from_ce_json(metadata.clone(), false, r2.to_string()) + .await + .expect("Failed to parse the cloudevent"); + + match a1.body { + Body::HashMap(body) => { + body.get("WEBHOOK__METADATA__").expect( + "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", + ); + } + _ => panic!("Expected a HashMap"), + } + + match a2.body { + Body::HashMap(body) => { + assert_eq!( + body + .get("body") + .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") + .to_string(), + "1.5" + ); + } + _ => panic!("Expected a HashMap"), + } + } +} diff --git a/backend/windmill-triggers/src/capture_ext.rs b/backend/windmill-triggers/src/capture_ext.rs new file mode 100644 index 0000000000..492d4909a1 --- /dev/null +++ b/backend/windmill-triggers/src/capture_ext.rs @@ -0,0 +1,87 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_common::{ + error::Result, + triggers::TriggerKind, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_queue::{PushArgs, PushArgsOwned}; + +const KEEP_LAST: i64 = 20; + +async fn clear_captures_history(db: &DB, w_id: &str) -> Result<()> { + if *CLOUD_HOSTED { + sqlx::query!( + r#" + DELETE FROM + capture + WHERE + workspace_id = $1 + AND created_at <= ( + SELECT + created_at + FROM + capture + WHERE + workspace_id = $1 + ORDER BY + created_at DESC + OFFSET $2 + LIMIT 1 + ) + "#, + &w_id, + KEEP_LAST, + ) + .execute(db) + .await?; + } + Ok(()) +} + +pub async fn insert_capture_payload( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, + trigger_kind: &TriggerKind, + main_args: PushArgsOwned, + preprocessor_args: PushArgsOwned, + owner: &str, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO + capture ( + workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + "#, + &w_id, + path, + is_flow, + trigger_kind as &TriggerKind, + sqlx::types::Json(PushArgs { args: &main_args.args, extra: main_args.extra }) + as sqlx::types::Json, + sqlx::types::Json(PushArgs { + args: &preprocessor_args.args, + extra: preprocessor_args.extra + }) as sqlx::types::Json, + owner, + ) + .execute(db) + .await?; + + clear_captures_history(db, &w_id).await?; + + Ok(()) +} diff --git a/backend/windmill-triggers/src/flow_ext.rs b/backend/windmill-triggers/src/flow_ext.rs new file mode 100644 index 0000000000..67126913f2 --- /dev/null +++ b/backend/windmill-triggers/src/flow_ext.rs @@ -0,0 +1,24 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::ApiAuthed; +use windmill_common::{error::Result, DB}; + +use crate::require_is_writer_internal; + +pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { + require_is_writer_internal( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM flow WHERE path = $1 AND workspace_id = $2", + "flow", + ) + .await +} diff --git a/backend/windmill-triggers/src/jobs_ext.rs b/backend/windmill-triggers/src/jobs_ext.rs new file mode 100644 index 0000000000..7fc9b9b380 --- /dev/null +++ b/backend/windmill-triggers/src/jobs_ext.rs @@ -0,0 +1,423 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Job operations trait for windmill-triggers. +//! +//! Complex job functions that have deep dependencies on windmill-api internals +//! are abstracted behind the `JobOps` trait. windmill-api provides the real +//! implementation at startup via `set_ops()`. + +use axum::response::Response; +use serde::Deserialize; +use serde_json::value::RawValue; +use sqlx::Postgres; +use std::sync::Arc; +use uuid::Uuid; +use windmill_api_auth::{ApiAuthed, OptTokened}; +use windmill_common::{ + db::UserDB, + error::{self, Error}, + jobs::check_tag_available_for_workspace_internal, + triggers::TriggerMetadata, + utils::StripPath, + DB, +}; +use windmill_queue::PushArgsOwned; + +#[derive(Debug, Deserialize, Clone, Default)] +pub struct RunJobQuery { + pub scheduled_for: Option>, + pub scheduled_in_secs: Option, + pub parent_job: Option, + pub root_job: Option, + pub invisible_to_owner: Option, + pub queue_limit: Option, + pub payload: Option, + pub job_id: Option, + pub tag: Option, + pub timeout: Option, + pub cache_ttl: Option, + pub cache_ignore_s3_path: Option, + pub skip_preprocessor: Option, + pub poll_delay_ms: Option, + pub memory_id: Option, + pub trigger_external_id: Option, + pub service_name: Option, + pub suspended_mode: Option, +} + +pub fn get_scope_tags(authed: &ApiAuthed) -> Option> { + authed.scopes.as_ref()?.iter().find_map(|s| { + if s.starts_with("if_jobs:filter_tags:") { + Some( + s.trim_start_matches("if_jobs:filter_tags:") + .split(",") + .collect::>(), + ) + } else { + None + } + }) +} + +pub async fn check_tag_available_for_workspace( + db: &DB, + w_id: &str, + tag: &Option, + authed: &ApiAuthed, +) -> error::Result<()> { + if let Some(tag) = tag.as_deref().filter(|t| !t.is_empty()) { + let tags = get_scope_tags(authed); + check_tag_available_for_workspace_internal(db, w_id, tag, &authed.email, tags).await + } else { + Ok(()) + } +} + +pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<(), Error> { + sqlx::query!( + "UPDATE v2_job SET args = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE v2_job_completed SET result = '{}'::jsonb WHERE id = $1", + job_uuid, + ) + .execute(db) + .await?; + sqlx::query!( + "UPDATE job_logs SET logs = '##DELETED##' WHERE job_id = $1", + job_uuid, + ) + .execute(db) + .await?; + Ok(()) +} + +#[cfg(feature = "enterprise")] +pub async fn check_license_key_valid() -> error::Result<()> { + use windmill_common::ee_oss::LICENSE_KEY_VALID; + + let valid = *LICENSE_KEY_VALID.read().await; + if !valid { + return Err(Error::BadRequest( + "License key is not valid. Go to your superadmin settings to update your license key." + .to_string(), + )); + } + Ok(()) +} + +/// Trait for complex job operations that stay in windmill-api. +/// windmill-api provides the implementation at startup via `set_ops()`. +#[axum::async_trait] +pub trait JobOps: Send + Sync + 'static { + async fn push_script_job_by_path_into_queue( + &self, + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, + ) -> error::Result<( + Uuid, + Option, + Option>, + )>; + + async fn push_flow_job_by_path_into_queue( + &self, + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, + ) -> error::Result<( + Uuid, + Option, + Option>, + )>; + + async fn run_wait_result_internal( + &self, + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, + ) -> error::Result<(Box, bool)>; + + fn result_to_response( + &self, + result: Box, + success: bool, + ) -> error::Result; + + async fn cancel_jobs( + &self, + jobs: Vec, + db: &DB, + username: &str, + w_id: &str, + force_cancel: bool, + ) -> error::JsonResult>; + + fn start_job_update_sse_stream( + &self, + opt_authed: Option, + opt_tokened: OptTokened, + db: DB, + w_id: String, + job_id: Uuid, + initial_log_offset: Option, + initial_stream_offset: Option, + get_progress: Option, + running: Option, + only_result: Option, + fast: Option, + no_logs: Option, + is_flow: Option, + tx: tokio::sync::mpsc::Sender, + poll_delay_ms: Option, + ); + + async fn try_get_resource_from_db( + &self, + authed: &ApiAuthed, + user_db: Option, + db: &DB, + resource_path: &str, + w_id: &str, + ) -> error::Result; + + async fn interpolate( + &self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + s: String, + ) -> Result; + + #[cfg(feature = "parquet")] + fn get_random_file_name(&self, ext: Option) -> String; + + #[cfg(feature = "parquet")] + async fn upload_file_internal( + &self, + s3_client: Arc, + file_key: &str, + bytes_stream: std::pin::Pin> + Send>>, + options: object_store::PutMultipartOpts, + ) -> error::Result<()>; + + #[cfg(feature = "parquet")] + async fn get_workspace_s3_resource( + &self, + authed: &ApiAuthed, + db: &DB, + user_db: Option, + w_id: &str, + storage: Option, + ) -> error::Result<(Option, Option)>; +} + +// Re-export types used by SSE streaming +#[derive(Debug, Clone, serde::Serialize)] +pub struct JobUpdate { + #[serde(skip_serializing_if = "Option::is_none")] + pub running: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub completed: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub new_logs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub log_offset: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub mem_peak: Option, +} + +#[derive(Debug, serde::Serialize)] +pub enum JobUpdateSSEStream { + Update(serde_json::Value), + Error { error: String }, + NotFound, + Timeout, + Ping, +} + +static JOB_OPS: std::sync::OnceLock> = std::sync::OnceLock::new(); + +/// Call this from windmill-api at startup to inject the real implementation. +pub fn set_ops(ops: Arc) { + let _ = JOB_OPS.set(ops); +} + +/// Get the job ops implementation. Panics if not initialized. +pub fn get_ops() -> &'static Arc { + JOB_OPS + .get() + .expect("JobOps not initialized. Call jobs_ext::set_ops() at startup.") +} + +// Convenience wrapper functions that delegate to the trait + +pub async fn push_script_job_by_path_into_queue( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + script_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + get_ops() + .push_script_job_by_path_into_queue( + authed, + db, + tx_o, + user_db, + w_id, + script_path, + run_query, + args, + trigger, + ) + .await +} + +pub async fn push_flow_job_by_path_into_queue( + authed: ApiAuthed, + db: DB, + tx_o: Option>, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<( + Uuid, + Option, + Option>, +)> { + get_ops() + .push_flow_job_by_path_into_queue( + authed, + db, + tx_o, + user_db, + w_id, + flow_path, + run_query, + args, + trigger, + ) + .await +} + +pub async fn run_wait_result_internal( + db: &DB, + uuid: Uuid, + w_id: &str, + node_id_for_empty_return: Option, + username: &str, +) -> error::Result<(Box, bool)> { + get_ops() + .run_wait_result_internal(db, uuid, w_id, node_id_for_empty_return, username) + .await +} + +pub fn result_to_response(result: Box, success: bool) -> error::Result { + get_ops().result_to_response(result, success) +} + +pub async fn cancel_jobs( + jobs: Vec, + db: &DB, + username: &str, + w_id: &str, + force_cancel: bool, +) -> error::JsonResult> { + get_ops() + .cancel_jobs(jobs, db, username, w_id, force_cancel) + .await +} + +pub fn start_job_update_sse_stream( + opt_authed: Option, + opt_tokened: OptTokened, + db: DB, + w_id: String, + job_id: Uuid, + initial_log_offset: Option, + initial_stream_offset: Option, + get_progress: Option, + running: Option, + only_result: Option, + fast: Option, + no_logs: Option, + is_flow: Option, + tx: tokio::sync::mpsc::Sender, + poll_delay_ms: Option, +) { + get_ops().start_job_update_sse_stream( + opt_authed, + opt_tokened, + db, + w_id, + job_id, + initial_log_offset, + initial_stream_offset, + get_progress, + running, + only_result, + fast, + no_logs, + is_flow, + tx, + poll_delay_ms, + ) +} + +pub async fn interpolate( + authed: &ApiAuthed, + db: &DB, + w_id: &str, + s: String, +) -> Result { + get_ops().interpolate(authed, db, w_id, s).await +} + +#[cfg(feature = "parquet")] +pub async fn get_workspace_s3_resource( + authed: &ApiAuthed, + db: &DB, + user_db: Option, + w_id: &str, + storage: Option, +) -> error::Result<(Option, Option)> { + get_ops() + .get_workspace_s3_resource(authed, db, user_db, w_id, storage) + .await +} diff --git a/backend/windmill-triggers/src/lib.rs b/backend/windmill-triggers/src/lib.rs new file mode 100644 index 0000000000..b591c7a1e3 --- /dev/null +++ b/backend/windmill-triggers/src/lib.rs @@ -0,0 +1,170 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +pub mod triggers; +#[cfg(feature = "native_trigger")] +pub mod native_triggers; + +pub mod args_ext; +pub mod capture_ext; +pub mod flow_ext; +pub mod jobs_ext; +pub mod resource_ext; +pub mod script_ext; +pub mod user_ext; + +// Shared permission helpers used by flow_ext, script_ext, and native_triggers + +use windmill_api_auth::ApiAuthed; +use windmill_common::error::{Error, Result}; +use windmill_common::DB; + +/// Check if the user is an owner of the given path. +pub fn require_owner_of_path(authed: &ApiAuthed, path: &str) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + let splitted = path.split("/").collect::>(); + if splitted[0] == "u" { + if splitted[1] == authed.username { + Ok(()) + } else { + Err(Error::BadRequest(format!( + "only the owner {} is authorized to perform this operation", + splitted[1] + ))) + } + } else if splitted[0] == "f" { + require_is_folder_owner(authed, splitted[1]) + } else { + Err(Error::BadRequest(format!( + "Not recognized path kind: {}", + path + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be owner of an empty path" + ))) + } +} + +fn is_folder_owner( + ApiAuthed { is_admin, folders, .. }: &ApiAuthed, + name: &str, +) -> bool { + if *is_admin { + true + } else { + folders.into_iter().any(|x| x.0 == name && x.2) + } +} + +fn require_is_folder_owner(authed: &ApiAuthed, name: &str) -> Result<()> { + if is_folder_owner(authed, name) { + Ok(()) + } else { + Err(Error::NotAuthorized(format!( + "You are not owner of the folder {}", + name + ))) + } +} + +fn get_perm_in_extra_perms_for_authed( + v: serde_json::Value, + authed: &ApiAuthed, +) -> Option { + match v { + serde_json::Value::Object(obj) => { + let mut keys = vec![format!("u/{}", authed.username)]; + for g in authed.groups.iter() { + keys.push(format!("g/{}", g)); + } + let mut res = None; + for k in keys { + if let Some(v) = obj.get(&k) { + if let Some(v) = v.as_bool() { + if v { + return Some(true); + } + res = Some(v); + } + } + } + res + } + _ => None, + } +} + +/// Generic require_is_writer with a configurable SQL query. +/// Used by flow_ext and script_ext. +pub async fn require_is_writer_internal( + authed: &ApiAuthed, + path: &str, + w_id: &str, + db: DB, + query: &str, + kind: &str, +) -> Result<()> { + if authed.is_admin { + return Ok(()); + } + if !path.is_empty() { + if require_owner_of_path(authed, path).is_ok() { + return Ok(()); + } + if path.starts_with("f/") && path.split('/').count() >= 2 { + let folder = path.split('/').nth(1).unwrap(); + let extra_perms = sqlx::query_scalar!( + "SELECT extra_perms FROM folder WHERE name = $1 AND workspace_id = $2", + folder, + w_id + ) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let is_folder_writer = + get_perm_in_extra_perms_for_authed(perms, authed).unwrap_or(false); + if is_folder_writer { + return Ok(()); + } + } + } + let extra_perms = sqlx::query_scalar(query) + .bind(path) + .bind(w_id) + .fetch_optional(&db) + .await?; + if let Some(perms) = extra_perms { + let perm = get_perm_in_extra_perms_for_authed(perms, authed); + match perm { + Some(true) => Ok(()), + Some(false) => Err(Error::BadRequest(format!( + "User {} is not a writer of {kind} path {path}", + authed.username + ))), + None => Err(Error::BadRequest(format!( + "User {} has neither read or write permission on {kind} {path}", + authed.username + ))), + } + } else { + Err(Error::BadRequest(format!( + "{path} does not exist yet and user {} is not an owner of the parent folder", + authed.username + ))) + } + } else { + Err(Error::BadRequest(format!( + "Cannot be writer of an empty path" + ))) + } +} diff --git a/backend/windmill-triggers/src/native_triggers/handler.rs b/backend/windmill-triggers/src/native_triggers/handler.rs new file mode 100644 index 0000000000..01584682d5 --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/handler.rs @@ -0,0 +1,565 @@ +use crate::{ + native_triggers::{ + delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, + get_workspace_integration, list_native_triggers, store_native_trigger, + update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, + NativeTriggerData, ServiceName, + }, + user_ext::{create_token_internal, NewToken}, +}; +use windmill_api_auth::{scopes::check_scopes, ApiAuthed}; +use axum::{ + extract::{Path, Query}, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use sqlx::PgConnection; +use std::sync::Arc; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::rd_string, + DB, +}; + +async fn require_is_writer_on_runnable( + authed: &ApiAuthed, + path: &str, + is_flow: bool, + w_id: &str, + db: DB, +) -> Result<()> { + if is_flow { + crate::flow_ext::require_is_writer(authed, path, w_id, db).await + } else { + crate::script_ext::require_is_writer(authed, path, w_id, db).await + } +} + +#[derive(Debug, Deserialize)] +pub struct ListQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, +} + +#[derive(Debug, Serialize)] +pub struct FullTriggerResponse { + #[serde(flatten)] + pub windmill_data: NativeTrigger, + pub external_data: T, +} + +#[derive(Debug, Serialize)] +pub struct CreateTriggerResponse { + pub external_id: String, +} + +async fn new_webhook_token( + tx: &mut PgConnection, + db: &DB, + authed: &ApiAuthed, + script_path: &str, + is_flow: bool, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let kind = if is_flow { "flows" } else { "scripts" }; + + let scopes = vec![format!("jobs:run:{kind}:{script_path}")]; + let label = format!("webhook-{}-{}", service_name.as_str(), rd_string(5)); + let token_config = NewToken::new( + Some(label), + None, + None, + Some(scopes), + Some(workspace_id.to_owned()), + ); + let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; + + Ok(token) +} + +async fn create_native_trigger( + Extension(handler): Extension>, + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(data): Json>, +) -> JsonResult { + check_scopes(&authed, || { + format!("native_triggers:write:{}", &data.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &data.script_path, + data.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let webhook_token = new_webhook_token( + &mut *tx, + &db, + &authed, + &data.script_path, + data.is_flow, + &workspace_id, + service_name, + ) + .await?; + + let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; + + let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse {} OAuth data: {}", + T::DISPLAY_NAME, + e + )) + })?; + + let resp = handler + .create( + &workspace_id, + &oauth_data, + &webhook_token, + &data, + &db, + &mut tx, + ) + .await?; + + let (external_id, _) = handler.external_id_and_metadata_from_response(&resp); + + // update the created external trigger with a new uri containing the external_id + handler + .update( + &workspace_id, + &oauth_data, + &external_id, + &webhook_token, + &data, + &db, + &mut tx, + ) + .await?; + + // Fetch the updated trigger data from the external service and extract service_config + let trigger_data = handler + .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) + .await?; + let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?; + + let config = NativeTriggerConfig { + script_path: data.script_path.clone(), + is_flow: data.is_flow, + webhook_token, + }; + + store_native_trigger( + &mut *tx, + &workspace_id, + service_name, + &external_id, + &config, + service_config, + ) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("native_triggers.{}.create", service_name), + ActionKind::Create, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(CreateTriggerResponse { external_id })) +} + +async fn update_native_trigger_handler( + Extension(handler): Extension>, + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, + Json(data): Json>, +) -> Result { + check_scopes(&authed, || { + format!("native_triggers:write:{}", &data.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &data.script_path, + data.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + // Look up the full token using the stored prefix (use db, not tx, for token table) + let webhook_token = match get_token_by_prefix(&db, &existing.webhook_token_prefix).await? { + Some(token) => token, + None => { + tracing::warn!( + "Webhook token not found for trigger {} (prefix: {}), recreating token", + external_id, + existing.webhook_token_prefix + ); + new_webhook_token( + &mut *tx, + &db, + &authed, + &data.script_path, + data.is_flow, + &workspace_id, + service_name, + ) + .await? + } + }; + + let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; + + let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse {} OAuth data: {}", + T::DISPLAY_NAME, + e + )) + })?; + + handler + .update( + &workspace_id, + &oauth_data, + &external_id, + &webhook_token, + &data, + &db, + &mut tx, + ) + .await?; + + // Fetch the updated trigger data from the external service and extract service_config + let trigger_data = handler + .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) + .await?; + let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?; + + let config = NativeTriggerConfig { + script_path: data.script_path.clone(), + is_flow: data.is_flow, + webhook_token, + }; + + store_native_trigger( + &mut *tx, + &workspace_id, + service_name, + &external_id, + &config, + service_config, + ) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("native_triggers.{}.update", service_name), + ActionKind::Update, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Native trigger updated")) +} + +async fn get_native_trigger_handler( + Extension(handler): Extension>, + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + + let windmill_trigger = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + check_scopes(&authed, || { + format!("native_triggers:read:{}", &windmill_trigger.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &windmill_trigger.script_path, + windmill_trigger.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; + + let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse {} OAuth data: {}", + T::DISPLAY_NAME, + e + )) + })?; + + let native_trigger = handler + .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) + .await; + + let native_trigger_config = match native_trigger { + Ok(native_cfg) => { + // Clear error if it was set + if windmill_trigger.error.is_some() { + update_native_trigger_error( + &mut *tx, + &workspace_id, + service_name, + &external_id, + None, + ) + .await?; + } + native_cfg + } + Err(Error::NotFound(_)) => { + let error_msg = "Trigger no longer exists on external service".to_string(); + tracing::warn!( + "Native trigger no longer exists on external service {}, setting error", + service_name + ); + + update_native_trigger_error( + &mut *tx, + &workspace_id, + service_name, + &external_id, + Some(&error_msg), + ) + .await?; + + tx.commit().await?; + + return Err(Error::NotFound(format!( + "Trigger '{}' no longer exists on external service {}", + external_id, service_name + ))); + } + Err(e) => return Err(e), + }; + + let full_resp = Json(FullTriggerResponse { + windmill_data: windmill_trigger, + external_data: native_trigger_config, + }); + + Ok(full_resp) +} + +async fn delete_native_trigger_handler( + Extension(handler): Extension>, + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, +) -> Result { + let mut tx = user_db.begin(&authed).await?; + + let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) + .await? + .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; + + check_scopes(&authed, || { + format!("native_triggers:write:{}", &existing.script_path) + })?; + require_is_writer_on_runnable( + &authed, + &existing.script_path, + existing.is_flow, + &workspace_id, + db.clone(), + ) + .await?; + + let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; + + let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse {} OAuth data: {}", + T::DISPLAY_NAME, + e + )) + })?; + + handler + .delete(&workspace_id, &oauth_data, &external_id, &db, &mut tx) + .await?; + + let deleted = + delete_native_trigger(&mut *tx, &workspace_id, service_name, &external_id).await?; + + if !deleted { + return Err(Error::NotFound(format!("Native trigger not found"))); + } + + // Delete the webhook token using its prefix + if !delete_token_by_prefix(&db, &existing.webhook_token_prefix).await? { + tracing::warn!( + "Webhook token not found when deleting trigger {} (prefix: {})", + external_id, + existing.webhook_token_prefix + ); + } + + audit_log( + &mut *tx, + &authed, + &format!("native_triggers.{}.delete", service_name), + ActionKind::Delete, + &workspace_id, + Some(&external_id), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Native trigger deleted")) +} + +async fn exists_native_trigger_handler( + Extension(service_name): Extension, + _authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, external_id)): Path<(String, String)>, +) -> JsonResult { + let exists = sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM native_trigger + WHERE + workspace_id = $1 AND + service_name = $2 AND + external_id = $3 + ) + "#, + workspace_id, + service_name as ServiceName, + external_id + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +async fn list_native_triggers_handler( + Extension(service_name): Extension, + authed: ApiAuthed, + Extension(user_db): Extension, + Path(workspace_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let triggers = list_native_triggers( + &mut *tx, + &workspace_id, + service_name, + query.page, + query.per_page, + query.path.as_deref(), + query.is_flow, + ) + .await?; + tx.commit().await?; + Ok(Json(triggers)) +} + +pub fn service_routes(handler: T) -> Router { + let additional_routes = handler.additional_routes(); + let service_name = T::SERVICE_NAME; + + let handler_arc = Arc::new(handler); + + let standard_routes = Router::new() + .route("/create", post(create_native_trigger::)) + .route("/list", get(list_native_triggers_handler::)) + .route("/get/:external_id", get(get_native_trigger_handler::)) + .route( + "/update/:external_id", + post(update_native_trigger_handler::), + ) + .route( + "/delete/:external_id", + delete(delete_native_trigger_handler::), + ) + .route( + "/exists/:external_id", + get(exists_native_trigger_handler::), + ); + + standard_routes + .merge(additional_routes) + .layer(Extension(handler_arc)) + .layer(Extension(service_name)) +} + +/// Generates routes for all registered native trigger services. +/// When adding a new service, add a new `.nest()` call here. +pub fn generate_native_trigger_routers() -> Router { + let router = Router::new(); + + #[cfg(feature = "native_trigger")] + { + use crate::native_triggers::nextcloud::NextCloud; + + // Register all service routes here + // When adding a new service: + // 1. Import the handler: use crate::native_triggers::newservice::NewServiceHandler; + // 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler)) + return router.nest("/nextcloud", service_routes(NextCloud)); + // Add new services here: + // .nest("/newservice", service_routes(NewServiceHandler)) + } + + #[cfg(not(feature = "native_trigger"))] + { + router + } +} diff --git a/backend/windmill-triggers/src/native_triggers/mod.rs b/backend/windmill-triggers/src/native_triggers/mod.rs new file mode 100644 index 0000000000..bf13543d17 --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/mod.rs @@ -0,0 +1,1053 @@ +//! Native Triggers Module +//! +//! This module provides integration with external services (like Nextcloud) that can +//! trigger Windmill scripts/flows via webhooks. +//! +//! ## Adding a New Native Trigger Service +//! +//! When adding a new service (e.g., "NewService"), you need to update the following locations: +//! +//! ### 1. This file (mod.rs): +//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block +//! - Add `NewService` variant to `ServiceName` enum +//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` +//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` +//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) +//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) +//! - Update `ServiceName::fmt()` (Display impl) - add match arm +//! +//! ### 2. sync.rs: +//! - Add `sync_service!()` macro call in `sync_all_triggers()` +//! +//! ### 3. handler.rs: +//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` +//! +//! ### 4. Database migration: +//! - Add `'newservice'` to the `native_trigger_service` enum type +//! +//! ### 5. windmill_common (if needed): +//! - Add `NewService` variant to `TriggerKind` enum +//! - Add `'newservice'` to `job_trigger_kind` enum type in migration +//! +//! The generic code (trait definitions, route handlers, database operations) does NOT +//! need modification when adding new services. + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use http::StatusCode; +use itertools::Itertools; +use reqwest::{Client, Method}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::json; +use serde_json::value::RawValue; +use sqlx::{FromRow, PgConnection, Postgres}; +use std::{collections::HashMap, fmt::Debug}; +use strum::{EnumIter, IntoEnumIterator}; +use tokio::task; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + variables::{build_crypt, decrypt, encrypt}, + DB, +}; +use windmill_queue::PushArgsOwned; + +#[cfg(feature = "native_trigger")] +use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; + +use windmill_api_auth::ApiAuthed; +pub mod handler; +pub mod sync; +pub mod workspace_integrations; + +// Service modules - add new services here: +#[cfg(feature = "native_trigger")] +pub mod nextcloud; +// #[cfg(feature = "native_trigger")] +// pub mod newservice; + +/// Enum of all supported native trigger services. +/// When adding a new service, add a variant here (e.g., `NewService`). +#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum ServiceName { + Nextcloud, + // Add new services here: + // NewService, +} + +impl TryFrom for ServiceName { + type Error = Error; + fn try_from(value: String) -> std::result::Result { + // Add new service match arms here: + let service = match value.as_str() { + "nextcloud" => ServiceName::Nextcloud, + // "newservice" => ServiceName::NewService, + _ => { + return Err(anyhow::anyhow!( + "Unknown service, currently supported services are: [{}]", + ServiceName::iter().join(",") + ) + .into()) + } + }; + + Ok(service) + } +} + +impl ServiceName { + /// Returns the lowercase string identifier for this service. + /// Add new service match arms here. + pub fn as_str(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "nextcloud", + // ServiceName::NewService => "newservice", + } + } + + /// Returns the corresponding TriggerKind for this service. + /// Requires adding the variant to TriggerKind in windmill_common. + pub fn as_trigger_kind(&self) -> TriggerKind { + match self { + ServiceName::Nextcloud => TriggerKind::Nextcloud, + // ServiceName::NewService => TriggerKind::NewService, + } + } + + /// Returns the corresponding JobTriggerKind for this service. + /// Requires adding the variant to JobTriggerKind in windmill_common. + pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { + match self { + ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, + // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, + } + } + + /// Returns the OAuth token endpoint path for this service. + /// Used for building OAuth clients dynamically. + pub fn token_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", + // ServiceName::NewService => "/oauth/token", + } + } + + /// Returns the OAuth authorization endpoint path for this service. + /// Used for building OAuth authorization URLs. + pub fn auth_endpoint(&self) -> &'static str { + match self { + ServiceName::Nextcloud => "/apps/oauth2/authorize", + // ServiceName::NewService => "/oauth/authorize", + } + } +} + +impl std::fmt::Display for ServiceName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct NativeTrigger { + pub external_id: String, + pub workspace_id: String, + pub service_name: ServiceName, + pub script_path: String, + pub is_flow: bool, + pub webhook_token_prefix: String, + pub service_config: Option, + pub error: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NativeTriggerConfig { + pub script_path: String, + pub is_flow: bool, + pub webhook_token: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct NativeTriggerData { + pub script_path: String, + pub is_flow: bool, + pub service_config: C, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WorkspaceIntegration { + pub workspace_id: String, + pub service_name: ServiceName, + pub oauth_data: serde_json::Value, + pub created_at: DateTime, + pub updated_at: DateTime, + pub created_by: String, +} + +#[async_trait] +pub trait External: Send + Sync + 'static { + type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + type TriggerData: Debug + Serialize + Send + Sync; + type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; + type CreateResponse: DeserializeOwned + Send + Sync; + + const SUPPORT_WEBHOOK: bool; + const SERVICE_NAME: ServiceName; + const DISPLAY_NAME: &'static str; + const TOKEN_ENDPOINT: &'static str; + const REFRESH_ENDPOINT: &'static str; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()>; + + #[allow(unused)] + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result; + + async fn list_all( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + db: &DB, + tx: &mut PgConnection, + ) -> Result>; + + async fn prepare_webhook( + &self, + _db: &DB, + _w_id: &str, + _header: HashMap, + _body: String, + _script_path: &str, + _is_flow: bool, + ) -> Result { + Ok(PushArgsOwned { extra: None, args: HashMap::new() }) + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option); + + fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; + + /// Extracts the service-specific config from trigger data (from external service). + /// Used for comparison during sync to detect config drift. + /// Default implementation converts the trigger data to a JSON value + /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. + fn extract_service_config_from_trigger_data( + &self, + data: &Self::TriggerData, + ) -> Result { + serde_json::to_value(data).map_err(|e| { + Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) + }) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn http_client_request( + &self, + url: &str, + method: Method, + workspace_id: &str, + tx: &mut PgConnection, + db: &DB, + headers: Option>, + body: Option<&B>, + ) -> Result { + let oauth_config: OAuthConfig = + decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; + + let result = make_http_request( + url, + method.clone(), + headers.clone(), + body.as_ref(), + &oauth_config.access_token, + ) + .await; + + match result { + Ok(response) => Ok(response), + Err(err) + if err.status() == Some(StatusCode::UNAUTHORIZED) + || err.status() == Some(StatusCode::FORBIDDEN) => + { + tracing::info!( + "HTTP auth error ({}), attempting token refresh", + err.status().unwrap() + ); + + let refreshed_oauth_config = + refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; + + task::spawn({ + let db_clone = db.clone(); + let workspace_id_clone = workspace_id.to_string(); + let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); + async move { + update_workspace_integration_tokens_helper( + db_clone, + workspace_id_clone, + Self::SERVICE_NAME, + refreshed_json, + ) + .await; + } + }); + + let response = make_http_request( + url, + method, + headers, + body.as_ref(), + &refreshed_oauth_config.access_token, + ) + .await + .map_err(to_anyhow)?; + Ok(response) + } + Err(e) => Err(to_anyhow(e).into()), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct OAuthConfig { + pub base_url: String, + pub access_token: String, + pub refresh_token: Option, + pub client_id: String, + pub client_secret: String, +} + +pub async fn make_http_request( + url: &str, + method: Method, + headers: Option>, + body: Option<&B>, + access_token: &str, +) -> std::result::Result { + let client = Client::new(); + let mut request = client.request(method, url); + + request = request + .header("Accept", "application/json") + .header("Authorization", format!("Bearer {}", access_token)); + + if body.is_some() { + request = request.header("Content-Type", "application/json"); + } + + if let Some(custom_headers) = headers { + for (key, value) in custom_headers { + request = request.header(key, value); + } + } + + if let Some(body_content) = body { + request = request.json(body_content); + } + + let response = request.send().await?.error_for_status()?; + + let response_json = response.json().await?; + + Ok(response_json) +} + +pub async fn decrypt_oauth_data< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, + T: DeserializeOwned, +>( + tx: E, + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = get_workspace_integration(tx, workspace_id, service_name).await?; + + let mc = build_crypt(db, workspace_id).await?; + let mut oauth_data: serde_json::Value = integration.oauth_data; + + if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { + let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; + oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); + } + + if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) + { + let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) + .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; + oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); + } + + serde_json::from_value(oauth_data) + .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) +} + +#[allow(unused)] +pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { + let base_url = oauth_data + .get("base_url") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? + .to_string(); + + let access_token = oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? + .to_string(); + + let refresh_token = oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let client_id = oauth_data + .get("client_id") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? + .to_string(); + + let client_secret = oauth_data + .get("client_secret") + .and_then(|v| v.as_str()) + .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? + .to_string(); + + Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) +} + +#[inline] +pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { + let mut json = json!({ + "base_url": config.base_url, + "access_token": config.access_token, + "client_id": config.client_id, + "client_secret": config.client_secret, + }); + + if let Some(refresh_token) = &config.refresh_token { + json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); + } + + json +} + +/// Token refresh response +#[cfg(feature = "native_trigger")] +#[derive(Debug, Deserialize)] +struct RefreshTokenResponse { + access_token: String, + refresh_token: Option, +} + +/// Refresh OAuth tokens using windmill-oauth. +#[cfg(feature = "native_trigger")] +pub async fn refresh_oauth_tokens( + oauth_config: &OAuthConfig, + refresh_endpoint: &str, +) -> Result { + let refresh_token_str = oauth_config + .refresh_token + .as_ref() + .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; + + // Build OAuth client for token refresh + // Auth URL is not used for refresh, but required by the client constructor + let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + + let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); + client.set_client_secret(oauth_config.client_secret.clone()); + + let token_response: RefreshTokenResponse = client + .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) + .with_client(&*OAUTH_HTTP_CLIENT) + .execute() + .await + .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; + + Ok(OAuthConfig { + base_url: oauth_config.base_url.clone(), + access_token: token_response.access_token, + refresh_token: token_response + .refresh_token + .or_else(|| oauth_config.refresh_token.clone()), + client_id: oauth_config.client_id.clone(), + client_secret: oauth_config.client_secret.clone(), + }) +} + +/// Fallback refresh without native_triggers feature +#[cfg(not(feature = "native_trigger"))] +pub async fn refresh_oauth_tokens( + _oauth_config: &OAuthConfig, + _refresh_endpoint: &str, +) -> Result { + Err(Error::InternalErr( + "Native triggers feature is not enabled".to_string(), + )) +} + +async fn update_workspace_integration_tokens_helper( + db: DB, + workspace_id: String, + service_name: ServiceName, + oauth_data: serde_json::Value, +) { + let result = async { + let mut tx = db.begin().await?; + let mc = build_crypt(&db, &workspace_id).await?; + let mut encrypted_oauth_data = oauth_data; + + if let Some(access_token) = encrypted_oauth_data + .get("access_token") + .and_then(|v| v.as_str()) + { + let encrypted_access_token = encrypt(&mc, access_token); + encrypted_oauth_data["access_token"] = + serde_json::Value::String(encrypted_access_token); + } + + if let Some(refresh_token) = encrypted_oauth_data + .get("refresh_token") + .and_then(|v| v.as_str()) + { + let encrypted_refresh_token = encrypt(&mc, refresh_token); + encrypted_oauth_data["refresh_token"] = + serde_json::Value::String(encrypted_refresh_token); + } + + sqlx::query!( + r#" + UPDATE workspace_integrations + SET oauth_data = $1, updated_at = now() + WHERE workspace_id = $2 AND service_name = $3 + "#, + encrypted_oauth_data, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok::<(), Error>(()) + } + .await; + + if let Err(e) = result { + tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", + service_name, workspace_id, e); + } +} + +/// Look up the full token from the token table using its prefix +pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result> { + let token = sqlx::query_scalar!( + r#" + SELECT token + FROM token + WHERE token LIKE concat($1::text, '%') + LIMIT 1 + "#, + token_prefix + ) + .fetch_optional(db) + .await?; + + Ok(token) +} + +/// Delete a token from the token table using its prefix +pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + token_prefix: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM token + WHERE token LIKE concat($1::text, '%') + "#, + token_prefix + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: C, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + INSERT INTO native_trigger ( + external_id, + workspace_id, + service_name, + script_path, + is_flow, + webhook_token_prefix, + service_config + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7 + ) + ON CONFLICT (external_id, workspace_id, service_name) + DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() + "#, + external_id, + workspace_id, + service_name as ServiceName, + config.script_path, + config.is_flow, + webhook_token_prefix, + sqlx::types::Json(service_config) as _, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + config: &NativeTriggerConfig, + service_config: Option<&RawValue>, +) -> Result<()> { + // Store only the first 10 characters of the webhook token as a prefix + let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); + + sqlx::query!( + r#" + UPDATE native_trigger + SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() + WHERE + workspace_id = $5 + AND service_name = $6 + AND external_id = $7 + "#, + config.script_path, + config.is_flow, + webhook_token_prefix, + service_config.map(sqlx::types::Json) as _, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await? + .rows_affected(); + + Ok(deleted > 0) +} +pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND external_id = $3 + "#, + workspace_id, + service_name as ServiceName, + external_id + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + script_path: &str, + is_flow: bool, +) -> Result> { + let trigger = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + external_id, + workspace_id, + service_name AS "service_name!: ServiceName", + script_path, + is_flow, + webhook_token_prefix, + service_config, + error, + created_at, + updated_at + FROM + native_trigger + WHERE + workspace_id = $1 + AND service_name = $2 + AND script_path = $3 + AND is_flow = $4 + LIMIT 1 + "#, + workspace_id, + service_name as ServiceName, + script_path, + is_flow + ) + .fetch_optional(db) + .await?; + + Ok(trigger) +} + +pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + page: Option, + per_page: Option, + path: Option<&str>, + is_flow: Option, +) -> Result> { + let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; + let limit = per_page.unwrap_or(100) as i64; + + let triggers = sqlx::query_as!( + NativeTrigger, + r#" + SELECT + nt.external_id, + nt.workspace_id, + nt.service_name AS "service_name!: ServiceName", + nt.script_path, + nt.is_flow, + nt.webhook_token_prefix, + nt.service_config, + nt.error, + nt.created_at, + nt.updated_at + FROM + native_trigger nt + WHERE + nt.workspace_id = $1 AND + nt.service_name = $2 AND + ($5::text IS NULL OR nt.script_path = $5) AND + ($6::bool IS NULL OR nt.is_flow = $6) AND + ( + (nt.is_flow = false AND EXISTS ( + SELECT 1 FROM script s + WHERE s.workspace_id = nt.workspace_id + AND s.path = nt.script_path + )) + OR + (nt.is_flow = true AND EXISTS ( + SELECT 1 FROM flow f + WHERE f.workspace_id = nt.workspace_id + AND f.path = nt.script_path + )) + ) + LIMIT $3 + OFFSET $4 + "#, + workspace_id, + service_name as ServiceName, + limit, + offset, + path, + is_flow + ) + .fetch_all(db) + .await?; + + Ok(triggers) +} + +pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + error: Option<&str>, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET error = $1 + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + error, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn update_native_trigger_service_config< + 'c, + E: sqlx::Executor<'c, Database = Postgres>, +>( + db: E, + workspace_id: &str, + service_name: ServiceName, + external_id: &str, + service_config: &serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + UPDATE native_trigger + SET service_config = $1, updated_at = NOW() + WHERE + workspace_id = $2 + AND service_name = $3 + AND external_id = $4 + "#, + service_config, + workspace_id, + service_name as ServiceName, + external_id, + ) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn store_workspace_integration( + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + service_name: ServiceName, + oauth_data: serde_json::Value, +) -> Result<()> { + sqlx::query!( + r#" + INSERT INTO workspace_integrations ( + workspace_id, + service_name, + oauth_data, + created_by, + created_at, + updated_at + ) VALUES ( + $1, $2, $3, $4, now(), now() + ) + ON CONFLICT (workspace_id, service_name) + DO UPDATE SET + oauth_data = $3, + updated_at = now() + "#, + workspace_id, + service_name as ServiceName, + oauth_data, + authed.username, + ) + .execute(&mut *tx) + .await?; + + Ok(()) +} + +pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( + db: E, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let integration = sqlx::query_as!( + WorkspaceIntegration, + r#" + SELECT + workspace_id, + service_name AS "service_name!: ServiceName", + oauth_data, + created_at, + updated_at, + created_by + FROM + workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .fetch_one(db) + .await?; + + Ok(integration) +} + +pub async fn delete_workspace_integration( + tx: &mut PgConnection, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let deleted = sqlx::query!( + r#" + DELETE FROM workspace_integrations + WHERE + workspace_id = $1 + AND service_name = $2 + "#, + workspace_id, + service_name as ServiceName, + ) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) +} + +/// Generates the webhook URL that external services will call. +/// +/// `external_id` is optional because during CREATE we don't have it yet +/// (it's returned by the external service). During UPDATE, we have it. +pub fn generate_webhook_service_url( + base_url: &str, + w_id: &str, + script_path: &str, + is_flow: bool, + external_id: Option<&str>, + service_name: ServiceName, + webhook_token: &str, +) -> String { + let runnable_prefix = if is_flow { "f" } else { "p" }; + + let mut url = format!( + "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", + base_url, + w_id, + runnable_prefix, + script_path, + &webhook_token, + service_name.as_str(), + ); + + if let Some(id) = external_id { + url.push_str(&format!("&trigger_external_id={}", id)); + } + + url +} diff --git a/backend/windmill-triggers/src/native_triggers/nextcloud/external.rs b/backend/windmill-triggers/src/native_triggers/nextcloud/external.rs new file mode 100644 index 0000000000..028ba131c3 --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/nextcloud/external.rs @@ -0,0 +1,304 @@ +use reqwest::Method; +use serde::{Deserialize, Serialize}; +use serde_json::value::to_raw_value; +use sqlx::PgConnection; +use std::collections::HashMap; +use windmill_common::{ + error::{Error, Result}, + BASE_URL, DB, +}; + +use crate::native_triggers::{ + generate_webhook_service_url, + nextcloud::{ + routes, NextCloud, NextCloudOAuthData, NextCloudTriggerData, NextcloudServiceConfig, + OcsResponse, + }, + External, NativeTriggerData, ServiceName, +}; + +lazy_static::lazy_static! { + pub static ref TOKEN_NEEDED: Box = to_raw_value(&serde_json::json!({ + "user_roles": ["owner", "trigger"] + })).unwrap(); +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct FullNextcloudPayload { + pub http_method: String, + pub uri: String, + pub token_needed: Box, + #[serde(flatten)] + service_config: NextcloudServiceConfig, +} + +impl FullNextcloudPayload { + async fn new( + w_id: &str, + external_id: Option<&str>, + webhook_token: &str, + data: &NativeTriggerData, + ) -> FullNextcloudPayload { + let base_url = &*BASE_URL.read().await; + let uri = generate_webhook_service_url( + base_url, + w_id, + &data.script_path, + data.is_flow, + external_id, + ServiceName::Nextcloud, + webhook_token, + ); + + FullNextcloudPayload { + http_method: http::Method::POST.to_string().to_uppercase(), + uri, + token_needed: TOKEN_NEEDED.clone(), + service_config: data.service_config.clone(), + } + } +} + +#[derive(Debug, Deserialize)] +pub struct RegisterWebhookResponse { + pub id: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct User { + pub uid: String, + #[serde(rename = "displayName")] + pub display_name: Option, +} +#[derive(Debug, Serialize, Deserialize)] +pub struct WebhookPayload { + pub event: EventPayload, + pub user: User, + pub time: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct EventPayload { + pub node: Node, + #[serde(rename = "class")] + pub class_name: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Node { + pub id: i64, + pub path: String, +} + +#[async_trait::async_trait] +impl External for NextCloud { + type ServiceConfig = NextcloudServiceConfig; + type TriggerData = NextCloudTriggerData; + type OAuthData = NextCloudOAuthData; + type CreateResponse = RegisterWebhookResponse; + const SERVICE_NAME: ServiceName = ServiceName::Nextcloud; + const DISPLAY_NAME: &'static str = "Nextcloud"; + const SUPPORT_WEBHOOK: bool = true; + const TOKEN_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token"; + const REFRESH_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token"; + + async fn create( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + // During create, we don't have external_id yet (it comes from NextCloud's response) + let full_nextcloud_payload = + FullNextcloudPayload::new(w_id, None, webhook_token, data).await; + + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks", + oauth_data.base_url + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let ocs_response = self + .http_client_request::, _>( + &url, + Method::POST, + w_id, + tx, + db, + Some(headers), + Some(&full_nextcloud_payload), + ) + .await?; + + Ok(ocs_response.ocs.data) + } + + async fn update( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + webhook_token: &str, + data: &NativeTriggerData, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()> { + // During update, we have the external_id so include it in the webhook URL + let full_nextcloud_payload = + FullNextcloudPayload::new(w_id, Some(external_id), webhook_token, data).await; + + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", + oauth_data.base_url, external_id + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let _ = self + .http_client_request::( + &url, + Method::POST, + w_id, + tx, + db, + Some(headers), + Some(&full_nextcloud_payload), + ) + .await?; + + Ok(()) + } + + async fn get( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", + oauth_data.base_url, external_id + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let ocs_response: OcsResponse = self + .http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, Some(headers), None) + .await?; + + Ok(ocs_response.ocs.data) + } + + async fn delete( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result<()> { + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", + oauth_data.base_url, external_id + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let _: serde_json::Value = self + .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, Some(headers), None) + .await + .or_else(|e| match &e { + Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), + _ => Err(e), + })?; + + Ok(()) + } + + async fn exists( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + external_id: &str, + db: &DB, + tx: &mut PgConnection, + ) -> Result { + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", + oauth_data.base_url, external_id + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let _ = self + .http_client_request::( + &url, + Method::GET, + w_id, + tx, + db, + Some(headers), + None, + ) + .await?; + + Ok(true) + } + + async fn list_all( + &self, + w_id: &str, + oauth_data: &Self::OAuthData, + db: &DB, + tx: &mut PgConnection, + ) -> Result> { + let url = format!( + "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks", + oauth_data.base_url + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let ocs_response = self + .http_client_request::>, ()>( + &url, + Method::GET, + w_id, + tx, + db, + Some(headers), + None, + ) + .await?; + + Ok(ocs_response.ocs.data) + } + + fn external_id_and_metadata_from_response( + &self, + resp: &Self::CreateResponse, + ) -> (String, Option) { + (resp.id.to_string(), None) + } + + fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String { + data.id.to_string() + } + + fn additional_routes(&self) -> axum::Router { + routes::nextcloud_routes(self.clone()) + } +} diff --git a/backend/windmill-triggers/src/native_triggers/nextcloud/mod.rs b/backend/windmill-triggers/src/native_triggers/nextcloud/mod.rs new file mode 100644 index 0000000000..fda8ad9f6e --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/nextcloud/mod.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; + +pub mod external; +mod routes; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct NextCloudOAuthData { + pub base_url: String, + pub access_token: String, + pub refresh_token: Option, + pub token_expires_at: Option>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct OcsResponse { + pub ocs: OcsData, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Meta { + status: String, + #[serde(rename = "statuscode")] + status_code: u16, + message: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct OcsData { + pub meta: Meta, + pub data: T, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NextCloudEventType { + pub name: String, + pub description: Option, + pub path: Option, + pub parameters: Option, +} + +#[derive(Copy, Clone)] +pub struct NextCloud; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NextcloudServiceConfig { + pub event: String, + pub event_filter: Option>, + pub user_id_filter: Option, + pub headers: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NextCloudTriggerData { + #[serde(skip_serializing)] + pub id: i64, + #[serde(skip_serializing)] + pub uri: String, + pub event: String, + pub event_filter: Option>, + pub user_id_filter: Option, + pub headers: Option>, +} diff --git a/backend/windmill-triggers/src/native_triggers/nextcloud/routes.rs b/backend/windmill-triggers/src/native_triggers/nextcloud/routes.rs new file mode 100644 index 0000000000..d9f8f37356 --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/nextcloud/routes.rs @@ -0,0 +1,65 @@ +use std::{collections::HashMap, sync::Arc}; + +use axum::{extract::Path, routing::get, Extension, Json, Router}; +use http::Method; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult}, + DB, +}; + +use windmill_api_auth::ApiAuthed; + +use crate::native_triggers::{ + get_workspace_integration, + nextcloud::{NextCloudEventType, OcsResponse}, + External, OAuthConfig, ServiceName, +}; + +async fn list_available_events( + authed: ApiAuthed, + Extension(handler): Extension>, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, +) -> JsonResult> { + let mut tx = user_db.clone().begin(&authed).await?; + let integration = + get_workspace_integration(&mut *tx, &workspace_id, ServiceName::Nextcloud).await?; + + let auth = serde_json::from_value::(integration.oauth_data) + .map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud OAuth data: {}", e)))?; + + let url = format!( + "{}/ocs/v2.php/apps/integration_windmill/api/v1/list/events", + &auth.base_url, + ); + + let mut headers = HashMap::new(); + headers.insert("OCS-APIRequest".to_string(), "true".to_string()); + + let ocs_response = handler + .http_client_request::( + &url, + Method::GET, + &workspace_id, + &mut *tx, + &db, + Some(headers), + None, + ) + .await?; + tx.commit().await?; + + let events = serde_json::from_str(&ocs_response.ocs.data) + .map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud events data: {}", e)))?; + + Ok(Json(events)) +} + +pub fn nextcloud_routes(service: T) -> Router { + let service = Arc::new(service); + Router::new() + .route("/events", get(list_available_events::)) + .layer(Extension(service)) +} diff --git a/backend/windmill-triggers/src/native_triggers/sync.rs b/backend/windmill-triggers/src/native_triggers/sync.rs new file mode 100644 index 0000000000..ca955630aa --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/sync.rs @@ -0,0 +1,416 @@ +use std::collections::HashMap; +use windmill_common::error::Result; +use windmill_common::DB; + +use serde::Serialize; + +use crate::native_triggers::{ + decrypt_oauth_data, list_native_triggers, update_native_trigger_error, + update_native_trigger_service_config, External, ServiceName, +}; + +#[derive(Debug, Serialize)] +pub struct TriggerSyncInfo { + pub external_id: String, + pub script_path: String, + pub action: SyncAction, +} + +#[derive(Debug, Serialize)] +pub enum SyncAction { + ErrorSet(String), + ErrorCleared, + ConfigUpdated, +} + +#[derive(Debug, Serialize)] +pub struct SyncError { + pub resource_path: String, + pub error_message: String, + pub error_type: String, +} + +#[derive(Debug)] +pub struct BackgroundSyncResult { + pub workspaces_processed: usize, + pub total_synced: usize, + pub total_errors: usize, + pub service_results: HashMap, +} + +#[derive(Debug)] +pub struct ServiceSyncResult { + pub synced_triggers: Vec, + pub errors: Vec, +} + +pub async fn sync_all_triggers(db: &DB) -> Result { + tracing::info!("Starting native triggers sync"); + + let mut service_results: HashMap = HashMap::new(); + let mut total_synced = 0; + let mut total_errors = 0; + let mut workspaces_processed = 0; + + // Sync all registered services + // Each service only syncs workspaces that have the corresponding integration configured + #[cfg(feature = "native_trigger")] + { + use crate::native_triggers::nextcloud::NextCloud; + + let (service_name, result) = sync_service_triggers(db, NextCloud).await; + total_synced += result.synced_triggers.len(); + total_errors += result.errors.len(); + service_results.insert(service_name, result); + + // Add new services here: + // use crate::native_triggers::newservice::NewService; + // let (service_name, result) = sync_service_triggers(db, NewService).await; + // total_synced += result.synced_triggers.len(); + // total_errors += result.errors.len(); + // service_results.insert(service_name, result); + } + + // Count unique workspaces processed across all services + for result in service_results.values() { + workspaces_processed += result + .synced_triggers + .iter() + .map(|t| &t.external_id) + .collect::>() + .len(); + } + + let result = + BackgroundSyncResult { workspaces_processed, total_synced, total_errors, service_results }; + + tracing::info!( + "Completed native triggers sync: {} updated, {} errors", + result.total_synced, + result.total_errors + ); + + Ok(result) +} + +async fn sync_service_triggers( + db: &DB, + handler: T, +) -> (ServiceName, ServiceSyncResult) { + let mut all_synced_triggers = Vec::new(); + let mut all_errors = Vec::new(); + + // Only sync workspaces that have the corresponding integration configured + let workspaces_with_integration = match sqlx::query_scalar!( + r#" + SELECT wi.workspace_id + FROM workspace_integrations wi + JOIN workspace w ON w.id = wi.workspace_id + WHERE wi.service_name = $1 + AND wi.oauth_data IS NOT NULL + AND w.deleted = false + "#, + T::SERVICE_NAME as ServiceName + ) + .fetch_all(db) + .await + { + Ok(workspaces) => workspaces, + Err(e) => { + tracing::error!( + "Error querying workspaces with {} integration: {:#}", + T::SERVICE_NAME.as_str(), + e + ); + all_errors.push(SyncError { + resource_path: "database".to_string(), + error_message: format!("Failed to query workspaces: {}", e), + error_type: "database_error".to_string(), + }); + return ( + T::SERVICE_NAME, + ServiceSyncResult { synced_triggers: Vec::new(), errors: all_errors }, + ); + } + }; + + if workspaces_with_integration.is_empty() { + tracing::debug!( + "No workspaces with {} integration configured, skipping sync", + T::SERVICE_NAME.as_str() + ); + return ( + T::SERVICE_NAME, + ServiceSyncResult { synced_triggers: Vec::new(), errors: Vec::new() }, + ); + } + + tracing::info!( + "Found {} workspaces with {} integration configured", + workspaces_with_integration.len(), + T::SERVICE_NAME.as_str() + ); + + for workspace_id in workspaces_with_integration { + let sync_result = sync_workspace_triggers::(db, &workspace_id, &handler).await; + + match sync_result { + Ok((synced_triggers, errors)) => { + all_synced_triggers.extend(synced_triggers); + all_errors.extend(errors); + } + Err(e) => { + tracing::error!( + "Error syncing {} triggers for workspace {}: {:#}", + T::SERVICE_NAME.as_str(), + workspace_id, + e + ); + all_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to sync workspace: {}", e), + error_type: "workspace_sync_error".to_string(), + }); + } + } + } + + ( + T::SERVICE_NAME, + ServiceSyncResult { synced_triggers: all_synced_triggers, errors: all_errors }, + ) +} + +#[cfg(feature = "native_trigger")] +pub async fn sync_workspace_triggers( + db: &DB, + workspace_id: &str, + handler: &T, +) -> Result<(Vec, Vec)> { + tracing::info!( + "Syncing {} triggers for workspace '{}'", + T::SERVICE_NAME.as_str(), + workspace_id + ); + + let windmill_triggers = + list_native_triggers(db, workspace_id, T::SERVICE_NAME, None, None, None, None).await?; + + if windmill_triggers.is_empty() { + tracing::info!( + "No {} triggers found for workspace '{}'", + T::SERVICE_NAME.as_str(), + workspace_id + ); + return Ok((Vec::new(), Vec::new())); + } + + let mut all_synced_triggers = Vec::new(); + let mut all_sync_errors = Vec::new(); + + let oauth_data = { + match decrypt_oauth_data(db, db, workspace_id, T::SERVICE_NAME).await { + Ok(oauth_data) => oauth_data, + Err(e) => { + tracing::error!( + "Failed to get workspace integration OAuth data for {}: {}", + workspace_id, + e + ); + all_sync_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to get workspace integration OAuth data: {}", e), + error_type: "oauth_error".to_string(), + }); + return Ok((Vec::new(), all_sync_errors)); + } + } + }; + + let mut tx = db.begin().await?; + let external_triggers = match handler + .list_all(workspace_id, &oauth_data, db, &mut tx) + .await + { + Ok(triggers) => triggers, + Err(e) => { + tracing::error!( + "Failed to fetch external triggers for {}: {}", + workspace_id, + e + ); + all_sync_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!("Failed to fetch external triggers: {}", e), + error_type: "external_service_error".to_string(), + }); + return Ok((Vec::new(), all_sync_errors)); + } + }; + tx.commit().await?; + + // Build a map of external trigger IDs to their data + let mut external_trigger_map: HashMap = HashMap::new(); + for external_trigger in &external_triggers { + let external_id = handler.get_external_id_from_trigger_data(external_trigger); + external_trigger_map.insert(external_id, external_trigger); + } + + for trigger in &windmill_triggers { + if !external_trigger_map.contains_key(&trigger.external_id) { + // Trigger no longer exists on external service - set error + let error_msg = "Trigger no longer exists on external service".to_string(); + + if trigger.error.as_deref() != Some(&error_msg) { + tracing::info!( + "Trigger (external_id: '{}', script_path: '{}') no longer exists in external service, setting error", + trigger.external_id, + trigger.script_path + ); + + match update_native_trigger_error( + db, + workspace_id, + T::SERVICE_NAME, + &trigger.external_id, + Some(&error_msg), + ) + .await + { + Ok(()) => { + all_synced_triggers.push(TriggerSyncInfo { + external_id: trigger.external_id.clone(), + script_path: trigger.script_path.clone(), + action: SyncAction::ErrorSet(error_msg), + }); + } + Err(e) => { + tracing::error!( + "Failed to update error for trigger (external_id: '{}'): {}", + trigger.external_id, + e + ); + all_sync_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!( + "Failed to update error for trigger (external_id: '{}'): {}", + trigger.external_id, e + ), + error_type: "database_update_error".to_string(), + }); + } + } + } + } else { + // Trigger exists on external service + let external_trigger_data = external_trigger_map.get(&trigger.external_id).unwrap(); + + // Clear error if it was set + if trigger.error.is_some() { + tracing::info!( + "Trigger (external_id: '{}', script_path: '{}') exists on external service, clearing error", + trigger.external_id, + trigger.script_path + ); + + match update_native_trigger_error( + db, + workspace_id, + T::SERVICE_NAME, + &trigger.external_id, + None, + ) + .await + { + Ok(()) => { + all_synced_triggers.push(TriggerSyncInfo { + external_id: trigger.external_id.clone(), + script_path: trigger.script_path.clone(), + action: SyncAction::ErrorCleared, + }); + } + Err(e) => { + tracing::error!( + "Failed to clear error for trigger (external_id: '{}'): {}", + trigger.external_id, + e + ); + all_sync_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!( + "Failed to clear error for trigger (external_id: '{}'): {}", + trigger.external_id, e + ), + error_type: "database_update_error".to_string(), + }); + } + } + } + + // Compare service_config and update if different + let external_service_config = + handler.extract_service_config_from_trigger_data(external_trigger_data)?; + let stored_service_config = trigger + .service_config + .clone() + .unwrap_or(serde_json::Value::Null); + + if external_service_config != stored_service_config { + tracing::info!( + "Trigger (external_id: '{}', script_path: '{}') config differs from external service, updating local config", + trigger.external_id, + trigger.script_path + ); + + match update_native_trigger_service_config( + db, + workspace_id, + T::SERVICE_NAME, + &trigger.external_id, + &external_service_config, + ) + .await + { + Ok(()) => { + all_synced_triggers.push(TriggerSyncInfo { + external_id: trigger.external_id.clone(), + script_path: trigger.script_path.clone(), + action: SyncAction::ConfigUpdated, + }); + } + Err(e) => { + tracing::error!( + "Failed to update config for trigger (external_id: '{}'): {}", + trigger.external_id, + e + ); + all_sync_errors.push(SyncError { + resource_path: format!("workspace:{}", workspace_id), + error_message: format!( + "Failed to update config for trigger (external_id: '{}'): {}", + trigger.external_id, e + ), + error_type: "database_update_error".to_string(), + }); + } + } + } else { + tracing::info!( + "Trigger (external_id: '{}', script_path: '{}') config is the same as external service, no update needed", + trigger.external_id, + trigger.script_path + ); + } + } + } + + tracing::info!( + "Sync completed for {} in workspace '{}'. Updated: {}, Errors: {}", + T::SERVICE_NAME.as_str(), + workspace_id, + all_synced_triggers.len(), + all_sync_errors.len() + ); + + Ok((all_synced_triggers, all_sync_errors)) +} diff --git a/backend/windmill-triggers/src/native_triggers/workspace_integrations.rs b/backend/windmill-triggers/src/native_triggers/workspace_integrations.rs new file mode 100644 index 0000000000..eea1f5faf4 --- /dev/null +++ b/backend/windmill-triggers/src/native_triggers/workspace_integrations.rs @@ -0,0 +1,522 @@ +use axum::{ + extract::Path, + routing::{delete, get, post}, + Extension, Json, Router, +}; + +#[cfg(feature = "native_trigger")] +use serde_json::to_value; +use sqlx::prelude::FromRow; +use strum::IntoEnumIterator; + +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +#[cfg(feature = "native_trigger")] +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::require_admin, + variables::{build_crypt, encrypt}, + DB, +}; + +use windmill_api_auth::ApiAuthed; + +use crate::native_triggers::ServiceName; +#[cfg(feature = "native_trigger")] +use crate::native_triggers::{delete_workspace_integration, store_workspace_integration}; + +#[cfg(feature = "native_trigger")] +use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT}; + +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +type HmacSha256 = Hmac; + +const STATE_EXPIRATION_SECONDS: i64 = 600; // 10 minutes + +/// Generate a signed OAuth state that is cluster-safe. +/// The state contains: workspace_id, service_name, timestamp, and nonce. +/// It's signed with HMAC-SHA256 using the workspace key. +#[cfg(feature = "native_trigger")] +async fn generate_signed_state( + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + use windmill_common::variables::get_workspace_key; + + let nonce = uuid::Uuid::new_v4().to_string(); + let timestamp = chrono::Utc::now().timestamp(); + let payload = format!( + "{}:{}:{}:{}", + workspace_id, + service_name.as_str(), + timestamp, + nonce + ); + + // Get workspace key for signing + let key = get_workspace_key(workspace_id, db).await?; + let mut mac = + HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + mac.update(payload.as_bytes()); + let signature = mac.finalize().into_bytes(); + + // Encode as: base64(payload):base64(signature) + let encoded_payload = URL_SAFE_NO_PAD.encode(payload.as_bytes()); + let encoded_signature = URL_SAFE_NO_PAD.encode(signature); + + Ok(format!("{}:{}", encoded_payload, encoded_signature)) +} + +/// Validate a signed OAuth state. +/// Returns true if the state is valid (correct signature and not expired). +#[cfg(feature = "native_trigger")] +async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Result { + use windmill_common::variables::get_workspace_key; + + let parts: Vec<&str> = state.split(':').collect(); + if parts.len() != 2 { + return Ok(false); + } + + let encoded_payload = parts[0]; + let encoded_signature = parts[1]; + + // Decode payload + let payload_bytes = match URL_SAFE_NO_PAD.decode(encoded_payload) { + Ok(bytes) => bytes, + Err(_) => return Ok(false), + }; + let payload = match String::from_utf8(payload_bytes) { + Ok(s) => s, + Err(_) => return Ok(false), + }; + + // Parse payload: workspace_id:service_name:timestamp:nonce + let payload_parts: Vec<&str> = payload.split(':').collect(); + if payload_parts.len() != 4 { + return Ok(false); + } + + let state_workspace_id = payload_parts[0]; + let timestamp: i64 = match payload_parts[2].parse() { + Ok(ts) => ts, + Err(_) => return Ok(false), + }; + + // Verify workspace_id matches + if state_workspace_id != workspace_id { + return Ok(false); + } + + // Check expiration + let now = chrono::Utc::now().timestamp(); + if now - timestamp > STATE_EXPIRATION_SECONDS { + return Ok(false); + } + + // Verify signature + let key = get_workspace_key(workspace_id, db).await?; + let mut mac = + HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; + mac.update(payload.as_bytes()); + + let received_signature = match URL_SAFE_NO_PAD.decode(encoded_signature) { + Ok(bytes) => bytes, + Err(_) => return Ok(false), + }; + + Ok(mac.verify_slice(&received_signature).is_ok()) +} + +#[derive(Debug, Serialize)] +pub struct IntegrationStatusResponse { + pub connected: bool, + pub service_name: ServiceName, + pub created_at: Option>, + pub created_by: Option, +} + +#[derive(Debug, Serialize)] +pub struct ListIntegrationsResponse { + pub integrations: Vec, +} + +#[derive(Debug, Serialize)] +pub struct ConnectIntegrationResponse { + pub auth_url: String, +} + +#[derive(FromRow, Debug, Clone, Serialize, Deserialize)] +pub struct WorkspaceOAuthConfig { + pub client_id: String, + pub client_secret: String, + pub base_url: String, + pub access_token: Option +} + +#[derive(Debug, Serialize)] +pub struct OAuthConfigResponse { + pub configured: bool, + pub base_url: Option, + pub redirect_uri: Option, +} + +#[cfg(feature = "native_trigger")] +async fn generate_connect_url( + authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, service_name)): Path<(String, ServiceName)>, + Json(RedirectUri { redirect_uri }): Json, +) -> JsonResult { + require_admin(authed.is_admin, &workspace_id)?; + + let oauth_config = + get_workspace_oauth_config_as_oauth_config(&db, &workspace_id, service_name).await?; + + // Generate a signed state that is cluster-safe + let state = generate_signed_state(&db, &workspace_id, service_name).await?; + let auth_url = build_authorization_url(&oauth_config, &state, &redirect_uri); + Ok(Json(auth_url)) +} + +#[cfg(feature = "native_trigger")] +async fn delete_integration( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, service_name)): Path<(String, ServiceName)>, +) -> JsonResult { + require_admin(authed.is_admin, &workspace_id)?; + + let mut tx = user_db.begin(&authed).await?; + + let deleted = delete_workspace_integration(&mut *tx, &workspace_id, service_name).await?; + + if !deleted { + return Err(Error::NotFound(format!( + "{} integration not found for workspace", + service_name + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!("workspace_integrations.{}.disconnect", service_name), + ActionKind::Delete, + &workspace_id, + Some(&format!("Disconnected {} integration", service_name)), + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(format!( + "{} integration disconnected successfully", + service_name + ))) +} + +#[derive(FromRow, Debug, Deserialize, Serialize)] +struct WorkspaceIntegrations { + service_name: ServiceName, + oauth_data: Option>, +} + +#[cfg(feature = "native_trigger")] +async fn list_integrations( + authed: ApiAuthed, + Extension(db): Extension, + Extension(_user_db): Extension, + Path(workspace_id): Path, +) -> JsonResult> { + require_admin(authed.is_admin, &workspace_id)?; + let mut tx = db.begin().await?; + let integrations = sqlx::query_as!( + WorkspaceIntegrations, + r#" + SELECT + oauth_data as "oauth_data!: sqlx::types::Json", + service_name as "service_name!: ServiceName" + FROM + workspace_integrations + WHERE + workspace_id = $1 + "#, + workspace_id + ) + .fetch_all(&mut *tx) + .await?; + + let key_value = integrations + .into_iter() + .map(|integration| (integration.service_name, integration.oauth_data)) + .collect::>(); + + let integrations = ServiceName::iter() + .map(|service_name| WorkspaceIntegrations { + service_name: service_name, + oauth_data: key_value.get(&service_name).cloned().flatten(), + }) + .collect::>(); + + tx.commit().await?; + + Ok(Json(integrations)) +} + +async fn integration_exist( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, service_name)): Path<(String, ServiceName)>, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + r#" + SELECT EXISTS ( + SELECT 1 + FROM workspace_integrations + WHERE workspace_id = $1 + AND service_name = $2 + AND oauth_data IS NOT NULL + ) + "#, + workspace_id, + service_name as ServiceName + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + + Ok(Json(exists)) +} + +#[derive(Debug, Deserialize)] +struct RedirectUri { + redirect_uri: String, +} + +#[cfg(feature = "native_trigger")] +async fn oauth_callback( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, service_name, code, state)): Path<(String, ServiceName, String, String)>, + Json(RedirectUri { redirect_uri }): Json, +) -> JsonResult { + require_admin(authed.is_admin, &workspace_id)?; + + // Validate the signed state (cluster-safe, no DB storage needed) + let state_was_valid = validate_signed_state(&db, &state, &workspace_id).await?; + + if !state_was_valid { + return Err(Error::BadRequest( + "Invalid or expired state parameter".to_string(), + )); + } + + let oauth_config = + get_workspace_oauth_config::(&db, &workspace_id, service_name) + .await?; + + let token_response = + exchange_code_for_token(&oauth_config, service_name, &code, &redirect_uri).await?; + + let mut tx = user_db.begin(&authed).await?; + + let mc = build_crypt(&db, &workspace_id).await?; + let mut oauth_data = serde_json::to_value(oauth_config).unwrap(); + + let encrypted_access_token = encrypt(&mc, &token_response.access_token); + oauth_data["access_token"] = serde_json::Value::String(encrypted_access_token); + + if let Some(refresh_token) = token_response.refresh_token { + let encrypted_refresh_token = encrypt(&mc, &refresh_token); + oauth_data["refresh_token"] = serde_json::Value::String(encrypted_refresh_token); + } + if let Some(expires_in) = token_response.expires_in { + let expires_at = chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64); + oauth_data["token_expires_at"] = serde_json::Value::String(expires_at.to_rfc3339()); + } + + store_workspace_integration(&mut *tx, &authed, &workspace_id, service_name, oauth_data).await?; + + audit_log( + &mut *tx, + &authed, + &format!("workspace_integrations.{}.connect", service_name), + ActionKind::Create, + &workspace_id, + Some(&format!("Connected {} integration via OAuth", service_name)), + None, + ) + .await?; + + tx.commit().await?; + + Ok(Json(format!( + "{} integration connected successfully via OAuth", + service_name + ))) +} + +/// Token response from OAuth token exchange +#[derive(Debug, Deserialize)] +struct TokenResponse { + access_token: String, + refresh_token: Option, + #[serde(default)] + expires_in: Option, +} + +/// Build an OAuth client for native trigger services using windmill-oauth. +#[cfg(feature = "native_trigger")] +fn build_native_oauth_client( + config: &WorkspaceOAuthConfig, + service_name: ServiceName, + redirect_uri: &str, +) -> Result { + let auth_url = Url::parse(&format!("{}{}", config.base_url, service_name.auth_endpoint())) + .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; + let token_url = Url::parse(&format!("{}{}", config.base_url, service_name.token_endpoint())) + .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; + let redirect = Url::parse(redirect_uri).map_err(|e| { + Error::BadRequest(format!( + "Invalid redirect URI '{}': {}. The redirect URI must be an absolute URL (e.g., https://example.com/callback)", + redirect_uri, e + )) + })?; + + let mut client = OClient::new(config.client_id.clone(), auth_url, token_url); + client.set_client_secret(config.client_secret.clone()); + client.set_redirect_url(redirect); + + Ok(client) +} + +/// Exchange authorization code for tokens using windmill-oauth. +#[cfg(feature = "native_trigger")] +async fn exchange_code_for_token( + config: &WorkspaceOAuthConfig, + service_name: ServiceName, + code: &str, + redirect_uri: &str, +) -> Result { + let client = build_native_oauth_client(config, service_name, redirect_uri)?; + + let token_response: TokenResponse = client + .exchange_code(code.to_string()) + .with_client(&*OAUTH_HTTP_CLIENT) + .execute() + .await + .map_err(|e| Error::InternalErr(format!("Failed to exchange code for token: {:?}", e)))?; + + Ok(token_response) +} + +async fn get_workspace_oauth_config( + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + let oauth_configs = sqlx::query_scalar!( + r#" + SELECT + oauth_data + FROM + workspace_integrations + WHERE + workspace_id = $1 AND + service_name = $2 + "#, + workspace_id, + service_name as ServiceName + ) + .fetch_optional(db) + .await? + .ok_or(Error::NotFound(format!( + "Integration for service {} not found", + service_name.as_str() + )))?; + + let config = serde_json::from_value::(oauth_configs) + .map_err(|e| Error::InternalErr(format!("Failed to parse OAuth config: {}", e)))?; + + Ok(config) +} + +#[cfg(feature = "native_trigger")] +pub async fn create_workspace_integration( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, service_name)): Path<(String, ServiceName)>, + Json(oauth_data): Json, +) -> Result<()> { + require_admin(authed.is_admin, &workspace_id)?; + + let mut tx = user_db.begin(&authed).await?; + + store_workspace_integration( + &mut tx, + &authed, + &workspace_id, + service_name, + to_value(oauth_data).unwrap(), + ) + .await?; + + tx.commit().await?; + + Ok(()) +} + +#[inline] +async fn get_workspace_oauth_config_as_oauth_config( + db: &DB, + workspace_id: &str, + service_name: ServiceName, +) -> Result { + get_workspace_oauth_config::(db, workspace_id, service_name).await +} + +fn build_authorization_url( + config: &WorkspaceOAuthConfig, + state: &str, + redirect_uri: &str, +) -> String { + let params = [ + ("response_type", "code"), + ("client_id", &config.client_id), + ("redirect_uri", redirect_uri), + ("state", state), + ("scope", "read write"), + ]; + + let query_string = params + .iter() + .map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(v))) + .collect::>() + .join("&"); + + format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string) +} + +pub fn workspaced_service() -> Router { + let router = Router::new() + .route("/list", get(list_integrations)) + .route("/:service_name/exists", get(integration_exist)) + .route("/:service_name/create", post(create_workspace_integration)) + .route( + "/:service_name/generate_connect_url", + post(generate_connect_url), + ) + .route("/:service_name/delete", delete(delete_integration)) + .route("/:service_name/callback/:code/:state", post(oauth_callback)); + + Router::new().nest("/integrations", router) +} diff --git a/backend/windmill-triggers/src/resource_ext.rs b/backend/windmill-triggers/src/resource_ext.rs new file mode 100644 index 0000000000..70f81ce19a --- /dev/null +++ b/backend/windmill-triggers/src/resource_ext.rs @@ -0,0 +1,36 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +// This module provides try_get_resource_from_db_as via the JobOps trait. +// The actual implementation lives in windmill-api and is injected at runtime. + +use windmill_api_auth::ApiAuthed; +use windmill_common::{db::UserDB, error::Result, DB}; + +use crate::jobs_ext::get_ops; + +pub async fn try_get_resource_from_db_as( + authed: &ApiAuthed, + user_db: Option, + db: &DB, + resource_path: &str, + w_id: &str, +) -> Result +where + T: serde::de::DeserializeOwned, +{ + let ops = get_ops(); + let value = ops + .try_get_resource_from_db(authed, user_db, db, resource_path, w_id) + .await?; + serde_json::from_value(value).map_err(|e| { + windmill_common::error::Error::internal_err(format!( + "Error deserializing resource {resource_path}: {e}" + )) + }) +} diff --git a/backend/windmill-triggers/src/script_ext.rs b/backend/windmill-triggers/src/script_ext.rs new file mode 100644 index 0000000000..04915e340c --- /dev/null +++ b/backend/windmill-triggers/src/script_ext.rs @@ -0,0 +1,24 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use windmill_api_auth::ApiAuthed; +use windmill_common::{error::Result, DB}; + +use crate::require_is_writer_internal; + +pub async fn require_is_writer(authed: &ApiAuthed, path: &str, w_id: &str, db: DB) -> Result<()> { + require_is_writer_internal( + authed, + path, + w_id, + db, + "SELECT extra_perms FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1", + "script", + ) + .await +} diff --git a/backend/windmill-triggers/src/triggers/email/handler_oss.rs b/backend/windmill-triggers/src/triggers/email/handler_oss.rs new file mode 100644 index 0000000000..b019fd3e14 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/email/handler_oss.rs @@ -0,0 +1,66 @@ +#[cfg(not(feature = "private"))] +use crate::triggers::TriggerData; + +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::EmailTrigger, + crate::triggers::TriggerCrud, + windmill_api_auth::ApiAuthed, + windmill_common::DB, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for EmailTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/email_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::EmailTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Email triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Email triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-triggers/src/triggers/email/mod.rs b/backend/windmill-triggers/src/triggers/email/mod.rs new file mode 100644 index 0000000000..836af77fa0 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/email/mod.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct EmailTrigger; diff --git a/backend/windmill-triggers/src/triggers/filter.rs b/backend/windmill-triggers/src/triggers/filter.rs new file mode 100644 index 0000000000..553e0a70f5 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/filter.rs @@ -0,0 +1,144 @@ +use serde::{ + de::{self, MapAccess, Visitor}, + Deserialize, Deserializer, +}; +use serde_json::Value; +use std::fmt; + +#[derive(Deserialize)] +pub struct JsonFilter { + pub key: String, + pub value: Value, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Filter { + JsonFilter(JsonFilter), +} + +struct SupersetVisitor<'a> { + key: &'a str, + value_to_check: &'a Value, +} + +impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object with a specific key at the top level") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: MapAccess<'de>, + { + let mut result = false; + let mut found = false; + + // Must consume entire map to satisfy deserializer contract + while let Some(key) = map.next_key::()? { + if !found && key == self.key { + let json_value: Value = map.next_value()?; + result = is_superset(&json_value, self.value_to_check); + found = true; + } else { + // Skip values we don't need (cheaper than full deserialization) + let _ = map.next_value::()?; + } + } + Ok(result) + } +} + +pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { + match (json_value, value_to_check) { + (Value::Object(json_map), Value::Object(check_map)) => { + check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }) + } + (Value::Array(json_array), Value::Array(check_array)) => { + check_array.iter().all(|check_item| { + json_array + .iter() + .any(|json_item| is_superset(json_item, check_item)) + }) + } + _ => json_value == value_to_check, + } +} + +pub fn is_value_superset<'a, 'de, D>( + deserializer: D, + key: &'a str, + value_to_check: &'a Value, +) -> std::result::Result +where + D: Deserializer<'de>, +{ + deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_filter_with_other_top_level_keys() { + let payload = r#"{"event_type": "test", "other": "data"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match when key exists with correct value"); + } + + #[test] + fn test_filter_with_key_not_first() { + let payload = r#"{"other": "data", "event_type": "test"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match even when key is not first"); + } + + #[test] + fn test_filter_with_nested_object() { + let payload = r#"{"data": {"status": "active", "count": 5}, "other": "value"}"#; + let key = "data"; + let value = json!({"status": "active"}); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(result, "Should match when nested object is superset"); + } + + #[test] + fn test_filter_no_match() { + let payload = r#"{"event_type": "other", "data": "value"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(!result, "Should not match when value differs"); + } + + #[test] + fn test_filter_key_not_found() { + let payload = r#"{"other": "data"}"#; + let key = "event_type"; + let value = json!("test"); + + let mut deserializer = serde_json::Deserializer::from_str(payload); + let result = is_value_superset(&mut deserializer, key, &value).unwrap(); + assert!(!result, "Should not match when key doesn't exist"); + } +} diff --git a/backend/windmill-triggers/src/triggers/gcp/handler_oss.rs b/backend/windmill-triggers/src/triggers/gcp/handler_oss.rs new file mode 100644 index 0000000000..1b0b8e7ebe --- /dev/null +++ b/backend/windmill-triggers/src/triggers/gcp/handler_oss.rs @@ -0,0 +1,63 @@ +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::GcpTrigger, + crate::triggers::{TriggerCrud, TriggerData}, + windmill_api_auth::ApiAuthed, + windmill_common::DB, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for GcpTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/gcp_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::GcpTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "GCP triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "GCP triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-triggers/src/triggers/gcp/listener_oss.rs b/backend/windmill-triggers/src/triggers/gcp/listener_oss.rs new file mode 100644 index 0000000000..a14168968a --- /dev/null +++ b/backend/windmill-triggers/src/triggers/gcp/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::GcpTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for GcpTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-triggers/src/triggers/gcp/mod.rs b/backend/windmill-triggers/src/triggers/gcp/mod.rs new file mode 100644 index 0000000000..dd2f2a65df --- /dev/null +++ b/backend/windmill-triggers/src/triggers/gcp/mod.rs @@ -0,0 +1,16 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Clone, Copy)] +pub struct GcpTrigger; diff --git a/backend/windmill-triggers/src/triggers/global_handler.rs b/backend/windmill-triggers/src/triggers/global_handler.rs new file mode 100644 index 0000000000..348fb1e95e --- /dev/null +++ b/backend/windmill-triggers/src/triggers/global_handler.rs @@ -0,0 +1,325 @@ +use crate::{ + jobs_ext::cancel_jobs, + triggers::trigger_helpers::trigger_runnable_inner, +}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; +use axum::{ + extract::{Extension, Path}, + response::Json, +}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::PgConnection; +use std::collections::HashMap; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error::{self, Error, Result}, + jobs::JobTriggerKind, + triggers::TriggerMetadata, +}; + +#[derive(sqlx::FromRow)] +pub struct SuspendedTrigger { + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: DateTime, + pub error_handler_path: Option, + pub error_handler_args: Option>>, + pub retry: Option>, +} + +async fn get_suspended_trigger( + tx: &mut PgConnection, + workspace_id: &str, + trigger_kind: &JobTriggerKind, + path: &str, +) -> Result { + match trigger_kind { + JobTriggerKind::Webhook | JobTriggerKind::Schedule => { + return Err(Error::BadRequest(format!( + "{} triggers do not support job reassignment", + trigger_kind + ))); + } + _ => {} + } + + let table_name = format!("{}_trigger", trigger_kind.to_string()); + + let fields = vec![ + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "error_handler_path", + "error_handler_args", + "retry", + ]; + + let sql = format!( + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND + path = $2 + "#, + fields.join(", "), + table_name + ); + + sqlx::query_as(&sql) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) +} + +struct JobWithArgs { + id: Uuid, + args: Option>>>, + created_at: chrono::DateTime, +} + +#[derive(Deserialize, Serialize, Default)] +pub struct ReassignJobsBody { + #[serde(default)] + pub job_ids: Option>, +} + +pub async fn resume_suspended_trigger_jobs( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>, + Json(body): Json, +) -> error::Result> { + let mut tx = user_db.clone().begin(&authed).await?; + + let trigger = get_suspended_trigger(&mut *tx, &w_id, &trigger_kind, &trigger_path).await?; + + let jobs = if let Some(job_ids) = body.job_ids.as_ref() { + if job_ids.is_empty() { + vec![] + } else { + sqlx::query_as!( + JobWithArgs, + r#" + SELECT + id, + args as "args: _", + created_at + FROM v2_job + WHERE workspace_id = $1 + AND ( + kind = 'unassigned_script'::JOB_KIND OR + kind = 'unassigned_flow'::JOB_KIND OR + kind = 'unassigned_singlestepflow'::JOB_KIND + ) + AND trigger_kind = $2 + AND trigger = $3 + AND id = ANY($4) + "#, + w_id, + trigger_kind as _, + trigger_path, + job_ids as _, + ) + .fetch_all(&mut *tx) + .await? + } + } else { + sqlx::query_as!( + JobWithArgs, + r#" + SELECT + id, + args as "args: _", + created_at + FROM v2_job + WHERE workspace_id = $1 + AND ( + kind = 'unassigned_script'::JOB_KIND OR + kind = 'unassigned_flow'::JOB_KIND OR + kind = 'unassigned_singlestepflow'::JOB_KIND + ) + AND trigger_kind = $2 + AND trigger = $3 + "#, + w_id, + trigger_kind as _, + trigger_path, + ) + .fetch_all(&mut *tx) + .await? + }; + + let trigger_metadata = TriggerMetadata::new(Some(trigger_path.clone()), trigger_kind); + + let l = jobs.len(); + + for job in jobs { + // If job was created before trigger was edited, simply update it to unsuspend + // instead of deleting and repushing + if job.created_at > trigger.edited_at { + let job_kind = if trigger.is_flow { + windmill_common::jobs::JobKind::Flow + } else { + windmill_common::jobs::JobKind::Script + }; + + sqlx::query!( + "UPDATE v2_job SET kind = $1 WHERE id = $2", + job_kind as _, + job.id + ) + .execute(&mut *tx) + .await?; + + // Update the job to unsuspend it and set the correct kind + sqlx::query!( + "UPDATE v2_job_queue SET scheduled_for = now() WHERE id = $1", + job.id + ) + .execute(&mut *tx) + .await?; + } else { + // Job was created after trigger edit - delete and repush with new configuration + // Pass the transaction to trigger_runnable_inner so everything is in the same transaction + let (_uuid, _delete_after_use, _early_return, tx_o) = trigger_runnable_inner( + &db, + Some(tx), + Some(user_db.clone()), + authed.clone(), + &w_id, + &trigger.script_path, + trigger.is_flow, + windmill_queue::PushArgsOwned { + extra: None, + args: job.args.map(|a| a.0).unwrap_or_default(), + }, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + trigger_path.clone(), + None, + trigger_metadata.clone(), + None, + ) + .await?; + + tx = match tx_o { + Some(tx) => tx, + None => { + return Err(error::Error::internal_err( + "Transaction should be returned when passed in".to_string(), + )); + } + }; + + // Delete the unassigned job from all related tables + sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM v2_job_runtime WHERE id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM concurrency_key WHERE job_id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM debounce_key WHERE job_id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM debounce_stale_data WHERE job_id = $1", job.id) + .execute(&mut *tx) + .await?; + + sqlx::query!("DELETE FROM v2_job WHERE id = $1", job.id) + .execute(&mut *tx) + .await?; + } + } + + tx.commit().await?; + + Ok(Json(format!("Reassigned {} jobs", l))) +} + +pub async fn cancel_suspended_trigger_jobs( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>, + Json(body): Json, +) -> error::Result> { + let mut tx = user_db.clone().begin(&authed).await?; + + // Get the list of job IDs to cancel + let jobs_to_cancel = if let Some(job_ids) = body.job_ids.as_ref() { + if job_ids.is_empty() { + vec![] + } else { + sqlx::query_scalar!( + "SELECT id FROM v2_job + WHERE workspace_id = $1 + AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND) + AND trigger_kind = $2 + AND trigger = $3 + AND id = ANY($4)", + w_id, + trigger_kind as _, + trigger_path, + job_ids as _, + ) + .fetch_all(&mut *tx) + .await? + } + } else { + sqlx::query_scalar!( + "SELECT id FROM v2_job + WHERE workspace_id = $1 + AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND) + AND trigger_kind = $2 + AND trigger = $3", + w_id, + trigger_kind as _, + trigger_path, + ) + .fetch_all(&mut *tx) + .await? + }; + + tx.commit().await?; + + let count = jobs_to_cancel.len(); + + if count > 0 { + let cancelled_jobs = cancel_jobs( + jobs_to_cancel, + &db, + authed.username.as_str(), + w_id.as_str(), + true, + ) + .await?; + Ok(Json(format!("Canceled {} jobs", cancelled_jobs.0.len()))) + } else { + Ok(Json(format!("No jobs to cancel"))) + } +} diff --git a/backend/windmill-triggers/src/triggers/handler.rs b/backend/windmill-triggers/src/triggers/handler.rs new file mode 100644 index 0000000000..8ff45c3105 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/handler.rs @@ -0,0 +1,1001 @@ +use crate::triggers::{StandardTriggerQuery, TriggerData, TriggerMode}; +use windmill_api_auth::ApiAuthed; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::{FromRow, PgConnection}; +use std::fmt::Debug; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{paginate, Pagination, StripPath}, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::DeployedObject; + +use axum::{ + extract::{Path, Query}, + http::StatusCode, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use std::sync::Arc; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_git_sync::handle_deployment_metadata; + +use windmill_api_auth::scopes::check_scopes; + +#[async_trait] +pub trait TriggerCrud: Send + Sync + 'static { + type Trigger: Serialize + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Send + + Sync + + Unpin; + + type TriggerConfig: Debug + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Serialize + + Send + + Sync + + Unpin; + + type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; + type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + + const TABLE_NAME: &'static str; + const TRIGGER_TYPE: &'static str; + const SUPPORTS_SERVER_STATE: bool; + const SUPPORTS_TEST_CONNECTION: bool; + const ROUTE_PREFIX: &'static str; + const DEPLOYMENT_NAME: &'static str; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; + const IS_ALLOWED_ON_CLOUD: bool; + + fn get_deployed_object(path: String) -> DeployedObject; + + async fn validate_new( + &self, + db: &DB, + workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + self.validate_config(db, new, workspace_id).await + } + + async fn validate_edit( + &self, + db: &DB, + workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + self.validate_config(db, edit, workspace_id).await + } + + async fn validate_config( + &self, + _db: &DB, + _config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + Ok(()) + } + + fn scope_domain_name() -> &'static str { + &Self::ROUTE_PREFIX[1..] + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn test_connection( + &self, + _db: &DB, + _authed: &ApiAuthed, + _user_db: &UserDB, + _workspace_id: &str, + _config: Self::TestConnectionConfig, + ) -> Result<()> { + Err( + anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) + .into(), + ) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn get_trigger_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let sql = format!( + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND + path = $2 + "#, + fields.join(", "), + Self::TABLE_NAME + ); + + sqlx::query_as(&sql) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) + } + + async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { + let exists = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .fetch_one(db) + .await?; + + Ok(exists) + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) + } + + async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { + Ok(()) + } + + async fn set_trigger_mode( + &self, + authed: &ApiAuthed, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + mode: &TriggerMode, + ) -> Result { + let updated = if Self::SUPPORTS_SERVER_STATE { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + } else { + sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = $1, + email = $2, + edited_by = $3, + edited_at = now() + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(mode) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected() + }; + + self.set_trigger_mode_extra_action(&mut *tx).await?; + + Ok(updated > 0) + } + + #[allow(unused)] + async fn trigger_count( + &self, + tx: &mut PgConnection, + workspace_id: &str, + is_flow: bool, + script_path: &str, + ) -> i64 { + let count = sqlx::query_scalar(&format!( + r#" + SELECT + COUNT(*) + FROM + {} + WHERE + workspace_id = $1 AND + is_flow = $2 AND + script_path = $3 + "#, + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(is_flow) + .bind(script_path) + .fetch_one(&mut *tx) + .await + .unwrap_or(0); + + count + } + + async fn list_triggers( + &self, + tx: &mut PgConnection, + workspace_id: &str, + query: Option<&StandardTriggerQuery>, + ) -> Result> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&workspace_id)); + + if let Some(query) = query { + let (per_page, offset) = + paginate(Pagination { per_page: query.per_page, page: query.page }); + if let Some(path) = &query.path { + sqlb.and_where_eq("script_path", "?".bind(path)); + } + + if let Some(is_flow) = query.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + + if let Some(path_start) = &query.path_start { + sqlb.and_where_like_left("path", path_start); + } + + sqlb.offset(offset).limit(per_page); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; + + Ok(triggers) + } +} + +pub fn trigger_routes() -> Router { + let mut router = Router::new() + .route("/create", post(create_trigger::)) + .route("/list", get(list_triggers::)) + .route("/get/*path", get(get_trigger::)) + .route("/update/*path", post(update_trigger::)) + .route("/delete/*path", delete(delete_trigger::)) + .route("/exists/*path", get(exists_trigger::)) + .route("/setmode/*path", post(set_trigger_mode::)); + + if T::SUPPORTS_TEST_CONNECTION { + router = router.route("/test", post(test_connection::)); + } + + router +} + +async fn create_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(new_trigger): Json>, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &new_trigger.base.path + ) + })?; + + if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { + return Err(Error::BadRequest(format!( + "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", + T::TRIGGER_TYPE + ))); + } + + handler + .validate_new(&db, &workspace_id, &new_trigger.config) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = new_trigger.base.path.clone(); + + handler + .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.create", T::TRIGGER_TYPE), + ActionKind::Create, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, new_path)) +} + +async fn list_triggers( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path(workspace_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let triggers = handler + .list_triggers(&mut *tx, &workspace_id, Some(&query)) + .await?; + tx.commit().await?; + + Ok(Json(triggers)) +} + +async fn get_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let trigger = handler + .get_trigger_by_path(&mut *tx, &workspace_id, path) + .await?; + + tx.commit().await?; + + Ok(Json(trigger)) +} + +async fn update_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(edit_trigger): Json>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &edit_trigger.base.path + ) + })?; + + handler + .validate_edit(&db, &workspace_id, &edit_trigger.config, path) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = edit_trigger.base.path.to_string(); + + handler + .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.update", T::TRIGGER_TYPE), + ActionKind::Update, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), + true, + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' updated", path)) +} + +async fn delete_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:write:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let deleted = handler + .delete_by_path(&mut *tx, &workspace_id, path) + .await?; + + if !deleted { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.delete", T::TRIGGER_TYPE), + ActionKind::Delete, + &workspace_id, + Some(&path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' deleted", path)) +} + +async fn exists_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), path) + })?; + let exists = handler.exists(&db, &workspace_id, path).await?; + + Ok(Json(exists)) +} + +#[derive(serde::Deserialize)] +struct SetTriggerModePayload { + mode: TriggerMode, +} + +async fn set_trigger_mode( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; + + let mut tx = user_db.begin(&authed).await?; + let updated = handler + .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) + .await?; + + if !updated { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(path.to_owned()), + Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), + true, + None, + ) + .await?; + + Ok(format!( + "Trigger '{}' {}", + path, + if payload.mode == TriggerMode::Enabled { + "enabled" + } else if payload.mode == TriggerMode::Disabled { + "disabled" + } else { + "suspended" + } + )) +} + +async fn test_connection( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(config): Json, +) -> Result<()> { + let connect_f = async move { + handler + .test_connection(&db, &authed, &user_db, &workspace_id, config) + .await + }; + + tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) + .await + .map_err(|_| { + Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) + })??; + Ok(()) +} + +#[allow(unused)] +pub fn complete_trigger_routes(handler: T) -> Router { + let standard_routes = trigger_routes::(); + + let additional_routes = handler.additional_routes(); + + standard_routes + .merge(additional_routes) + .layer(Extension(Arc::new(handler))) +} + +pub fn generate_trigger_routers() -> Router { + #[allow(unused_mut)] + let mut router = Router::new(); + + #[cfg(feature = "http_trigger")] + { + use crate::triggers::http::handler::HttpTrigger; + + router = router.nest( + HttpTrigger::ROUTE_PREFIX, + complete_trigger_routes(HttpTrigger), + ); + } + + #[cfg(feature = "websocket")] + { + use crate::triggers::websocket::WebsocketTrigger; + + router = router.nest( + WebsocketTrigger::ROUTE_PREFIX, + complete_trigger_routes(WebsocketTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] + { + use crate::triggers::kafka::KafkaTrigger; + + router = router.nest( + KafkaTrigger::ROUTE_PREFIX, + complete_trigger_routes(KafkaTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] + { + use crate::triggers::nats::NatsTrigger; + + router = router.nest( + NatsTrigger::ROUTE_PREFIX, + complete_trigger_routes(NatsTrigger), + ); + } + + #[cfg(feature = "mqtt_trigger")] + { + use crate::triggers::mqtt::MqttTrigger; + + router = router.nest( + MqttTrigger::ROUTE_PREFIX, + complete_trigger_routes(MqttTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] + { + use crate::triggers::sqs::SqsTrigger; + + router = router.nest( + SqsTrigger::ROUTE_PREFIX, + complete_trigger_routes(SqsTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] + { + use crate::triggers::gcp::GcpTrigger; + + router = router.nest( + GcpTrigger::ROUTE_PREFIX, + complete_trigger_routes(GcpTrigger), + ); + } + + #[cfg(feature = "postgres_trigger")] + { + use crate::triggers::postgres::PostgresTrigger; + + router = router.nest( + PostgresTrigger::ROUTE_PREFIX, + complete_trigger_routes(PostgresTrigger), + ); + } + + #[cfg(all(feature = "smtp", feature = "private"))] + { + use crate::triggers::email::EmailTrigger; + + router = router.nest( + EmailTrigger::ROUTE_PREFIX, + complete_trigger_routes(EmailTrigger), + ); + } + + { + use crate::triggers::global_handler::{ + cancel_suspended_trigger_jobs, resume_suspended_trigger_jobs, + }; + + router = router + .route( + "/trigger/:trigger_kind/resume_suspended_trigger_jobs/*trigger_path", + post(resume_suspended_trigger_jobs), + ) + .route( + "/trigger/:trigger_kind/cancel_suspended_trigger_jobs/*trigger_path", + post(cancel_suspended_trigger_jobs), + ); + } + + router +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggerPrimarySchedule { + schedule: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggersCount { + primary_schedule: Option, + schedule_count: i64, + http_routes_count: i64, + webhook_count: i64, + email_count: i64, + default_email_count: i64, + websocket_count: i64, + kafka_count: i64, + nats_count: i64, + postgres_count: i64, + mqtt_count: i64, + sqs_count: i64, + gcp_count: i64, + nextcloud_count: i64, +} + +pub async fn get_triggers_count_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult { + let primary_schedule = sqlx::query_scalar!( + "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_optional(db) + .await?; + + let schedule_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + #[allow(unused)] + let mut tx = db.begin().await?; + + #[cfg(feature = "http_trigger")] + let http_routes_count = { + use crate::triggers::http::handler::HttpTrigger; + let count = HttpTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "http_trigger"))] + let http_routes_count = 0; + + #[cfg(feature = "websocket")] + let websocket_count = { + use crate::triggers::websocket::WebsocketTrigger; + let count = WebsocketTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "websocket"))] + let websocket_count = 0; + + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] + let kafka_count = { + use crate::triggers::kafka::KafkaTrigger; + let count = KafkaTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(all(feature = "kafka", feature = "enterprise", feature = "private")))] + let kafka_count = 0; + + #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] + let nats_count = { + use crate::triggers::nats::NatsTrigger; + let count = NatsTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(all(feature = "nats", feature = "enterprise", feature = "private")))] + let nats_count = 0; + + #[cfg(feature = "postgres_trigger")] + let postgres_count = { + use crate::triggers::postgres::PostgresTrigger; + let count = PostgresTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "postgres_trigger"))] + let postgres_count = 0; + + #[cfg(feature = "mqtt_trigger")] + let mqtt_count = { + use crate::triggers::mqtt::MqttTrigger; + let count = MqttTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "mqtt_trigger"))] + let mqtt_count = 0; + + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] + let sqs_count = { + use crate::triggers::sqs::SqsTrigger; + let count = SqsTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; + count + }; + #[cfg(not(all(feature = "sqs_trigger", feature = "enterprise", feature = "private")))] + let sqs_count = 0; + + #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] + let gcp_count = { + use crate::triggers::gcp::GcpTrigger; + let count = GcpTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; + count + }; + #[cfg(not(all(feature = "gcp_trigger", feature = "enterprise", feature = "private")))] + let gcp_count = 0; + + #[cfg(all(feature = "smtp", feature = "enterprise", feature = "private"))] + let email_count = { + use crate::triggers::email::EmailTrigger; + let count = EmailTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(all(feature = "smtp", feature = "enterprise", feature = "private")))] + let email_count = 0; + + tx.commit().await?; + + let webhook_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + let default_email_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + let nextcloud_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM native_trigger WHERE workspace_id = $1 AND script_path = $2 AND is_flow = $3 AND service_name = 'nextcloud'", + w_id, + path, + is_flow, + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + Ok(Json(TriggersCount { + primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), + schedule_count, + http_routes_count, + webhook_count, + default_email_count, + email_count, + websocket_count, + kafka_count, + nats_count, + postgres_count, + mqtt_count, + gcp_count, + sqs_count, + nextcloud_count, + })) +} diff --git a/backend/windmill-triggers/src/triggers/http/handler.rs b/backend/windmill-triggers/src/triggers/http/handler.rs new file mode 100644 index 0000000000..116bec33bc --- /dev/null +++ b/backend/windmill-triggers/src/triggers/http/handler.rs @@ -0,0 +1,1183 @@ +use super::{ + http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, RequestType, + TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, +}; +use crate::{ + jobs_ext::start_job_update_sse_stream, + resource_ext::try_get_resource_from_db_as, + triggers::{ + http::{ + refresh_routers, validate_authentication_method, HttpConfig, HttpConfigRequest, + RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, + }, + trigger_helpers::{ + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, + trigger_runnable_inner, RunnableId, + }, + Trigger, TriggerCrud, TriggerData, TriggerMode, + }, +}; +use windmill_api_auth::{fetch_api_authed, scopes::check_scopes, ApiAuthed, AuthCache, OptTokened}; +use windmill_common::{utils::ExpiringCacheEntry, DB}; +use axum::{ + async_trait, + extract::Path, + response::{IntoResponse, Response}, + routing::{get, post}, + Extension, Json, Router, +}; +use futures::StreamExt; +use http::{HeaderMap, StatusCode}; +use sqlx::PgConnection; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::Arc, +}; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + utils::{not_found_if_none, require_admin, StripPath}, + worker::CLOUD_HOSTED, +}; +use windmill_git_sync::handle_deployment_metadata; + +#[cfg(feature = "parquet")] +use { + crate::jobs_ext::get_workspace_s3_resource, + windmill_common::s3_helpers::build_object_store_client, +}; + +use windmill_git_sync::DeployedObject; + +pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { + sqlx::query!("SELECT nextval('http_trigger_version_seq')") + .fetch_one(tx) + .await?; + Ok(()) +} + +pub fn generate_route_path_key(route_path: &str) -> String { + ROUTE_PATH_KEY_RE + .replace_all(route_path, "${1}${2}key") + .to_string() +} + +pub async fn route_path_key_exists( + route_path_key: &str, + http_method: &HttpMethod, + w_id: &str, + trigger_path: Option<&str>, + workspaced_route: Option, + db: &DB, +) -> Result { + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + route_path_key = $1 + AND workspace_id = $2 + AND http_method = $3 + AND ($4::TEXT IS NULL OR path != $4) + ) + "#, + &route_path_key, + w_id, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + } else { + let route_path_key = match workspaced_route { + Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), + _ => Cow::Borrowed(route_path_key), + }; + + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) + OR (workspaced_route IS FALSE AND route_path_key = $1)) + AND http_method = $2 + AND ($3::TEXT IS NULL OR path != $3) + ) + "#, + &route_path_key, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + + Ok(exists) +} + +pub async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< + RouteExists, + >, +) -> Result> { + let route_path_key = generate_route_path_key(&route_path); + + let exists = route_path_key_exists( + &route_path_key, + &http_method, + &w_id, + trigger_path.as_deref(), + workspaced_route, + &db, + ) + .await?; + + Ok(Json(exists)) +} + +fn check_no_duplicates<'trigger>( + new_http_triggers: &[TriggerData], + route_path_key: &[String], +) -> Result<()> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert(( + &route_path_key[i], + trigger.config.http_method, + trigger.config.workspaced_route, + )) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.config.route_path + ))); + } + } + + Ok(()) +} + +pub async fn insert_new_trigger_into_db( + authed: &ApiAuthed, + tx: &mut PgConnection, + w_id: &str, + trigger: &TriggerData, + route_path_key: &str, +) -> Result<()> { + require_admin(authed.is_admin, &authed.username)?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + INSERT INTO http_trigger ( + workspace_id, + path, + route_path, + route_path_key, + workspaced_route, + authentication_resource_path, + wrap_body, + raw_string, + script_path, + summary, + description, + is_flow, + mode, + request_type, + authentication_method, + http_method, + static_asset_config, + edited_by, + email, + edited_at, + is_static_website, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 + ) + "#, + w_id, + trigger.base.path, + trigger.config.route_path, + route_path_key, + trigger.config.workspaced_route.unwrap_or(false), + trigger.config.authentication_resource_path, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.base.script_path, + trigger.config.summary, + trigger.config.description, + trigger.base.is_flow, + trigger.base.mode() as _, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_many_http_triggers( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>>, +) -> Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let handler = HttpTrigger; + + let error_wrapper = |route_path: &str, error: Error| -> Error { + anyhow::anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + route_path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + handler + .validate_new(&db, &w_id, &new_http_trigger.config) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + let route_path_key = + check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; + + route_path_keys.push(route_path_key.clone()); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { + insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + audit_log( + &mut *tx, + &authed, + "http_trigger.create", + ActionKind::Create, + &w_id, + Some(&new_http_trigger.base.path), + None, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + + increase_trigger_version(&mut tx) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, + Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), + true, + None, + ) + .await + .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; + } + Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) +} + +async fn check_if_route_exist( + db: &DB, + config: &HttpConfigRequest, + workspace_id: &str, + trigger_path: Option<&str>, +) -> Result { + let route_path_key = generate_route_path_key(&config.route_path); + + let exists = route_path_key_exists( + &route_path_key, + &config.http_method, + workspace_id, + trigger_path, + config.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + Ok(route_path_key) +} + +pub struct HttpTrigger; + +#[async_trait] +impl TriggerCrud for HttpTrigger { + type TriggerConfig = HttpConfig; + type Trigger = Trigger; + type TriggerConfigRequest = HttpConfigRequest; + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = "http_trigger"; + const TRIGGER_TYPE: &'static str = "http"; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/http_triggers"; + const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; + const IS_ALLOWED_ON_CLOUD: bool = true; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "route_path", + "route_path_key", + "request_type", + "authentication_method", + "http_method", + "summary", + "description", + "static_asset_config", + "is_static_website", + "authentication_resource_path", + "workspaced_route", + "wrap_body", + "raw_string", + ]; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::HttpTrigger { path } + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/create_many", post(create_many_http_triggers)) + .route("/route_exists", post(exists_route)) + } + + async fn validate_new( + &self, + _db: &DB, + _workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method(new.authentication_method, new.raw_string)?; + + Ok(()) + } + + async fn validate_edit( + &self, + _db: &DB, + _workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + validate_authentication_method(edit.authentication_method, edit.raw_string)?; + + Ok(()) + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; + + insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + if authed.is_admin { + if trigger.config.route_path.is_empty() { + return Err(Error::BadRequest("route_path is required".to_string())); + }; + + let route_path = &trigger.config.route_path; + if !VALID_ROUTE_PATH_RE.is_match(route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + let route_path_key = + check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; + + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + route_path = $1, + route_path_key = $2, + workspaced_route = $3, + wrap_body = $4, + raw_string = $5, + authentication_resource_path = $6, + script_path = $7, + path = $8, + is_flow = $9, + mode = $10, + http_method = $11, + static_asset_config = $12, + edited_by = $13, + email = $14, + request_type = $15, + authentication_method = $16, + summary = $17, + description = $18, + edited_at = now(), + is_static_website = $19, + error_handler_path = $20, + error_handler_args = $21, + retry = $22 + WHERE + workspace_id = $23 AND + path = $24 + "#, + route_path, + &route_path_key, + trigger.config.workspaced_route, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } else { + let request_type = trigger.config.request_type; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + wrap_body = $1, + raw_string = $2, + authentication_resource_path = $3, + script_path = $4, + path = $5, + is_flow = $6, + mode = $7, + http_method = $8, + static_asset_config = $9, + edited_by = $10, + email = $11, + request_type = $12, + authentication_method = $13, + summary = $14, + description = $15, + edited_at = now(), + is_static_website = $16, + error_handler_path = $17, + error_handler_args = $18, + retry = $19 + WHERE + workspace_id = $20 AND + path = $21 + "#, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.base.mode() as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + request_type as _, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { + increase_trigger_version(tx).await + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + increase_trigger_version(tx).await?; + + Ok(deleted > 0) + } +} + +async fn conditional_cors_middleware( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + let mut response = next.run(req).await; + + let headers = response.headers_mut(); + + // Check existing headers first to determine what not to insert + let mut not_insert_origin = false; + let mut not_insert_methods = false; + let mut not_insert_headers = false; + + for key in headers.keys() { + if !not_insert_origin && key == http::header::ACCESS_CONTROL_ALLOW_ORIGIN { + not_insert_origin = true; + } + if !not_insert_methods && key == http::header::ACCESS_CONTROL_ALLOW_METHODS { + not_insert_methods = true; + } + if !not_insert_headers && key == http::header::ACCESS_CONTROL_ALLOW_HEADERS { + not_insert_headers = true; + } + + // Early exit if all headers are already present + if not_insert_origin && not_insert_methods && not_insert_headers { + break; + } + } + + // Insert only the missing headers + if !not_insert_origin { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_ORIGIN, + http::HeaderValue::from_static("*"), + ); + } + + if !not_insert_methods { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_METHODS, + http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), + ); + } + + if !not_insert_headers { + headers.insert( + http::header::ACCESS_CONTROL_ALLOW_HEADERS, + http::HeaderValue::from_static("content-type, authorization"), + ); + } + + response +} + +pub fn http_route_trigger_handler() -> Router { + Router::new() + .route( + "/*path", + get(route_job) + .post(route_job) + .delete(route_job) + .put(route_job) + .patch(route_job) + .head(|| async { "" }) + .options(|| async { "" }), + ) + .layer(axum::middleware::from_fn(conditional_cors_middleware)) +} + +async fn get_http_route_trigger( + route_path: &str, + auth_cache: &Arc, + token: Option<&String>, + db: &DB, + user_db: UserDB, + method: &http::Method, +) -> Result<(TriggerRoute, String, HashMap, ApiAuthed)> { + let http_method: HttpMethod = method.try_into()?; + + let requested_path = format!("/{}", route_path); + + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + + let routers_cache = if routers_cache.routers.is_empty() { + tracing::warn!("HTTP routers are not loaded, loading from db"); + let (_, routers_cache) = refresh_routers(db).await?; + routers_cache + } else { + routers_cache + }; + + let router = routers_cache + .routers + .get(&http_method) + .ok_or(Error::internal_err( + "HTTP routers could not be loaded".to_string(), + ))?; + + let trigger_match = router.at(requested_path.as_str()).ok(); + + let matchit::Match { value: trigger, params } = + not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?; + + let params: HashMap = params + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let username_override = if let AuthenticationMethod::Windmill = trigger.authentication_method { + let opt_authed = if let Some(token) = token { + auth_cache + .get_authed(Some(trigger.workspace_id.clone()), token) + .await + } else { + None + }; + if let Some(authed) = opt_authed { + check_scopes(&authed, || format!("http_triggers:read:{}", &trigger.path))?; + + // check that the user has access to the trigger + let cache_key = ( + trigger.workspace_id.clone(), + trigger.path.clone(), + authed.clone(), + ); + let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { + Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { + tracing::debug!("HTTP access cache hit for route {}", trigger.path); + true + } + _ => { + tracing::debug!("HTTP access cache miss for route {}", trigger.path); + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM + http_trigger + WHERE + workspace_id = $1 AND + path = $2 + ) + "#, + trigger.workspace_id, + trigger.path + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + if exists { + HTTP_ACCESS_CACHE.insert( + cache_key, + ExpiringCacheEntry { + value: (), + expiry: std::time::Instant::now() + + std::time::Duration::from_secs(10), + }, + ); + } + exists + } + }; + if exists { + Some(authed.display_username().to_owned()) + } else { + return Err(Error::NotAuthorized("Unauthorized".to_string())); + } + } else { + return Err(Error::NotAuthorized("Requires authentication".to_string())); + } + } else { + None + }; + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + &db, + Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), + ) + .await?; + + Ok((trigger.clone(), route_path.to_string(), params, authed)) +} + +async fn route_job( + Extension(db): Extension, + Extension(user_db): Extension, + Extension(auth_cache): Extension>, + OptTokened { token }: OptTokened, + Path(route_path): Path, + headers: HeaderMap, + args: RawHttpTriggerArgs, +) -> std::result::Result { + let route_path = route_path.to_path().trim_end_matches("/"); + + let (trigger, called_path, params, authed) = get_http_route_trigger( + route_path, + &auth_cache, + token.as_ref(), + &db, + user_db.clone(), + &args.0.metadata.method, + ) + .await + .map_err(|e| e.into_response())?; + + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { + return Err(Error::NotFound(format!( + "Runnable path of HTTP route at path: {}", + trigger.path + )) + .into_response()); + } + + let args = args + .process_args( + &authed, + &db, + &trigger.workspace_id, + match trigger.authentication_method { + AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, + _ => trigger.raw_string, + }, + ) + .await + .map_err(|e| e.into_response())?; + + match trigger.authentication_method { + AuthenticationMethod::None + | AuthenticationMethod::Windmill + | AuthenticationMethod::CustomScript => {} + _ => { + let resource_path = match trigger.authentication_resource_path { + Some(resource_path) => resource_path, + None => { + return Err(Error::BadRequest( + "Missing authentication resource path".to_string(), + ) + .into_response()) + } + }; + + let cache_key = ( + trigger.workspace_id.clone(), + resource_path.clone(), + authed.clone(), + ); + + let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { + Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { + tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); + cache_entry.value + } + _ => { + tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); + let auth_method = try_get_resource_from_db_as::< + super::http_trigger_auth::AuthenticationMethod, + >( + &authed, + Some(user_db.clone()), + &db, + &resource_path, + &trigger.workspace_id, + ) + .await + .map_err(|e| e.into_response())?; + HTTP_AUTH_CACHE.insert( + cache_key, + ExpiringCacheEntry { + value: auth_method.clone(), + expiry: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + ); + auth_method + } + }; + + let raw_payload = args.0.metadata.raw_string.as_ref(); + + let response = authentication_method + .authenticate_http_request(&headers, raw_payload) + .map_err(|e| e.into_response())?; + + if let Some(response) = response { + return Ok(response); + } + } + } + + #[cfg(not(feature = "parquet"))] + if trigger.static_asset_config.is_some() { + return Err(Error::internal_err( + "Static asset configuration is not supported in this build".to_string(), + ) + .into_response()); + } + + #[cfg(feature = "parquet")] + if let Some(sqlx::types::Json(config)) = trigger.static_asset_config { + let build_static_response_f = async { + let (_, s3_resource_opt) = get_workspace_s3_resource( + &authed, + &db, + None, + &trigger.workspace_id, + config.storage, + ) + .await?; + let s3_resource = s3_resource_opt.ok_or(Error::internal_err( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_object_store_client(&s3_resource).await?; + + let path = if trigger.is_static_website { + let subpath = params + .get("wm_subpath") + .cloned() + .unwrap_or("index.html".to_string()); + tracing::info!("subpath: {}", subpath); + format!("{}/{}", config.s3.trim_end_matches('/'), subpath) + } else { + config.s3.clone() + }; + let path = object_store::path::Path::from(path); + let s3_object = s3_client.get(&path).await; + + let s3_object = match s3_object { + Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { + // fallback to index.html if the file is not found + let path = object_store::path::Path::from(format!( + "{}/index.html", + config.s3.trim_end_matches('/') + )); + s3_client.get(&path).await + } + r => r, + }; + + let s3_object = s3_object.map_err(|err| { + tracing::warn!("Error retrieving file from S3: {:?}", err); + let mut msg = format!("Error retrieving file: {err}"); + let mut source = std::error::Error::source(&err); + while let Some(e) = source { + msg.push_str(&format!("\n caused by: {e}")); + source = e.source(); + } + Error::internal_err(msg) + })?; + + let mut response_headers = http::HeaderMap::new(); + if let Some(ref e_tag) = s3_object.meta.e_tag { + if let Some(if_none_match) = headers.get(http::header::IF_NONE_MATCH) { + if if_none_match == e_tag { + return Ok::<_, Error>(( + StatusCode::NOT_MODIFIED, + response_headers, + axum::body::Body::empty(), + )); + } + } + if let Ok(e_tag) = e_tag.parse() { + response_headers.insert("etag", e_tag); + } + } + response_headers.insert( + "content-type", + s3_object + .attributes + .get(&object_store::Attribute::ContentType) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("application/octet-stream".parse().unwrap()), + ); + if !trigger.is_static_website { + response_headers.insert( + "content-disposition", + config.filename.as_ref().map_or_else( + || { + s3_object + .attributes + .get(&object_store::Attribute::ContentDisposition) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("inline".parse().unwrap()) + }, + |filename| { + format!("inline; filename=\"{}\"", filename) + .parse() + .unwrap_or("inline".parse().unwrap()) + }, + ), + ); + } + + let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); + Ok::<_, Error>((StatusCode::OK, response_headers, body_stream)) + }; + match build_static_response_f.await { + Ok((status, headers, body_stream)) => { + return Ok((status, headers, body_stream).into_response()) + } + Err(e) => return Err(e.into_response()), + } + } + + let runnable_format = get_runnable_format( + if trigger.is_flow { + RunnableId::from_flow_path(&trigger.script_path) + } else { + RunnableId::from_script_path(&trigger.script_path) + }, + &trigger.workspace_id, + &db, + &TriggerKind::Http, + ) + .await + .map_err(|e| e.into_response())?; + + let args = args + .to_args_from_format( + &trigger.route_path, + &called_path, + ¶ms, + runnable_format, + trigger.wrap_body, + ) + .map_err(|e| e.into_response())?; + + let trigger_info = TriggerMetadata::new(Some(trigger.path.clone()), JobTriggerKind::Http); + if trigger.mode == TriggerMode::Suspended { + let _ = trigger_runnable( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + None, + true, + trigger_info, + ) + .await + .map_err(|e| e.into_response())?; + + return Ok(( + StatusCode::OK, + format!( + "Trigger {} is in suspended mode, jobs are added to the queue but suspended", + &trigger.path + ), + ) + .into_response()); + } + + // Handle execution based on the execution mode + match trigger.request_type { + RequestType::SyncSse => { + // Trigger the job (always async when streaming) + let (uuid, _, _, _) = trigger_runnable_inner( + &db, + None, + Some(user_db.clone()), + authed.clone(), + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + None, + trigger_info, + None, + ) + .await + .map_err(|e| e.into_response())?; + + // Set up SSE stream + let opt_authed = Some(authed.clone()); + let opt_tokened = OptTokened { token: None }; + let (tx, rx) = tokio::sync::mpsc::channel(32); + + let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { + format!( + "data: {}\n\n", + serde_json::to_string(&x).unwrap_or_default() + ) + }); + + start_job_update_sse_stream( + opt_authed, + opt_tokened, + db.clone(), + trigger.workspace_id.clone(), + uuid, + None, + None, + None, + None, + Some(true), + Some(true), + None, + None, + tx, + None, + ); + + let body = axum::body::Body::from_stream( + stream.map(std::result::Result::<_, std::convert::Infallible>::Ok), + ); + + Ok(Response::builder() + .status(200) + .header("Content-Type", "text/event-stream") + .header("Cache-Control", "no-cache") + .body(body) + .map_err(|e| Error::internal_err(e.to_string()).into_response())?) + } + RequestType::Async => trigger_runnable( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + None, + false, + trigger_info, + ) + .await + .map_err(|e| e.into_response()), + RequestType::Sync => trigger_runnable_and_wait_for_result( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + trigger_info, + ) + .await + .map_err(|e| e.into_response()), + } +} diff --git a/backend/windmill-triggers/src/triggers/http/http_trigger_args.rs b/backend/windmill-triggers/src/triggers/http/http_trigger_args.rs new file mode 100644 index 0000000000..e7873d3fbe --- /dev/null +++ b/backend/windmill-triggers/src/triggers/http/http_trigger_args.rs @@ -0,0 +1,223 @@ +use std::collections::HashMap; + +use axum::{ + extract::{FromRequest, Request}, + response::Response, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use windmill_common::{ + error::Error, + triggers::{RunnableFormat, RunnableFormatVersion}, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +use crate::args_ext::{ + build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs, + WebhookArgsMetadata, +}; +use windmill_api_auth::ApiAuthed; + +pub struct RawHttpTriggerArgs(pub RawWebhookArgs); + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[axum::async_trait] +impl FromRequest for RawHttpTriggerArgs +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let args = try_from_request_body(request, _state, true).await?; + + Ok(Self(args)) + } +} + +#[derive(Debug, Clone)] +pub struct HttpTriggerArgs(pub WebhookArgs); + +impl RawHttpTriggerArgs { + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + use_raw: bool, + ) -> Result { + if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body { + return Err(Error::BadRequest( + "Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(), + ) + .into()); + } + + let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?; + + Ok(HttpTriggerArgs(args)) + } +} + +#[derive(Serialize)] +struct HttpTriggerPreprocessorEvent<'a> { + kind: String, + route: &'a str, + path: &'a str, + body: Box, + raw_string: Option, + params: &'a HashMap, + headers: HashMap>, + query: HashMap>, + method: HttpMethod, +} + +#[derive(Serialize)] +struct HttpTriggerWmTrigger<'a> { + route: &'a str, + path: &'a str, + params: &'a HashMap, + query: HashMap>, + headers: HashMap>, + method: HttpMethod, +} + +impl HttpTriggerArgs { + pub fn to_main_args(self, wrap_body: bool) -> Result { + let mut extra = HashMap::new(); + + let WebhookArgsMetadata { raw_string, .. } = self.0.metadata; + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.0.body { + Body::HashMap(mut body) => { + if wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); + } + Ok(PushArgsOwned { args: body, extra }) + } + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) + } + } + } + + pub fn to_args_from_format( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + format: RunnableFormat, + wrap_body: bool, + ) -> Result { + let headers = build_headers(&self.0.metadata.headers, None, true); + let query = build_query(self.0.metadata.query.as_deref(), None, true); + match format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + // we don't care about wrap_body in v2 + self.to_v2_preprocessor_args(route_path, called_path, params, headers, query) + } + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => self + .to_v1_preprocessor_args( + route_path, + called_path, + params, + wrap_body, + headers, + query, + ), + RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body), + } + } + + fn to_v1_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + wrap_body: bool, + headers: HashMap>, + query: HashMap>, + ) -> Result { + let mut extra = HashMap::new(); + let mut wm_trigger = HashMap::new(); + wm_trigger.insert("kind".to_string(), to_raw_value(&"http".to_string())); + wm_trigger.insert( + "http".to_string(), + to_raw_value(&HttpTriggerWmTrigger { + route: route_path, + path: called_path, + method: (&self.0.metadata.method).try_into()?, + params, + query, + headers, + }), + ); + extra.insert("wm_trigger".to_string(), to_raw_value(&wm_trigger)); + + let mut args = self.to_main_args(wrap_body)?; + + args.extra.get_or_insert_default().extend(extra); + + Ok(args) + } + + pub fn to_v2_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + headers: HashMap>, + query: HashMap>, + ) -> Result { + let mut args = HashMap::new(); + args.insert( + "event".to_string(), + to_raw_value(&HttpTriggerPreprocessorEvent { + kind: "http".to_string(), + body: to_raw_value(&self.0.body), + raw_string: self.0.metadata.raw_string, + headers, + query, + method: (&self.0.metadata.method).try_into()?, + route: route_path, + path: called_path, + params, + }), + ); + Ok(PushArgsOwned { args, extra: None }) + } +} diff --git a/backend/windmill-triggers/src/triggers/http/http_trigger_auth.rs b/backend/windmill-triggers/src/triggers/http/http_trigger_auth.rs new file mode 100644 index 0000000000..85288d87b8 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/http/http_trigger_auth.rs @@ -0,0 +1,751 @@ +use axum::response::{IntoResponse, Response}; +use base64::{ + prelude::{BASE64_STANDARD, BASE64_URL_SAFE}, + Engine, +}; +use hmac::{Hmac, Mac}; +use http::{header, HeaderMap, HeaderValue, StatusCode}; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_json::json; +use sha1::Sha1; +use sha2::{Sha256, Sha512}; +use std::{borrow::Cow, collections::HashMap}; + +pub type HmacSha256 = Hmac; +pub type HmacSha512 = Hmac; +pub type HmacSha1 = Hmac; + +mod github { + use super::*; + pub struct Github; + + impl WebhookHandler for Github { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let github_secret_header = headers.try_get_webhook_header("X-Hub-Signature-256")?; + + let authentication_data = SignatureAuthenticationData::new( + Cow::Borrowed(raw_payload), + github_secret_header, + Some("sha256="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + ); + + Ok(authentication_data) + } + } +} + +mod slack { + use super::*; + pub struct Slack; + + impl WebhookHandler for Slack { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let slack_secret_signature = headers.try_get_webhook_header("X-Slack-Signature")?; + let slack_timestamp_header = + headers.try_get_webhook_header("X-Slack-Request-Timestamp")?; + let signed_payload = format!("v0:{}:{}", slack_timestamp_header, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + slack_secret_signature, + Some("v0="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod stripe { + use super::*; + + pub struct Stripe; + + impl WebhookHandler for Stripe { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let stripe_signature_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?; + + let stripe_signature = parse_signature(stripe_signature_header, (",", "=")); + + let timestamp = *stripe_signature + .get("t") + .ok_or(AuthenticationError::InvalidTimestamp)?; + let v1 = *stripe_signature + .get("v1") + .ok_or(AuthenticationError::InvalidSignature)?; + + let signed_payload = format!("{}.{}", timestamp, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + v1, + None, + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod tiktok { + use super::*; + + pub struct TikTok; + + impl WebhookHandler for TikTok { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + _: &SignatureConfigData, + _: &str, + ) -> Result, AuthenticationError> { + Ok(None) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let tiktok_secret_signature = headers.try_get_webhook_header("TikTok-Signature")?; + + let stripe_signature = parse_signature(tiktok_secret_signature, (",", "=")); + + let timestamp = *stripe_signature + .get("t") + .ok_or(AuthenticationError::InvalidTimestamp)?; + let s = *stripe_signature + .get("s") + .ok_or(AuthenticationError::InvalidSignature)?; + + let signed_payload = format!("{}.{}", timestamp, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(signed_payload), + s, + None, + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +mod twitch { + use super::*; + use http::header; + use serde_json::value::RawValue; + #[derive(Debug, Deserialize)] + struct TwitchCrcBody { + challenge: String, + #[allow(unused)] + subscription: Box, + } + + pub struct Twitch; + + impl WebhookHandler for Twitch { + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let twitch_secret_signature = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Signature")?; + let twitch_message_id_header = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Id")?; + let twitch_timestamp_header = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?; + + let message = format!( + "{}{}{}", + twitch_message_id_header, twitch_timestamp_header, raw_payload + ); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(message), + twitch_secret_signature, + Some("sha256="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + + fn handle_challenge_request<'header>( + &self, + headers: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError> { + let authentication_data = self.get_hmac_authentication_data(headers, raw_payload)?; + verify_hmac_signature(authentication_data, &signature_config_data.secret_key)?; + + let twitch_eventsub_message_type = + headers.try_get_webhook_header("Twitch-Eventsub-Message-Type")?; + + if twitch_eventsub_message_type != "webhook_callback_verification" { + return Ok(None); + } + let twitch_crc_body = + serde_json::from_str::(raw_payload).map_err(|e| { + AuthenticationError::InvalidChallengeResponse(format!( + "Twitch :{}", + e.to_string() + )) + })?; + + let response = ( + StatusCode::OK, + [(header::CONTENT_TYPE, "text/plain")], + twitch_crc_body.challenge.to_string(), + ); + + Ok(Some(response.into_response())) + } + } +} + +mod zoom { + use axum::Json; + + use super::*; + + #[derive(Debug, Deserialize)] + struct ZoomPayload { + #[serde(rename = "plainToken")] + plain_token: String, + } + + #[derive(Debug, Deserialize)] + #[allow(unused)] + struct ZoomChallengeResponse { + payload: ZoomPayload, + event_ts: u64, + event: String, + } + + pub struct Zoom; + + impl WebhookHandler for Zoom { + fn handle_challenge_request<'header>( + &self, + _: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError> { + let Ok(zoom_request_body) = serde_json::from_str::(raw_payload) + else { + return Ok(None); + }; + + if zoom_request_body.event != "endpoint.url_validation" { + return Ok(None); + } + + let hmac_signature = calculate_hmac_signature( + HmacAlgorithm::Sha256, + &signature_config_data.secret_key, + &zoom_request_body.payload.plain_token, + ); + + let encoded_hmac_signature = encode_hmac_signature(Encoding::Hex, &hmac_signature); + + let response = ( + StatusCode::OK, + Json(json!({ + "plainToken": zoom_request_body.payload.plain_token, + "encryptedToken": encoded_hmac_signature + })), + ); + + Ok(Some(response.into_response())) + } + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError> + { + let zoom_signature_header = headers.try_get_webhook_header("x-zm-signature")?; + let zoom_timestamp_header = headers.try_get_webhook_header("x-zm-request-timestamp")?; + + let message = format!("v0:{}:{}", zoom_timestamp_header, raw_payload); + + Ok(SignatureAuthenticationData::new( + Cow::Owned(message), + zoom_signature_header, + Some("v0="), + SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), + )) + } + } +} + +use constant_time_eq::constant_time_eq; +use github::Github; +use slack::Slack; +use stripe::Stripe; +use tiktok::TikTok; +use twitch::Twitch; +use zoom::Zoom; + +#[derive(Debug)] +pub struct SignatureAuthenticationDetails { + pub algorithm_to_use: HmacAlgorithm, + pub header_key_encoding: Encoding, +} + +impl SignatureAuthenticationDetails { + #[inline] + fn new(algorithm_to_use: HmacAlgorithm, header_key_encoding: Encoding) -> Self { + Self { algorithm_to_use, header_key_encoding } + } +} + +fn parse_signature<'header>( + signature: &'header str, + splitters: (&str, &str), +) -> HashMap<&'header str, &'header str> { + let headers: HashMap<&str, &str> = signature + .split(splitters.0) + .map(|header| { + let mut key_and_value = header.split(splitters.1); + let key = key_and_value.next(); + let value = key_and_value.next(); + (key, value) + }) + .filter_map(|(key, value)| match (key, value) { + (Some(key), Some(value)) => Some((key, value)), + _ => None, + }) + .collect(); + headers +} + +#[derive(Debug)] +pub struct SignatureAuthenticationData<'payload, 'header, 'prefix> { + pub signed_payload: Cow<'payload, str>, + pub header_key_value: &'header str, + pub signature_prefix: Option<&'prefix str>, + pub config: SignatureAuthenticationDetails, +} + +impl<'payload, 'header, 'prefix> SignatureAuthenticationData<'payload, 'header, 'prefix> { + pub fn new( + signed_payload: Cow<'payload, str>, + header_key_value: &'header str, + signature_prefix: Option<&'prefix str>, + config: SignatureAuthenticationDetails, + ) -> Self { + Self { signed_payload, header_key_value, signature_prefix, config } + } +} + +pub trait WebhookHandler { + fn handle_challenge_request<'header>( + &self, + headers: &'header HeaderMap, + signature_config_data: &SignatureConfigData, + raw_payload: &str, + ) -> Result, AuthenticationError>; + + fn get_hmac_authentication_data<'payload, 'header, 'prefix>( + &self, + headers: &'header HeaderMap, + raw_payload: &'payload str, + ) -> Result, AuthenticationError>; +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HmacAlgorithm { + Sha1, + Sha256, + Sha512, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Encoding { + Base64, + Base64Uri, + Hex, +} +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SignatureAuthenticationMethod { + algorithm: HmacAlgorithm, + encoding: Encoding, + signature_header_name: String, + signature_prefix: Option, +} + +pub struct SignatureConfigData<'config> { + secret_key: &'config str, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct SignatureAuthentication { + signature_provider: WebhookType, + secret_key: String, + authentication_config: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct BasicAuthAuthentication { + pub username: String, + pub password: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ApiKeyAuthentication { + pub api_key_header: String, + pub api_key_secret: String, +} + +#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] +#[non_exhaustive] +pub enum WebhookType { + Github, + Slack, + Stripe, + TikTok, + Twitch, + Zoom, + Custom, +} + +impl WebhookType { + pub fn get_webhook_handler(&self) -> Option<&'static dyn WebhookHandler> { + let handler: &'static dyn WebhookHandler = match *self { + WebhookType::Github => &Github, + WebhookType::Slack => &Slack, + WebhookType::Stripe => &Stripe, + WebhookType::TikTok => &TikTok, + WebhookType::Twitch => &Twitch, + WebhookType::Zoom => &Zoom, + WebhookType::Custom => return None, + }; + Some(handler) + } +} + +trait TryGetWebhookHeader { + fn try_get_webhook_header<'header>( + &'header self, + header_name: &str, + ) -> Result<&'header str, AuthenticationError>; +} + +impl TryGetWebhookHeader for HeaderMap { + fn try_get_webhook_header<'header>( + &'header self, + header_name: &str, + ) -> Result<&'header str, AuthenticationError> { + let Some(signature_header) = self.get(header_name) else { + return Err(AuthenticationError::MissingHeader(header_name.to_string())); + }; + let Some(signature_header) = signature_header.to_str().ok() else { + return Err(AuthenticationError::InvalidHeader(header_name.to_string())); + }; + + Ok(signature_header) + } +} + +pub fn calculate_hmac_signature(algorithm: HmacAlgorithm, secret: &str, payload: &str) -> Vec { + match algorithm { + HmacAlgorithm::Sha1 => { + let mut mac = + HmacSha1::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + HmacAlgorithm::Sha256 => { + let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) + .expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + HmacAlgorithm::Sha512 => { + let mut mac = HmacSha512::new_from_slice(secret.as_bytes()) + .expect("HMAC can take key of any size"); + mac.update(payload.as_bytes()); + mac.finalize().into_bytes().to_vec() + } + } +} + +pub fn encode_hmac_signature(encoding: Encoding, hmac_signature: &[u8]) -> String { + match encoding { + Encoding::Hex => hex::encode(hmac_signature), + Encoding::Base64 => BASE64_STANDARD.encode(hmac_signature), + Encoding::Base64Uri => BASE64_URL_SAFE.encode(hmac_signature), + } +} + +pub fn verify_hmac_signature( + authentication_data: SignatureAuthenticationData, + webhook_signing_secret: &str, +) -> Result<(), AuthenticationError> { + let hmac_signature = calculate_hmac_signature( + authentication_data.config.algorithm_to_use, + &webhook_signing_secret, + &authentication_data.signed_payload, + ); + + let encoded_signature = encode_hmac_signature( + authentication_data.config.header_key_encoding, + &hmac_signature, + ); + + let final_expected_signature = + if let Some(signature_prefix) = authentication_data.signature_prefix { + format!("{}{}", signature_prefix, encoded_signature) + } else { + encoded_signature + }; + + if !constant_time_eq( + final_expected_signature.as_bytes(), + authentication_data.header_key_value.as_bytes(), + ) { + return Err(AuthenticationError::InvalidSignature); + } + + Ok(()) +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(untagged)] +pub enum AuthenticationMethod { + Signature(SignatureAuthentication), + BasicAuth(BasicAuthAuthentication), + ApiKey(ApiKeyAuthentication), +} + +impl AuthenticationMethod { + pub fn authenticate_http_request( + &self, + headers: &HeaderMap, + raw_payload: Option<&String>, + ) -> Result, AuthenticationError> { + match self { + AuthenticationMethod::Signature(SignatureAuthentication { + secret_key, + authentication_config, + signature_provider, + }) => { + let raw_payload = raw_payload.ok_or(AuthenticationError::InvalidPayload)?; + let config_data = SignatureConfigData { secret_key: &secret_key }; + let handler = signature_provider.get_webhook_handler(); + let challenge_response = handler + .map(|handler| { + handler.handle_challenge_request(headers, &config_data, raw_payload) + }) + .transpose()? + .flatten(); + + if let Some(challenge_response) = challenge_response { + return Ok(Some(challenge_response)); + } + + let authentication_data = match handler { + Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?, + None => { + let authentication_config = authentication_config + .as_ref() + .ok_or(AuthenticationError::InvalidCustomConfig)?; + let signature_header_value = headers + .try_get_webhook_header(&authentication_config.signature_header_name)?; + SignatureAuthenticationData::new( + Cow::Borrowed(raw_payload), + signature_header_value, + authentication_config.signature_prefix.as_deref(), + SignatureAuthenticationDetails::new( + authentication_config.algorithm, + authentication_config.encoding, + ), + ) + } + }; + + verify_hmac_signature(authentication_data, &secret_key)?; + } + AuthenticationMethod::ApiKey(ApiKeyAuthentication { + api_key_header, + api_key_secret, + }) => { + let api_key_to_cmp = headers + .try_get_webhook_header(&api_key_header) + .map_err(|_| AuthenticationError::InvalidApiKey)?; + if api_key_to_cmp != api_key_secret { + return Err(AuthenticationError::InvalidApiKey); + } + } + AuthenticationMethod::BasicAuth(BasicAuthAuthentication { username, password }) => { + let mut credentials_store = headers + .try_get_webhook_header("Authorization") + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)? + .split(' '); + + let _ = credentials_store + .next() + .filter(|r#type| *r#type == "Basic") + .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_as_base64 = credentials_store + .next() + .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_from_base64_as_bytes = BASE64_STANDARD + .decode(credentials_as_base64.as_bytes()) + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials_separated_with_colon = + String::from_utf8(credentials_from_base64_as_bytes) + .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; + + let credentials = credentials_separated_with_colon.split(':').collect_vec(); + + if credentials.len() != 2 { + return Err(AuthenticationError::UnauthorizedBasicHttpAuth); + } + + if credentials.get(0).unwrap() != username + || credentials.get(1).unwrap() != password + { + return Err(AuthenticationError::UnauthorizedBasicHttpAuth); + } + } + } + + Ok(None) + } +} + +#[derive(thiserror::Error, Debug)] +#[allow(unused)] +pub enum AuthenticationError { + #[error("failed to parse timestamp")] + InvalidTimestamp, + + #[error("invalid secret")] + InvalidSecret(#[from] base64::DecodeError), + + #[error("invalid header `{0}`")] + InvalidHeader(String), + + #[error("signature timestamp too old")] + TimestampTooOldError, + + #[error("signature timestamp too far in future")] + FutureTimestampError, + + #[error("missing header {0}")] + MissingHeader(String), + + #[error("signature invalid")] + InvalidSignature, + + #[error("payload invalid")] + InvalidPayload, + + #[error("invalid custom config")] + InvalidCustomConfig, + + #[error("invalid auth header: {0}")] + InvalidAuthHeader(String), + + #[error("invalid api key")] + InvalidApiKey, + + #[error("invalid challenge response: {0}")] + InvalidChallengeResponse(String), + + #[error("")] + UnauthorizedBasicHttpAuth, +} + +impl IntoResponse for AuthenticationError { + fn into_response(self) -> Response { + let (status, error_message) = match &self { + AuthenticationError::InvalidTimestamp + | AuthenticationError::InvalidPayload + | AuthenticationError::InvalidHeader(_) + | AuthenticationError::MissingHeader(_) + | AuthenticationError::TimestampTooOldError + | AuthenticationError::FutureTimestampError + | AuthenticationError::InvalidCustomConfig + | AuthenticationError::InvalidChallengeResponse(_) => { + (StatusCode::BAD_REQUEST, self.to_string()) + } + + AuthenticationError::InvalidSecret(_) + | AuthenticationError::InvalidSignature + | AuthenticationError::InvalidAuthHeader(_) => { + (StatusCode::UNAUTHORIZED, self.to_string()) + } + AuthenticationError::UnauthorizedBasicHttpAuth => { + return ( + StatusCode::UNAUTHORIZED, + [(header::WWW_AUTHENTICATE, r#"Basic realm="Restricted Area""#)], + "Unauthorized", + ) + .into_response() + } + AuthenticationError::InvalidApiKey => { + return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response() + } + }; + + let body = json!({ "error": error_message }); + + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", HeaderValue::from_static("application/json")); + + (status, headers, body.to_string()).into_response() + } +} diff --git a/backend/windmill-triggers/src/triggers/http/mod.rs b/backend/windmill-triggers/src/triggers/http/mod.rs new file mode 100644 index 0000000000..b0359d477b --- /dev/null +++ b/backend/windmill-triggers/src/triggers/http/mod.rs @@ -0,0 +1,412 @@ +use std::collections::HashMap; + +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use tokio::sync::{RwLock, RwLockReadGuard}; +use windmill_common::{ + error::{Error, Result}, + flows::Retry, + s3_helpers::S3Object, + worker::CLOUD_HOSTED, + DB, +}; + +use crate::triggers::TriggerMode; +use windmill_api_auth::ApiAuthed; +use windmill_common::utils::ExpiringCacheEntry; + +pub mod handler; +pub mod http_trigger_args; +pub mod http_trigger_auth; + +lazy_static::lazy_static! { + static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); + static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); + + static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { + routers: HashMap::new(), + version: 0, + }); +} + +#[derive(Debug, Deserialize, Clone)] +pub struct TriggerRoute { + path: String, + script_path: String, + is_flow: bool, + route_path: String, + workspace_id: String, + request_type: RequestType, + authentication_method: AuthenticationMethod, + edited_by: String, + email: String, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: bool, + wrap_body: bool, + raw_string: bool, + error_handler_path: Option, + error_handler_args: Option>>, + retry: Option>, + mode: TriggerMode, +} + +pub struct RoutersCache { + routers: HashMap>, + version: i64, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub enum RequestType { + Sync, + Async, + SyncSse, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] +#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] +pub enum AuthenticationMethod { + None, + Windmill, + ApiKey, + BasicHttp, + CustomScript, + Signature, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct HttpConfig { + pub route_path: String, + pub route_path_key: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct HttpConfigRequest { + #[serde(default)] + pub route_path: String, + pub request_type: RequestType, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: Option, + pub wrap_body: Option, + pub raw_string: Option, +} + +#[derive(Deserialize)] +struct HttpConfigRequestHelper { + #[serde(default)] + route_path: String, + request_type: Option, + is_async: Option, + authentication_method: AuthenticationMethod, + http_method: HttpMethod, + summary: Option, + description: Option, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: Option, + wrap_body: Option, + raw_string: Option, +} + +impl<'de> Deserialize<'de> for HttpConfigRequest { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + let helper = HttpConfigRequestHelper::deserialize(deserializer)?; + + // Determine request_type with backward compatibility + let request_type = if let Some(mode) = helper.request_type { + mode + } else if let Some(is_async) = helper.is_async { + if is_async { + RequestType::Async + } else { + RequestType::Sync + } + } else { + RequestType::Sync + }; + + Ok(HttpConfigRequest { + route_path: helper.route_path, + request_type, + authentication_method: helper.authentication_method, + http_method: helper.http_method, + summary: helper.summary, + description: helper.description, + static_asset_config: helper.static_asset_config, + is_static_website: helper.is_static_website, + authentication_resource_path: helper.authentication_resource_path, + workspaced_route: helper.workspaced_route, + wrap_body: helper.wrap_body, + raw_string: helper.raw_string, + }) + } +} + +// Regex patterns for route validation +lazy_static::lazy_static! { + // Matches named params like :id or wildcards like :* or * + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); + static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); +} + +#[derive(Deserialize)] +pub struct RouteExists { + pub route_path: String, + pub http_method: HttpMethod, + pub trigger_path: Option, + pub workspaced_route: Option, +} + +pub fn validate_authentication_method( + authentication_method: AuthenticationMethod, + raw_string: Option, +) -> Result<()> { + match (authentication_method, raw_string) { + (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { + Err(Error::BadRequest( + "To use custom script authentication, please enable the raw body option." + .to_string(), + )) + } + _ => Ok(()), + } +} + +pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) + .fetch_one(db) + .await?; + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + if routers_cache.version == 0 || version > routers_cache.version { + drop(routers_cache); + let mut routers = HashMap::new(); + + for http_method in [ + HttpMethod::Get, + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, + ] { + let triggers = sqlx::query_as!( + TriggerRoute, + r#" + SELECT + path, + script_path, + is_flow, + route_path, + authentication_resource_path, + workspace_id, + request_type AS "request_type: _", + authentication_method AS "authentication_method: _", + edited_by, + email, + static_asset_config AS "static_asset_config: _", + wrap_body, + raw_string, + workspaced_route, + is_static_website, + error_handler_path, + error_handler_args as "error_handler_args: _", + retry as "retry: _", + mode as "mode: _" + FROM + http_trigger + WHERE + http_method = $1 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + "#, + &http_method as &HttpMethod + ) + .fetch_all(db) + .await?; + + let mut router = matchit::Router::new(); + + for trigger in triggers { + let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { + format!("/{}/{}", trigger.workspace_id, trigger.route_path) + } else { + format!("/{}", trigger.route_path) + }; + + if trigger.is_static_website { + router + .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider HTTP route {}/*wm_subpath: {:?}", + full_path, + e, + ); + }); + } + router + .insert(full_path.clone(), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); + }); + } + + routers.insert(http_method, router); + } + + let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; + *routers_cache = RoutersCache { routers, version }; + + Ok((true, routers_cache.downgrade())) + } else { + tracing::debug!("No HTTP routers refresh needed"); + Ok((false, routers_cache)) + } +} + +pub async fn refresh_routers_loop( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + match refresh_routers(db).await { + Ok(_) => { + tracing::info!("Loaded HTTP routers"); + } + Err(err) => { + tracing::error!("Error loading HTTP routers: {err:#}"); + } + }; + let db = db.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = killpill_rx.recv() => { + break; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + match refresh_routers(&db).await { + Ok((true, _)) => { + tracing::info!("Refreshed HTTP routers"); + } + Err(err) => { + tracing::error!("Error refreshing HTTP routers: {err:#}"); + } + _ => {} + } + } + } + } + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_request_type_backward_compatibility() { + // Test with new request_type field + let json_new = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + + // Test with legacy is_async = true + let json_legacy_async = r#"{ + "route_path": "/test", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); + assert_eq!(config.request_type, RequestType::Async); + + // Test with legacy is_async = false + let json_legacy_sync = r#"{ + "route_path": "/test", + "is_async": false, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test with neither field (default to sync) + let json_default = r#"{ + "route_path": "/test", + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); + assert_eq!(config.request_type, RequestType::Sync); + + // Test that request_type takes precedence over is_async + let json_both = r#"{ + "route_path": "/test", + "request_type": "sync_sse", + "is_async": true, + "authentication_method": "none", + "http_method": "get", + "is_static_website": false + }"#; + let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); + assert_eq!(config.request_type, RequestType::SyncSse); + } +} diff --git a/backend/windmill-triggers/src/triggers/kafka/handler_oss.rs b/backend/windmill-triggers/src/triggers/kafka/handler_oss.rs new file mode 100644 index 0000000000..313e1e449e --- /dev/null +++ b/backend/windmill-triggers/src/triggers/kafka/handler_oss.rs @@ -0,0 +1,66 @@ +#[cfg(not(feature = "private"))] +use crate::triggers::TriggerData; + +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::KafkaTrigger, + crate::triggers::TriggerCrud, + windmill_api_auth::ApiAuthed, + windmill_common::DB, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for KafkaTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/kafka_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::KafkaTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Kafka triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Kafka triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-triggers/src/triggers/kafka/listener_oss.rs b/backend/windmill-triggers/src/triggers/kafka/listener_oss.rs new file mode 100644 index 0000000000..c5dca4ab37 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/kafka/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::KafkaTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for KafkaTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-triggers/src/triggers/kafka/mod.rs b/backend/windmill-triggers/src/triggers/kafka/mod.rs new file mode 100644 index 0000000000..f09b2283ff --- /dev/null +++ b/backend/windmill-triggers/src/triggers/kafka/mod.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct KafkaTrigger; diff --git a/backend/windmill-triggers/src/triggers/listener.rs b/backend/windmill-triggers/src/triggers/listener.rs new file mode 100644 index 0000000000..aa330182ec --- /dev/null +++ b/backend/windmill-triggers/src/triggers/listener.rs @@ -0,0 +1,927 @@ +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +use crate::{ + capture_ext::insert_capture_payload, + triggers::{ + handler::TriggerCrud, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, + Trigger, TriggerErrorHandling, TriggerMode, + }, +}; +use windmill_api_auth::{fetch_api_authed, ApiAuthed}; +use async_trait::async_trait; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sql_builder::SqlBuilder; +use sqlx::{FromRow, Row}; +use tokio::sync::RwLock; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerKind, TriggerMetadata}, + utils::report_critical_error, + DB, INSTANCE_NAME, +}; + +#[allow(unused)] +#[async_trait] +pub trait Listener: TriggerCrud + TriggerJobArgs { + type Consumer: Send; + type Extra: Send + Sync; + type ExtraState: Send + Sync; + + const JOB_TRIGGER_KIND: JobTriggerKind; + const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; + const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result>; + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + extra: Option<&Self::ExtraState>, + ); + async fn fetch_enabled_unlistened_triggers( + &self, + db: &DB, + ) -> Result>> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + "mode", + "error_handler_path", + "error_handler_args", + "retry", + ]; + + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + ); + + for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let triggers = triggers + .into_iter() + .map(|trigger| ListeningTrigger { + path: trigger.base.path, + workspace_id: trigger.base.workspace_id, + is_flow: trigger.base.is_flow, + username: trigger.base.edited_by, + email: trigger.base.email, + script_path: trigger.base.script_path, + trigger_config: trigger.config, + error_handling: Some(trigger.error_handling), + trigger_mode: true, + suspended_mode: trigger.base.mode == TriggerMode::Suspended, + }) + .collect_vec(); + + Ok(triggers) + } + + async fn fetch_unlistened_captures( + &self, + db: &DB, + ) -> Result>> { + let fields = vec![ + "path", + "is_flow", + "workspace_id", + "owner AS username", + "email", + "trigger_config", + ]; + + let mut sqlb = SqlBuilder::select_from("capture_config"); + sqlb.fields(&fields) + .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) + .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") + .and_where("trigger_config IS NOT NULL") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", + ); + + for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb.sql().expect("failed to build SQL"); + + let captures: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let captures = captures + .into_iter() + .map(|capture| ListeningTrigger { + username: capture.username, + path: capture.path, + workspace_id: capture.workspace_id, + script_path: "".to_string(), + email: capture.email, + trigger_config: capture.trigger_config, + trigger_mode: false, + is_flow: capture.is_flow, + error_handling: None, + suspended_mode: false, + }) + .collect_vec(); + + Ok(captures) + } + + async fn get_extra_state(&self) -> Option { + None + } + + async fn cleanup( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _extra: Option<&Self::ExtraState>, + ) -> Result<()> { + Ok(()) + } + + async fn loop_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + status: Arc>>, + error_message: Option, + ) { + update_rw_lock(status.clone(), error_message).await; + loop { + if let None = self + .update_ping(db, listening_trigger, status.read().await.as_deref()) + .await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn update_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + if listening_trigger.trigger_mode { + self.update_trigger_ping(db, listening_trigger, error).await + } else { + self.update_capture_ping(db, listening_trigger, error).await + } + } + + async fn update_ping_and_loop_ping_status( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + loop_ping_status: Arc>>, + error: Option, + ) -> Option<()> { + // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status + update_rw_lock(loop_ping_status.clone(), error.clone()).await; + if let None = self + .update_ping(db, listening_trigger, error.as_deref()) + .await + { + return None; + } + Some(()) + } + + async fn update_trigger_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar::<_, i32>(&format!( + r#" + UPDATE + {} + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + server_id = $4 AND + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + RETURNING 1 + "#, + Self::TABLE_NAME + )) + .bind(error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .bind(&*INSTANCE_NAME) + .fetch_optional(db) + .await; + + self.handle_ping_result(updated, db, listening_trigger, "trigger") + .await + } + + async fn update_capture_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + server_id = $6 AND + last_client_ping > NOW() - INTERVAL '10 seconds' + RETURNING 1 + "#, + error, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind, + &*INSTANCE_NAME + ) + .fetch_optional(db) + .await + .map(|result| result.flatten()); + + self.handle_ping_result(updated, db, listening_trigger, "capture") + .await + } + + async fn handle_ping_result( + &self, + result: sqlx::Result>, + db: &DB, + listening_trigger: &ListeningTrigger, + entity_type: &str, + ) -> Option<()> { + match result { + Ok(updated) => { + if updated.is_none() { + self.reset_ping_for_restart(db, listening_trigger).await; + tracing::info!( + "{} {} {} changed, disabled, or deleted, stopping...", + Self::TRIGGER_KIND, + entity_type, + listening_trigger.path + ); + return None; + } + } + Err(error) => { + tracing::warn!( + "Error updating ping of {} {} {}: {:?}", + Self::TRIGGER_KIND, + entity_type, + &listening_trigger.path, + error + ); + } + } + + Some(()) + } + + async fn reset_ping_for_restart( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) { + if listening_trigger.trigger_mode { + let _ = sqlx::query(&format!( + r#" + UPDATE + {} + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + server_id IS NULL + "#, + Self::TABLE_NAME + )) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + } else { + let _ = sqlx::query!( + r#" + UPDATE + capture_config + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + is_flow = $3 AND + trigger_kind = $4 AND + server_id IS NULL + "#, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + } + } + + async fn disable_with_error( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: String, + ) { + if listening_trigger.trigger_mode { + let report_status = sqlx::query(&format!( + r#" + UPDATE + {} + SET + mode = 'disabled'::TRIGGER_MODE, + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 + "#, + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + + match report_status { + Ok(_) => { + report_critical_error( + format!( + "Disabling {} trigger {} because of error: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + error + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + } + Err(disable_err) => { + report_critical_error( + format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ).await; + } + } + return; + } + + let report_status = sqlx::query!( + r#" + UPDATE + capture_config + SET + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 + "#, + error, + listening_trigger.workspace_id, + listening_trigger.path, + listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + + if let Err(disable_err) = report_status { + tracing::error!( + "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", + Self::TRIGGER_KIND, + listening_trigger.path, + listening_trigger.workspace_id, + disable_err, + error + ) + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + _extra: Option, + ) -> Result<()> { + let args = Self::build_job_args( + &listening_trigger.script_path, + listening_trigger.is_flow, + &listening_trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let (retry, error_handler_path, error_handler_args) = + match listening_trigger.error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + + tracing::debug!( + "Triggering job from {} event {} with args {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + args + ); + + trigger_runnable( + db, + None, + authed, + &listening_trigger.workspace_id, + &listening_trigger.script_path, + listening_trigger.is_flow, + args, + retry, + error_handler_path.as_deref(), + error_handler_args, + format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), + None, + listening_trigger.suspended_mode, + TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), + ) + .await?; + + Ok(()) + } + + async fn handle_event( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + if listening_trigger.trigger_mode { + if let Err(err) = self + .handle_trigger(db, listening_trigger, payload, trigger_info, extra) + .await + { + report_critical_error( + format!( + "Failed to trigger job from {} event {}: {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + return Err(err); + }; + return Ok(()); + } + + let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); + if let Err(err) = insert_capture_payload( + db, + &listening_trigger.workspace_id, + &listening_trigger.path, + listening_trigger.is_flow, + &Self::TRIGGER_KIND, + main_args, + preprocessor_args, + &listening_trigger.username, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); + return Err(err); + } + Ok(()) + } +} + +#[allow(unused)] +async fn listening( + db: DB, + listener: T, + listening_trigger: ListeningTrigger, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let killpill_rx_consumer = killpill_rx.resubscribe(); + let killpill_rx_get_consumer = killpill_rx.resubscribe(); + + let loop_ping_status = Arc::new(RwLock::new(None)); + let extra_state = listener.get_extra_state().await; + let path = listening_trigger.path.clone(); + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + } + consumer = { + tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) + } => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { + tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + _ = async { + match consumer { + Ok(Some(consumer)) => { + listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; + tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); + listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; + tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); + } + Err(error) => { + tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); + listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; + } + Ok(None) => { + tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); + } + } + } => { + let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; + return; + } + } + } + } +} + +#[allow(unused)] +async fn listen_to_unlistened_events( + listener: T, + db: DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; + + match unlistend_enabled_triggers { + Ok(mut unlistend_enabled_triggers) => { + unlistend_enabled_triggers.shuffle(&mut rand::rng()); + for trigger in unlistend_enabled_triggers { + let has_lock = sqlx::query_scalar(&format!( + r#" + UPDATE + {} + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + T::TABLE_NAME, + )) + .bind(&*INSTANCE_NAME) + .bind(&trigger.workspace_id) + .bind(&trigger.path) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!( + "Spawning new task to listen for {} event", + T::TABLE_NAME + ); + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, trigger, killpill_rx).await } + }); + } else { + tracing::info!( + "{} trigger {} already being listened to", + T::TRIGGER_KIND, + trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for {} trigger {}: {:?}", + T::TRIGGER_KIND, + trigger.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); + } + } + + let unlisted_captures = listener.fetch_unlistened_captures(&db).await; + + match unlisted_captures { + Ok(unlistened_captures) => { + for capture in unlistened_captures { + let has_lock = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + last_client_ping > NOW() - INTERVAL '10 seconds' AND + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') + RETURNING true + "#, + *INSTANCE_NAME, + &capture.workspace_id, + &capture.path, + &capture.is_flow, + T::TRIGGER_KIND as TriggerKind + ) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, capture, killpill_rx).await } + }); + } else { + tracing::info!( + "{} capture {} already being listened to", + T::TRIGGER_KIND.to_string(), + capture.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for capture {} {}: {:?}", + T::TRIGGER_KIND, + capture.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!( + "Error fetching captures {} triggers: {:?}", + T::TRIGGER_KIND, + err + ); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + path: String, + is_flow: bool, + workspace_id: String, + username: String, + email: String, + #[serde(flatten)] + trigger_config: T, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let trigger_config_value = row.try_get("trigger_config")?; + let trigger_config: T = serde_json::from_value(trigger_config_value) + .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; + Ok(Capture { + path: row.try_get("path")?, + is_flow: row.try_get("is_flow")?, + workspace_id: row.try_get("workspace_id")?, + username: row.try_get("username")?, + email: row.try_get("email")?, + trigger_config, + }) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListeningTrigger { + pub path: String, + pub is_flow: bool, + pub workspace_id: String, + pub username: String, + pub email: String, + pub trigger_config: T, + pub script_path: String, + pub trigger_mode: bool, + pub error_handling: Option, + pub suspended_mode: bool, +} + +impl ListeningTrigger { + pub async fn authed(&self, db: &DB, username: &str) -> Result { + fetch_api_authed( + self.username.clone(), + self.email.clone(), + &self.workspace_id, + db, + Some(format!("{}-{}", username, self.path)), + ) + .await + } +} + +#[allow(unused)] +pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { + let mut w = lock.write().await; + *w = value; +} + +#[allow(unused)] +fn listen_to( + trigger: T, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tokio::spawn(async move { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await + } + } + } + }); +} + +#[allow(unused)] +pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>) { + tracing::info!("Starting trigger listeners based on available features..."); + + #[cfg(feature = "postgres_trigger")] + { + let postgres_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::postgres::PostgresTrigger; + + listen_to(PostgresTrigger, db.clone(), postgres_killpill_rx) + } + + #[cfg(feature = "mqtt_trigger")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::mqtt::MqttTrigger; + + listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(feature = "websocket")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::websocket::WebsocketTrigger; + + listen_to(WebsocketTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::gcp::GcpTrigger; + + listen_to(GcpTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::sqs::SqsTrigger; + + listen_to(SqsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::nats::NatsTrigger; + + listen_to(NatsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::kafka::KafkaTrigger; + + listen_to(KafkaTrigger, db.clone(), gcp_killpill_rx); + } + + tracing::info!("All available trigger listeners have been started"); +} diff --git a/backend/windmill-triggers/src/triggers/mod.rs b/backend/windmill-triggers/src/triggers/mod.rs new file mode 100644 index 0000000000..ce8538de2d --- /dev/null +++ b/backend/windmill-triggers/src/triggers/mod.rs @@ -0,0 +1,182 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use std::{collections::HashMap, fmt::Debug}; +use windmill_common::jobs::JobTriggerKind; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum HandlerAction { + Trigger { path: String, trigger_kind: JobTriggerKind }, + // Future variants can be added here (e.g., Script, Flow, etc.) +} + +#[cfg(all(feature = "smtp", feature = "private"))] +pub mod email; +#[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] +pub mod gcp; +#[cfg(feature = "http_trigger")] +pub mod http; +#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] +pub mod kafka; +#[cfg(feature = "mqtt_trigger")] +pub mod mqtt; +#[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] +pub mod nats; +#[cfg(feature = "postgres_trigger")] +pub mod postgres; +#[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] +pub mod sqs; +#[cfg(feature = "websocket")] +pub mod websocket; + +pub mod filter; +pub mod global_handler; +mod handler; +mod listener; +pub mod trigger_helpers; + +#[allow(unused)] +pub(crate) use handler::TriggerCrud; +pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; +pub use listener::start_all_listeners; +#[allow(unused)] +pub(crate) use listener::Listener; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StandardTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct BaseTrigger { + pub workspace_id: String, + pub path: String, + pub script_path: String, + pub mode: TriggerMode, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: DateTime, + pub extra_perms: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct ServerState { + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct TriggerErrorHandling { + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + #[serde(flatten)] + pub base: BaseTrigger, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub server_state: Option, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let base = BaseTrigger::from_row(row)?; + + Ok(Trigger { + base, + config: T::from_row(row)?, + server_state: ServerState::from_row(row).ok(), + error_handling: TriggerErrorHandling::from_row(row)?, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BaseTriggerData { + pub path: String, + pub script_path: String, + pub is_flow: bool, + #[deprecated(note = "Use mode instead")] + enabled: Option, // Kept for backwards compatibility, use mode instead + mode: Option, +} + +impl BaseTriggerData { + pub fn mode(&self) -> &TriggerMode { + self.mode.as_ref().unwrap_or( + #[allow(deprecated)] + if self.enabled.unwrap_or(true) { + &TriggerMode::Enabled + } else { + &TriggerMode::Disabled + }, + ) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TriggerData { + #[serde(flatten)] + pub base: BaseTriggerData, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl StandardTriggerQuery { + pub fn offset(&self) -> i64 { + let page = self.page.unwrap_or(0); + let per_page = self.per_page.unwrap_or(100); + (page * per_page) as i64 + } + + pub fn limit(&self) -> i64 { + self.per_page.unwrap_or(100) as i64 + } +} + +impl Default for StandardTriggerQuery { + fn default() -> Self { + Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] +#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] +#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] +pub enum TriggerMode { + Enabled, + Disabled, + Suspended, +} diff --git a/backend/windmill-triggers/src/triggers/mqtt/handler.rs b/backend/windmill-triggers/src/triggers/mqtt/handler.rs new file mode 100644 index 0000000000..edd326d6e3 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/mqtt/handler.rs @@ -0,0 +1,238 @@ +use crate::{ + resource_ext::try_get_resource_from_db_as, + triggers::{Trigger, TriggerCrud, TriggerData}, +}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; +use axum::async_trait; +use itertools::Itertools; +use sqlx::{types::Json as SqlxJson, PgConnection}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, +}; +use windmill_git_sync::DeployedObject; + +use super::{ + MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger, + MqttV3Config, MqttV5Config, SubscribeTopic, TestMqttConfig, +}; + +#[async_trait] +impl TriggerCrud for MqttTrigger { + type TriggerConfig = MqttConfig; + type Trigger = Trigger; + type TriggerConfigRequest = MqttConfigRequest; + type TestConnectionConfig = TestMqttConfig; + + const TABLE_NAME: &'static str = "mqtt_trigger"; + const TRIGGER_TYPE: &'static str = "mqtt"; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/mqtt_triggers"; + const DEPLOYMENT_NAME: &'static str = "MQTT trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "mqtt_resource_path", + "subscribe_topics", + "v3_config", + "v5_config", + "client_id", + "client_version", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::MqttTrigger { path } + } + + async fn validate_config( + &self, + _db: &DB, + config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + if config.mqtt_resource_path.trim().is_empty() { + return Err(Error::BadRequest( + "MQTT resource path cannot be empty".to_string(), + )); + } + + if config.subscribe_topics.is_empty() { + return Err(Error::BadRequest( + "At least one subscribe topic must be specified".to_string(), + )); + } + + Ok(()) + } + + async fn create_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let subscribe_topics = trigger + .config + .subscribe_topics + .into_iter() + .map(SqlxJson) + .collect_vec(); + let v3_config = trigger.config.v3_config.map(SqlxJson); + let v5_config = trigger.config.v5_config.map(SqlxJson); + + sqlx::query!( + r#" + INSERT INTO mqtt_trigger ( + mqtt_resource_path, + subscribe_topics, + client_version, + client_id, + v3_config, + v5_config, + workspace_id, + path, + script_path, + is_flow, + email, + mode, + edited_by, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 + )"#, + trigger.config.mqtt_resource_path, + subscribe_topics.as_slice() as &[SqlxJson], + trigger.config.client_version as Option, + trigger.config.client_id, + v3_config as Option>, + v5_config as Option>, + w_id, + trigger.base.path, + trigger.base.script_path, + trigger.base.is_flow, + authed.email, + trigger.base.mode() as _, + authed.username, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn update_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let subscribe_topics = trigger + .config + .subscribe_topics + .into_iter() + .map(SqlxJson) + .collect_vec(); + let v3_config = trigger.config.v3_config.map(SqlxJson); + let v5_config = trigger.config.v5_config.map(SqlxJson); + + // Important to set server_id to NULL to stop current mqtt listener + sqlx::query!( + r#" + UPDATE + mqtt_trigger + SET + mqtt_resource_path = $1, + subscribe_topics = $2, + client_version = $3, + client_id = $4, + v3_config = $5, + v5_config = $6, + is_flow = $7, + edited_by = $8, + email = $9, + script_path = $10, + path = $11, + edited_at = now(), + error = NULL, + server_id = NULL, + error_handler_path = $14, + error_handler_args = $15, + retry = $16 + WHERE + workspace_id = $12 AND + path = $13 + "#, + trigger.config.mqtt_resource_path, + subscribe_topics.as_slice() as &[SqlxJson], + trigger.config.client_version as Option, + trigger.config.client_id, + v3_config as Option>, + v5_config as Option>, + trigger.base.is_flow, + authed.username, + authed.email, + trigger.base.script_path, + trigger.base.path, + workspace_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let mqtt_resource = try_get_resource_from_db_as::( + authed, + Some(user_db.clone()), + db, + &config.mqtt_resource_path, + workspace_id, + ) + .await?; + + let connect_f = async { + let client_builder = MqttClientBuilder::new( + mqtt_resource, + Some(""), + vec![], + config.v3_config.as_ref(), + config.v5_config.as_ref(), + config.client_version.as_ref(), + ); + + client_builder.build_client().await.map_err(|err| { + Error::BadConfig(format!( + "Error connecting to mqtt broker: {}", + err.to_string() + )) + }) + }; + + connect_f.await?; + Ok(()) + } +} diff --git a/backend/windmill-triggers/src/triggers/mqtt/listener.rs b/backend/windmill-triggers/src/triggers/mqtt/listener.rs new file mode 100644 index 0000000000..76c4bdbb9b --- /dev/null +++ b/backend/windmill-triggers/src/triggers/mqtt/listener.rs @@ -0,0 +1,376 @@ +use std::{collections::HashMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use rumqttc::{ + v5::{ + mqttbytes::v5::PublishProperties, Event as V5Event, EventLoop as V5EventLoop, + Incoming as V5Incoming, + }, + Event as V3Event, EventLoop as V3EventLoop, Incoming as V3Incoming, +}; +use std::time::Duration; +use tokio::sync::RwLock; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resource_ext::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + mqtt::{ + MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, + V3MqttHandler, V5MqttHandler, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +#[async_trait] +impl Listener for MqttTrigger { + type Consumer = MqttClientResult; + type Extra = (); + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let MqttConfig { + mqtt_resource_path, + subscribe_topics, + v3_config, + v5_config, + client_id, + client_version, + .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let mqtt_resource = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + mqtt_resource_path, + workspace_id, + ) + .await?; + + let subscribe_topics = subscribe_topics + .iter() + .map(|topic| topic.0.clone()) + .collect(); + + let client_builder = MqttClientBuilder::new( + mqtt_resource, + client_id.as_deref(), + subscribe_topics, + v3_config.as_ref().map(|c| &c.0), + v5_config.as_ref().map(|c| &c.0), + client_version.as_ref(), + ); + + let client_result = client_builder + .build_client() + .await + .map_err(|e| Error::BadConfig(format!("Failed to build MQTT client: {}", e)))?; + + Ok(Some(client_result)) + } + + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + tracing::info!( + "Starting to listen for MQTT trigger {}", + &listening_trigger.path + ); + + match consumer { + MqttClientResult::V3((v3_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v3_handler, event_loop).await + } + MqttClientResult::V5((v5_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v5_handler, event_loop).await + } + } + } +} + +const TIMEOUT_DURATION: u64 = 10; +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +fn convert_disconnect_packet_into_string( + disconnect: rumqttc::v5::mqttbytes::v5::Disconnect, +) -> String { + let err_message = disconnect + .properties + .map(|properties| properties.reason_string) + .flatten(); + let reason_code = disconnect.reason_code as u8; + format!( + "Disconnected by the broker, reason code: {}, {}", + reason_code, + err_message + .map(|err| format!("message: {}", err)) + .unwrap_or("".to_string()) + ) +} + +#[async_trait] +pub trait EventLoop { + type Event; + type Error; + + async fn poll(&mut self) -> Result; + async fn verify_connection(&mut self) -> Result<()>; +} + +#[async_trait] +impl EventLoop for V5EventLoop { + type Event = V5Event; + type Error = rumqttc::v5::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), + Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { + return Err(Error::BadConfig(convert_disconnect_packet_into_string( + disconnect, + ))); + } + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +#[async_trait] +impl EventLoop for V3EventLoop { + type Event = V3Event; + type Error = rumqttc::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +async fn handle_event( + db: &DB, + listener: &T, + listening_trigger: &ListeningTrigger, + handler: H, + mut event_loop: E, +) -> () +where + T: Listener, + H: MqttEvent, + E: EventLoop, + E::Error: ToString, + ::Payload: From, +{ + loop { + let event = event_loop.poll().await; + + match event { + Ok(event) => { + let publish_data = handler.handle_event(event); + if let Ok(Some((payload, publish_data))) = publish_data { + let trigger_info = HashMap::from([ + ("topic".to_string(), to_raw_value(&publish_data.topic)), + ("retain".to_string(), to_raw_value(&publish_data.retain)), + ("pkid".to_string(), to_raw_value(&publish_data.pkid)), + ("qos".to_string(), to_raw_value(&publish_data.qos)), + ( + "v5".to_string(), + to_raw_value(&publish_data.v5.map(|properties| { + serde_json::json!({ + "payload_format_indicator": properties.payload_format_indicator, + "topic_alias": properties.topic_alias, + "response_topic": properties.response_topic, + "correlation_data": properties.correlation_data.as_deref(), + "user_properties": properties.user_properties, + "subscription_identifiers": properties.subscription_identifiers, + "content_type": properties.content_type, + }) + })), + ), + ]); + let _ = listener + .handle_event(db, listening_trigger, payload.into(), trigger_info, None) + .await; + } + } + Err(err) => { + let error = err.to_string(); + tracing::debug!("Error: {}", &err); + listener + .disable_with_error(db, listening_trigger, error) + .await; + return; + } + } + } +} + +#[derive(Clone)] +#[allow(unused)] +pub struct PublishData { + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, +} + +impl PublishData { + pub fn new( + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, + ) -> PublishData { + PublishData { topic, retain, pkid, v5, qos } + } +} + +trait MqttEvent { + type IncomingPacket; + type PublishPacket; + type Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; + fn handle_event(&self, event: Self::Event) -> Result>; +} + +impl MqttEvent for V5MqttHandler { + type IncomingPacket = V5Incoming; + type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; + type Event = V5Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), + publish_packet.retain, + publish_packet.pkid, + publish_packet.properties, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V5 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + Self::IncomingPacket::Disconnect(disconnect) => { + return Err( + anyhow::anyhow!(convert_disconnect_packet_into_string(disconnect)).into(), + ); + } + packet => { + tracing::debug!("Received = {:#?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:#?}", packet); + } + } + + Ok(None) + } +} + +impl MqttEvent for V3MqttHandler { + type IncomingPacket = V3Incoming; + type PublishPacket = rumqttc::mqttbytes::v4::Publish; + type Event = V3Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + publish_packet.topic, + publish_packet.retain, + publish_packet.pkid, + None, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V3 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + packet => { + tracing::debug!("Received = {:?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:?}", packet); + } + } + + Ok(None) + } +} diff --git a/backend/windmill-triggers/src/triggers/mqtt/mod.rs b/backend/windmill-triggers/src/triggers/mqtt/mod.rs new file mode 100644 index 0000000000..cfc8e36900 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/mqtt/mod.rs @@ -0,0 +1,346 @@ +use base64::{engine, prelude::*}; +use itertools::Itertools; +use rumqttc::{ + v5::{ + mqttbytes::{ + v5::{ConnectProperties, Filter}, + QoS as V5QoS, + }, + AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, + }, + AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, + QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow, Type}; +use std::{collections::HashMap, time::Duration}; +use windmill_common::{ + error::{to_anyhow, Error}, + triggers::TriggerKind, + worker::to_raw_value, +}; + +use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; + +pub mod handler; +pub mod listener; + +#[derive(Clone, Copy)] +pub struct MqttTrigger; + +impl TriggerJobArgs for MqttTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[serde(rename_all = "lowercase")] +pub enum QualityOfService { + Qos0, + Qos1, + Qos2, +} + +impl From for V3QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V3QoS::AtMostOnce, + QualityOfService::Qos1 => V3QoS::AtLeastOnce, + QualityOfService::Qos2 => V3QoS::ExactlyOnce, + } + } +} + +impl From for V5QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V5QoS::AtMostOnce, + QualityOfService::Qos1 => V5QoS::AtLeastOnce, + QualityOfService::Qos2 => V5QoS::ExactlyOnce, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV3Config { + clean_session: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV5Config { + clean_start: Option, + session_expiry_interval: Option, + topic_alias_maximum: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[sqlx(type_name = "MQTT_CLIENT_VERSION")] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MqttClientVersion { + V3, + V5, +} + +#[derive(Debug, Deserialize)] +pub struct Tls { + enabled: bool, + ca_certificate: String, + pkcs12_client_certificate: Option, + pkcs12_certificate_password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Credentials { + username: Option, + password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct MqttResource { + broker: String, + port: u16, + credentials: Option, + tls: Option, +} + +#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] +pub struct SubscribeTopic { + qos: QualityOfService, + topic: String, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct MqttConfig { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec>, + pub v3_config: Option>, + pub v5_config: Option>, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttConfigRequest { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec, + pub v3_config: Option, + pub v5_config: Option, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestMqttConfig { + pub mqtt_resource_path: String, + pub client_version: Option, + pub v3_config: Option, + pub v5_config: Option, +} + +// Constants +pub const KEEP_ALIVE: u64 = 60; +pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; +pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; +pub const TIMEOUT_DURATION: u64 = 10; +pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +pub struct V3MqttHandler; +pub struct V5MqttHandler; + +pub enum MqttClientResult { + V3((V3MqttHandler, V3EventLoop)), + V5((V5MqttHandler, V5EventLoop)), +} + +#[derive(Debug, thiserror::Error)] +pub enum MqttError { + #[error("{0}")] + Common(#[from] Error), + #[error("{0}")] + V5RumqttClient(#[from] rumqttc::v5::ClientError), + #[error("{0}")] + V5ConnectionError(#[from] rumqttc::v5::ConnectionError), + #[error("{0}")] + V3RumqttClient(#[from] rumqttc::ClientError), + #[error("{0}")] + V3ConnectionError(#[from] rumqttc::ConnectionError), + #[error("{0}")] + Base64Decode(#[from] base64::DecodeError), +} + +pub struct MqttClientBuilder<'client> { + mqtt_resource: MqttResource, + client_id: &'client str, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, +} + +impl<'client> MqttClientBuilder<'client> { + pub fn new( + mqtt_resource: MqttResource, + client_id: Option<&'client str>, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, + ) -> Self { + Self { + mqtt_resource, + client_id: client_id.unwrap_or(""), + subscribe_topics, + v3_config, + v5_config, + mqtt_client_version, + } + } + + pub async fn build_client(&self) -> Result { + match self.mqtt_client_version { + Some(MqttClientVersion::V5) | None => self.build_v5_client().await, + Some(MqttClientVersion::V3) => self.build_v3_client().await, + } + } + + fn get_tls_configuration(&self) -> Result, Error> { + let transport = match self.mqtt_resource.tls { + Some(ref tls) if tls.enabled => { + let transport = match tls.ca_certificate.trim().is_empty() { + true => rumqttc::Transport::Tls(TlsConfiguration::Native), + false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { + ca: tls.ca_certificate.as_bytes().to_vec(), + client_auth: { + match tls.pkcs12_client_certificate.as_ref() { + Some(client_certificate) + if !client_certificate.trim().is_empty() => + { + let client_certificate = BASE64_STANDARD + .decode(client_certificate) + .map_err(to_anyhow)?; + let password = tls + .pkcs12_certificate_password + .clone() + .unwrap_or("".to_string()); + Some((client_certificate, password)) + } + _ => None, + } + }, + }), + }; + + Some(transport) + } + _ => None, + }; + + Ok(transport) + } + + async fn build_v5_client(&self) -> Result { + let mut mqtt_options = V5MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + + mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); + + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + + if let Some(v5_config) = self.v5_config { + mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); + mqtt_options.set_connect_properties(ConnectProperties { + session_expiry_interval: v5_config.session_expiry_interval, + receive_maximum: None, + max_packet_size: None, + topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), + request_response_info: None, + request_problem_info: None, + user_properties: vec![], + authentication_method: None, + authentication_data: None, + }); + } + + let (async_client, mut event_loop) = + V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) + } + + async fn build_v3_client(&self) -> Result { + let mut mqtt_options = V3MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + if let Some(v3_config) = self.v3_config { + mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); + } + + let (async_client, mut event_loop) = + V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) + } +} diff --git a/backend/windmill-triggers/src/triggers/nats/handler_oss.rs b/backend/windmill-triggers/src/triggers/nats/handler_oss.rs new file mode 100644 index 0000000000..a90fc3f1a9 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/nats/handler_oss.rs @@ -0,0 +1,63 @@ +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::NatsTrigger, + crate::triggers::{TriggerCrud, TriggerData}, + windmill_api_auth::ApiAuthed, + windmill_common::DB, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for NatsTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/nats_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::NatsTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "NATS triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "NATS triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-triggers/src/triggers/nats/listener_oss.rs b/backend/windmill-triggers/src/triggers/nats/listener_oss.rs new file mode 100644 index 0000000000..80f1befded --- /dev/null +++ b/backend/windmill-triggers/src/triggers/nats/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::NatsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for NatsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-triggers/src/triggers/nats/mod.rs b/backend/windmill-triggers/src/triggers/nats/mod.rs new file mode 100644 index 0000000000..668df073c9 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/nats/mod.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct NatsTrigger; diff --git a/backend/windmill-triggers/src/triggers/postgres/bool.rs b/backend/windmill-triggers/src/triggers/postgres/bool.rs new file mode 100644 index 0000000000..13b0a8fa68 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/bool.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs +* +*/ + +#[derive(Debug, Error)] +pub enum ParseBoolError { + #[error("invalid input value: {0}")] + InvalidInput(String), +} + +pub fn parse_bool(s: &str) -> Result { + match s { + "t" => Ok(true), + "f" => Ok(false), + _ => Err(ParseBoolError::InvalidInput(s.to_string())), + } +} diff --git a/backend/windmill-triggers/src/triggers/postgres/converter.rs b/backend/windmill-triggers/src/triggers/postgres/converter.rs new file mode 100644 index 0000000000..f57268c4b8 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/converter.rs @@ -0,0 +1,255 @@ +use core::str; +use std::{ + num::{ParseFloatError, ParseIntError}, + str::FromStr, +}; + +use super::{ + bool::{parse_bool, ParseBoolError}, + hex::{from_bytea_hex, ByteaHexParseError}, +}; +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use rust_decimal::Decimal; +use rust_postgres::types::Type; +use serde_json::{to_value, Number, Value}; +use thiserror::Error; +use uuid::Uuid; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs +* +*/ + +#[derive(Debug, Error)] +pub enum ConverterError { + #[error("invalid bool value")] + InvalidBool(#[from] ParseBoolError), + + #[error("invalid int value")] + InvalidInt(#[from] ParseIntError), + + #[error("invalid float value")] + InvalidFloat(#[from] ParseFloatError), + + #[error("invalid numeric: {0}")] + InvalidNumeric(#[from] rust_decimal::Error), + + #[error("invalid bytea: {0}")] + InvalidBytea(#[from] ByteaHexParseError), + + #[error("invalid uuid: {0}")] + InvalidUuid(#[from] uuid::Error), + + #[error("invalid json: {0}")] + InvalidJson(#[from] serde_json::Error), + + #[error("invalid timestamp: {0} ")] + InvalidTimestamp(#[from] chrono::ParseError), + + #[error("invalid array: {0}")] + InvalidArray(#[from] ArrayParseError), + + #[error("{0}")] + Custom(String), +} + +fn convert_into(number: T) -> Number +where + T: Sized, + serde_json::Number: From, +{ + serde_json::Number::from(number) +} + +pub struct Converter; + +#[derive(Debug, Error)] +pub enum ArrayParseError { + #[error("input too short")] + InputTooShort, + + #[error("missing braces")] + MissingBraces, +} + +fn f64_to_json_number(raw_val: f64) -> Result { + let temp = serde_json::Number::from_f64(raw_val.into()) + .ok_or(ConverterError::Custom("invalid json-float".to_string()))?; + Ok(Value::Number(temp)) +} + +impl Converter { + pub fn try_from_str(typ: Option, str: &str) -> Result { + let value = match typ.unwrap_or(Type::TEXT) { + Type::BOOL => Value::Bool(parse_bool(str)?), + Type::BOOL_ARRAY => { + Converter::parse_array(str, |str| Ok(Value::Bool(parse_bool(str)?)))? + } + Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => { + Value::String(str.to_string()) + } + Type::CHAR_ARRAY + | Type::BPCHAR_ARRAY + | Type::VARCHAR_ARRAY + | Type::NAME_ARRAY + | Type::TEXT_ARRAY => { + Converter::parse_array(str, |str| Ok(Value::String(str.to_string())))? + } + Type::INT2 => Value::Number(convert_into(str.parse::()?)), + Type::INT2_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::INT4 => Value::Number(convert_into(str.parse::()?)), + Type::INT4_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::INT8 => Value::Number(convert_into(str.parse::()?)), + Type::INT8_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::FLOAT4 => f64_to_json_number(str.parse::()?)?, + Type::FLOAT4_ARRAY => { + Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? + } + Type::FLOAT8 => f64_to_json_number(str.parse::()?)?, + Type::FLOAT8_ARRAY => { + Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? + } + Type::NUMERIC => serde_json::json!(Decimal::from_str(str)?), + Type::NUMERIC_ARRAY => { + Converter::parse_array(str, |str| Ok(serde_json::json!(Decimal::from_str(str)?)))? + } + Type::BYTEA => to_value(from_bytea_hex(str)?).unwrap(), + Type::BYTEA_ARRAY => { + Converter::parse_array(str, |str| Ok(to_value(from_bytea_hex(str)?).unwrap()))? + } + Type::DATE => { + let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; + Value::String(date.to_string()) + } + Type::DATE_ARRAY => Converter::parse_array(str, |str| { + let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; + Ok(Value::String(date.to_string())) + })?, + Type::TIME => { + let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; + Value::String(time.to_string()) + } + Type::TIME_ARRAY => Converter::parse_array(str, |str| { + let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; + Ok(Value::String(time.to_string())) + })?, + Type::TIMESTAMP => { + let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; + Value::String(timestamp.to_string()) + } + Type::TIMESTAMP_ARRAY => Converter::parse_array(str, |str| { + let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; + Ok(Value::String(timestamp.to_string())) + })?, + Type::TIMESTAMPTZ => { + let val = + match DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z") { + Ok(val) => val, + Err(_) => { + DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%:z")? + } + }; + let utc: DateTime = val.into(); + Value::String(utc.to_string()) + } + Type::TIMESTAMPTZ_ARRAY => { + match Converter::parse_array(str, |str| { + let utc: DateTime = + DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z")? + .into(); + Ok(Value::String(utc.to_string())) + }) { + Ok(val) => val, + Err(_) => Converter::parse_array(str, |str| { + let utc: DateTime = DateTime::::parse_from_str( + str, + "%Y-%m-%d %H:%M:%S%.f%#z", + )? + .into(); + Ok(Value::String(utc.to_string())) + })?, + } + } + Type::UUID => Value::String(Uuid::parse_str(str)?.to_string()), + Type::UUID_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::String(Uuid::parse_str(str)?.to_string())) + })?, + Type::JSON | Type::JSONB => serde_json::from_str::(str)?, + Type::JSON_ARRAY | Type::JSONB_ARRAY => Converter::parse_array(str, |str| { + Ok(serde_json::from_str::(str)?) + })?, + Type::OID => Value::Number(convert_into(str.parse::()?)), + Type::OID_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + _ => Value::String(str.to_string()), + }; + + Ok(value) + } + + fn parse_array

(str: &str, mut parse: P) -> Result + where + P: FnMut(&str) -> Result, + { + if str.len() < 2 { + return Err(ArrayParseError::InputTooShort.into()); + } + + if !str.starts_with('{') || !str.ends_with('}') { + return Err(ArrayParseError::MissingBraces.into()); + } + + let mut res = vec![]; + let str = &str[1..(str.len() - 1)]; + let mut val_str = String::with_capacity(10); + let mut in_quotes = false; + let mut in_escape = false; + let mut chars = str.chars(); + let mut done = str.is_empty(); + + while !done { + loop { + match chars.next() { + Some(c) => match c { + c if in_escape => { + val_str.push(c); + in_escape = false; + } + '"' => in_quotes = !in_quotes, + '\\' => in_escape = true, + ',' if !in_quotes => { + break; + } + c => { + val_str.push(c); + } + }, + None => { + done = true; + break; + } + } + } + let val = if val_str.to_lowercase() == "null" { + Value::Null + } else { + parse(&val_str)? + }; + res.push(val); + val_str.clear(); + } + let arr = Value::Array(res); + Ok(arr) + } +} diff --git a/backend/windmill-triggers/src/triggers/postgres/handler.rs b/backend/windmill-triggers/src/triggers/postgres/handler.rs new file mode 100644 index 0000000000..58b0d175e6 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/handler.rs @@ -0,0 +1,1102 @@ +use std::collections::HashMap; + +use axum::{ + async_trait, + extract::Path, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use itertools::Itertools; +use lazy_static::lazy_static; +use pg_escape::{quote_identifier, quote_literal}; +use quick_cache::sync::Cache; +use rust_postgres::{types::Type, Client}; +use sqlx::PgConnection; +use uuid; +use windmill_common::{ + db::UserDB, + error::{self, to_anyhow, Error, Result}, +}; +use windmill_git_sync::DeployedObject; + +use crate::triggers::{postgres::PostgresTrigger, Trigger, TriggerCrud, TriggerData}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; + +use super::{ + check_if_valid_publication_for_postgres_version, create_logical_replication_slot, + create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + mapper::{Mapper, MappingInfo}, + PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PublicationData, + Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, + ERROR_PUBLICATION_NAME_NOT_EXISTS, +}; + +// Lazy static template cache +lazy_static! { + pub static ref TEMPLATE: Cache = Cache::new(50); +} + +#[async_trait] +impl TriggerCrud for PostgresTrigger { + type TriggerConfig = PostgresConfig; + type Trigger = Trigger; + type TriggerConfigRequest = PostgresConfigRequest; + type TestConnectionConfig = TestPostgresConfig; + + const TABLE_NAME: &'static str = "postgres_trigger"; + const TRIGGER_TYPE: &'static str = "postgres"; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/postgres_triggers"; + const DEPLOYMENT_NAME: &'static str = "PostgreSQL trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "postgres_resource_path", + "replication_slot_name", + "publication_name", + "NULL::text AS basic_mode", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::PostgresTrigger { path } + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let Self::TriggerConfigRequest { + postgres_resource_path, + publication_name, + replication_slot_name, + publication, + } = trigger.config; + + let (pub_name, slot_name) = + if publication_name.is_empty() && replication_slot_name.is_empty() { + if publication.is_none() { + return Err(Error::BadRequest("publication must be set".to_string())); + } + + let PostgresPublicationReplication { publication_name, replication_slot_name } = + create_custom_slot_and_publication_inner( + authed.clone(), + UserDB::new(db.clone()), + &db, + &postgres_resource_path, + &w_id, + &publication.unwrap(), + ) + .await?; + + (publication_name, replication_slot_name) + } else { + if publication_name.is_empty() { + return Err(Error::BadRequest( + "Publication name must not be empty".to_string(), + )); + } else if replication_slot_name.is_empty() { + return Err(Error::BadRequest( + "Replication slot name must not be empty".to_string(), + )); + } + (publication_name, replication_slot_name) + }; + + sqlx::query!( + r#" + INSERT INTO postgres_trigger ( + workspace_id, + path, + postgres_resource_path, + replication_slot_name, + publication_name, + script_path, + is_flow, + mode, + edited_by, + email, + edited_at, + error_handler_path, + error_handler_args, + retry + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13 + ) + "#, + w_id, + trigger.base.path, + postgres_resource_path, + slot_name, + pub_name, + trigger.base.script_path, + trigger.base.is_flow, + trigger.base.mode() as _, + authed.username, + authed.email, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + Ok(()) + } + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let Self::TriggerConfigRequest { + replication_slot_name, + publication_name, + postgres_resource_path, + publication, + } = trigger.config; + + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(UserDB::new(db.clone())), + db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let exists = + check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name) + .await?; + + let remote_db_tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + if !exists { + tracing::debug!( + "Logical replication slot named: {} does not exists creating it...", + &replication_slot_name + ); + create_logical_replication_slot(remote_db_tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; + } + + if let Some(publication) = publication { + let publication_data = + get_publication_scope_and_transaction(remote_db_tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; + + update_pg_publication( + remote_db_tx.client(), + &publication_name, + publication, + publication_data.map(|publication| publication.0), + ) + .await + .map_err(to_anyhow)?; + } + + remote_db_tx.commit().await.map_err(to_anyhow)?; + + sqlx::query!( + r#" + UPDATE postgres_trigger + SET + postgres_resource_path = $1, + replication_slot_name = $2, + publication_name = $3, + script_path = $4, + path = $5, + is_flow = $6, + edited_by = $7, + email = $8, + edited_at = now(), + server_id = NULL, + error = NULL, + error_handler_path = $11, + error_handler_args = $12, + retry = $13 + WHERE + workspace_id = $9 AND path = $10 + "#, + postgres_resource_path, + replication_slot_name, + publication_name, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + authed.username, + authed.email, + w_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let connect_f = async { + get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + db, + &config.postgres_resource_path, + workspace_id, + ) + .await + .map_err(|err| { + Error::BadConfig(format!("Error connecting to postgres: {}", err.to_string())) + })?; + + Ok::<(), Error>(()) + }; + + connect_f.await?; + Ok(()) + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/get_template_script/:id", get(get_template_script)) + .route("/create_template_script", post(create_template_script)) + .route( + "/is_valid_postgres_configuration/*path", + get(is_database_in_logical_level), + ) + .nest("/publication", publication_service()) + .nest("/slot", slot_service()) + .nest("/postgres", postgres_service()) + } +} + +fn publication_service() -> Router { + Router::new() + .route("/get/:publication_name/*path", get(get_publication_info)) + .route("/create/:publication_name/*path", post(create_publication)) + .route("/update/:publication_name/*path", post(alter_publication)) + .route( + "/delete/:publication_name/*path", + delete(delete_publication), + ) + .route("/list/*path", get(list_database_publication)) +} + +fn slot_service() -> Router { + Router::new() + .route("/list/*path", get(list_slot_name)) + .route("/create/*path", post(create_slot)) + .route("/delete/*path", delete(drop_slot_name)) +} + +fn postgres_service() -> Router { + Router::new().route("/version/*path", get(get_postgres_version)) +} + +async fn check_if_logical_replication_slot_exist( + pg_connection: &mut Client, + replication_slot_name: &str, +) -> Result { + let row = pg_connection + .query_opt( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", + &[&replication_slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(row.is_some()) +} + +async fn create_custom_slot_and_publication_inner( + authed: ApiAuthed, + user_db: UserDB, + db: &DB, + postgres_resource_path: &str, + w_id: &str, + publication: &PublicationData, +) -> Result { + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + let publication_name = format!("windmill_trigger_{}", generate_random_string()); + let replication_slot_name = publication_name.clone(); + + create_logical_replication_slot(tx.client(), &replication_slot_name).await?; + create_pg_publication( + &tx.client(), + &publication_name, + publication.table_to_track.as_deref(), + &publication.transaction_to_track, + ) + .await?; + + tx.commit().await.map_err(to_anyhow)?; + + Ok(PostgresPublicationReplication::new( + publication_name, + replication_slot_name, + )) +} + +pub async fn get_postgres_version_internal(pg_connection: &Client) -> Result { + let row = pg_connection + .query_one("SHOW server_version;", &[]) + .await + .map_err(to_anyhow)?; + + let postgres_version: String = row.get(0); + + Ok(postgres_version) +} + +pub async fn get_postgres_version( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> Result { + let pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let postgres_version = get_postgres_version_internal(&pg_connection).await?; + + Ok(postgres_version) +} + +pub async fn list_slot_name( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> Result>> { + let pg_connection: Client = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let rows = pg_connection + .query( + r#" + SELECT + slot_name, + active + FROM + pg_replication_slots + WHERE + plugin = 'pgoutput' AND + slot_type = 'logical'; + "#, + &[], + ) + .await + .map_err(to_anyhow)?; + + let slots = rows + .into_iter() + .map(|row| SlotList { slot_name: row.get("slot_name"), active: row.get("active") }) + .collect(); + + Ok(Json(slots)) +} + +pub async fn create_slot( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, + Json(Slot { name }): Json, +) -> Result { + let pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + create_logical_replication_slot(&pg_connection, &name).await?; + + Ok(format!("Replication slot {} created!", name)) +} + +pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { + let row = pg_connection + .query_opt( + r#" + SELECT + active_pid + FROM + pg_replication_slots + WHERE + slot_name = $1 + "#, + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + + let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); + + if let Some(pid) = active_pid { + pg_connection + .execute("SELECT pg_terminate_backend($1)", &[&pid]) + .await + .map_err(to_anyhow)?; + } + + pg_connection + .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + +pub async fn drop_slot_name( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, + Json(Slot { name }): Json, +) -> Result { + let pg_connection = + get_default_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id) + .await + .map_err(to_anyhow)?; + + drop_logical_replication_slot(&pg_connection, &name) + .await + .map_err(to_anyhow)?; + + Ok(format!("Replication slot {} deleted!", name)) +} + +pub async fn list_database_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> Result>> { + let pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let rows = pg_connection + .query( + "SELECT pubname AS publication_name FROM pg_publication;", + &[], + ) + .await + .map_err(to_anyhow)?; + + let publications = rows + .into_iter() + .map(|row| row.get::<_, String>("publication_name")) + .collect_vec(); + + Ok(Json(publications)) +} + +pub async fn get_publication_info( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, +) -> Result> { + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let publication_data = + get_publication_scope_and_transaction(&mut pg_connection, &publication_name).await; + + let (all_table, transaction_to_track) = match publication_data { + Ok(Some(pub_data)) => pub_data, + Ok(None) => { + return Err(Error::NotFound( + ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), + )) + } + Err(e) => return Err(e), + }; + + let table_to_track = if !all_table { + Some(get_tracked_relations(&mut pg_connection, &publication_name).await?) + } else { + None + }; + Ok(Json(PublicationData::new( + table_to_track, + transaction_to_track, + ))) +} + +pub async fn create_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, + Json(publication_data): Json, +) -> Result { + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let PublicationData { table_to_track, transaction_to_track } = publication_data; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + create_pg_publication( + tx.client(), + &publication_name, + table_to_track.as_deref(), + &transaction_to_track, + ) + .await?; + + tx.commit().await.map_err(to_anyhow)?; + + Ok(format!( + "Publication {} successfully created!", + publication_name + )) +} + +pub async fn delete_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, +) -> Result { + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + drop_publication(&mut pg_connection, &publication_name).await?; + + Ok(format!( + "Publication {} successfully deleted!", + publication_name + )) +} + +pub async fn update_pg_publication( + pg_connection: &Client, + publication_name: &str, + PublicationData { table_to_track, transaction_to_track }: PublicationData, + all_table: Option, +) -> Result<()> { + let quoted_publication_name = quote_identifier(publication_name); + let transaction_to_track_as_str = transaction_to_track.iter().join(","); + match table_to_track { + Some(ref relations) if !relations.is_empty() => { + // If all_table is None, the publication does not exist yet + if all_table.unwrap_or(true) { + if all_table.is_some_and(|all_table| all_table) { + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; + } + create_pg_publication( + pg_connection, + publication_name, + table_to_track.as_deref(), + &transaction_to_track, + ) + .await + .map_err(to_anyhow)?; + } else { + let pg_14 = check_if_valid_publication_for_postgres_version( + pg_connection, + table_to_track.as_deref(), + ) + .await + .map_err(to_anyhow)?; + + let mut query = format!("ALTER PUBLICATION {} SET ", quoted_publication_name); + let mut first = true; + + for (i, schema) in relations.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push_str("TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); + } else { + if pg_14 && first { + query.push_str("TABLE ONLY "); + first = false; + } else if !pg_14 { + query.push_str("TABLE ONLY "); + } + + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", schema_name, table_name); + query.push_str(&full_name); + + if let Some(columns) = table.columns_name.as_ref() { + let cols = + columns.iter().map(|col| quote_identifier(col)).join(", "); + query.push_str(&format!(" ({})", cols)); + } + + if let Some(where_clause) = &table.where_clause { + query.push_str(&format!(" WHERE ({})", where_clause)); + } + + if j + 1 != schema.table_to_track.len() { + query.push_str(", "); + } + } + } + + if i + 1 != relations.len() { + query.push_str(", "); + } + } + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + let publish_query = format!( + "ALTER PUBLICATION {} SET (publish = '{}');", + quoted_publication_name, transaction_to_track_as_str + ); + pg_connection + .execute(&publish_query, &[]) + .await + .map_err(to_anyhow)?; + } + } + _ => { + drop_publication(pg_connection, publication_name) + .await + .map_err(to_anyhow)?; + let create_all_query = format!( + "CREATE PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');", + quoted_publication_name, transaction_to_track_as_str + ); + pg_connection + .execute(&create_all_query, &[]) + .await + .map_err(to_anyhow)?; + } + } + + Ok(()) +} + +pub async fn alter_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, + Json(publication_data): Json, +) -> Result { + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let tx = pg_connection.transaction().await.map_err(to_anyhow)?; + + let publication = get_publication_scope_and_transaction(tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; + + update_pg_publication( + tx.client(), + &publication_name, + publication_data, + publication.map(|publication| publication.0), + ) + .await + .map_err(to_anyhow)?; + + tx.commit().await.map_err(to_anyhow)?; + + Ok(format!( + "Publication {} updated with success", + publication_name + )) +} + +pub async fn get_publication_scope_and_transaction( + pg_connection: &Client, + publication_name: &str, +) -> Result)>> { + let row_opt = pg_connection + .query_opt( + r#" + SELECT + puballtables AS all_table, + pubinsert AS insert, + pubupdate AS update, + pubdelete AS delete + FROM + pg_publication + WHERE + pubname = $1 + "#, + &[&publication_name], + ) + .await + .map_err(to_anyhow)?; + + let row = match row_opt { + Some(r) => r, + None => return Ok(None), + }; + + let all_table: bool = row.get("all_table"); + let pub_insert: bool = row.get("insert"); + let pub_update: bool = row.get("update"); + let pub_delete: bool = row.get("delete"); + + let mut transaction_to_track = Vec::with_capacity(3); + if pub_insert { + transaction_to_track.push("insert".to_string()); + } + if pub_update { + transaction_to_track.push("update".to_string()); + } + if pub_delete { + transaction_to_track.push("delete".to_string()); + } + + Ok(Some((all_table, transaction_to_track))) +} + +pub async fn get_tracked_relations( + pg_connection: &Client, + publication_name: &str, +) -> Result> { + let pg_version = get_postgres_version_internal(pg_connection).await?; + + let query = if pg_version.starts_with("14") { + r#" + SELECT + schemaname AS schema_name, + tablename AS table_name, + NULL::text[] AS columns, + NULL::text AS where_clause + FROM + pg_publication_tables + WHERE + pubname = $1; + "# + } else { + r#" + SELECT + schemaname AS schema_name, + tablename AS table_name, + attnames AS columns, + rowfilter AS where_clause + FROM + pg_publication_tables + WHERE + pubname = $1; + "# + }; + + let rows = pg_connection + .query(query, &[&publication_name]) + .await + .map_err(to_anyhow)?; + + let mut table_to_track: HashMap = HashMap::new(); + + for row in rows { + let schema_name: Option = row.get("schema_name"); + let table_name: Option = row.get("table_name"); + let columns: Option> = row.get("columns"); + let where_clause: Option = row.get("where_clause"); + + let schema_name = schema_name.ok_or_else::(|| { + anyhow::anyhow!( + "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.", + publication_name, + ) + .into() + })?; + + let table_name = table_name.ok_or_else::(|| { + anyhow::anyhow!( + "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.", + schema_name, + publication_name, + ) + .into() + })?; + + let entry = table_to_track.entry(schema_name.clone()); + let table_to_track_item = TableToTrack::new(table_name, where_clause, columns); + + match entry { + std::collections::hash_map::Entry::Occupied(mut occupied) => { + occupied.get_mut().add_new_table(table_to_track_item); + } + std::collections::hash_map::Entry::Vacant(vacant) => { + vacant.insert(Relations::new(schema_name, vec![table_to_track_item])); + } + } + } + + Ok(table_to_track.into_values().collect_vec()) +} + +pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> Result { + let template = if let Some((_, template)) = TEMPLATE.remove(&id) { + template + } else { + "".to_string() + }; + Ok(template) +} + +pub async fn create_template_script( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(template_script): Json, +) -> Result { + let TemplateScript { postgres_resource_path, relations, language } = template_script; + + let relations = match relations { + Some(r) => r, + None => { + return Err( + anyhow::anyhow!("You must at least choose schema to fetch table from").into(), + ) + } + }; + + let pg_connection: Client = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let mut schema_or_fully_qualified_name = Vec::with_capacity(relations.len()); + let mut columns_list = Vec::with_capacity(relations.len()); + + for relation in relations { + if !relation.table_to_track.is_empty() { + for table in relation.table_to_track { + let fully_qualified_name = format!("{}.{}", relation.schema_name, table.table_name); + schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name)); + let columns = table + .columns_name + .map(|c| quote_literal(&c.join(","))) + .unwrap_or_else(|| "''".to_string()); + columns_list.push(columns); + } + } else { + schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); + columns_list.push("''".to_string()); + } + } + + let tables_name = schema_or_fully_qualified_name.join(","); + let columns_list = columns_list.join(","); + + let query = format!( + r#" + WITH table_column_mapping AS ( + SELECT + unnest(ARRAY[{}]) AS table_name, + unnest(ARRAY[{}]) AS column_list + ), + parsed_columns AS ( + SELECT + tcm.table_name, + CASE + WHEN tcm.column_list = '' THEN NULL + ELSE string_to_array(tcm.column_list, ',') + END AS columns + FROM table_column_mapping tcm + ) + SELECT + ns.nspname AS table_schema, + cls.relname AS table_name, + attr.attname AS column_name, + attr.atttypid AS oid, + attr.attnotnull AS is_nullable + FROM pg_attribute attr + JOIN pg_class cls ON attr.attrelid = cls.oid + JOIN pg_namespace ns ON cls.relnamespace = ns.oid + JOIN parsed_columns pc + ON ns.nspname || '.' || cls.relname = pc.table_name + OR ns.nspname = pc.table_name + WHERE + attr.attnum > 0 + AND NOT attr.attisdropped + AND cls.relkind = 'r' + AND ( + pc.columns IS NULL + OR attr.attname = ANY(pc.columns) + ); + "#, + tables_name, columns_list + ); + + let rows = pg_connection.query(&query, &[]).await.map_err(to_anyhow)?; + + let mut schema_map: HashMap>> = HashMap::new(); + + #[derive(Debug)] + struct ColumnInfo { + table_schema: String, + table_name: String, + column_name: String, + oid: u32, + is_nullable: bool, + } + + for row in rows { + let info = ColumnInfo { + table_schema: row.get("table_schema"), + table_name: row.get("table_name"), + column_name: row.get("column_name"), + oid: row.get::<_, u32>("oid"), + is_nullable: row.get::<_, bool>("is_nullable"), + }; + + let mapped_info = + MappingInfo::new(info.column_name, Type::from_oid(info.oid), info.is_nullable); + + match schema_map.entry(info.table_schema) { + std::collections::hash_map::Entry::Occupied(mut schema_entry) => { + match schema_entry.get_mut().entry(info.table_name) { + std::collections::hash_map::Entry::Occupied(mut table_entry) => { + table_entry.get_mut().push(mapped_info); + } + std::collections::hash_map::Entry::Vacant(v) => { + v.insert(vec![mapped_info]); + } + } + } + std::collections::hash_map::Entry::Vacant(schema_vacant) => { + let mut table_map = HashMap::new(); + table_map.insert(info.table_name, vec![mapped_info]); + schema_vacant.insert(table_map); + } + } + } + + let mapper = Mapper::new(schema_map, language); + let template = mapper.get_template(); + + let id = format!("{}-{}", w_id, uuid::Uuid::new_v4()); + + TEMPLATE.insert(id.clone(), template); + + Ok(id) +} + +pub async fn is_database_in_logical_level( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> error::JsonResult { + let pg_connection = get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; + + let row_opt = pg_connection + .query_opt("SHOW wal_level;", &[]) + .await + .map_err(to_anyhow)?; + + let wal_level: Option = row_opt.map(|row| row.get(0)); + + let is_logical = matches!(wal_level.as_deref(), Some("logical")); + + Ok(Json(is_logical)) +} diff --git a/backend/windmill-triggers/src/triggers/postgres/hex.rs b/backend/windmill-triggers/src/triggers/postgres/hex.rs new file mode 100644 index 0000000000..2ada70dd35 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/hex.rs @@ -0,0 +1,45 @@ +use std::num::ParseIntError; + + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs +* +*/ + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ByteaHexParseError { + #[error("missing prefix '\\x'")] + InvalidPrefix, + + #[error("invalid byte")] + OddNumerOfDigits, + + #[error("parse int result: {0}")] + ParseInt(#[from] ParseIntError), +} + +pub fn from_bytea_hex(s: &str) -> Result, ByteaHexParseError> { + if s.len() < 2 || &s[..2] != "\\x" { + return Err(ByteaHexParseError::InvalidPrefix); + } + + let mut result = Vec::with_capacity((s.len() - 2) / 2); + let s = &s[2..]; + + if s.len() % 2 != 0 { + return Err(ByteaHexParseError::OddNumerOfDigits); + } + + for i in (0..s.len()).step_by(2) { + let val = u8::from_str_radix(&s[i..i + 2], 16)?; + result.push(val); + } + + Ok(result) +} diff --git a/backend/windmill-triggers/src/triggers/postgres/listener.rs b/backend/windmill-triggers/src/triggers/postgres/listener.rs new file mode 100644 index 0000000000..cae8b713b5 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/listener.rs @@ -0,0 +1,441 @@ +use std::{collections::HashMap, pin::Pin, sync::Arc}; + +use bytes::{BufMut, Bytes, BytesMut}; +use chrono::TimeZone; +use futures::{pin_mut, SinkExt}; +use pg_escape::{quote_identifier, quote_literal}; +use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; +use tokio::sync::RwLock; +use tokio_stream::StreamExt; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resource_ext::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + postgres::{ + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::drop_logical_replication_slot, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{ + Begin, Commit, Delete, Insert, Relation, Type, Update, + }, + PrimaryKeepAliveBody, ReplicationMessage, + }, + Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +pub struct LogicalReplicationSettings { + pub streaming: bool, +} + +impl LogicalReplicationSettings { + pub fn new(streaming: bool) -> Self { + Self { streaming } + } +} + +pub struct PostgresSimpleClient(Client); + +trait RowExist { + fn row_exist(&self) -> bool; +} + +impl RowExist for Vec { + fn row_exist(&self) -> bool { + self.iter() + .find_map(|element| { + if let SimpleQueryMessage::CommandComplete(value) = element { + Some(*value) + } else { + None + } + }) + .is_some_and(|value| value > 0) + } +} + +impl PostgresSimpleClient { + async fn new(database: &Postgres) -> Result { + let client = get_raw_postgres_connection(database, true).await?; + + Ok(PostgresSimpleClient(client)) + } + + async fn execute_query( + &self, + query: &str, + ) -> std::result::Result, rust_postgres::Error> { + self.0.simple_query(query).await + } + + async fn get_logical_replication_stream( + &self, + publication_name: &str, + logical_replication_slot_name: &str, + ) -> Result<(CopyBothDuplex, LogicalReplicationSettings)> { + let options = format!( + r#"("proto_version" '2', "publication_names" {})"#, + quote_literal(publication_name), + ); + + let query = format!( + r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, + quote_identifier(logical_replication_slot_name), + options + ); + + Ok(( + self.0 + .copy_both_simple::(query.as_str()) + .await + .map_err(to_anyhow)?, + LogicalReplicationSettings::new(false), + )) + } + + async fn send_status_update( + primary_keep_alive: PrimaryKeepAliveBody, + copy_both_stream: &mut Pin<&mut CopyBothDuplex>, + ) { + let mut buf = BytesMut::new(); + let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let ts = chrono::Utc::now() + .signed_duration_since(ts) + .num_microseconds() + .unwrap_or(0); + + buf.put_u8(b'r'); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_i64(ts); + buf.put_u8(0); + copy_both_stream.send(buf.freeze()).await.unwrap(); + tracing::debug!("Send update status message"); + } +} + +#[async_trait::async_trait] +impl Listener for PostgresTrigger { + type Consumer = (CopyBothDuplex, LogicalReplicationSettings); + type Extra = (); + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let PostgresConfig { + postgres_resource_path, publication_name, replication_slot_name, .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let database = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + postgres_resource_path, + workspace_id, + ) + .await?; + + let client = PostgresSimpleClient::new(&database).await?; + + let publication = client + .execute_query(&format!( + "SELECT pubname FROM pg_publication WHERE pubname = {}", + quote_literal(&publication_name) + )) + .await + .map_err(to_anyhow)?; + + if !publication.row_exist() { + return Err(Error::BadConfig( + ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), + )); + } + + let replication_slot = client + .execute_query(&format!( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", + quote_literal(&replication_slot_name) + )) + .await + .map_err(to_anyhow)?; + + if !replication_slot.row_exist() { + return Err(Error::BadConfig( + ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), + )); + } + + let (logical_replication_stream, logical_replication_settings) = client + .get_logical_replication_stream(&publication_name, &replication_slot_name) + .await + .map_err(to_anyhow)?; + + Ok(Some(( + logical_replication_stream, + logical_replication_settings, + ))) + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + let (logical_replication_stream, logical_replication_settings) = consumer; + pin_mut!(logical_replication_stream); + let mut relations = RelationConverter::new(); + tracing::info!( + "Starting to listen for postgres trigger {}", + &listening_trigger.path + ); + loop { + let message = logical_replication_stream.next().await; + let message = match message { + Some(message) => message, + None => { + tracing::error!( + "Stream for postgres trigger {} closed", + &listening_trigger.path + ); + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some("Stream closed".to_string()), + ) + .await + { + return; + } + return; + } + }; + + let message = match message { + Ok(message) => message, + Err(err) => { + let err = format!( + "Postgres trigger named {} had an error while receiving a message : {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + let logical_message = match ReplicationMessage::parse(message) { + Ok(logical_message) => logical_message, + Err(err) => { + let err = format!( + "Postgres trigger named: {} had an error while parsing message: {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + match logical_message { + ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { + if primary_keep_alive.reply { + PostgresSimpleClient::send_status_update( + primary_keep_alive, + &mut logical_replication_stream, + ) + .await; + } + } + ReplicationMessage::XLogData(x_log_data) => { + let logical_replication_message = match x_log_data + .parse(&logical_replication_settings) + { + Ok(logical_replication_message) => logical_replication_message, + Err(err) => { + tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); + continue; + } + }; + + let json = match logical_replication_message { + Relation(relation_body) => { + relations.add_relation(relation_body); + None + } + Begin | Type | Commit => None, + Insert(insert) => Some(( + insert.o_id, + Ok(None), + relations.row_to_json((insert.o_id, insert.tuple)), + "insert", + )), + Update(update) => { + let old_row = update + .old_tuple + .map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))) + .transpose(); + let row = relations.row_to_json((update.o_id, update.new_tuple)); + Some((update.o_id, old_row, row, "update")) + } + Delete(delete) => { + let row = delete + .old_tuple + .unwrap_or_else(|| delete.key_tuple.unwrap()); + Some(( + delete.o_id, + Ok(None), + relations.row_to_json((delete.o_id, row)), + "delete", + )) + } + }; + match json { + Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + let database_info = HashMap::from([ + ("schema_name".to_string(), to_raw_value(&relation.namespace)), + ("table_name".to_string(), to_raw_value(&relation.name)), + ( + "transaction_type".to_string(), + to_raw_value(&transaction_type), + ), + ("old_row".to_string(), to_raw_value(&old_row)), + ("row".to_string(), to_raw_value(&row)), + ]); + let _ = self + .handle_event( + db, + listening_trigger, + database_info, + HashMap::new(), + None, + ) + .await; + } + Some((o_id, old_row, row, transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + + if let Err(err) = old_row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode OLD row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + + if let Err(err) = row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode NEW row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + } + _ => {} + } + } + } + } + } + + async fn cleanup( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _extra_state: Option<&Self::ExtraState>, + ) -> Result<()> { + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let user_db = UserDB::new(db.clone()); + + let mut pg_connection = get_default_pg_connection( + authed, + Some(user_db), + &db, + &listening_trigger.trigger_config.postgres_resource_path, + &listening_trigger.workspace_id, + ) + .await?; + + if listening_trigger.trigger_config.basic_mode.unwrap_or(false) { + drop_logical_replication_slot( + &mut pg_connection, + &listening_trigger.trigger_config.replication_slot_name, + ) + .await?; + + drop_publication( + &mut pg_connection, + &listening_trigger.trigger_config.publication_name, + ) + .await?; + } + + Ok(()) + } +} diff --git a/backend/windmill-triggers/src/triggers/postgres/mapper.rs b/backend/windmill-triggers/src/triggers/postgres/mapper.rs new file mode 100644 index 0000000000..a24fb65378 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/mapper.rs @@ -0,0 +1,136 @@ +use std::collections::HashMap; + +use rust_postgres::types::Type; + +use super::Language; + +fn postgres_to_typescript_type(postgres_type: Option) -> String { + let data_type = match postgres_type { + Some(postgres_type) => match postgres_type { + Type::BOOL => "boolean", + Type::BOOL_ARRAY => "Array", + Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => "string", + Type::CHAR_ARRAY + | Type::BPCHAR_ARRAY + | Type::VARCHAR_ARRAY + | Type::NAME_ARRAY + | Type::TEXT_ARRAY => "Array", + Type::INT2 | Type::INT4 | Type::INT8 | Type::NUMERIC => "number", + Type::INT2_ARRAY | Type::INT4_ARRAY | Type::INT8_ARRAY => "Array", + Type::FLOAT4 | Type::FLOAT8 => "number", + Type::FLOAT8_ARRAY | Type::FLOAT4_ARRAY => "Array", + Type::NUMERIC_ARRAY => "Array", + Type::BYTEA => "Array", + Type::BYTEA_ARRAY => "Array>", + Type::DATE => "string", + Type::DATE_ARRAY => "Array", + Type::TIME => "string", + Type::TIME_ARRAY => "Array", + Type::TIMESTAMPTZ | Type::TIMESTAMP => "string", + Type::TIMESTAMPTZ_ARRAY | Type::TIMESTAMP_ARRAY => "Array", + Type::UUID => "string", + Type::UUID_ARRAY => "Array", + Type::JSON | Type::JSONB | Type::JSON_ARRAY | Type::JSONB_ARRAY => "unknown", + Type::OID => "number", + Type::OID_ARRAY => "Array", + _ => "string", + }, + None => "string", + }; + + data_type.to_string() +} + +fn into_body_struct(language: Language, mapped_info: Vec) -> String { + let mut block = String::new(); + match language { + Language::Typescript => { + block.push_str("{\r\n"); + for field in mapped_info { + let typescript_type = postgres_to_typescript_type(field.data_type); + let mut key = field.column_name; + if field.is_nullable { + key.push('?'); + } + let full_field = format!("\t\t{}: {},\r\n", key, typescript_type); + block.push_str(&full_field); + } + block.push_str("\t}"); + } + } + block +} + +#[derive(Debug)] +pub struct MappingInfo { + data_type: Option, + is_nullable: bool, + column_name: String, +} + +impl MappingInfo { + pub fn new(column_name: String, data_type: Option, is_nullable: bool) -> Self { + Self { column_name, data_type, is_nullable } + } +} + +pub struct Mapper { + to_template: HashMap>>, + language: Language, +} + +impl Mapper { + pub fn new( + to_template: HashMap>>, + language: Language, + ) -> Self { + Self { to_template, language } + } + + fn into_typescript_template(self) -> Vec { + let mut struct_definitions = Vec::new(); + for (_, mapping_info) in self.to_template { + let last_elem = mapping_info.len() - 1; + for (i, (_, mapped_info)) in mapping_info.into_iter().enumerate() { + let mut struct_body = into_body_struct(Language::Typescript, mapped_info); + let struct_body = if i != last_elem { + struct_body.push_str("\r\n"); + struct_body + } else { + struct_body + }; + struct_definitions.push(struct_body); + } + } + struct_definitions + } + + pub fn get_template(self) -> String { + let struct_definition = match self.language { + Language::Typescript => self.into_typescript_template(), + }; + + let struct_definition = if struct_definition.is_empty() { + "any".to_string() + } else { + struct_definition.join("\t| ") + }; + + format!( + r#" + + +export async function main( + transaction_type: "insert" | "update" | "delete", + schema_name: string, + table_name: string, + row: {}, + old_row?: {} +) {{ +}} + "#, + &struct_definition, + &struct_definition + ) + } +} diff --git a/backend/windmill-triggers/src/triggers/postgres/mod.rs b/backend/windmill-triggers/src/triggers/postgres/mod.rs new file mode 100644 index 0000000000..85d6f37435 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/mod.rs @@ -0,0 +1,554 @@ +use std::collections::HashMap; + +use crate::{ + resource_ext::try_get_resource_from_db_as, + triggers::trigger_helpers::TriggerJobArgs, +}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; +use chrono::Utc; +use itertools::Itertools; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; +use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::value::RawValue; +use sqlx::FromRow; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + utils::empty_as_none, +}; + +mod bool; +mod converter; +pub mod handler; +mod hex; +pub mod listener; +mod mapper; +mod relation; +mod replication_message; + +#[derive(Clone, Copy)] +pub struct PostgresTrigger; + +impl TriggerJobArgs for PostgresTrigger { + type Payload = HashMap>; + const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; + fn v1_payload_fn(payload: &HashMap>) -> HashMap> { + payload.to_owned() + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct PostgresConfig { + pub postgres_resource_path: String, + pub replication_slot_name: String, + pub publication_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub basic_mode: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PostgresConfigRequest { + postgres_resource_path: String, + #[serde(default)] + replication_slot_name: String, + #[serde(default)] + publication_name: String, + publication: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestPostgresConfig { + pub postgres_resource_path: String, +} + +fn check_if_valid_relation<'de, D>( + relations: D, +) -> std::result::Result>, D::Error> +where + D: Deserializer<'de>, +{ + let relations: Option> = Option::deserialize(relations)?; + let mut track_all_table_in_schema = false; + let mut track_specific_columns_in_table = false; + match relations { + Some(relations) => { + for relation in relations.iter() { + if relation.schema_name.is_empty() { + return Err(serde::de::Error::custom( + "Schema Name must not be empty".to_string(), + )); + } + + if !track_all_table_in_schema && relation.table_to_track.is_empty() { + track_all_table_in_schema = true; + continue; + } + + for table_to_track in relation.table_to_track.iter() { + if table_to_track.table_name.trim().is_empty() { + return Err(serde::de::Error::custom( + "Table name must not be empty".to_string(), + )); + } + + if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { + track_specific_columns_in_table = true; + } + } + + if track_all_table_in_schema && track_specific_columns_in_table { + return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); + } + } + + if !relations + .iter() + .map(|relation| relation.schema_name.as_str()) + .all_unique() + { + return Err(serde::de::Error::custom( + "You cannot choose a schema more than one time".to_string(), + )); + } + + Ok(Some(relations)) + } + None => Ok(None), + } +} + +fn check_if_valid_transaction_type<'de, D>( + transaction_type: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; + if transaction_type.len() > 3 { + return Err(serde::de::Error::custom( + "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" + .to_string(), + )); + } + transaction_type.sort_unstable(); + transaction_type.dedup(); + + for transaction in transaction_type.iter() { + match transaction.to_lowercase().as_ref() { + "insert" => {}, + "update" => {}, + "delete" => {}, + _ => { + return Err(serde::de::Error::custom( + "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" + .to_string(), + )) + } + } + } + + Ok(transaction_type) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PublicationData { + #[serde(default, deserialize_with = "check_if_valid_relation")] + pub table_to_track: Option>, + #[serde(deserialize_with = "check_if_valid_transaction_type")] + pub transaction_to_track: Vec, +} + +impl PublicationData { + pub fn new( + table_to_track: Option>, + transaction_to_track: Vec, + ) -> PublicationData { + PublicationData { table_to_track, transaction_to_track } + } +} + +// Slot list struct +#[derive(FromRow, Debug, Serialize)] +pub struct SlotList { + pub slot_name: Option, + pub active: Option, +} + +// Slot struct +#[derive(Debug, Serialize, Deserialize)] +pub struct Slot { + pub name: String, +} + +// Template script struct +#[derive(Debug, Deserialize)] +pub struct TemplateScript { + pub postgres_resource_path: String, + #[serde(deserialize_with = "check_if_valid_relation")] + pub relations: Option>, + pub language: Language, +} + +// Language enum +#[derive(Deserialize, Debug)] +pub enum Language { + #[serde(rename = "typescript", alias = "Typescript")] + Typescript, +} + +// Test postgres struct +#[derive(Serialize, Deserialize)] +pub struct TestPostgres { + pub postgres_resource_path: String, +} + +// PostgreSQL publication replication struct +#[derive(Serialize, Deserialize)] +pub struct PostgresPublicationReplication { + pub publication_name: String, + pub replication_slot_name: String, +} + +impl PostgresPublicationReplication { + pub fn new( + publication_name: String, + replication_slot_name: String, + ) -> PostgresPublicationReplication { + PostgresPublicationReplication { publication_name, replication_slot_name } + } +} + +pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Postgres { + pub user: String, + pub password: String, + pub host: String, + pub port: Option, + pub dbname: String, + #[serde(default)] + pub sslmode: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub root_certificate_pem: Option, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct TableToTrack { + pub table_name: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub where_clause: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub columns_name: Option>, +} + +impl TableToTrack { + pub fn new( + table_name: String, + where_clause: Option, + columns_name: Option>, + ) -> TableToTrack { + TableToTrack { table_name, where_clause, columns_name } + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct Relations { + pub schema_name: String, + pub table_to_track: Vec, +} + +impl Relations { + pub fn new(schema_name: String, table_to_track: Vec) -> Relations { + Relations { schema_name, table_to_track } + } + + pub fn add_new_table(&mut self, table_to_track: TableToTrack) { + self.table_to_track.push(table_to_track); + } +} + +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); + if let Err(e) = connection.await { + tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); + }; + tracing::info!("PostgreSQL trigger connection closed"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + +pub async fn get_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, + logical_mode: bool, +) -> Result { + let database = + try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) + .await?; + + Ok(get_raw_postgres_connection(&database, logical_mode).await?) +} + +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await +} + +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn check_if_valid_publication_for_postgres_version( + pg_connection: &Client, + table_to_track: Option<&[Relations]>, +) -> Result { + use crate::triggers::postgres::handler::get_postgres_version_internal; + + let postgres_version = get_postgres_version_internal(pg_connection).await?; + + let pg_14 = postgres_version.starts_with("14"); + if pg_14 { + let unsupported_publication = table_to_track + .and_then(|relations| { + relations.iter().find(|relation| { + let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { + table_to_track.where_clause.is_some() + || table_to_track.columns_name.is_some() + }); + + relation.table_to_track.is_empty() || invalid_relation.is_some() + }) + }) + .is_some(); + + if unsupported_publication { + return Err(Error::BadRequest( + "Your PostgreSQL database is running version 14, which does not support the following publication features: \ + - WHERE clause filtering, \ + - selective column tracking, and \ + - tracking all tables within a schema.\n\ + These features are only available in PostgreSQL 15 and above.".to_string(), + )); + } + } + Ok(pg_14) +} + +pub async fn create_pg_publication( + pg_connection: &Client, + publication_name: &str, + table_to_track: Option<&[Relations]>, + transaction_to_track: &[String], +) -> Result<()> { + let pg_14 = + check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; + let mut query = String::from("CREATE PUBLICATION "); + + query.push_str("e_identifier(publication_name)); + + match table_to_track { + Some(database_component) if !database_component.is_empty() => { + query.push_str(" FOR"); + let mut first = true; + for (i, schema) in database_component.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push_str(" TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); + } else { + if pg_14 && first { + query.push_str(" TABLE ONLY "); + first = false + } else if !pg_14 { + query.push_str(" TABLE ONLY "); + } + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push_str(&full_name); + if let Some(columns) = table.columns_name.as_ref() { + query.push_str(" ("); + let columns = columns + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push_str(&columns); + query.push_str(")"); + } + + if let Some(where_clause) = &table.where_clause { + query.push_str(" WHERE ("); + query.push_str(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push_str(", "); + } + } + } + if i < database_component.len() - 1 { + query.push_str(", "); + } + } + } + _ => { + query.push_str(" FOR ALL TABLES "); + } + }; + + if !transaction_to_track.is_empty() { + let transactions = || transaction_to_track.iter().join(", "); + query.push_str(" WITH (publish = '"); + query.push_str(&transactions()); + query.push_str("');"); + } + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { + let mut query = String::from("DROP PUBLICATION IF EXISTS "); + let quoted_publication_name = quote_identifier(publication_name); + query.push_str("ed_publication_name); + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + +pub fn generate_random_string() -> String { + let timestamp = Utc::now().timestamp_millis().to_string(); + let mut rng = rand::rng(); + let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; + + let random_part = (0..10) + .map(|_| { + charset + .chars() + .nth(rng.random_range(0..charset.len())) + .unwrap() + }) + .collect::(); + + format!("{}_{}", timestamp, random_part) +} diff --git a/backend/windmill-triggers/src/triggers/postgres/relation.rs b/backend/windmill-triggers/src/triggers/postgres/relation.rs new file mode 100644 index 0000000000..f313efb893 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/relation.rs @@ -0,0 +1,74 @@ +use core::str; + +use serde_json::{Map, Value}; +use std::{collections::HashMap, str::Utf8Error}; + +use super::{ + converter::{Converter, ConverterError}, + replication_message::{Columns, RelationBody, TupleData}, +}; +use rust_postgres::types::Oid; +#[derive(Debug, thiserror::Error)] +pub enum RelationConversionError { + #[error("Could not find matching table")] + FailToFindMatchingTable, + + #[error("Binary data not supported")] + BinaryFormatNotSupported, + + #[error("decode error: {0}")] + FromBytes(#[from] ConverterError), + + #[error("invalid string value")] + InvalidStr(#[from] Utf8Error), +} + +pub struct RelationConverter(HashMap); + +impl RelationConverter { + pub fn new() -> Self { + Self(HashMap::new()) + } + + pub fn add_relation(&mut self, relation: RelationBody) { + self.0.insert(relation.o_id, relation); + } + + pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> { + self.0 + .get(&o_id) + .map(|relation_body| &relation_body.columns) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> { + self.0 + .get(&o_id) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn row_to_json( + &self, + to_decode: (Oid, Vec), + ) -> Result, RelationConversionError> { + let (o_id, tuple_data) = to_decode; + let mut object: Map = Map::new(); + let columns = self.get_columns(o_id)?; + + for (i, column) in columns.iter().enumerate() { + let value = match &tuple_data[i] { + TupleData::Null | TupleData::UnchangedToast => Value::Null, + TupleData::Binary(_) => { + return Err(RelationConversionError::BinaryFormatNotSupported) + } + TupleData::Text(bytes) => { + let str = str::from_utf8(&bytes[..])?; + Converter::try_from_str(column.type_o_id.clone(), str)? + } + }; + + object.insert(column.name.clone(), value); + } + Ok(object) + } +} diff --git a/backend/windmill-triggers/src/triggers/postgres/replication_message.rs b/backend/windmill-triggers/src/triggers/postgres/replication_message.rs new file mode 100644 index 0000000000..5ae7f8cf2f --- /dev/null +++ b/backend/windmill-triggers/src/triggers/postgres/replication_message.rs @@ -0,0 +1,510 @@ +#![allow(unused)] + +use core::str; +use std::{ + cmp, + io::{self, Cursor, Read}, + str::Utf8Error, +}; + +use byteorder::{BigEndian, ReadBytesExt}; +use bytes::Bytes; +use rust_postgres::types::{Oid, Type}; +use thiserror::Error; + +use super::listener::LogicalReplicationSettings; +const PRIMARY_KEEPALIVE_BYTE: u8 = b'k'; +const X_LOG_DATA_BYTE: u8 = b'w'; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/cdc_event.rs +* +*/ + +#[derive(Debug)] +pub struct PrimaryKeepAliveBody { + pub wal_end: u64, + pub timestamp: i64, + pub reply: bool, +} + +impl PrimaryKeepAliveBody { + pub fn new(wal_end: u64, timestamp: i64, reply: bool) -> PrimaryKeepAliveBody { + PrimaryKeepAliveBody { wal_end, timestamp, reply } + } +} + +const BEGIN_BYTE: u8 = b'B'; +const COMMIT_BYTE: u8 = b'C'; +const ORIGIN_BYTE: u8 = b'O'; +const RELATION_BYTE: u8 = b'R'; +const TYPE_BYTE: u8 = b'Y'; +const INSERT_BYTE: u8 = b'I'; +const UPDATE_BYTE: u8 = b'U'; +const DELETE_BYTE: u8 = b'D'; +const TUPLE_NEW_BYTE: u8 = b'N'; +const TUPLE_KEY_BYTE: u8 = b'K'; +const TUPLE_OLD_BYTE: u8 = b'O'; +const TUPLE_DATA_NULL_BYTE: u8 = b'n'; +const TUPLE_DATA_TOAST_BYTE: u8 = b'u'; +const TUPLE_DATA_TEXT_BYTE: u8 = b't'; +const TUPLE_DATA_BINARY_BYTE: u8 = b'b'; + +const REPLICA_IDENTITY_DEFAULT_BYTE: i8 = 0x64; +const REPLICA_IDENTITY_NOTHING_BYTE: i8 = 0x6E; +const REPLICA_IDENTITY_FULL_BYTE: i8 = 0x66; +const REPLICA_IDENTITY_INDEX_BYTE: i8 = 0x69; + +#[derive(Debug)] +pub enum ReplicaIdentity { + Default, + Nothing, + Full, + Index, +} + +#[derive(Debug)] +pub struct Column { + pub flags: i8, + pub name: String, + pub type_o_id: Option, + pub type_modifier: i32, +} + +impl Column { + pub fn new(flags: i8, name: String, type_o_id: Option, type_modifier: i32) -> Self { + Self { flags, name, type_o_id, type_modifier } + } +} + +pub type Columns = Vec; + +#[derive(Debug)] +pub struct RelationBody { + pub transaction_id: Option, + pub o_id: Oid, + pub namespace: String, + pub name: String, + pub replica_identity: ReplicaIdentity, + pub columns: Columns, +} + +impl RelationBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + namespace: String, + name: String, + replica_identity: ReplicaIdentity, + columns: Columns, + ) -> Self { + Self { transaction_id, o_id, namespace, name, replica_identity, columns } + } +} + +#[derive(Debug)] +pub struct InsertBody { + pub transaction_id: Option, + pub o_id: Oid, + pub tuple: Vec, +} + +impl InsertBody { + pub fn new(transaction_id: Option, o_id: Oid, tuple: Vec) -> Self { + Self { transaction_id, o_id, tuple } + } +} + +#[derive(Debug)] +pub struct UpdateBody { + transaction_id: Option, + pub o_id: Oid, + pub old_tuple: Option>, + pub key_tuple: Option>, + pub new_tuple: Vec, +} + +impl UpdateBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + old_tuple: Option>, + key_tuple: Option>, + new_tuple: Vec, + ) -> Self { + Self { transaction_id, o_id, old_tuple, key_tuple, new_tuple } + } +} + +#[derive(Debug)] +pub struct DeleteBody { + transaction_id: Option, + pub o_id: Oid, + pub old_tuple: Option>, + pub key_tuple: Option>, +} + +impl DeleteBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + old_tuple: Option>, + key_tuple: Option>, + ) -> Self { + Self { transaction_id, o_id, old_tuple, key_tuple } + } +} + +#[derive(Debug)] +pub enum TupleData { + Null, + UnchangedToast, + Text(Bytes), + Binary(Bytes), +} + +impl TupleData { + fn parse(buf: &mut Buffer) -> Result, ConversionError> { + let number_of_columns = buf.read_i16::()?; + let mut tuples = Vec::with_capacity(number_of_columns as usize); + for _ in 0..number_of_columns { + let byte = buf.read_u8()?; + let tuple_data = match byte { + TUPLE_DATA_NULL_BYTE => TupleData::Null, + TUPLE_DATA_TOAST_BYTE => TupleData::UnchangedToast, + TUPLE_DATA_TEXT_BYTE => { + let len = buf.read_i32::()?; + let mut data = vec![0; len as usize]; + buf.read_exact(&mut data)?; + TupleData::Text(data.into()) + } + TUPLE_DATA_BINARY_BYTE => { + let len = buf.read_i32::()?; + let mut data = vec![0; len as usize]; + buf.read_exact(&mut data)?; + TupleData::Binary(data.into()) + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message byte `{}`", byte), + ))); + } + }; + + tuples.push(tuple_data); + } + + Ok(tuples) + } +} + +#[derive(Debug)] +pub enum TransactionBody { + Insert(InsertBody), + Update(UpdateBody), + Delete(DeleteBody), +} + +#[non_exhaustive] +#[derive(Debug)] +pub enum LogicalReplicationMessage { + Begin, + Commit, + Relation(RelationBody), + Type, + Insert(InsertBody), + Update(UpdateBody), + Delete(DeleteBody), +} + +#[derive(Debug)] +pub struct XLogDataBody { + pub wal_start: u64, + pub wal_end: u64, + pub timestamp: i64, + pub data: Bytes, +} + +#[derive(Error, Debug)] +pub enum ConversionError { + #[error("Error: {0}")] + Io(#[from] io::Error), + #[error("Utf8Error conversion: {0}")] + Utf8(#[from] Utf8Error), +} + +struct Buffer { + bytes: Bytes, + idx: usize, +} + +impl Buffer { + pub fn new(bytes: Bytes, idx: usize) -> Buffer { + Buffer { bytes, idx } + } + + fn slice(&self) -> &[u8] { + &self.bytes[self.idx..] + } + + fn read_cstr(&mut self) -> Result { + match self.slice().iter().position(|&x| x == 0) { + Some(pos) => { + let start = self.idx; + let end = start + pos; + let cstr = str::from_utf8(&self.bytes[start..end])?.to_owned(); + self.idx = end + 1; + Ok(cstr) + } + None => Err(ConversionError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF", + ))), + } + } +} + +impl Read for Buffer { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = { + let slice = self.slice(); + let len = cmp::min(slice.len(), buf.len()); + buf[..len].copy_from_slice(&slice[..len]); + len + }; + self.idx += len; + Ok(len) + } +} + +impl XLogDataBody { + pub fn new(wal_start: u64, wal_end: u64, timestamp: i64, data: Bytes) -> XLogDataBody { + XLogDataBody { wal_start, wal_end, timestamp, data } + } + + pub fn parse( + self, + logical_replication_settings: &LogicalReplicationSettings, + ) -> Result { + let mut buf = Buffer::new(self.data.clone(), 0); + let byte = buf.read_u8()?; + + let logical_replication_message = match byte { + BEGIN_BYTE => { + buf.read_i64::()?; + buf.read_i64::()?; + buf.read_i32::()?; + + LogicalReplicationMessage::Begin + } + COMMIT_BYTE => { + buf.read_i8()?; + buf.read_u64::()?; + buf.read_u64::()?; + buf.read_i64::()?; + LogicalReplicationMessage::Commit + } + RELATION_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + + let o_id = buf.read_u32::()?; + let namespace = buf.read_cstr()?; + let name = buf.read_cstr()?; + let replica_identity = match buf.read_i8()? { + REPLICA_IDENTITY_DEFAULT_BYTE => ReplicaIdentity::Default, + REPLICA_IDENTITY_NOTHING_BYTE => ReplicaIdentity::Nothing, + REPLICA_IDENTITY_FULL_BYTE => ReplicaIdentity::Full, + REPLICA_IDENTITY_INDEX_BYTE => ReplicaIdentity::Index, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replica identity byte `{}`", byte), + ))); + } + }; + + let num_of_column = buf.read_i16::()?; + + let mut columns = Vec::with_capacity(num_of_column as usize); + for _ in 0..num_of_column { + let flags = buf.read_i8()?; + let name = buf.read_cstr()?; + let o_id = buf.read_u32::()?; + let type_modifier = buf.read_i32::()?; + let type_o_id = Type::from_oid(o_id); + let column = Column::new(flags, name, type_o_id, type_modifier); + + columns.push(column); + } + + LogicalReplicationMessage::Relation(RelationBody::new( + transaction_id, + o_id, + namespace, + name, + replica_identity, + columns, + )) + } + TYPE_BYTE => { + buf.read_u32::()?; + buf.read_cstr()?; + buf.read_cstr()?; + + LogicalReplicationMessage::Type + } + INSERT_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let byte = buf.read_u8()?; + + let tuple = match byte { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected tuple byte `{}`", byte), + ))); + } + }; + + LogicalReplicationMessage::Insert(InsertBody::new(transaction_id, o_id, tuple)) + } + UPDATE_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let byte = buf.read_u8()?; + let mut key_tuple = None; + let mut old_tuple = None; + + let new_tuple = match byte { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + TUPLE_OLD_BYTE | TUPLE_KEY_BYTE => { + if byte == TUPLE_OLD_BYTE { + old_tuple = Some(TupleData::parse(&mut buf)?); + } else { + key_tuple = Some(TupleData::parse(&mut buf)?); + } + match buf.read_u8()? { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected tuple byte `{}`", byte), + ))); + } + } + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown tuple byte `{}`", byte), + ))); + } + }; + + LogicalReplicationMessage::Update(UpdateBody::new( + transaction_id, + o_id, + old_tuple, + key_tuple, + new_tuple, + )) + } + DELETE_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let tag = buf.read_u8()?; + + let mut key_tuple = None; + let mut old_tuple = None; + + match tag { + TUPLE_OLD_BYTE => old_tuple = Some(TupleData::parse(&mut buf)?), + TUPLE_KEY_BYTE => key_tuple = Some(TupleData::parse(&mut buf)?), + tag => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown tuple tag `{}`", tag), + ))); + } + } + + LogicalReplicationMessage::Delete(DeleteBody::new( + transaction_id, + o_id, + old_tuple, + key_tuple, + )) + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message tag `{}`", byte), + ))); + } + }; + + Ok(logical_replication_message) + } +} + +#[non_exhaustive] +#[derive(Debug)] +pub enum ReplicationMessage { + XLogData(XLogDataBody), + PrimaryKeepAlive(PrimaryKeepAliveBody), +} + +impl ReplicationMessage { + pub fn parse(buf: Bytes) -> io::Result { + let (byte, mut message) = buf.split_first().unwrap(); + + let replication_message = match *byte { + X_LOG_DATA_BYTE => { + let len = buf.len(); + let wal_start = message.read_u64::()?; + let wal_end = message.read_u64::()?; + let timestamp = message.read_i64::()?; + let len = len - message.len(); + let data = buf.slice(len..); + ReplicationMessage::XLogData(XLogDataBody::new(wal_start, wal_end, timestamp, data)) + } + PRIMARY_KEEPALIVE_BYTE => { + let wal_end = message.read_u64::()?; + let timestamp = message.read_i64::()?; + let reply = message.read_u8()?; + ReplicationMessage::PrimaryKeepAlive(PrimaryKeepAliveBody::new( + wal_end, + timestamp, + reply == 1, + )) + } + byte => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message byte `{}`", byte), + )); + } + }; + + Ok(replication_message) + } +} diff --git a/backend/windmill-triggers/src/triggers/sqs/handler_oss.rs b/backend/windmill-triggers/src/triggers/sqs/handler_oss.rs new file mode 100644 index 0000000000..2751fc4d83 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/sqs/handler_oss.rs @@ -0,0 +1,64 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::SqsTrigger, + crate::triggers::{Trigger, TriggerCrud, TriggerData}, + windmill_api_auth::ApiAuthed, + windmill_common::DB, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +#[cfg(not(feature = "private"))] +impl TriggerCrud for SqsTrigger { + type Trigger = Trigger; + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/sqs_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::SqsTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "SQS triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "SQS triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-triggers/src/triggers/sqs/listener_oss.rs b/backend/windmill-triggers/src/triggers/sqs/listener_oss.rs new file mode 100644 index 0000000000..0aaf5360c3 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/sqs/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::SqsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for SqsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-triggers/src/triggers/sqs/mod.rs b/backend/windmill-triggers/src/triggers/sqs/mod.rs new file mode 100644 index 0000000000..8c7217725c --- /dev/null +++ b/backend/windmill-triggers/src/triggers/sqs/mod.rs @@ -0,0 +1,15 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct SqsTrigger; diff --git a/backend/windmill-triggers/src/triggers/trigger_helpers.rs b/backend/windmill-triggers/src/triggers/trigger_helpers.rs new file mode 100644 index 0000000000..218ad71264 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/trigger_helpers.rs @@ -0,0 +1,957 @@ +use anyhow::Context; +use axum::response::IntoResponse; +use http::StatusCode; +use serde::Deserialize; +use serde_json::value::RawValue; +use sqlx::types::Json; +use std::collections::HashMap; +use std::future::Future; +use uuid::Uuid; +use windmill_common::{ + db::{UserDB, UserDbWithAuthed}, + error::Result, + flows::{FlowModuleValue, Retry}, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, + scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, + triggers::{ + HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, TriggerMetadata, + RUNNABLE_FORMAT_VERSION_CACHE, + }, + users::username_to_permissioned_as, + utils::StripPath, + worker::to_raw_value, +}; +use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; + +#[cfg(feature = "enterprise")] +use crate::jobs_ext::check_license_key_valid; +use crate::jobs_ext::{ + check_tag_available_for_workspace, delete_job_metadata_after_use, + push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response, + run_wait_result_internal, RunJobQuery, +}; +use windmill_api_auth::{scopes::check_scopes, ApiAuthed}; +use windmill_common::utils::HTTP_CLIENT; +use windmill_common::DB; + +struct ScriptInfo { + has_preprocessor: Option, + language: ScriptLang, + content: String, + schema: Option>, +} + +#[derive(Debug, Deserialize)] +struct PropertyDefinition { + r#type: Option>, +} + +#[derive(Debug, Deserialize)] +struct PartialSchema { + properties: Option>, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum RunnableId { + FlowId(FlowId), + ScriptId(ScriptId), + HubScript(String), +} + +impl RunnableId { + pub fn from_script_hash(hash: ScriptHash) -> Self { + Self::ScriptId(ScriptId::ScriptHash(hash)) + } + + pub fn from_script_path(path: &str) -> Self { + if path.starts_with("hub/") { + Self::HubScript(path.to_string()) + } else { + Self::ScriptId(ScriptId::ScriptPath(path.to_string())) + } + } + + pub fn from_flow_path(path: &str) -> Self { + Self::FlowId(FlowId::FlowPath(path.to_string())) + } + + pub fn from_flow_version(version: i64) -> Self { + Self::FlowId(FlowId::FlowVersion(version)) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum FlowId { + FlowPath(String), + FlowVersion(i64), +} + +impl FlowId { + async fn get_flow_version_id(self, workspace_id: &str, db: &DB) -> Result { + let version_id = match self { + FlowId::FlowPath(path) => { + let info = + get_latest_flow_version_info_for_path(None, db, workspace_id, &path, true) + .await?; + info.version + } + FlowId::FlowVersion(version) => version, + }; + + Ok(version_id) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum ScriptId { + ScriptPath(String), + ScriptHash(ScriptHash), +} + +impl ScriptId { + async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { + let hash = match self { + ScriptId::ScriptPath(path) => { + let info = get_latest_deployed_hash_for_path(None, db.clone(), workspace_id, &path) + .await?; + info.hash + } + ScriptId::ScriptHash(hash) => hash.0, + }; + + Ok(hash) + } +} + +async fn get_script_info( + db: &DB, + workspace_id: &str, + hash: i64, +) -> std::result::Result { + sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) + .fetch_one(db) + .await +} + +fn runnable_format_from_schema_without_preprocessor( + trigger_kind: &TriggerKind, + schema: Option>, +) -> RunnableFormat { + match trigger_kind { + TriggerKind::Mqtt + if schema.as_ref().is_some_and(|schema| { + schema.properties.as_ref().is_some_and(|properties| { + properties.iter().any(|(key, def)| { + key == "payload" + && def.r#type.as_ref().is_some_and(|t| { + let typ = t.get().trim(); + typ == "array" || (typ.starts_with('[') && typ.ends_with(']')) + }) + }) + }) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } + } + TriggerKind::Kafka | TriggerKind::Nats + if schema.as_ref().is_some_and(|schema| { + schema + .properties + .as_ref() + .is_some_and(|properties| properties.keys().any(|key| key == "msg")) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } + } + _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false }, + } +} + +fn runnable_format_from_preprocessor_args( + args: Option>, +) -> RunnableFormat { + if let Some(args) = args { + if args.iter().any(|arg| arg.name == "wm_trigger") + || (args.len() > 0 && args.iter().all(|arg| arg.name != "event")) + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } + } + } else { + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } + } +} + +enum PreprocessorInfo { + Preprocessor { content: String, language: ScriptLang }, + NoPreprocessor { schema: Option> }, +} + +#[derive(Debug, Deserialize)] +struct FlowInfo { + preprocessor_module: Option>, + schema: Option>, +} + +fn get_preprocessor_args_from_content_and_language( + content: &str, + language: &ScriptLang, +) -> Result>> { + let args = match language { + ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature( + &content, + true, + false, + Some("preprocessor".to_string()), + )?; + Some(args.args) + } + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature( + &content, + Some("preprocessor".to_string()), + false, + )?; + Some(args.args) + } + _ => None, + }; + Ok(args) +} + +pub async fn get_runnable_format( + runnable_id: RunnableId, + workspace_id: &str, + db: &DB, + trigger_kind: &TriggerKind, +) -> Result { + let (key, preprocessor_info) = match runnable_id { + RunnableId::HubScript(path) => { + let Some(version) = path.split("/").nth(1) else { + return Err(windmill_common::error::Error::internal_err( + "Invalid hub script path".to_string(), + )); + }; + + let version = match version.parse::() { + Ok(version) => version, + Err(_) => { + return Err(windmill_common::error::Error::internal_err( + "Invalid hub script version".to_string(), + )); + } + }; + + let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone()); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for hub script {path}"); + return Ok(runnable_format); + } + + let hub_script = + get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db)) + .await?; + + let has_preprocessor = get_has_preprocessor_from_content_and_lang( + &hub_script.content, + &hub_script.language, + )?; + + let partial_schema = serde_json::from_str(hub_script.schema.get())?; + + ( + key, + if has_preprocessor { + PreprocessorInfo::Preprocessor { + content: hub_script.content, + language: hub_script.language, + } + } else { + PreprocessorInfo::NoPreprocessor { + schema: Some(sqlx::types::Json(partial_schema)), + } + }, + ) + } + RunnableId::FlowId(flow_id) => { + let version = flow_id.get_flow_version_id(workspace_id, db).await?; + + let key = ( + HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), + version, + trigger_kind.clone(), + ); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for flow version {version}"); + return Ok(runnable_format); + } + + let flow_info = sqlx::query_as!( + FlowInfo, + "SELECT + value->'preprocessor_module'->'value' as \"preprocessor_module: _\", + schema as \"schema: _\" + FROM flow_version + WHERE + id = $1 + AND workspace_id = $2", + version, + workspace_id, + ) + .fetch_one(db) + .await?; + + if let Some(preprocessor_module) = flow_info.preprocessor_module { + match preprocessor_module.0 { + FlowModuleValue::RawScript { content, language, .. } => { + (key, PreprocessorInfo::Preprocessor { content, language }) + } + FlowModuleValue::Script { path, hash, .. } => { + let hash = if let Some(hash) = hash { + hash.0 + } else { + let script_hash = get_latest_deployed_hash_for_path( + None, + db.clone(), + workspace_id, + &path, + ) + .await?; + script_hash.hash + }; + let script_info = get_script_info(db, workspace_id, hash).await?; + ( + key, + PreprocessorInfo::Preprocessor { + content: script_info.content, + language: script_info.language, + }, + ) + } + _ => { + return Err(windmill_common::error::Error::internal_err( + "Unsupported preprocessor module".to_string(), + )); + } + } + } else { + ( + key, + PreprocessorInfo::NoPreprocessor { schema: flow_info.schema }, + ) + } + } + RunnableId::ScriptId(script_id) => { + let hash = script_id.get_script_hash(workspace_id, db).await?; + let key = ( + HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), + hash, + trigger_kind.clone(), + ); + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for script {hash}"); + return Ok(runnable_format); + } + + let script_info = get_script_info(db, workspace_id, hash).await?; + + if script_info.has_preprocessor.unwrap_or(false) { + ( + key, + PreprocessorInfo::Preprocessor { + content: script_info.content, + language: script_info.language, + }, + ) + } else { + ( + key, + PreprocessorInfo::NoPreprocessor { schema: script_info.schema }, + ) + } + } + }; + + let runnable_format = match preprocessor_info { + PreprocessorInfo::Preprocessor { content, language } => { + let args = get_preprocessor_args_from_content_and_language(&content, &language)?; + runnable_format_from_preprocessor_args(args) + } + PreprocessorInfo::NoPreprocessor { schema } => { + runnable_format_from_schema_without_preprocessor(trigger_kind, schema) + } + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) +} + +#[allow(dead_code)] + +pub trait TriggerJobArgs { + type Payload: Send + Sync; + const TRIGGER_KIND: TriggerKind; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap>; + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + Self::v1_payload_fn(payload) + } + + fn build_job_args_v2( + has_preprocessor: bool, + payload: &Self::Payload, + info: HashMap>, + ) -> PushArgsOwned { + let mut args = Self::v2_payload_fn(payload); + if has_preprocessor { + args.insert( + "kind".to_string(), + to_raw_value(&Self::TRIGGER_KIND.to_key()), + ); + args.extend(info); + let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); + PushArgsOwned { args, extra: None } + } else { + PushArgsOwned { args, extra: None } + } + } + + fn build_job_args_v1( + has_preprocessor: bool, + payload: &Self::Payload, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_key = Self::TRIGGER_KIND.to_key(); + let args = Self::v1_payload_fn(payload); + let extra = if has_preprocessor { + Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": trigger_key, + trigger_key: info + })), + )])) + } else { + None + }; + PushArgsOwned { args, extra } + } + + fn build_job_args( + runnable_path: &str, + is_flow: bool, + w_id: &str, + db: &DB, + payload: Self::Payload, + info: HashMap>, + ) -> impl Future> + Send { + async move { + let runnable_id = if is_flow { + RunnableId::from_flow_path(runnable_path) + } else { + RunnableId::from_script_path(runnable_path) + }; + Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await + } + } + + fn build_job_args_from_runnable_id( + runnable_id: RunnableId, + w_id: &str, + db: &DB, + payload: Self::Payload, + trigger_info: HashMap>, + ) -> impl Future> + Send { + async move { + tracing::debug!("Building job args for {runnable_id:?}"); + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &Self::TRIGGER_KIND).await?; + let job_args = match runnable_format { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { + Self::build_job_args_v1(has_preprocessor, &payload, trigger_info) + } + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { + Self::build_job_args_v2(has_preprocessor, &payload, trigger_info) + } + }; + + Ok(job_args) + } + } + + fn build_capture_payloads( + payload: &Self::Payload, + info: HashMap>, + ) -> (PushArgsOwned, PushArgsOwned) { + let main_args = Self::build_job_args_v2(false, payload, info.clone()); + let preprocessor_args = Self::build_job_args_v2(true, payload, info); + (main_args, preprocessor_args) + } +} + +#[allow(dead_code)] +pub async fn trigger_runnable_inner( + db: &DB, + tx_o: Option>, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, + job_id: Option, + trigger: TriggerMetadata, + suspended_mode: Option, +) -> Result<( + Uuid, + Option, + Option, + Option>, +)> { + let error_handler_args = error_handler_args.map(|args| { + let args = args + .0 + .iter() + .map(|(key, value)| (key.to_owned(), to_raw_value(&value))) + .collect::>>(); + Json(args) + }); + + let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); + let (uuid, delete_after_use, early_return, tx_out) = if is_flow { + let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; + let path = StripPath(runnable_path.to_string()); + let (uuid, early_return, tx_out) = push_flow_job_by_path_into_queue( + authed, + db.clone(), + tx_o, + user_db, + workspace_id.to_string(), + path, + run_query, + args, + Some(trigger), + ) + .await?; + (uuid, None, early_return, tx_out) + } else { + let (uuid, delete_after_use, tx_out) = trigger_script_internal( + db, + tx_o, + user_db, + authed, + workspace_id, + runnable_path, + args, + retry, + error_handler_path, + error_handler_args.as_ref(), + trigger_path, + job_id, + trigger, + suspended_mode, + ) + .await?; + (uuid, delete_after_use, None, tx_out) + }; + + Ok((uuid, delete_after_use, early_return, tx_out)) +} + +#[allow(dead_code)] +pub async fn trigger_runnable( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, + job_id: Option, + suspended_mode: bool, + trigger: TriggerMetadata, +) -> Result { + let uuid = trigger_runnable_inner( + db, + None, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + job_id, + trigger, + Some(suspended_mode), + ) + .await? + .0; + Ok((StatusCode::CREATED, uuid.to_string()).into_response()) +} + +#[allow(dead_code)] +pub async fn trigger_runnable_and_wait_for_result( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, + trigger: TriggerMetadata, +) -> Result { + let username = authed.username.clone(); + let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( + db, + None, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + None, + trigger, + None, + ) + .await?; + let (result, success) = + run_wait_result_internal(db, uuid, &workspace_id, early_return, &username).await?; + + if delete_after_use.unwrap_or(false) { + delete_job_metadata_after_use(&db, uuid).await?; + } + + result_to_response(result, success) +} + +#[allow(dead_code)] +pub async fn trigger_runnable_and_wait_for_raw_result( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, + trigger: TriggerMetadata, +) -> Result<(Box, bool)> { + let username = authed.username.clone(); + let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( + db, + None, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + None, + trigger, + None, + ) + .await?; + + let (result, success) = + run_wait_result_internal(db, uuid, &workspace_id, early_return, &username) + .await + .with_context(|| { + format!( + "Error fetching job result for {} {}", + if is_flow { "flow" } else { "script" }, + runnable_path + ) + })?; + + if delete_after_use.unwrap_or(false) { + delete_job_metadata_after_use(&db, uuid).await?; + } + + Ok((result, success)) +} + +pub async fn trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, + trigger: TriggerMetadata, +) -> Result> { + let (result, success) = trigger_runnable_and_wait_for_raw_result( + db, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + trigger, + ) + .await?; + + if !success { + Err(windmill_common::error::Error::internal_err(format!( + "{} {runnable_path} failed: {:?}", + if is_flow { "Flow" } else { "Script" }, + result + ))) + } else { + Ok(result) + } +} + +async fn trigger_script_internal( + db: &DB, + tx_o: Option>, + user_db: UserDB, + authed: ApiAuthed, + workspace_id: &str, + script_path: &str, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>>, + trigger_path: String, + job_id: Option, + trigger: TriggerMetadata, + suspended_mode: Option, +) -> Result<( + Uuid, + Option, + Option>, +)> { + if retry.is_none() && error_handler_path.is_none() { + let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; + let path = StripPath(script_path.to_string()); + let (uuid, delete_after_use, tx_out) = push_script_job_by_path_into_queue( + authed, + db.clone(), + tx_o, + user_db, + workspace_id.to_string(), + path, + run_query, + args, + Some(trigger), + ) + .await?; + Ok((uuid, delete_after_use, tx_out)) + } else { + let (uuid, delete_after_use, tx_out) = trigger_script_with_retry_and_error_handler( + db, + tx_o, + user_db, + authed, + workspace_id, + script_path, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + job_id, + trigger, + suspended_mode, + ) + .await?; + Ok((uuid, delete_after_use, tx_out)) + } +} + +async fn trigger_script_with_retry_and_error_handler( + db: &DB, + tx_o: Option>, + user_db: UserDB, + authed: ApiAuthed, + workspace_id: &str, + script_path: &str, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>>, + trigger_path: String, + job_id: Option, + trigger: TriggerMetadata, + suspended_mode: Option, +) -> Result<( + Uuid, + Option, + Option>, +)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + + let retry = retry.map(|r| r.0.clone()); + let error_handler_path = error_handler_path.map(|p| p.to_string()); + let error_handler_args = error_handler_args.map(|args| args.0.clone()); + + let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { + let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + script_path_to_payload( + script_path, + Some(db_authed), + db.clone(), + &workspace_id, + Some(false), + ) + .await? + }; + + check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; + + let return_tx = tx_o.is_some(); + + let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Transaction(tx), + ) + } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { + ( + on_behalf_of.email.as_str(), + on_behalf_of.permissioned_as.clone(), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db, authed.clone().into()), + ) + }; + + let push_args = PushArgs { args: &args.args, extra: args.extra }; + + let retryable_job_payload = match job_payload { + JobPayload::ScriptHash { + hash, + path, + concurrency_settings, + debouncing_settings, + cache_ttl, + cache_ignore_s3_path, + priority, + apply_preprocessor, + .. + } => JobPayload::SingleStepFlow { + path, + hash: Some(hash), + flow_version: None, + args: HashMap::from(&push_args), + retry, + error_handler_path, + error_handler_args, + skip_handler: None, + cache_ttl, + cache_ignore_s3_path, + priority, + tag_override: tag.clone(), + apply_preprocessor, + trigger_path: Some(trigger_path.clone()), + concurrency_settings, + debouncing_settings, + }, + _ => { + return Err(windmill_common::error::Error::internal_err(format!( + "Unsupported job payload: {:?}", + job_payload + ))) + } + }; + let (uuid, tx) = push( + &db, + tx, + &workspace_id, + retryable_job_payload, + push_args, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + job_id, + false, + false, + None, + true, + tag, + timeout, + None, + None, + push_authed.as_ref(), + false, + None, + Some(trigger), + suspended_mode, + ) + .await?; + + // If we were given a transaction, return it; otherwise commit it + if return_tx { + Ok((uuid, delete_after_use, Some(tx))) + } else { + tx.commit().await?; + Ok((uuid, delete_after_use, None)) + } +} diff --git a/backend/windmill-triggers/src/triggers/websocket/handler.rs b/backend/windmill-triggers/src/triggers/websocket/handler.rs new file mode 100644 index 0000000000..de258e0fe6 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/websocket/handler.rs @@ -0,0 +1,262 @@ +use std::borrow::Cow; + +use crate::triggers::{Trigger, TriggerCrud, TriggerData}; +use windmill_api_auth::ApiAuthed; +use windmill_common::DB; +use axum::async_trait; +use itertools::Itertools; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, PgConnection}; +use tokio_tungstenite::connect_async; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + worker::to_raw_value, +}; +use windmill_git_sync::DeployedObject; + +use super::{ + get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, + WebsocketTrigger, +}; + +#[async_trait] +impl TriggerCrud for WebsocketTrigger { + type TriggerConfig = WebsocketConfig; + type Trigger = Trigger; + type TriggerConfigRequest = WebsocketConfigRequest; + type TestConnectionConfig = TestWebsocketConfig; + + const TABLE_NAME: &'static str = "websocket_trigger"; + const TRIGGER_TYPE: &'static str = "websocket"; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/websocket_triggers"; + const DEPLOYMENT_NAME: &'static str = "WebSocket trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "url", + "filters", + "initial_messages", + "url_runnable_args", + "can_return_message", + "can_return_error_result", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::WebsocketTrigger { path } + } + + async fn validate_config( + &self, + _db: &DB, + config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + if config.url.trim().is_empty() { + return Err(Error::BadRequest( + "WebSocket URL cannot be empty".to_string(), + )); + } + + if let Some(args) = &config.url_runnable_args { + if !args.is_object() { + return Err(Error::BadRequest( + "url_runnable_args must be an object".to_string(), + )); + } + } + + Ok(()) + } + + async fn create_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let filters = trigger + .config + .filters + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + let initial_messages = trigger + .config + .initial_messages + .unwrap_or_default() + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + workspace_id, + path, + url, + script_path, + is_flow, + mode, + filters, + initial_messages, + url_runnable_args, + edited_by, + can_return_message, + can_return_error_result, + email, + edited_at, + error_handler_path, + error_handler_args, + retry + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16 + ) + "#, + w_id, + trigger.base.path, + trigger.config.url, + trigger.base.script_path, + trigger.base.is_flow, + trigger.base.mode() as _, + &filters as _, + &initial_messages as _, + trigger + .config + .url_runnable_args + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _, + authed.username, + trigger.config.can_return_message, + trigger.config.can_return_error_result, + authed.email, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) + } + + async fn update_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let filters = trigger + .config + .filters + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + let initial_messages = trigger + .config + .initial_messages + .unwrap_or_default() + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + + // important to update server_id to NULL to stop current websocket listener + sqlx::query!( + " + UPDATE + websocket_trigger + SET + url = $1, + script_path = $2, + path = $3, + is_flow = $4, + filters = $5, + initial_messages = $6, + url_runnable_args = $7, + edited_by = $8, + email = $9, + can_return_message = $10, + can_return_error_result = $11, + edited_at = now(), + server_id = NULL, + error = NULL, + error_handler_path = $14, + error_handler_args = $15, + retry = $16 + WHERE + workspace_id = $12 AND path = $13 + ", + trigger.config.url, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + filters.as_slice() as &[SqlxJson>], + initial_messages.as_slice() as &[SqlxJson>], + trigger + .config + .url_runnable_args + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + as Option>>, + &authed.username, + &authed.email, + trigger.config.can_return_message, + trigger.config.can_return_error_result, + w_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + _user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let url = &config.url; + + let connect_url: Cow = if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + let path = url.splitn(2, ':').nth(1).unwrap(); + Cow::Owned( + get_url_from_runnable_value( + path, + url.starts_with("$flow:"), + &db, + authed.clone(), + config.url_runnable_args.as_ref().map(to_raw_value).as_ref(), + &workspace_id, + ) + .await?, + ) + } else { + return Err(Error::BadConfig(format!( + "Invalid WebSocket runnable path: {}", + url + ))); + } + } else { + Cow::Borrowed(&url) + }; + + connect_async(&*connect_url).await.map_err(|err| { + Error::BadConfig(format!( + "Error connecting to WebSocket: {}", + err.to_string() + )) + })?; + + Ok(()) + } +} diff --git a/backend/windmill-triggers/src/triggers/websocket/listener.rs b/backend/windmill-triggers/src/triggers/websocket/listener.rs new file mode 100644 index 0000000000..0f22dffdc3 --- /dev/null +++ b/backend/windmill-triggers/src/triggers/websocket/listener.rs @@ -0,0 +1,477 @@ +use super::WebsocketTrigger; +use crate::triggers::{ + filter::{is_value_superset, Filter, JsonFilter}, + listener::ListeningTrigger, + trigger_helpers::{ + trigger_runnable, trigger_runnable_and_wait_for_raw_result, + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, + }, + websocket::{get_url_from_runnable_value, WebsocketConfig}, + Listener, +}; +use anyhow::Context; +use async_trait::async_trait; +use futures::{stream::SplitSink, SinkExt, StreamExt}; +use http::Response; +use itertools::Itertools; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::{borrow::Cow, collections::HashMap, sync::Arc}; +use tokio::{net::TcpStream, sync::RwLock}; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + triggers::TriggerMetadata, + utils::report_critical_error, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +impl ListeningTrigger { + async fn send_initial_messages( + &self, + writer: &mut SplitSink>, Message>, + db: &DB, + ) -> Result<()> { + let initial_messages: Vec = self + .trigger_config + .initial_messages + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); + + let WebsocketConfig { ref url, .. } = self.trigger_config; + let runnable_kind = if self.is_flow { "flow" } else { "script" }; + let mut authed_o = None; + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to WebSocket {}: {}", + url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) + .await + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running {} {} for initial message to WebSocket {}", + runnable_kind, + path, + url, + ); + + let args = raw_value_to_args_hashmap(Some(&args))?; + + if authed_o.is_none() { + authed_o = Some(self.authed(db, "ws").await?); + } + let authed = authed_o.clone().unwrap(); + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed.clone(), + &self.workspace_id, + &path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new(Some(self.path.to_owned()), JobTriggerKind::Websocket), + ) + .await + .map(|r| r.get().to_owned())?; + + tracing::info!( + "Sending {} {} result to WebSocket {}", + runnable_kind, + path, + url + ); + + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| { + format!("Failed to send {} {} result", runnable_kind, path) + })?; + } + } + } + Ok(()) + } +} + +#[async_trait] +impl Listener for WebsocketTrigger { + type Consumer = ( + WebSocketStream>, + Response>>, + ); + type Extra = ReturnMessageChannels; + type ExtraState = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let url = &listening_trigger.trigger_config.url; + let connect_url: Cow = if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return Ok(None); + }, + _ = self.loop_ping(&db, listening_trigger, err_message.clone(), Some( + "Waiting on runnable to return WebSocket URL...".to_string() + )) => { + return Ok(None); + }, + url_result = { + let authed = listening_trigger.authed(db, "ws").await?; + let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0); + let path = url.splitn(2, ':').nth(1).unwrap(); + get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id) + } => match url_result { + Ok(url) => Cow::Owned(url), + Err(err) => { + return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into()); + } + }, + } + } else { + return Err(anyhow::anyhow!("Invalid WebSocket runnable path: {}", url).into()); + } + } else { + Cow::Borrowed(&url) + }; + + let connection = connect_async(&*connect_url) + .await + .map(|conn| Some(conn)) + .map_err(|err| to_anyhow(err).into()); + + connection + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + _extra_state: Option<&Self::ExtraState>, + ) { + let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; + + tracing::info!("Connected to WebSocket {}", url); + + let (ws_stream, _) = consumer; + + let (mut writer, mut reader) = ws_stream.split(); + + // send initial messages + if listening_trigger.trigger_mode { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { + return; + }, + result = listening_trigger.send_initial_messages(&mut writer, &db) => { + if let Err(err) = result { + self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; + return + } else { + tracing::debug!("Initial messages sent successfully to WebSocket {}", url); + } + } + } + } + + let (return_message_channels, message_sender_handle) = if listening_trigger.trigger_mode + && listening_trigger.trigger_config.can_return_message + { + let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); + let w_id = listening_trigger.workspace_id.clone(); + let url = url.clone(); + let db = db.clone(); + let handle = tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Err(err) = writer + .send(tokio_tungstenite::tungstenite::Message::Text(message)) + .await + { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; + } + } + }); + + let killpill_rx = killpill_rx.resubscribe(); + + let return_message_channels = ReturnMessageChannels { send_message_tx, killpill_rx }; + + (Some(return_message_channels), Some(handle)) + } else { + (None, None) + }; + + tokio::select! { + biased; + _ = killpill_rx.recv() => { + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), None) => { + }, + _ = async { + let filters: Vec = if listening_trigger.trigger_mode { + listening_trigger + .trigger_config + .filters + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec() + } else { + vec![] + }; + loop { + if let Some(msg) = reader.next().await { + match msg { + Ok(msg) => { + match msg { + tokio_tungstenite::tungstenite::Message::Text(text) => { + tracing::debug!("Received text message from WebSocket {}: {}", url, text); + let mut should_handle = true; + for filter in &filters { + match filter { + Filter::JsonFilter(JsonFilter { key, value }) => { + let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); + should_handle = match is_value_superset(&mut deserializer, key, &value) { + Ok(filter_match) => { + filter_match + }, + Err(err) => { + tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); + false + } + }; + } + } + if !should_handle { + break; + } + } + if should_handle { + let trigger_info = HashMap::from([ + ("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)), + ]); + let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await; + } + }, + a @ _ => { + tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); + } + } + }, + Err(err) => { + tracing::error!("Error reading from WebSocket {}: {:?}", url, err); + } + } + } else { + tracing::error!("WebSocket {} closed", url); + self.update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), Some("WebSocket closed".to_string())).await; + break; + } + } + } => {} + } + // make sure to stop return message handler + if let Some(message_sender_handle) = message_sender_handle { + message_sender_handle.abort(); + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + let ListeningTrigger { + path, + is_flow, + workspace_id, + trigger_config, + script_path, + error_handling, + suspended_mode, + .. + } = listening_trigger; + + let WebsocketConfig { url, .. } = trigger_config; + + let args = WebsocketTrigger::build_job_args( + &script_path, + *is_flow, + workspace_id, + db, + payload, + trigger_info, + ) + .await; + + let args = match args { + Ok(args) => args, + Err(err) => { + return Err(err); + } + }; + + let authed = listening_trigger.authed(db, "ws").await?; + + let (retry, error_handler_path, error_handler_args) = match error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + let trigger = TriggerMetadata::new(Some(path.to_owned()), Self::JOB_TRIGGER_KIND); + if *suspended_mode || extra.is_none() { + trigger_runnable( + db, + None, + authed, + &workspace_id, + &script_path, + *is_flow, + args, + retry, + error_handler_path, + error_handler_args, + format!("websocket_trigger/{}", listening_trigger.path), + None, + *suspended_mode, + trigger, + ) + .await?; + } else if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = extra { + let db_ = db.clone(); + let url = url.to_owned(); + let script_path = script_path.to_owned(); + let is_flow = *is_flow; + let w_id = workspace_id.to_owned(); + let retry = retry.cloned(); + let error_handler_path = error_handler_path.map(|s| s.to_string()); + let error_handler_args = error_handler_args.cloned(); + let trigger_path = path.clone(); + let can_return_error_result = trigger_config.can_return_error_result; + let handle_response_f = async move { + tokio::select! { + _ = killpill_rx.recv() => { + return; + }, + result = trigger_runnable_and_wait_for_raw_result( + &db_, + None, + authed, + &w_id, + &script_path, + is_flow, + args, + retry.as_ref(), + error_handler_path.as_deref(), + error_handler_args.as_ref(), + format!("websocket_trigger/{}", trigger_path), + trigger, + ) => { + if let Ok((result, success)) = result { + if !success && !can_return_error_result { + return; + } + let result = result.get().to_owned(); + // only send the result if it's not null + if result != "null" { + tracing::info!("Sending job result to WebSocket {}", url); + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + if let Err(err) = send_message_tx.send(result).await { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; + } + } + } + } + }; + }; + + tokio::spawn(handle_response_f); + } + + Ok(()) + } +} + +pub struct ReturnMessageChannels { + send_message_tx: tokio::sync::mpsc::Sender, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +} + +impl Clone for ReturnMessageChannels { + fn clone(&self) -> Self { + Self { + send_message_tx: self.send_message_tx.clone(), + killpill_rx: self.killpill_rx.resubscribe(), + } + } +} + +#[derive(Debug, Deserialize)] +enum InitialMessage { + #[serde(rename = "raw_message")] + RawMessage(String), + #[serde(rename = "runnable_result")] + RunnableResult { path: String, args: Box, is_flow: bool }, +} + +fn raw_value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + serde_json::from_str::>>>(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new) + } else { + HashMap::new() + }; + Ok(args) +} diff --git a/backend/windmill-triggers/src/triggers/websocket/mod.rs b/backend/windmill-triggers/src/triggers/websocket/mod.rs new file mode 100644 index 0000000000..9c99e92cdd --- /dev/null +++ b/backend/windmill-triggers/src/triggers/websocket/mod.rs @@ -0,0 +1,126 @@ +use std::collections::HashMap; + +use crate::triggers::trigger_helpers::{ + trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, +}; +use windmill_api_auth::ApiAuthed; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow}; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::{TriggerMetadata, TriggerKind}, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +mod handler; +mod listener; + +#[derive(Copy, Clone)] +pub struct WebsocketTrigger; + +impl TriggerJobArgs for WebsocketTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WebsocketConfig { + pub url: String, + #[serde(default)] + pub filters: Vec>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_messages: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub url_runnable_args: Option>>, + #[serde(default)] + pub can_return_message: bool, + #[serde(default)] + pub can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebsocketConfigRequest { + url: String, + filters: Vec, + initial_messages: Option>, + url_runnable_args: Option, + can_return_message: bool, + can_return_error_result: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestWebsocketConfig { + url: String, + url_runnable_args: Option, +} + +pub fn value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + let args_map: Option> = serde_json::from_str(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; + + args_map + .unwrap_or_else(HashMap::new) + .into_iter() + .map(|(k, v)| { + let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { + Error::BadRequest(format!("failed to convert to raw value: {}", e)) + })?; + Ok((k, raw_value)) + }) + .collect::>>>() + } else { + Ok(HashMap::new()) + }?; + Ok(args) +} + +pub async fn get_url_from_runnable_value( + path: &str, + is_flow: bool, + db: &DB, + authed: ApiAuthed, + args: Option<&Box>, + workspace_id: &str, +) -> Result { + tracing::info!( + "Running {} {} to get WebSocket URL", + if is_flow { "flow" } else { "script" }, + path + ); + + let args = value_to_args_hashmap(args)?; + + let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( + db, + None, + authed, + workspace_id, + path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), + ) + .await?; + + serde_json::from_str::(result.get()).map_err(|_| { + Error::BadConfig(format!( + "{} {} did not return a string", + if is_flow { "Flow" } else { "Script" }, + path, + )) + }) +} diff --git a/backend/windmill-triggers/src/user_ext.rs b/backend/windmill-triggers/src/user_ext.rs new file mode 100644 index 0000000000..2596cb2800 --- /dev/null +++ b/backend/windmill-triggers/src/user_ext.rs @@ -0,0 +1,96 @@ +/* + * Author: Windmill Labs, Inc + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use sqlx::PgConnection; +use tracing::Instrument; +use windmill_api_auth::ApiAuthed; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + error::{Error, Result}, + utils::rd_string, + worker::CLOUD_HOSTED, + DB, +}; + +#[derive(serde::Deserialize)] +pub struct NewToken { + pub label: Option, + pub expiration: Option>, + pub impersonate_email: Option, + pub scopes: Option>, + pub workspace_id: Option, +} + +impl NewToken { + pub fn new( + label: Option, + expiration: Option>, + impersonate_email: Option, + scopes: Option>, + workspace_id: Option, + ) -> NewToken { + NewToken { label, expiration, impersonate_email, scopes, workspace_id } + } +} + +pub async fn create_token_internal( + tx: &mut PgConnection, + db: &DB, + authed: &ApiAuthed, + token_config: NewToken, +) -> Result { + let token = rd_string(32); + + let is_super_admin = sqlx::query_scalar!( + "SELECT super_admin FROM password WHERE email = $1", + authed.email + ) + .fetch_optional(&mut *tx) + .await? + .unwrap_or(false); + if *CLOUD_HOSTED { + let nb_tokens = + sqlx::query_scalar!("SELECT COUNT(*) FROM token WHERE email = $1", &authed.email) + .fetch_one(db) + .await?; + if nb_tokens.unwrap_or(0) >= 10000 { + return Err(Error::BadRequest( + "You have reached the maximum number of tokens (10000) on cloud. Contact support@windmill.dev to increase the limit" + .to_string(), + )); + } + } + sqlx::query!( + "INSERT INTO token + (token, email, label, expiration, super_admin, scopes, workspace_id) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + token, + authed.email, + token_config.label, + token_config.expiration, + is_super_admin, + token_config.scopes.as_ref().map(|x| x.as_slice()), + token_config.workspace_id, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + authed, + "users.token.create", + ActionKind::Create, + &"global", + Some(&token[0..10]), + None, + ) + .instrument(tracing::info_span!("token", email = &authed.email)) + .await?; + + Ok(token) +}