From 6776b2b162b8a1081e5181852638f725e8c44e4b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 24 Jan 2025 00:21:54 +0100 Subject: [PATCH 1/6] pin cargo-chef --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f6ce5525d8..cb539abbf0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get -y update \ RUN rustup component add rustfmt -RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version ^0.1 +RUN CARGO_NET_GIT_FETCH_WITH_CLI=true cargo install cargo-chef --version 0.1.68 RUN cargo install sccache --version ^0.8 ENV RUSTC_WRAPPER=sccache SCCACHE_DIR=/backend/sccache From e47dd697f96883f8f97bf469e69d3344f12d9633 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 24 Jan 2025 13:42:48 +0300 Subject: [PATCH 2/6] feat(python): Multiple runtime versions (#4579) * feat: Handle `pip install` by `uv` Dirty and untested, but already something working * Integrate with NSJAIL and prepare fallbacks * Refactor fallback no_uv disable compile and install where no_uv_install and no_uv_compile are a bit more specific * Remove `--disable-pip-version-check` Reason: warning: pip's `--disable-pip-version-check` has no effect * Fix backend compilation error * Pip fallback overwrite UV's cache * Initially refactor cache (No S3) * Support S3 * Remove unused import * Handle flags for NSJAIL * Return deleted flag * Remove verbose mode and enable link-mode=copy * Granural migration of lockfiles Before i realized we dont need it :) * Initial draft (not-working) * Add fallback * Fix bug preventing uv from installing deps '\n' - Love it * Add verbosity indicator * Iterate on feature - Added instance python version - Rework logic * Fix EE build error error[E0599]: no method named `iter` found for tuple `(PyVersion, std::vec::Vec)` in the current scope * Support S3 * Support NSJAIL * Refactor `get_python` * Make NSJAIL work [Unsafe] config file missed /proc mount causing install phase to fail * Trigger CI * Clean up * Make Actions build it * Trigger CI #2 * Update Dockerfile and clean up * Change fallbacks now there is only no_uv and NOUV * Expose INSTANCE_PYTHON_VERSION through env variable * Change namings * Include py-version to requirements.in Also add comments and make code much cleaner * Use const for python installation dir It was hardcoded before * Pin preinstalled version * Update python_executor.rs * Up to date branch * Create PYCACHE dirs TODO: PY_TAR_DIRS * Fix after merge * Make it safer * Implement USE_SYSTEM_PYTHON * Implement latest_stable option * Load INSTANCE_PYTHON_VERSION on startup * Check for multiple annotations used * Fix Latest Stable button not pressed if selected * Proper error handling for conflict on multiple annotations * Fix merge conflicts * Preinstall 3.11 and Latest Stable * Preinstall latest stable in non-blocking manner * Fix Warning * Gate preinstall logic behind "python" feature * Handle raw_deps properly * Make it work with nsjail * Revert docker-image.yml * Revert Dockerfile * Cleanup + Fixing * Add windows support --------- Co-authored-by: Ruben Fiszel --- Dockerfile | 12 + .../windmill-parser-py-imports/src/lib.rs | 77 ++- .../windmill-parser-py-imports/tests/tests.rs | 3 + backend/src/main.rs | 23 +- backend/src/monitor.rs | 23 +- backend/windmill-common/src/error.rs | 2 + .../windmill-common/src/global_settings.rs | 4 +- backend/windmill-common/src/worker.rs | 6 +- .../nsjail/download.py.config.proto | 6 + .../nsjail/download_deps.py.sh | 1 + .../nsjail/run.python3.config.proto | 7 + .../windmill-worker/src/ansible_executor.rs | 5 +- backend/windmill-worker/src/global_cache.rs | 18 +- .../windmill-worker/src/python_executor.rs | 551 ++++++++++++++++-- backend/windmill-worker/src/worker.rs | 52 +- .../windmill-worker/src/worker_lockfiles.rs | 46 +- .../src/lib/components/InstanceSetting.svelte | 28 +- .../src/lib/components/instanceSettings.ts | 42 +- shell.nix | 2 +- 19 files changed, 802 insertions(+), 106 deletions(-) diff --git a/Dockerfile b/Dockerfile index cb539abbf0..8e45b675a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,6 +95,14 @@ ARG WITH_KUBECTL=true ARG WITH_HELM=true ARG WITH_GIT=true +# To change latest stable version: +# 1. Change placeholder in instanceSettings.ts +# 2. Change LATEST_STABLE_PY in dockerfile +# 3. Change #[default] annotation for PyVersion in backend +ARG LATEST_STABLE_PY=3.11.10 +ENV UV_PYTHON_INSTALL_DIR=/tmp/windmill/cache/py_runtime +ENV UV_PYTHON_PREFERENCE=only-managed + RUN pip install --upgrade pip==24.2 RUN apt-get update \ @@ -161,6 +169,10 @@ ENV GO_PATH=/usr/local/go/bin/go # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.5.15/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +# Preinstall python runtimes +RUN uv python install 3.11.10 +RUN uv python install $LATEST_STABLE_PY + RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl procps nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index edd7221a0f..d0a5c9622b 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -21,7 +21,7 @@ use rustpython_parser::{ Parse, }; use sqlx::{Pool, Postgres}; -use windmill_common::error; +use windmill_common::{error, worker::PythonAnnotations}; const DEF_MAIN: &str = "def main("; @@ -171,14 +171,75 @@ fn parse_code_for_imports(code: &str, path: &str) -> error::Result> return Ok(nimports); } -#[async_recursion] pub async fn parse_python_imports( code: &str, w_id: &str, path: &str, db: &Pool, already_visited: &mut Vec, + annotated_pyv_numeric: &mut Option, ) -> error::Result> { + parse_python_imports_inner( + code, + w_id, + path, + db, + already_visited, + annotated_pyv_numeric, + &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())), + ) + .await +} + +#[async_recursion] +async fn parse_python_imports_inner( + code: &str, + w_id: &str, + path: &str, + db: &Pool, + already_visited: &mut Vec, + annotated_pyv_numeric: &mut Option, + path_where_annotated_pyv: &mut Option, +) -> error::Result> { + let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + + // we pass only if there is none or only one annotation + + // Naive: + // 1. Check if there are multiple annotated version + // 2. If no, take one and compare with annotated version + // 3. We continue if same or replace none with new one + + // Optimized: + // 1. Iterate over all annotations compare each with annotated_pyv and replace on flight + // 2. If annotated_pyv is different version, throw and error + + // This way we make sure there is no multiple annotations for same script + // and we get detailed span on conflicting versions + + let mut check = |is_py_xyz, numeric| -> error::Result<()> { + if is_py_xyz { + if let Some(v) = annotated_pyv_numeric { + if *v != numeric { + return Err(error::Error::from(anyhow::anyhow!( + "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.", + path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned()) + ))); + } + } else { + *annotated_pyv_numeric = Some(numeric); + } + + *path_where_annotated_pyv = Some(path.to_owned()); + } + Ok(()) + }; + + check(py310, 310)?; + check(py311, 311)?; + check(py312, 312)?; + check(py313, 313)?; + let find_requirements = code .lines() .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); @@ -225,11 +286,21 @@ pub async fn parse_python_imports( .fetch_optional(db) .await? .unwrap_or_else(|| "".to_string()); + if already_visited.contains(&rpath) { vec![] } else { already_visited.push(rpath.clone()); - parse_python_imports(&code, w_id, &rpath, db, already_visited).await? + parse_python_imports_inner( + &code, + w_id, + &rpath, + db, + already_visited, + annotated_pyv_numeric, + path_where_annotated_pyv, + ) + .await? } } else { vec![replace_import(n.to_string())] diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index 436493e132..a634f247dd 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -25,6 +25,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; // println!("{}", serde_json::to_string(&r)?); @@ -57,6 +58,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; println!("{}", serde_json::to_string(&r)?); @@ -87,6 +89,7 @@ def main(): "f/foo/bar", &db, &mut already_visited, + &mut None, ) .await?; println!("{}", serde_json::to_string(&r)?); diff --git a/backend/src/main.rs b/backend/src/main.rs index c262d7d8ab..fca33c9b82 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -9,8 +9,9 @@ use anyhow::Context; use monitor::{ load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, - reload_nuget_config_setting, reload_timeout_wait_result_setting, - send_current_log_file_to_object_store, send_logs_to_object_store, + reload_instance_python_version_setting, reload_nuget_config_setting, + reload_timeout_wait_result_setting, send_current_log_file_to_object_store, + send_logs_to_object_store, }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -35,7 +36,7 @@ use windmill_common::{ CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, @@ -69,8 +70,9 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, - POWERSHELL_CACHE_DIR, PY311_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR, - TMP_LOGS_DIR, UV_CACHE_DIR, + POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, + TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -765,6 +767,9 @@ Windmill Community Edition {GIT_VERSION} PIP_INDEX_URL_SETTING => { reload_pip_index_url_setting(&db).await }, + INSTANCE_PYTHON_VERSION_SETTING => { + reload_instance_python_version_setting(&db).await + }, NPM_CONFIG_REGISTRY_SETTING => { reload_npm_config_registry_setting(&db).await }, @@ -1016,12 +1021,18 @@ pub async fn run_workers( TMP_LOGS_DIR, UV_CACHE_DIR, TAR_PIP_CACHE_DIR, - TAR_PY311_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, BUN_CACHE_DIR, + PY310_CACHE_DIR, PY311_CACHE_DIR, + PY312_CACHE_DIR, + PY313_CACHE_DIR, + TAR_PY310_CACHE_DIR, + TAR_PY311_CACHE_DIR, + TAR_PY312_CACHE_DIR, + TAR_PY313_CACHE_DIR, PIP_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 59a625fc2a..587a25d64a 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -41,10 +41,10 @@ use windmill_common::{ BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, - EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, + NUGET_CONFIG_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -68,8 +68,8 @@ use windmill_common::{ use windmill_queue::cancel_job; use windmill_worker::{ create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, - SendResult, BUNFIG_INSTALL_SCOPES, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, NPM_CONFIG_REGISTRY, - NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, + SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, + NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY, }; #[cfg(feature = "parquet")] @@ -198,6 +198,7 @@ pub async fn initial_load( reload_pip_index_url_setting(&db).await; reload_npm_config_registry_setting(&db).await; reload_bunfig_install_scopes_setting(&db).await; + reload_instance_python_version_setting(&db).await; reload_nuget_config_setting(&db).await; } } @@ -908,6 +909,16 @@ pub async fn reload_pip_index_url_setting(db: &DB) { .await; } +pub async fn reload_instance_python_version_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + INSTANCE_PYTHON_VERSION_SETTING, + "INSTANCE_PYTHON_VERSION", + INSTANCE_PYTHON_VERSION.clone(), + ) + .await; +} + pub async fn reload_npm_config_registry_setting(db: &DB) { reload_option_setting_with_tracing( db, diff --git a/backend/windmill-common/src/error.rs b/backend/windmill-common/src/error.rs index 0080568f5e..426354a13d 100644 --- a/backend/windmill-common/src/error.rs +++ b/backend/windmill-common/src/error.rs @@ -66,6 +66,8 @@ pub enum Error { AiError(String), #[error("{0}")] AlreadyCompleted(String), + #[error("Find python error: {0}")] + FindPythonError(String), #[error("{0}")] Utf8(#[from] std::string::FromUtf8Error), #[error("Encoding/decoding error: {0}")] diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 01ca59752f..3829826da6 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -14,6 +14,7 @@ pub const NUGET_CONFIG_SETTING: &str = "nuget_config"; pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; +pub const INSTANCE_PYTHON_VERSION_SETTING: &str = "instance_python_version"; pub const SCIM_TOKEN_SETTING: &str = "scim_token"; pub const SAML_METADATA_SETTING: &str = "saml_metadata"; pub const SMTP_SETTING: &str = "smtp_settings"; @@ -39,7 +40,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 54] = [ +pub const ENV_SETTINGS: [&str; 55] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -62,6 +63,7 @@ pub const ENV_SETTINGS: [&str; 54] = [ "GOPRIVATE", "GOPROXY", "NETRC", + "INSTANCE_PYTHON_VERSION", "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "PIP_TRUSTED_HOST", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index f1c51bd870..36d9f14804 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -333,14 +333,18 @@ fn parse_file(path: &str) -> Option { .flatten() } +#[derive(Copy, Clone)] #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, pub no_uv: bool, pub no_uv_install: bool, pub no_uv_compile: bool, - pub no_postinstall: bool, + pub py310: bool, + pub py311: bool, + pub py312: bool, + pub py313: bool, } #[annotations("//")] diff --git a/backend/windmill-worker/nsjail/download.py.config.proto b/backend/windmill-worker/nsjail/download.py.config.proto index c718f0ec30..b910e1c690 100644 --- a/backend/windmill-worker/nsjail/download.py.config.proto +++ b/backend/windmill-worker/nsjail/download.py.config.proto @@ -20,6 +20,7 @@ clone_newuser: {CLONE_NEWUSER} keep_caps: true keep_env: true +mount_proc: true mount { src: "/bin" @@ -79,6 +80,11 @@ mount { is_bind: true rw: true } +mount { + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" + is_bind: true +} mount { src: "/dev/urandom" diff --git a/backend/windmill-worker/nsjail/download_deps.py.sh b/backend/windmill-worker/nsjail/download_deps.py.sh index 1d8e911f74..06a61ac19f 100755 --- a/backend/windmill-worker/nsjail/download_deps.py.sh +++ b/backend/windmill-worker/nsjail/download_deps.py.sh @@ -27,6 +27,7 @@ CMD="/usr/local/bin/uv pip install --no-color --no-deps --link-mode=copy +$PY_PATH $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG --index-strategy unsafe-best-match --system diff --git a/backend/windmill-worker/nsjail/run.python3.config.proto b/backend/windmill-worker/nsjail/run.python3.config.proto index 06ced731c3..a9d61dc8b1 100644 --- a/backend/windmill-worker/nsjail/run.python3.config.proto +++ b/backend/windmill-worker/nsjail/run.python3.config.proto @@ -16,6 +16,7 @@ clone_newuser: {CLONE_NEWUSER} keep_caps: false keep_env: true +mount_proc: true mount { src: "/bin" @@ -110,6 +111,12 @@ mount { is_bind: true } +mount { + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" + is_bind: true +} + mount { src: "/dev/urandom" dst: "/dev/urandom" diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 4be27de5b0..e21728943e 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -32,7 +32,7 @@ use crate::{ check_executor_binary_exists, get_reserved_variables, read_and_check_result, start_child_process, transform_json, OccupancyMetrics }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; @@ -88,6 +88,7 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), + PyVersion::Py311, false, false, ) @@ -115,7 +116,7 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), - false, + crate::python_executor::PyVersion::Py311, false, ) .await?; diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index f65a0bbb80..57ded13ee7 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -17,19 +17,24 @@ use std::sync::Arc; pub async fn build_tar_and_push( s3_client: Arc, folder: String, + // python_311 + python_xyz: String, no_uv: bool, ) -> error::Result<()> { use object_store::path::Path; - use crate::{TAR_PIP_CACHE_DIR, TAR_PY311_CACHE_DIR}; + use crate::{TAR_PIP_CACHE_DIR, TAR_PYBASE_CACHE_DIR}; tracing::info!("Started building and pushing piptar {folder}"); let start = Instant::now(); + + // e.g. tiny==1.0.0 let folder_name = folder.split("/").last().unwrap(); + let prefix = if no_uv { TAR_PIP_CACHE_DIR } else { - TAR_PY311_CACHE_DIR + &format!("{TAR_PYBASE_CACHE_DIR}/{}", python_xyz) }; let tar_path = format!("{prefix}/{folder_name}_tar.tar",); @@ -53,7 +58,7 @@ pub async fn build_tar_and_push( .put( &Path::from(format!( "/tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } + if no_uv { "pip" } else { &python_xyz } )), std::fs::read(&tar_path)?.into(), ) @@ -82,6 +87,8 @@ pub async fn build_tar_and_push( pub async fn pull_from_tar( client: Arc, folder: String, + // python_311 + python_xyz: String, no_uv: bool, ) -> error::Result<()> { use windmill_common::s3_helpers::attempt_fetch_bytes; @@ -91,14 +98,13 @@ pub async fn pull_from_tar( tracing::info!("Attempting to pull piptar {folder_name} from bucket"); let start = Instant::now(); + let tar_path = format!( "tar/{}/{folder_name}.tar", - if no_uv { "pip" } else { "python_311" } + if no_uv { "pip".to_owned() } else { python_xyz } ); let bytes = attempt_fetch_bytes(client, &tar_path).await?; - // tracing::info!("B: {target} {folder}"); - extract_tar(bytes, &folder).await.map_err(|e| { tracing::error!("Failed to extract piptar {folder_name}. Error: {:?}", e); e diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 68ba52c701..4d3a946926 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -22,7 +22,10 @@ use uuid::Uuid; #[cfg(all(feature = "enterprise", feature = "parquet", unix))] use windmill_common::ee::{get_license_plan, LicensePlan}; use windmill_common::{ - error::{self, Error}, + error::{ + self, + Error::{self}, + }, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, worker::{write_file, PythonAnnotations, WORKER_CONFIG}, @@ -51,18 +54,18 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); + pub static ref USE_SYSTEM_PYTHON: bool = std::env::var("USE_SYSTEM_PYTHON") + .ok().map(|flag| flag == "true").unwrap_or(false); + pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); - /// Use pip install pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); - static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); static ref EPHEMERAL_TOKEN_CMD: Option = std::env::var("EPHEMERAL_TOKEN_CMD").ok(); - } const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto"); @@ -82,12 +85,292 @@ use crate::{ create_args_and_out_file, get_main_override, get_reserved_variables, read_file, read_result, start_child_process, OccupancyMetrics, }, - handle_child::{get_mem_peak, handle_child}, - AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR, - NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, - PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR, + handle_child::handle_child, + AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, + LOCK_CACHE_DIR, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; +// To change latest stable version: +// 1. Change placeholder in instanceSettings.ts +// 2. Change LATEST_STABLE_PY in dockerfile +// 3. Change #[default] annotation for PyVersion in backend +#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] +pub enum PyVersion { + Py310, + #[default] + Py311, + Py312, + Py313, +} + +impl PyVersion { + pub async fn from_instance_version() -> Self { + match INSTANCE_PYTHON_VERSION.read().await.clone() { + Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| { + let v = PyVersion::default(); + tracing::error!( + "Cannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", + *INSTANCE_PYTHON_VERSION + ); + v + }), + // Use latest stable + None => PyVersion::default(), + } + } + /// e.g.: `/tmp/windmill/cache/python_3xy` + pub fn to_cache_dir(&self) -> String { + use windmill_common::worker::ROOT_CACHE_DIR; + format!("{ROOT_CACHE_DIR}{}", &self.to_cache_dir_top_level()) + } + /// e.g.: `python_3xy` + pub fn to_cache_dir_top_level(&self) -> String { + format!("python_{}", self.to_string_no_dot()) + } + /// e.g.: `3xy` + pub fn to_string_no_dot(&self) -> String { + self.to_string_with_dot().replace('.', "") + } + /// e.g.: `3.xy` + pub fn to_string_with_dot(&self) -> &str { + use PyVersion::*; + match self { + Py310 => "3.10", + Py311 => "3.11", + Py312 => "3.12", + Py313 => "3.13", + } + } + pub fn from_string_with_dots(value: &str) -> Option { + use PyVersion::*; + match value { + "3.10" => Some(Py310), + "3.11" => Some(Py311), + "3.12" => Some(Py312), + "3.13" => Some(Py313), + "default" => Some(PyVersion::default()), + _ => { + tracing::warn!( + "Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz" + ); + None + } + } + } + pub fn from_string_no_dots(value: &str) -> Option { + use PyVersion::*; + match value { + "310" => Some(Py310), + "311" => Some(Py311), + "312" => Some(Py312), + "313" => Some(Py313), + "default" => Some(PyVersion::default()), + _ => { + tracing::warn!( + "Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz" + ); + None + } + } + } + /// e.g.: `# py3xy` -> `PyVersion::Py3XY` + pub fn parse_version(line: &str) -> Option { + Self::from_string_no_dots(line.replace(" ", "").replace("#py", "").as_str()) + } + pub fn from_py_annotations(a: PythonAnnotations) -> Option { + let PythonAnnotations { py310, py311, py312, py313, .. } = a; + use PyVersion::*; + if py313 { + Some(Py313) + } else if py312 { + Some(Py312) + } else if py311 { + Some(Py311) + } else if py310 { + Some(Py310) + } else { + None + } + } + pub fn from_numeric(n: u32) -> Option { + use PyVersion::*; + match n { + 310 => Some(Py310), + 311 => Some(Py311), + 312 => Some(Py312), + 313 => Some(Py313), + _ => None, + } + } + pub fn to_numeric(&self) -> u32 { + use PyVersion::*; + match self { + Py310 => 310, + Py311 => 311, + Py312 => 312, + Py313 => 313, + } + } + pub async fn get_python( + &self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result> { + // lazy_static::lazy_static! { + // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); + // } + + let res = self + .get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await; + + if let Err(ref e) = res { + tracing::error!( + "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n + Error while getting python from uv, falling back to system python: {e:?}" + ); + } + res + } + async fn get_python_inner( + self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result> { + let py_path = self.find_python().await; + + // Runtime is not installed + if py_path.is_err() { + // Install it + if let Err(err) = self + .install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await + { + tracing::error!("Cannot install python: {err}"); + return Err(err); + } else { + // Try to find one more time + let py_path = self.find_python().await; + + if let Err(err) = py_path { + tracing::error!("Cannot find python version {err}"); + return Err(err); + } + + // TODO: Cache the result + py_path + } + } else { + py_path + } + } + async fn install_python( + self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + db: &Pool, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result<()> { + let v = self.to_string_with_dot(); + append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await; + // Create dirs for newly installed python + // If we dont do this, NSJAIL will not be able to mount cache + // For the default version directory created during startup (main.rs) + DirBuilder::new() + .recursive(true) + .create(self.to_cache_dir()) + .await + .expect("could not create initial worker dir"); + + let logs = String::new(); + + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); + child_cmd + .args(["python", "install", v, "--python-preference=only-managed"]) + // TODO: Do we need these? + .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let child_process = start_child_process(child_cmd, "uv").await?; + + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + &mut None, + child_process, + false, + worker_name, + &w_id, + "uv", + None, + false, + occupancy_metrics, + ) + .await + } + async fn find_python(self) -> error::Result> { + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); + let output = child_cmd + // .current_dir(job_dir) + .args([ + "python", + "find", + self.to_string_with_dot(), + "--python-preference=only-managed", + ]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), + ("UV_PYTHON_PREFERENCE", "only-managed"), + ]) + // .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await?; + + // Check if the command was successful + if output.status.success() { + // Convert the output to a String + let stdout = + String::from_utf8(output.stdout).expect("Failed to convert output to String"); + return Ok(Some(stdout.replace('\n', ""))); + } else { + // If the command failed, print the error + let stderr = + String::from_utf8(output.stderr).expect("Failed to convert error output to String"); + return Err(error::Error::FindPythonError(stderr)); + } + } +} + #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -120,6 +403,10 @@ pub fn handle_ephemeral_token(x: String) -> String { x } +// This function only invoked during deployment of script or test run. +// And never for already deployed scripts, these have their lockfiles in PostgreSQL +// thus this function call is skipped. +/// Returns lockfile and python version pub async fn uv_pip_compile( job_id: &Uuid, requirements: &str, @@ -130,14 +417,16 @@ pub async fn uv_pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - // Fallback to pip-compile. Will be removed in future - mut no_uv: bool, + py_version: PyVersion, // Debug-only flag no_cache: bool, + // Fallback to pip-compile. Will be removed in future + mut no_uv: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); logs.push_str(&format!("\ncontent of requirements:\n{}\n", requirements)); + let requirements = if let Some(pip_local_dependencies) = WORKER_CONFIG.read().await.pip_local_dependencies.as_ref() { @@ -167,6 +456,11 @@ pub async fn uv_pip_compile( requirements.to_string() }; + // Include python version to requirements.in + // We need it because same hash based on requirements.in can get calculated even for different python versions + // To prevent from overwriting same requirements.in but with different python versions, we include version to hash + let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements); + #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; @@ -186,15 +480,21 @@ pub async fn uv_pip_compile( if !no_cache { if let Some(cached) = sqlx::query_scalar!( "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + // Python version is included in hash, + // hash will be the different for every python version req_hash ) .fetch_optional(db) .await? { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); + logs.push_str(&format!( + "\nFound cached resolution: {req_hash}, on python version: {}", + py_version.to_string_with_dot() + )); return Ok(cached); } } + let file = "requirements.in"; write_file(job_dir, file, &requirements)?; @@ -269,6 +569,11 @@ pub async fn uv_pip_compile( .await .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; } else { + // Make sure we have python runtime installed + py_version + .get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics) + .await?; + let mut args = vec![ "pip", "compile", @@ -286,11 +591,15 @@ pub async fn uv_pip_compile( // Target to /tmp/windmill/cache/uv "--cache-dir", UV_CACHE_DIR, - // We dont want UV to manage python installations - "--python-preference", - "only-system", - "--no-python-downloads", ]; + + args.extend([ + "-p", + &py_version.to_string_with_dot(), + "--python-preference", + "only-managed", + ]); + if no_cache { args.extend(["--no-cache"]); } @@ -332,6 +641,7 @@ pub async fn uv_pip_compile( .env_clear() .env("HOME", HOME_ENV.to_string()) .env("PATH", PATH_ENV.to_string()) + .env("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR.to_string()) .envs(PROXY_ENVS.clone()) .args(&args) .stdout(Stdio::piped()) @@ -378,17 +688,22 @@ pub async fn uv_pip_compile( let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; - let lockfile = req_content - .lines() - .filter(|x| !x.trim_start().starts_with('#')) - .map(|x| x.to_string()) - .collect::>() - .join("\n"); + let lockfile = format!( + "# py{}\n{}", + py_version.to_string_no_dot(), + req_content + .lines() + .filter(|x| !x.trim_start().starts_with('#')) + .map(|x| x.to_string()) + .collect::>() + .join("\n") + ); sqlx::query!( "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2", req_hash, lockfile ).fetch_optional(db).await?; + Ok(lockfile) } @@ -538,7 +853,8 @@ pub async fn handle_python_job( occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); - let mut additional_python_paths = handle_python_deps( + + let (py_version, mut additional_python_paths) = handle_python_deps( job_dir, requirements_o, inner_content, @@ -554,23 +870,53 @@ pub async fn handle_python_job( ) .await?; + let PythonAnnotations { no_uv, no_postinstall, .. } = PythonAnnotations::parse(inner_content); tracing::debug!("Finished handling python dependencies"); + let python_path = if no_uv { + PYTHON_PATH.clone() + } else if let Some(python_path) = py_version + .get_python( + &job.id, + mem_peak, + db, + worker_name, + &job.workspace_id, + &mut Some(occupancy_metrics), + ) + .await? + { + python_path + } else { + PYTHON_PATH.clone() + }; - if !PythonAnnotations::parse(inner_content).no_postinstall { + if !no_postinstall { if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await { tracing::error!("Postinstall stage has failed. Reason: {e}"); } tracing::debug!("Finished deps postinstall stage"); } - append_logs( - &job.id, - &job.workspace_id, - "\n\n--- PYTHON CODE EXECUTION ---\n".to_string(), - db, - ) - .await; - + if no_uv { + append_logs( + &job.id, + &job.workspace_id, + format!("\n\n--- SYSTEM PYTHON (Fallback) CODE EXECUTION ---\n",), + db, + ) + .await; + } else { + append_logs( + &job.id, + &job.workspace_id, + format!( + "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", + py_version.to_string_with_dot() + ), + db, + ) + .await; + } let ( import_loader, import_base64, @@ -725,6 +1071,7 @@ mount {{ "run.config.proto", &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT .replace("{JOB_DIR}", job_dir) + .replace("{PY_INSTALL_DIR}", PY_INSTALL_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) .replace("{SHARED_MOUNT}", shared_mount) .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) @@ -743,6 +1090,7 @@ mount {{ "started python code execution {}", job.id ); + let child = if !*DISABLE_NSJAIL { let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); nsjail_cmd @@ -759,7 +1107,7 @@ mount {{ "--config", "run.config.proto", "--", - PYTHON_PATH.as_str(), + &python_path, "-u", "-m", "wrapper", @@ -768,7 +1116,9 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let mut python_cmd = Command::new(PYTHON_PATH.as_str()); + let mut python_cmd = Command::new(&python_path); + + let args = vec!["-u", "-m", "wrapper"]; python_cmd .current_dir(job_dir) .env_clear() @@ -778,7 +1128,7 @@ mount {{ .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(vec!["-u", "-m", "wrapper"]) + .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -788,7 +1138,7 @@ mount {{ python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); } - start_child_process(python_cmd, PYTHON_PATH.as_str()).await? + start_child_process(python_cmd, &python_path).await? }; handle_child( @@ -1073,7 +1423,7 @@ async fn handle_python_deps( mem_peak: &mut i32, canceled_by: &mut Option, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, -) -> error::Result> { +) -> error::Result<(PyVersion, Vec)> { create_dependencies_dir(job_dir).await; let mut additional_python_paths: Vec = WORKER_CONFIG @@ -1084,8 +1434,12 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut requirements; + let mut annotated_pyv = None; + let mut annotated_pyv_numeric = None; + let is_deployed = requirements_o.is_some(); + let instance_pyv = PyVersion::from_instance_version().await; + let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content); let requirements = match requirements_o { Some(r) => r, None => { @@ -1097,9 +1451,13 @@ async fn handle_python_deps( script_path, db, &mut already_visited, + &mut annotated_pyv_numeric, ) .await? .join("\n"); + + annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v)); + if !requirements.is_empty() { requirements = uv_pip_compile( job_id, @@ -1111,8 +1469,9 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, - annotations.no_uv || annotations.no_uv_compile, + annotated_pyv.unwrap_or(instance_pyv), annotations.no_cache, + annotations.no_uv || annotations.no_uv_compile, ) .await .map_err(|e| { @@ -1123,12 +1482,47 @@ async fn handle_python_deps( } }; + let requirements_lines: Vec<&str> = if requirements.len() > 0 { + requirements + .split("\n") + .filter(|x| !x.starts_with("--") && !x.trim().is_empty()) + .collect() + } else { + vec![] + }; + + /* + For deployed scripts we want to find out version in following order: + 1. Assigned version (written in lockfile) + 2. 3.11 + + For Previews: + 1. Annotated version + 2. Instance version + 3. Latest Stable + */ + let final_version = if is_deployed { + // If script is deployed we can try to parse first line to get assigned version + if let Some(v) = requirements_lines + .get(0) + .and_then(|line| PyVersion::parse_version(line)) + { + // We have valid assigned version, we use it + v + } else { + // If there is no assigned version in lockfile we automatically fallback to 3.11 + // In this case we have dependencies, but no associated python version + // This is the case for old deployed scripts + PyVersion::Py311 + } + } else { + // This is not deployed script, meaning we test run it (Preview) + annotated_pyv.unwrap_or(instance_pyv) + }; + // If len > 0 it means there is atleast one dependency or assigned python version if requirements.len() > 0 { let mut venv_path = handle_python_reqs( - requirements - .split("\n") - .filter(|x| !x.starts_with("--") && !x.trim().is_empty()) - .collect(), + requirements_lines, job_id, w_id, mem_peak, @@ -1138,13 +1532,14 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, + final_version, annotations.no_uv || annotations.no_uv_install, - false, ) .await?; additional_python_paths.append(&mut venv_path); } - Ok(additional_python_paths) + + Ok((final_version, additional_python_paths)) } lazy_static::lazy_static! { @@ -1160,6 +1555,8 @@ async fn spawn_uv_install( venv_p: &str, job_dir: &str, (pip_extra_index_url, pip_index_url): (Option, Option), + // If none, it is system python + py_path: Option, no_uv_install: bool, ) -> Result { if !*DISABLE_NSJAIL { @@ -1181,7 +1578,14 @@ async fn spawn_uv_install( if let Some(host) = PIP_TRUSTED_HOST.as_ref() { vars.push(("TRUSTED_HOST", host)); } - + let _owner; + if let Some(py_path) = py_path.as_ref() { + _owner = format!( + "-p {} --python-preference only-managed", + py_path.as_str() // + ); + vars.push(("PY_PATH", &_owner)); + } vars.push(("REQ", &req)); vars.push(("TARGET", venv_p)); @@ -1231,8 +1635,6 @@ async fn spawn_uv_install( &req, "--no-deps", "--no-color", - // "-p", - // "3.11", // Prevent uv from discovering configuration files. "--no-config", "--link-mode=copy", @@ -1250,6 +1652,22 @@ async fn spawn_uv_install( ] }; + if !no_uv_install { + if let Some(py_path) = py_path.as_ref() { + command_args.extend([ + "-p", + py_path.as_str(), + "--python-preference", + "only-managed", // + ]); + } else { + command_args.extend([ + "--python-preference", + "only-system", // + ]); + } + } + if let Some(url) = pip_extra_index_url.as_ref() { url.split(",").for_each(|url| { command_args.extend(["--extra-index-url", url]); @@ -1339,7 +1757,7 @@ fn pad_string(value: &str, total_length: usize) -> String { } } -/// pip install, include cached or pull from S3 +/// uv pip install, include cached or pull from S3 pub async fn handle_python_reqs( requirements: Vec<&str>, job_id: &Uuid, @@ -1351,9 +1769,9 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, _occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + py_version: PyVersion, // TODO: Remove (Deprecated) mut no_uv_install: bool, - is_ansible: bool, ) -> error::Result> { let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: @@ -1405,7 +1823,7 @@ pub async fn handle_python_reqs( } no_uv_install |= *USE_PIP_INSTALL; - if no_uv_install && !is_ansible { + if no_uv_install { append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await; tracing::warn!("Fallback to pip"); } @@ -1449,13 +1867,14 @@ pub async fn handle_python_reqs( NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT }) .replace("{WORKER_DIR}", &worker_dir) + .replace("{PY_INSTALL_DIR}", &PY_INSTALL_DIR) .replace( "{CACHE_DIR}", - if no_uv_install { - PIP_CACHE_DIR + &(if no_uv_install { + PIP_CACHE_DIR.to_owned() } else { - PY311_CACHE_DIR - }, + py_version.to_cache_dir() + }), ) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()), )?; @@ -1473,11 +1892,10 @@ pub async fn handle_python_reqs( if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - // TODO: Remove let py_prefix = if no_uv_install { PIP_CACHE_DIR } else { - PY311_CACHE_DIR + &py_version.to_cache_dir() }; let venv_p = format!( @@ -1528,7 +1946,7 @@ pub async fn handle_python_reqs( let mut local_mem_peak = 0; for pid_o in pids.lock().await.iter() { if pid_o.is_some(){ - let mem = get_mem_peak(*pid_o, !*DISABLE_NSJAIL).await; + let mem = crate::handle_child::get_mem_peak(*pid_o, !*DISABLE_NSJAIL).await; if mem < 0 { tracing::warn!( workspace_id = %w_id_2, @@ -1659,6 +2077,14 @@ pub async fn handle_python_reqs( let is_not_pro = !matches!(get_license_plan().await, LicensePlan::Pro); let total_time = std::time::Instant::now(); + let py_path = if no_uv_install { + None + } else { + py_version + .get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics) + .await? + }; + let has_work = req_with_penv.len() > 0; for ((i, (req, venv_p)), mut kill_rx) in req_with_penv.iter().enumerate().zip(kill_rxs.into_iter()) @@ -1688,6 +2114,7 @@ pub async fn handle_python_reqs( let venv_p = venv_p.clone(); let counter_arc = counter_arc.clone(); let pip_indexes = pip_indexes.clone(); + let py_path = py_path.clone(); let pids = pids.clone(); handles.push(task::spawn(async move { @@ -1711,7 +2138,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), no_uv_install) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -1744,7 +2171,8 @@ pub async fn handle_python_reqs( &venv_p, &job_dir, pip_indexes, - no_uv_install, + py_path, + no_uv_install ).await { Ok(r) => r, Err(e) => { @@ -1828,7 +2256,6 @@ pub async fn handle_python_reqs( #[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))] let s3_push = false; - print_success( false, s3_push, @@ -1846,7 +2273,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), no_uv_install)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), no_uv_install)); } } @@ -1955,7 +2382,7 @@ pub async fn start_worker( .to_vec(); let context_envs = build_envs_map(context).await; - let additional_python_paths = handle_python_deps( + let (_, additional_python_paths) = handle_python_deps( job_dir, requirements_o, inner_content, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0a0599047f..9499dd9eec 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -105,7 +105,7 @@ use crate::rust_executor::handle_rust_job; use crate::php_executor::handle_php_job; #[cfg(feature = "python")] -use crate::python_executor::handle_python_job; +use crate::python_executor::{handle_python_job, PyVersion}; #[cfg(feature = "python")] use crate::ansible_executor::handle_ansible_job; @@ -256,14 +256,20 @@ pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); // Used as fallback now pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); -// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); + +pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); +pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); +pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); +pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); +pub const PY_INSTALL_DIR: &str = concatcp!(ROOT_CACHE_DIR, "py_runtime"); +pub const TAR_PYBASE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); -pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); pub const DENO_CACHE_DIR_NPM: &str = concatcp!(ROOT_CACHE_DIR, "deno/npm"); @@ -399,6 +405,7 @@ lazy_static::lazy_static! { pub static ref PIP_EXTRA_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); pub static ref PIP_INDEX_URL: Arc>> = Arc::new(RwLock::new(None)); + pub static ref INSTANCE_PYTHON_VERSION: Arc>> = Arc::new(RwLock::new(None)); pub static ref JOB_DEFAULT_TIMEOUT: Arc>> = Arc::new(RwLock::new(None)); static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT") @@ -765,6 +772,41 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); + #[cfg(feature = "python")] + { + let (db, worker_name, hostname, worker_dir) = ( + db.clone(), + worker_name.clone(), + hostname.to_owned(), + worker_dir.clone(), + ); + tokio::spawn(async move { + if let Err(e) = PyVersion::from_instance_version() + .await + .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .await + { + tracing::error!( + worker = %worker_name, + hostname = %hostname, + worker_dir = %worker_dir, + "Cannot preinstall or find Instance Python version to worker: {e}"// + ); + } + if let Err(e) = PyVersion::Py311 + .get_python(&Uuid::nil(), &mut 0, &db, &worker_name, "", &mut None) + .await + { + tracing::error!( + worker = %worker_name, + hostname = %hostname, + worker_dir = %worker_dir, + "Cannot preinstall or find default 311 version to worker: {e}"// + ); + } + }); + } + if let Some(ref netrc) = *NETRC { tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str()); write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 06b7e4f0d2..d4df729910 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -39,7 +39,8 @@ use crate::csharp_executor::generate_nuget_lockfile; use crate::php_executor::{composer_install, parse_php_imports}; #[cfg(feature = "python")] use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL, + create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, USE_PIP_COMPILE, + USE_PIP_INSTALL, }; #[cfg(feature = "rust")] use crate::rust_executor::generate_cargo_lockfile; @@ -1595,10 +1596,28 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + annotated_pyv_numeric: Option, + annotations: PythonAnnotations, no_uv_compile: bool, no_uv_install: bool, ) -> std::result::Result { create_dependencies_dir(job_dir).await; + + /* + Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) + This one used specifically for deploying scripts + So we can get final_version right away and include in lockfile + And the precendence is following: + + 1. Annotation version + 2. Instance version + 3. Latest Stable + */ + + let final_version = annotated_pyv_numeric + .and_then(|pyv| PyVersion::from_numeric(pyv)) + .unwrap_or(PyVersion::from_instance_version().await); + let req: std::result::Result = uv_pip_compile( job_id, &reqs, @@ -1609,8 +1628,9 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, + final_version, + annotations.no_cache, no_uv_compile, - false, ) .await; // install the dependencies to pre-fill the cache @@ -1626,8 +1646,8 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, + final_version, no_uv_install, - false, ) .await; @@ -1665,10 +1685,18 @@ async fn capture_dependency_job( return Err(Error::InternalErr( "Python requires the python feature to be enabled".to_string(), )); - #[cfg(feature = "python")] { + let anns = PythonAnnotations::parse(job_raw_code); + let mut annotated_pyv_numeric = None; + let reqs = if raw_deps { + // `wmill script generate-metadata` + // should also respect annotated pyversion + // can be annotated in script itself + // or in requirements.txt if present + annotated_pyv_numeric = + PyVersion::from_py_annotations(anns).map(|v| v.to_numeric()); job_raw_code.to_string() } else { let mut already_visited = vec![]; @@ -1679,14 +1707,12 @@ async fn capture_dependency_job( script_path, &db, &mut already_visited, + &mut annotated_pyv_numeric, ) .await? .join("\n") }; - - let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = - PythonAnnotations::parse(job_raw_code); - + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = anns; if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { if let Err(e) = sqlx::query!( r#" @@ -1713,6 +1739,8 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + annotated_pyv_numeric, + anns, no_uv_compile | no_uv, no_uv_install | no_uv, ) @@ -1761,6 +1789,8 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + None, + PythonAnnotations::default(), false, false, ) diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index a7e41f5876..079d8fc378 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -27,15 +27,20 @@ import { createEventDispatcher } from 'svelte' import { fade } from 'svelte/transition' import { base } from '$lib/base' + import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import SimpleEditor from './SimpleEditor.svelte' export let setting: Setting export let version: string export let values: Writable> export let loading = true - const dispatch = createEventDispatcher() + if (setting.fieldType == 'select' && $values[setting.key] == undefined){ + $values[setting.key] = "default"; + } + let latestKeyRenewalAttempt: { result: string attempted_at: string @@ -166,6 +171,24 @@ EE only {#if setting.ee_only != ''}{setting.ee_only}{/if} {/if} + {#if setting.fieldType == 'select'} +
+ + + + {#each (setting.select_items ?? []) as item } + + {/each} + +
+ {:else} + {/if} {/if} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index a0d7982c81..75af6e568c 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -6,6 +6,13 @@ export interface Setting { ee_only?: string tooltip?: string key: string + // If value is not specified for first element, it will automatcally use undefined + select_items?: { + label: string, + tooltip?: string, + // If not specified, label will be used + value?: any, + }[], fieldType: | 'text' | 'number' @@ -74,9 +81,9 @@ export const settings: Record = { isValid: (value: string | undefined) => value ? value?.startsWith('http') && - value.includes('://') && - !value?.endsWith('/') && - !value?.endsWith(' ') + value.includes('://') && + !value?.endsWith('/') && + !value?.endsWith(' ') : false }, { @@ -219,6 +226,35 @@ export const settings: Record = { ], 'Auth/OAuth': [], Registries: [ + { + label: 'Instance Python Version', + description: 'Default python version for newly deployed scripts', + key: 'instance_python_version', + fieldType: 'select', + // To change latest stable version: + // 1. Change placeholder in instanceSettings.ts + // 2. Change LATEST_STABLE_PY in dockerfile + // 3. Change #[default] annotation for PyVersion in backend + placeholder: '3.10,3.11,3.12,3.13', + select_items: [{ + label: "Latest Stable", + value: "default", + tooltip: "python-3.11", + }, + { + label: "3.10", + }, + { + label: "3.11", + }, + { + label: "3.12", + }, + { + label: "3.13", + }], + storage: 'setting', + }, { label: 'Pip index url', description: 'Add private Pip registry', diff --git a/shell.nix b/shell.nix index 149e8bdae8..79ac12c49e 100644 --- a/shell.nix +++ b/shell.nix @@ -29,7 +29,7 @@ in pkgs.mkShell { postgresql watchexec # used in client's dev.nu poetry # for python client - uv + # uv python312Packages.pip-tools # pip-compile ]; From 316a2167e7648052afb3d7456c5ab9324348e56c Mon Sep 17 00:00:00 2001 From: dieriba Date: Fri, 24 Jan 2025 15:12:41 +0100 Subject: [PATCH 3/6] feat: postgres triggers (#4860) * feat: init database triggers * feat: :sparkles: wip: database_triggers * feat: :sparkles: add database triggers front view * feat: :construction: database_triggers * feat: :construction: add definition in yaml, updated backend code and added migration * feat: :construction: updated migration file, update openapi.yml, updated database_triggers page and backend function * fix: struct rust * feat: :construction: update migrate, database trigger backend function fixed * feat: :construction: add resource picker front, update backend function * feat: :construction: edit inner database inner, update triggers * feat: :construction: database_triggers * feat: :construction: update openapi yaml, prettied websocker trigger * feat: :construction: database_triggers * feat: :construction: add resource module, update variable file, working on main loop for database_trigger * feat: :bug: working sqlx query * feat: :construction: fix query with sqlx, added main loop * feat: :construction: add new column database_trigg * feat: :passport_control: run jobs works * feat: :construction: handling slot name and replication * feat: restructring triggers, decoding trigger message on work * feat: :construction: database_trigger * feat: :construction: * feat: :construction: converter done, work on custom script * feat: :construction: multiple trigger * feat: :construction: adding new argument function * feat: :construction: database_trigger * feat: :construction: add generate template for front, update script picker * feat: :construction: template script fix bug, work on restructing backend logic * feat: :construction: update autogenerated script, add persistence state for template script * feat: :construction: update structure client * feat: :construction: rewrited crud function * feat: :construction: added publication handler * feat: :construction: new ui finished * feat: :construction: added slot function hanlder finish front ux * feat: :sparkles: ux improvement done, backend logic done * feat: :bug: fix where clause * feat: * feat: * chore: update .sqlx and remove empty package * fix: update publication and remove unneccessary print * feat: finish converter to json, remove save button, remove unused crate * feat: update migration for database trigger, fixed query, fixed frontend error with missing feature on back * chore: update .sqlx * chore: update sqlx * chore: .sqlx * chore: add unused * fix: use right database resource in back and front * fix build * refactor: * fix sqlx * nits: retrieve all script from database trigger editor * fix: template script object, fix route name * merge main * feat: rework ux * feat/finish ux and update backend logic * feat: database_trigger * feat: database_trigger * chore: update sqlx * chore: fix ci * nits: fix worker.rs * feat: database_trigger * Update frontend/src/lib/components/triggers/database/DatabaseTriggersPanel.svelte Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> * feat: * fix ci * nits: database to postgres * nits: fix .sqlx * feat: add link to original code and open tab functionallity * nits: update .sqlx * nits: fix request type and yaml file * nits: add abel suggestion * fix: update .sqlx * fix: update .sqlx * nits: add comment to hex.rs * nits: remove features, rename database to postgres name * nits: replace database name by postgres * nits: database name to postgres * nits: update .sqlx * nits: update .sqlx * nit * nit * nits * nit * nits * nits * nits * nits * fix sqlx * nits: update .vscode --------- Co-authored-by: HugoCasa Co-authored-by: ellipsis-dev[bot] <65095814+ellipsis-dev[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel --- ...3fad4878f48db8c17c6d58590bd5df2e3350a.json | 24 + ...4665a84325e34ae04bf2795fafb7bb6f2f0c6.json | 23 + ...34bbabdd8703bfdf4d43df2c65e50d4ca2c85.json | 40 + ...6505a642c8f04b883c278b07e7ef5956ce121.json | 46 + ...48af970aed2638a3f767e152b654aebf68e48.json | 20 + ...f126f339d2e3cf97f363be8921084b16dac45.json | 20 + ...821940cb5064e3f155b9630b131c09c9dba13.json | 26 + ...f4968ecebec1614580e55de3d5895f0595e52.json | 15 + ...7f5033b9c9afc344d9c3e385ba20a3ad2197a.json | 2 +- ...fe682b023d1868a182b7cac16ce799433c257.json | 25 + ...1771f0137ecab6671d632febadede729e7783.json | 40 + ...052d2064fcc8f373fecfebd40914768b2b1cf.json | 24 + ...87434abd3e213b8c3c488166ca58fe3321147.json | 23 + ...587d8cad3b67d4dfa9de52777d4ea9490b6b7.json | 15 + ...c08bfc262f99c875bf16c25bfb5e987efe978.json | 104 ++ ...6197f02ca1cf898d7f2686419749b1cb0679e.json | 29 + ...4efe29311aabfba6e09efa10bab6a551d658b.json | 16 + ...f4a3600c1ccfe6847bd526a751bc7baf825c5.json | 23 + ...b00147fc8bcd42f3df37ef81b9ded80974bd0.json | 2 +- ...d4b2ce906fffc3c51e77d083121dbd68dadd4.json | 107 ++ ...05a50d1ed4f69629c43de7d874769c340d909.json | 26 + backend/.vscode/settings.json | 11 +- backend/Cargo.lock | 80 +- backend/Cargo.toml | 11 +- .../20241123152203_postgres_triggers.down.sql | 2 + .../20241123152203_postgres_triggers.up.sql | 69 + .../windmill-parser-csharp/src/wasm_libc.rs | 5 +- backend/tests/worker.rs | 2 +- backend/windmill-api/Cargo.toml | 7 +- backend/windmill-api/openapi.yaml | 785 +++++++-- backend/windmill-api/src/ai.rs | 35 +- backend/windmill-api/src/lib.rs | 18 +- .../src/postgres_triggers/bool.rs | 24 + .../src/postgres_triggers/converter.rs | 255 +++ .../src/postgres_triggers/handler.rs | 1458 +++++++++++++++++ .../windmill-api/src/postgres_triggers/hex.rs | 45 + .../src/postgres_triggers/mapper.rs | 134 ++ .../windmill-api/src/postgres_triggers/mod.rs | 155 ++ .../src/postgres_triggers/relation.rs | 74 + .../postgres_triggers/replication_message.rs | 510 ++++++ .../src/postgres_triggers/trigger.rs | 523 ++++++ backend/windmill-api/src/slack_approvals.rs | 14 +- backend/windmill-api/src/triggers.rs | 12 + backend/windmill-api/src/variables.rs | 25 + .../windmill-api/src/websocket_triggers.rs | 2 +- backend/windmill-api/src/workspaces.rs | 19 +- backend/windmill-api/src/workspaces_export.rs | 10 +- backend/windmill-common/Cargo.toml | 1 + backend/windmill-common/src/lib.rs | 20 +- backend/windmill-common/src/variables.rs | 15 +- backend/windmill-indexer/src/indexer_ee.rs | 1 + backend/windmill-queue/Cargo.toml | 1 - .../windmill-worker/src/ansible_executor.rs | 16 +- backend/windmill-worker/src/lib.rs | 4 +- .../windmill-worker/src/oracledb_executor.rs | 8 +- backend/windmill-worker/src/php_executor.rs | 3 +- backend/windmill-worker/src/worker.rs | 21 +- frontend/{.prettierrc => .prettierrc.json} | 0 .../src/lib/components/ApiConnectForm.svelte | 6 +- .../src/lib/components/AppConnectInner.svelte | 1 - .../src/lib/components/FlowBuilder.svelte | 10 +- frontend/src/lib/components/Path.svelte | 7 + .../src/lib/components/ScriptBuilder.svelte | 32 +- .../src/lib/components/ScriptPicker.svelte | 5 +- frontend/src/lib/components/Section.svelte | 3 +- .../details/DetailPageDetailPanel.svelte | 2 + .../details/DetailPageLayout.svelte | 3 + .../details/DetailPageTriggerPanel.svelte | 9 + .../renderers/triggers/TriggersBadge.svelte | 6 +- .../components/sidebar/OperatorMenu.svelte | 18 +- .../components/sidebar/SidebarContent.svelte | 13 +- frontend/src/lib/components/triggers.ts | 1 + .../components/triggers/CaptureWrapper.svelte | 10 +- .../components/triggers/TriggersEditor.svelte | 18 +- .../triggers/TriggersWrapper.svelte | 10 +- .../triggers/{ => http}/RouteEditor.svelte | 0 .../RouteEditorConfigSection.svelte | 8 +- .../{ => http}/RouteEditorInner.svelte | 14 +- .../triggers/{ => http}/RoutesPanel.svelte | 11 +- .../{ => kafka}/KafkaTriggerEditor.svelte | 0 .../KafkaTriggerEditorInner.svelte | 2 +- .../KafkaTriggersConfigSection.svelte | 13 +- .../{ => kafka}/KafkaTriggersPanel.svelte | 10 +- .../{ => nats}/NatsTriggerEditor.svelte | 0 .../{ => nats}/NatsTriggerEditorInner.svelte | 2 +- .../NatsTriggersConfigSection.svelte | 12 +- .../{ => nats}/NatsTriggersPanel.svelte | 8 +- .../postgres/PostgresTriggerEditor.svelte | 23 + .../PostgresTriggerEditorInner.svelte | 596 +++++++ .../postgres/PostgresTriggersPanel.svelte | 103 ++ .../postgres/PublicationPicker.svelte | 132 ++ .../triggers/postgres/RelationPicker.svelte | 174 ++ .../triggers/postgres/SlotPicker.svelte | 98 ++ .../{ => scheduled}/ScheduledPollPanel.svelte | 0 .../WebhooksConfigSection.svelte | 14 +- .../{ => webhook}/WebhooksPanel.svelte | 6 +- .../WebsocketEditorConfigSection.svelte | 4 +- .../WebsocketTriggerEditor.svelte | 0 .../WebsocketTriggerEditorInner.svelte | 14 +- .../WebsocketTriggersPanel.svelte | 9 +- frontend/src/lib/script_helpers.ts | 27 +- frontend/src/lib/stores.ts | 6 +- frontend/src/lib/utils.ts | 7 +- .../src/routes/(root)/(logged)/+layout.svelte | 13 +- .../(logged)/flows/get/[...path]/+page.svelte | 16 +- .../(logged)/kafka_triggers/+page.svelte | 2 +- .../(logged)/nats_triggers/+page.svelte | 6 +- .../(logged)/postgres_triggers/+page.js | 5 + .../(logged)/postgres_triggers/+page.svelte | 424 +++++ .../(root)/(logged)/routes/+page.svelte | 2 +- .../scripts/get/[...hash]/+page.svelte | 22 +- .../(logged)/websocket_triggers/+page.svelte | 4 +- 112 files changed, 6567 insertions(+), 399 deletions(-) create mode 100644 backend/.sqlx/query-01e92a4ba3074f1dce6ec98bc6c3fad4878f48db8c17c6d58590bd5df2e3350a.json create mode 100644 backend/.sqlx/query-124b27de35b49fbdb13a1f772044665a84325e34ae04bf2795fafb7bb6f2f0c6.json create mode 100644 backend/.sqlx/query-199a76c04e3f0891ad09af27b9534bbabdd8703bfdf4d43df2c65e50d4ca2c85.json create mode 100644 backend/.sqlx/query-24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121.json create mode 100644 backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json create mode 100644 backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json create mode 100644 backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json create mode 100644 backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json create mode 100644 backend/.sqlx/query-833a4ecec12dfe67f28016a135ffe682b023d1868a182b7cac16ce799433c257.json create mode 100644 backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json create mode 100644 backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json create mode 100644 backend/.sqlx/query-946977f0d525abf6267bf02e7a887434abd3e213b8c3c488166ca58fe3321147.json create mode 100644 backend/.sqlx/query-95e420b60fba20b36b2c6675998587d8cad3b67d4dfa9de52777d4ea9490b6b7.json create mode 100644 backend/.sqlx/query-9cb31818d4db8a0e294884ab3dec08bfc262f99c875bf16c25bfb5e987efe978.json create mode 100644 backend/.sqlx/query-a5a03b9235b25bca359235f2e546197f02ca1cf898d7f2686419749b1cb0679e.json create mode 100644 backend/.sqlx/query-a6c168c60bc8c42f70b18565e824efe29311aabfba6e09efa10bab6a551d658b.json create mode 100644 backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json create mode 100644 backend/.sqlx/query-fcc11a9353ea101109aec30f8bdd4b2ce906fffc3c51e77d083121dbd68dadd4.json create mode 100644 backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json create mode 100644 backend/migrations/20241123152203_postgres_triggers.down.sql create mode 100644 backend/migrations/20241123152203_postgres_triggers.up.sql create mode 100644 backend/windmill-api/src/postgres_triggers/bool.rs create mode 100644 backend/windmill-api/src/postgres_triggers/converter.rs create mode 100644 backend/windmill-api/src/postgres_triggers/handler.rs create mode 100644 backend/windmill-api/src/postgres_triggers/hex.rs create mode 100644 backend/windmill-api/src/postgres_triggers/mapper.rs create mode 100644 backend/windmill-api/src/postgres_triggers/mod.rs create mode 100644 backend/windmill-api/src/postgres_triggers/relation.rs create mode 100644 backend/windmill-api/src/postgres_triggers/replication_message.rs create mode 100644 backend/windmill-api/src/postgres_triggers/trigger.rs rename frontend/{.prettierrc => .prettierrc.json} (100%) rename frontend/src/lib/components/triggers/{ => http}/RouteEditor.svelte (100%) rename frontend/src/lib/components/triggers/{ => http}/RouteEditorConfigSection.svelte (96%) rename frontend/src/lib/components/triggers/{ => http}/RouteEditorInner.svelte (95%) rename frontend/src/lib/components/triggers/{ => http}/RoutesPanel.svelte (92%) rename frontend/src/lib/components/triggers/{ => kafka}/KafkaTriggerEditor.svelte (100%) rename frontend/src/lib/components/triggers/{ => kafka}/KafkaTriggerEditorInner.svelte (99%) rename frontend/src/lib/components/triggers/{ => kafka}/KafkaTriggersConfigSection.svelte (93%) rename frontend/src/lib/components/triggers/{ => kafka}/KafkaTriggersPanel.svelte (93%) rename frontend/src/lib/components/triggers/{ => nats}/NatsTriggerEditor.svelte (100%) rename frontend/src/lib/components/triggers/{ => nats}/NatsTriggerEditorInner.svelte (99%) rename frontend/src/lib/components/triggers/{ => nats}/NatsTriggersConfigSection.svelte (94%) rename frontend/src/lib/components/triggers/{ => nats}/NatsTriggersPanel.svelte (93%) create mode 100644 frontend/src/lib/components/triggers/postgres/PostgresTriggerEditor.svelte create mode 100644 frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte create mode 100644 frontend/src/lib/components/triggers/postgres/PostgresTriggersPanel.svelte create mode 100644 frontend/src/lib/components/triggers/postgres/PublicationPicker.svelte create mode 100644 frontend/src/lib/components/triggers/postgres/RelationPicker.svelte create mode 100644 frontend/src/lib/components/triggers/postgres/SlotPicker.svelte rename frontend/src/lib/components/triggers/{ => scheduled}/ScheduledPollPanel.svelte (100%) rename frontend/src/lib/components/triggers/{ => webhook}/WebhooksConfigSection.svelte (96%) rename frontend/src/lib/components/triggers/{ => webhook}/WebhooksPanel.svelte (87%) rename frontend/src/lib/components/triggers/{ => websocket}/WebsocketEditorConfigSection.svelte (97%) rename frontend/src/lib/components/triggers/{ => websocket}/WebsocketTriggerEditor.svelte (100%) rename frontend/src/lib/components/triggers/{ => websocket}/WebsocketTriggerEditorInner.svelte (96%) rename frontend/src/lib/components/triggers/{ => websocket}/WebsocketTriggersPanel.svelte (92%) create mode 100644 frontend/src/routes/(root)/(logged)/postgres_triggers/+page.js create mode 100644 frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte diff --git a/backend/.sqlx/query-01e92a4ba3074f1dce6ec98bc6c3fad4878f48db8c17c6d58590bd5df2e3350a.json b/backend/.sqlx/query-01e92a4ba3074f1dce6ec98bc6c3fad4878f48db8c17c6d58590bd5df2e3350a.json new file mode 100644 index 0000000000..a6f202a16f --- /dev/null +++ b/backend/.sqlx/query-01e92a4ba3074f1dce6ec98bc6c3fad4878f48db8c17c6d58590bd5df2e3350a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "01e92a4ba3074f1dce6ec98bc6c3fad4878f48db8c17c6d58590bd5df2e3350a" +} diff --git a/backend/.sqlx/query-124b27de35b49fbdb13a1f772044665a84325e34ae04bf2795fafb7bb6f2f0c6.json b/backend/.sqlx/query-124b27de35b49fbdb13a1f772044665a84325e34ae04bf2795fafb7bb6f2f0c6.json new file mode 100644 index 0000000000..18cd8fbc0b --- /dev/null +++ b/backend/.sqlx/query-124b27de35b49fbdb13a1f772044665a84325e34ae04bf2795fafb7bb6f2f0c6.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO postgres_trigger (\n publication_name,\n replication_slot_name,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n postgres_resource_path, \n edited_by\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7, \n $8, \n $9, \n $10\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "124b27de35b49fbdb13a1f772044665a84325e34ae04bf2795fafb7bb6f2f0c6" +} diff --git a/backend/.sqlx/query-199a76c04e3f0891ad09af27b9534bbabdd8703bfdf4d43df2c65e50d4ca2c85.json b/backend/.sqlx/query-199a76c04e3f0891ad09af27b9534bbabdd8703bfdf4d43df2c65e50d4ca2c85.json new file mode 100644 index 0000000000..7dfa56ad40 --- /dev/null +++ b/backend/.sqlx/query-199a76c04e3f0891ad09af27b9534bbabdd8703bfdf4d43df2c65e50d4ca2c85.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n schemaname AS schema_name,\n tablename AS table_name,\n attnames AS columns,\n rowfilter AS where_clause\n FROM\n pg_publication_tables\n WHERE\n pubname = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "schema_name", + "type_info": "Name" + }, + { + "ordinal": 1, + "name": "table_name", + "type_info": "Name" + }, + { + "ordinal": 2, + "name": "columns", + "type_info": "NameArray" + }, + { + "ordinal": 3, + "name": "where_clause", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + true, + true, + true, + true + ] + }, + "hash": "199a76c04e3f0891ad09af27b9534bbabdd8703bfdf4d43df2c65e50d4ca2c85" +} diff --git a/backend/.sqlx/query-24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121.json b/backend/.sqlx/query-24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121.json new file mode 100644 index 0000000000..d8cca020fe --- /dev/null +++ b/backend/.sqlx/query-24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n \n EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS \"websocket_used!\", \n \n EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS \"http_routes_used!\",\n EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as \"kafka_used!\",\n EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as \"nats_used!\",\n EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS \"postgres_used!\"\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "websocket_used!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "http_routes_used!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "kafka_used!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "nats_used!", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "postgres_used!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null, + null + ] + }, + "hash": "24178c21aadc1aed90f31e9362c6505a642c8f04b883c278b07e7ef5956ce121" +} diff --git a/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json b/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json new file mode 100644 index 0000000000..da7d13f71c --- /dev/null +++ b/backend/.sqlx/query-2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SHOW WAL_LEVEL;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "wal_level", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "2ef25599ea0c9ef946d6cc70ae048af970aed2638a3f767e152b654aebf68e48" +} diff --git a/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json b/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json new file mode 100644 index 0000000000..46a8d2e4f0 --- /dev/null +++ b/backend/.sqlx/query-4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pubname AS publication_name FROM pg_publication;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "publication_name", + "type_info": "Name" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "4469ee6c206c46951980ea1bc73f126f339d2e3cf97f363be8921084b16dac45" +} diff --git a/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json b/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json new file mode 100644 index 0000000000..52136dd33e --- /dev/null +++ b/backend/.sqlx/query-4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n slot_name,\n active\n FROM\n pg_replication_slots \n WHERE \n plugin = 'pgoutput' AND\n slot_type = 'logical';\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slot_name", + "type_info": "Name" + }, + { + "ordinal": 1, + "name": "active", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + true, + true + ] + }, + "hash": "4ee0017771f46f0272817d18edb821940cb5064e3f155b9630b131c09c9dba13" +} diff --git a/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json b/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json new file mode 100644 index 0000000000..4a95ebaf74 --- /dev/null +++ b/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM postgres_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52" +} diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 1fa370e682..75b8108281 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-833a4ecec12dfe67f28016a135ffe682b023d1868a182b7cac16ce799433c257.json b/backend/.sqlx/query-833a4ecec12dfe67f28016a135ffe682b023d1868a182b7cac16ce799433c257.json new file mode 100644 index 0000000000..44653c2265 --- /dev/null +++ b/backend/.sqlx/query-833a4ecec12dfe67f28016a135ffe682b023d1868a182b7cac16ce799433c257.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n postgres_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "833a4ecec12dfe67f28016a135ffe682b023d1868a182b7cac16ce799433c257" +} diff --git a/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json b/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json new file mode 100644 index 0000000000..cbf85f6b0d --- /dev/null +++ b/backend/.sqlx/query-86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n puballtables AS all_table,\n pubinsert AS insert,\n pubupdate AS update,\n pubdelete AS delete\n FROM\n pg_publication\n WHERE\n pubname = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "all_table", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "insert", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "update", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "delete", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Name" + ] + }, + "nullable": [ + false, + false, + false, + false + ] + }, + "hash": "86ae16175ace0179e784aacfd381771f0137ecab6671d632febadede729e7783" +} diff --git a/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json b/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json new file mode 100644 index 0000000000..a6780ad262 --- /dev/null +++ b/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM postgres_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf" +} diff --git a/backend/.sqlx/query-946977f0d525abf6267bf02e7a887434abd3e213b8c3c488166ca58fe3321147.json b/backend/.sqlx/query-946977f0d525abf6267bf02e7a887434abd3e213b8c3c488166ca58fe3321147.json new file mode 100644 index 0000000000..94c8a3b5b4 --- /dev/null +++ b/backend/.sqlx/query-946977f0d525abf6267bf02e7a887434abd3e213b8c3c488166ca58fe3321147.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger \n SET \n script_path = $1, \n path = $2, \n is_flow = $3, \n edited_by = $4, \n email = $5, \n postgres_resource_path = $6, \n replication_slot_name = $7,\n publication_name = $8,\n edited_at = now(), \n error = NULL,\n server_id = NULL\n WHERE \n workspace_id = $9 AND \n path = $10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "946977f0d525abf6267bf02e7a887434abd3e213b8c3c488166ca58fe3321147" +} diff --git a/backend/.sqlx/query-95e420b60fba20b36b2c6675998587d8cad3b67d4dfa9de52777d4ea9490b6b7.json b/backend/.sqlx/query-95e420b60fba20b36b2c6675998587d8cad3b67d4dfa9de52777d4ea9490b6b7.json new file mode 100644 index 0000000000..74f7f38890 --- /dev/null +++ b/backend/.sqlx/query-95e420b60fba20b36b2c6675998587d8cad3b67d4dfa9de52777d4ea9490b6b7.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n postgres_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "95e420b60fba20b36b2c6675998587d8cad3b67d4dfa9de52777d4ea9490b6b7" +} diff --git a/backend/.sqlx/query-9cb31818d4db8a0e294884ab3dec08bfc262f99c875bf16c25bfb5e987efe978.json b/backend/.sqlx/query-9cb31818d4db8a0e294884ab3dec08bfc262f99c875bf16c25bfb5e987efe978.json new file mode 100644 index 0000000000..d46b709083 --- /dev/null +++ b/backend/.sqlx/query-9cb31818d4db8a0e294884ab3dec08bfc262f99c875bf16c25bfb5e987efe978.json @@ -0,0 +1,104 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n path,\n script_path,\n replication_slot_name,\n publication_name,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n postgres_resource_path\n FROM\n postgres_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "replication_slot_name", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "publication_name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 10, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 11, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 12, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 13, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 14, + "name": "postgres_resource_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false, + false + ] + }, + "hash": "9cb31818d4db8a0e294884ab3dec08bfc262f99c875bf16c25bfb5e987efe978" +} diff --git a/backend/.sqlx/query-a5a03b9235b25bca359235f2e546197f02ca1cf898d7f2686419749b1cb0679e.json b/backend/.sqlx/query-a5a03b9235b25bca359235f2e546197f02ca1cf898d7f2686419749b1cb0679e.json new file mode 100644 index 0000000000..3bcb7d31de --- /dev/null +++ b/backend/.sqlx/query-a5a03b9235b25bca359235f2e546197f02ca1cf898d7f2686419749b1cb0679e.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value, is_secret \n FROM variable \n WHERE path = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "is_secret", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "a5a03b9235b25bca359235f2e546197f02ca1cf898d7f2686419749b1cb0679e" +} diff --git a/backend/.sqlx/query-a6c168c60bc8c42f70b18565e824efe29311aabfba6e09efa10bab6a551d658b.json b/backend/.sqlx/query-a6c168c60bc8c42f70b18565e824efe29311aabfba6e09efa10bab6a551d658b.json new file mode 100644 index 0000000000..ce9457cc6a --- /dev/null +++ b/backend/.sqlx/query-a6c168c60bc8c42f70b18565e824efe29311aabfba6e09efa10bab6a551d658b.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE postgres_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a6c168c60bc8c42f70b18565e824efe29311aabfba6e09efa10bab6a551d658b" +} diff --git a/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json b/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json new file mode 100644 index 0000000000..9b94f6bab7 --- /dev/null +++ b/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM postgres_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5" +} diff --git a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json index b378d68f5b..4843d959c1 100644 --- a/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json +++ b/backend/.sqlx/query-bfff3d8df18db198d6ebba8a049b00147fc8bcd42f3df37ef81b9ded80974bd0.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-fcc11a9353ea101109aec30f8bdd4b2ce906fffc3c51e77d083121dbd68dadd4.json b/backend/.sqlx/query-fcc11a9353ea101109aec30f8bdd4b2ce906fffc3c51e77d083121dbd68dadd4.json new file mode 100644 index 0000000000..5af907c6b1 --- /dev/null +++ b/backend/.sqlx/query-fcc11a9353ea101109aec30f8bdd4b2ce906fffc3c51e77d083121dbd68dadd4.json @@ -0,0 +1,107 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n replication_slot_name,\n publication_name,\n postgres_resource_path\n FROM \n postgres_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "server_id", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "last_server_ping", + "type_info": "Timestamptz" + }, + { + "ordinal": 9, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 10, + "name": "error", + "type_info": "Text" + }, + { + "ordinal": 11, + "name": "enabled", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "replication_slot_name", + "type_info": "Varchar" + }, + { + "ordinal": 13, + "name": "publication_name", + "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "postgres_resource_path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + true, + true, + true, + true, + false, + false, + false, + false + ] + }, + "hash": "fcc11a9353ea101109aec30f8bdd4b2ce906fffc3c51e77d083121dbd68dadd4" +} diff --git a/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json b/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json new file mode 100644 index 0000000000..06cf05a25d --- /dev/null +++ b/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Bool", + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909" +} diff --git a/backend/.vscode/settings.json b/backend/.vscode/settings.json index e2f6a75faa..ab8340f5c8 100644 --- a/backend/.vscode/settings.json +++ b/backend/.vscode/settings.json @@ -1,8 +1,6 @@ { "python.analysis.typeCheckingMode": "basic", - "rust-analyzer.linkedProjects": [ - "./windmill-common/Cargo.toml" - ], + "rust-analyzer.linkedProjects": ["./windmill-common/Cargo.toml"], "rust-analyzer.showUnlinkedFileNotification": false, "remote.portsAttributes": { "8000": { @@ -10,5 +8,8 @@ "onAutoForward": "openPreview" } }, - "remote.autoForwardPorts": true -} \ No newline at end of file + "remote.autoForwardPorts": true, + "conventionalCommits.scopes": [ + "restructring triggers, decoding trigger message on work" + ], +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 76c4ea6ca0..aa79cf5e13 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -6123,6 +6123,15 @@ dependencies = [ "indexmap 2.7.1", ] +[[package]] +name = "pg_escape" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c7bc82ccbe2c7ef7ceed38dcac90d7ff46681e061e9d7310cbcd409113e303" +dependencies = [ + "phf", +] + [[package]] name = "phf" version = "0.11.3" @@ -6262,7 +6271,7 @@ dependencies = [ "native-tls", "tokio", "tokio-native-tls", - "tokio-postgres", + "tokio-postgres 0.7.12", ] [[package]] @@ -6283,6 +6292,33 @@ dependencies = [ "stringprep", ] +[[package]] +name = "postgres-protocol" +version = "0.6.7" +source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b#20265ef38e32a06f76b6f9b678e2077fc2211f6b" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5 0.10.6", + "memchr", + "rand 0.8.5", + "sha2 0.10.8", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.7" +source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b#20265ef38e32a06f76b6f9b678e2077fc2211f6b" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol 0.6.7 (git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b)", +] + [[package]] name = "postgres-types" version = "0.2.8" @@ -6294,7 +6330,7 @@ dependencies = [ "bytes", "chrono", "fallible-iterator", - "postgres-protocol", + "postgres-protocol 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde", "serde_json", "uuid 1.12.1", @@ -7212,7 +7248,7 @@ dependencies = [ "borsh", "bytes", "num-traits", - "postgres-types", + "postgres-types 0.2.8", "rand 0.8.5", "rkyv", "serde", @@ -9512,6 +9548,31 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-postgres" +version = "0.7.11" +source = "git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b#20265ef38e32a06f76b6f9b678e2077fc2211f6b" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol 0.6.7 (git+https://github.com/imor/rust-postgres?rev=20265ef38e32a06f76b6f9b678e2077fc2211f6b)", + "postgres-types 0.2.7", + "rand 0.8.5", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-postgres" version = "0.7.12" @@ -9529,8 +9590,8 @@ dependencies = [ "percent-encoding", "phf", "pin-project-lite", - "postgres-protocol", - "postgres-types", + "postgres-protocol 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", + "postgres-types 0.2.8", "rand 0.8.5", "socket2", "tokio", @@ -10736,6 +10797,7 @@ dependencies = [ "gethostname", "git-version", "lazy_static", + "memchr", "object_store", "once_cell", "prometheus", @@ -10778,6 +10840,7 @@ dependencies = [ "axum", "base32", "base64 0.22.1", + "byteorder", "bytes", "candle-core", "candle-nn", @@ -10807,6 +10870,7 @@ dependencies = [ "object_store", "openidconnect", "openssl", + "pg_escape", "pin-project", "prometheus", "quick_cache", @@ -10816,6 +10880,7 @@ dependencies = [ "reqwest 0.12.9", "rsa", "rust-embed", + "rust_decimal", "samael", "serde", "serde_json", @@ -10824,11 +10889,13 @@ dependencies = [ "sql-builder", "sqlx", "tempfile", + "thiserror 2.0.11", "time", "tinyvector", "tokenizers", "tokio", "tokio-native-tls", + "tokio-postgres 0.7.11", "tokio-tar", "tokio-tungstenite", "tokio-util", @@ -11187,7 +11254,6 @@ dependencies = [ "async-recursion", "axum", "backon", - "bigdecimal", "chrono", "chrono-tz 0.10.1", "cron", @@ -11278,7 +11344,7 @@ dependencies = [ "tar", "tiberius", "tokio", - "tokio-postgres", + "tokio-postgres 0.7.12", "tokio-util", "tracing", "urlencoding", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5759f010be..2ef62c9f57 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -73,6 +73,7 @@ oracledb = ["windmill-worker/oracledb"] mssql = ["windmill-worker/mssql"] bigquery = ["windmill-worker/bigquery"] websocket = ["windmill-api/websocket"] +postgres_trigger = ["windmill-api/postgres_trigger"] python = ["windmill-worker/python"] smtp = ["windmill-api/smtp", "windmill-common/smtp"] csharp = ["windmill-worker/csharp"] @@ -114,6 +115,8 @@ serde.workspace = true deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } quote.workspace = true +memchr.workspace = true + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } @@ -154,6 +157,7 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" } windmill-parser-php = { path = "./parsers/windmill-parser-php" } windmill-api-client = { path = "./windmill-api-client" } +memchr = "2.7.4" axum = { version = "^0.7", features = ["multipart"] } headers = "^0" hyper = { version = "^1", features = ["full"] } @@ -234,7 +238,7 @@ sqlx = { version = "0.8.0", features = [ "runtime-tokio-rustls", "bigdecimal" ] } -bigdecimal = "^0" +bigdecimal = {version = "^0"} dotenv = "^0" ulid = { version = "^1", features = ["uuid"] } futures = "^0" @@ -260,6 +264,7 @@ wasm-bindgen-test = "0.3.42" convert_case = "0.6.0" getrandom = "0.2" tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]} +rust-postgres = { package = "tokio-postgres", git = "https://github.com/imor/rust-postgres", rev = "20265ef38e32a06f76b6f9b678e2077fc2211f6b"} bit-vec = "=0.6.3" mappable-rc = "^0" mysql_async = { version = "*", default-features = false, features = ["minimal", "default", "native-tls-tls", "rust_decimal"]} @@ -269,7 +274,7 @@ native-tls = "^0" # samael = { git="https://github.com/njaremko/samael", rev="464d015e3ae393e4b5dd00b4d6baa1b617de0dd6", features = ["xmlsec"] } samael = { version="0.0.14", features = ["xmlsec"] } gcp_auth = "0.9.0" -rust_decimal = { version = "^1", features = ["db-postgres"]} +rust_decimal = { version = "^1", features = ["db-postgres", "serde-float"]} jsonwebtoken = "8.3.0" pem = "3.0.1" nix = { version = "0.27.1", features = ["process", "signal"] } @@ -287,6 +292,7 @@ openssl = "=0.10" mail-parser = "^0" matchit = "=0.7.3" rdkafka = { version = "0.36.2", features = ["cmake-build", "ssl-vendored"] } +pg_escape = "0.1.1" async-nats = "0.38.0" nkeys = "0.4.4" @@ -311,6 +317,7 @@ opentelemetry-semantic-conventions = { version = "*", features = ["semconv_exper bollard = "0.18.1" tonic = { version = "^0", features = ["tls-native-roots"] } +byteorder = "1.5.0" tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } diff --git a/backend/migrations/20241123152203_postgres_triggers.down.sql b/backend/migrations/20241123152203_postgres_triggers.down.sql new file mode 100644 index 0000000000..66431d8887 --- /dev/null +++ b/backend/migrations/20241123152203_postgres_triggers.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +DROP TABLE IF EXISTS postgres_trigger; \ No newline at end of file diff --git a/backend/migrations/20241123152203_postgres_triggers.up.sql b/backend/migrations/20241123152203_postgres_triggers.up.sql new file mode 100644 index 0000000000..d6f927560d --- /dev/null +++ b/backend/migrations/20241123152203_postgres_triggers.up.sql @@ -0,0 +1,69 @@ +-- Add up migration script here +CREATE TABLE postgres_trigger( + path VARCHAR(255) NOT NULL, + script_path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + edited_by VARCHAR(50) NOT NULL, + email VARCHAR(255) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + extra_perms JSONB NULL, + postgres_resource_path VARCHAR(255) NOT NULL, + error TEXT NULL, + server_id VARCHAR(50) NULL, + last_server_ping TIMESTAMPTZ NULL, + replication_slot_name VARCHAR(255) NOT NULL, + publication_name VARCHAR(255) NOT NULL, + enabled BOOLEAN NOT NULL, + CONSTRAINT PK_postgres_trigger PRIMARY KEY (path,workspace_id), + CONSTRAINT fk_postgres_trigger_workspace FOREIGN KEY (workspace_id) + REFERENCES workspace(id) ON DELETE CASCADE +); + +GRANT ALL ON postgres_trigger TO windmill_user; +GRANT ALL ON postgres_trigger TO windmill_admin; + +ALTER TABLE postgres_trigger ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON postgres_trigger FOR ALL TO windmill_admin USING (true); + +CREATE POLICY see_folder_extra_perms_user_select ON postgres_trigger FOR SELECT TO windmill_user +USING (SPLIT_PART(postgres_trigger.path, '/', 1) = 'f' AND SPLIT_PART(postgres_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_insert ON postgres_trigger FOR INSERT TO windmill_user +WITH CHECK (SPLIT_PART(postgres_trigger.path, '/', 1) = 'f' AND SPLIT_PART(postgres_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON postgres_trigger FOR UPDATE TO windmill_user +USING (SPLIT_PART(postgres_trigger.path, '/', 1) = 'f' AND SPLIT_PART(postgres_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_delete ON postgres_trigger FOR DELETE TO windmill_user +USING (SPLIT_PART(postgres_trigger.path, '/', 1) = 'f' AND SPLIT_PART(postgres_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); + +CREATE POLICY see_own ON postgres_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(postgres_trigger.path, '/', 1) = 'u' AND SPLIT_PART(postgres_trigger.path, '/', 2) = current_setting('session.user')); +CREATE POLICY see_member ON postgres_trigger FOR ALL TO windmill_user +USING (SPLIT_PART(postgres_trigger.path, '/', 1) = 'g' AND SPLIT_PART(postgres_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[])); + +CREATE POLICY see_extra_perms_user_select ON postgres_trigger FOR SELECT TO windmill_user +USING (extra_perms ? CONCAT('u/', current_setting('session.user'))); +CREATE POLICY see_extra_perms_user_insert ON postgres_trigger FOR INSERT TO windmill_user +WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); +CREATE POLICY see_extra_perms_user_update ON postgres_trigger FOR UPDATE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); +CREATE POLICY see_extra_perms_user_delete ON postgres_trigger FOR DELETE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON postgres_trigger FOR SELECT TO windmill_user +USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]); +CREATE POLICY see_extra_perms_groups_insert ON postgres_trigger FOR INSERT TO windmill_user +WITH CHECK (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_update ON postgres_trigger FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); +CREATE POLICY see_extra_perms_groups_delete ON postgres_trigger FOR DELETE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); \ No newline at end of file diff --git a/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs b/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs index bb703fb131..b67c3a0768 100644 --- a/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs +++ b/backend/parsers/windmill-parser-csharp/src/wasm_libc.rs @@ -1,11 +1,11 @@ +use std::collections::BTreeMap; +use std::sync::{Mutex, OnceLock}; use std::{ alloc::{self, Layout}, ffi::{c_char, c_int, c_void}, mem::align_of, ptr, }; -use std::collections::BTreeMap; -use std::sync::{Mutex, OnceLock}; use wasm_bindgen::prelude::*; /* -------------------------------- stdlib.h -------------------------------- */ @@ -71,7 +71,6 @@ pub unsafe extern "C" fn free(buf: *mut c_void) { alloc::dealloc(buf, layout); } - // In all these allocations, we store the layout before the data for later retrieval. // This is because we need to know the layout when deallocating the memory. // Here are some helper methods for that: diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 7bb71416c7..001b15ac45 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -1,7 +1,7 @@ use serde::de::DeserializeOwned; use std::future::Future; use std::{str::FromStr, sync::Arc}; -use windmill_api_client::types::{NewScript, NewScriptLanguage}; +use windmill_api_client::types::{NewScript, ScriptLang as NewScriptLanguage}; #[cfg(feature = "enterprise")] use chrono::Timelike; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 3bf133c020..2033132b47 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -28,6 +28,7 @@ zip = ["dep:async_zip"] oauth2 = ["dep:async-oauth2"] http_trigger = ["dep:matchit"] static_frontend = ["dep:rust-embed"] +postgres_trigger = ["dep:rust-postgres", "dep:pg_escape", "dep:byteorder", "dep:thiserror", "dep:rust_decimal"] [dependencies] windmill-queue.workspace = true @@ -107,8 +108,12 @@ rdkafka = { workspace = true, optional = true } async-nats = { workspace = true, optional = true } nkeys = { workspace = true, optional = true } const_format.workspace = true - pin-project.workspace = true http.workspace = true async-stream.workspace = true ulid.workspace = true +rust-postgres = { workspace = true, optional = true } +pg_escape = { workspace = true, optional = true } +byteorder = { workspace = true, optional = true } +thiserror = { workspace = true, optional = true } +rust_decimal = { workspace = true, optional = true } \ No newline at end of file diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index dd7fc1877c..82bd7a9c0e 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2429,12 +2429,14 @@ paths: type: boolean nats_used: type: boolean + postgres_used: + type: boolean required: - http_routes_used - websocket_used - kafka_used - nats_used - + - postgres_used /w/{workspace}/users/list: get: summary: list users @@ -8240,6 +8242,396 @@ paths: schema: type: string + /w/{workspace}/postgres_triggers/is_valid_postgres_configuration/{path}: + get: + summary: check if postgres configuration is set to logical + operationId: isValidPostgresConfiguration + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: boolean that indicates if postgres is set to logical level or not + content: + application/json: + schema: + type: boolean + + /w/{workspace}/postgres_triggers/create_template_script: + post: + summary: create template script + operationId: createTemplateScript + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: template script + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TemplateScript" + responses: + "200": + description: custom id to retrieve template script + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/get_template_script/{id}: + get: + summary: get template script + operationId: getTemplateScript + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Id" + responses: + "200": + description: template script + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/slot/list/{path}: + get: + summary: list postgres replication slot + operationId: listPostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: list postgres slot + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/SlotList" + + /w/{workspace}/postgres_triggers/slot/create/{path}: + post: + summary: create replication slot for postgres + operationId: createPostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: new slot for postgres + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Slot" + responses: + "201": + description: slot created + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/slot/delete/{path}: + delete: + summary: delete postgres replication slot + operationId: deletePostgresReplicationSlot + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: replication slot of postgres + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/Slot" + responses: + "200": + description: postgres replication slot deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/publication/list/{path}: + get: + summary: list postgres publication + operationId: listPostgresPublication + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: database publication list + content: + application/json: + schema: + type: array + items: + type: string + + /w/{workspace}/postgres_triggers/publication/get/{publication}/{path}: + get: + summary: get postgres publication + operationId: getPostgresPublication + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/PublicationName" + responses: + "200": + description: postgres publication get + content: + application/json: + schema: + $ref: "#/components/schemas/PublicationData" + + /w/{workspace}/postgres_triggers/publication/create/{publication}/{path}: + post: + summary: create publication for postgres + operationId: createPostgresPublication + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/PublicationName" + requestBody: + description: new publication for postgres + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublicationData" + responses: + "201": + description: publication created + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/publication/update/{publication}/{path}: + post: + summary: update publication for postgres + operationId: updatePostgresPublication + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/PublicationName" + requestBody: + description: update publication for postgres + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/PublicationData" + responses: + "201": + description: publication updated + content: + text/plain: + schema: + type: string + + + /w/{workspace}/postgres_triggers/publication/delete/{publication}/{path}: + delete: + summary: delete postgres publication + operationId: deletePostgresPublication + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + - $ref: "#/components/parameters/PublicationName" + responses: + "200": + description: postgres publication deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/create: + post: + summary: create postgres trigger + operationId: createPostgresTrigger + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new postgres trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewPostgresTrigger" + responses: + "201": + description: postgres trigger created + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/update/{path}: + post: + summary: update postgres trigger + operationId: updatePostgresTrigger + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EditPostgresTrigger" + responses: + "200": + description: postgres trigger updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/delete/{path}: + delete: + summary: delete postgres trigger + operationId: deletePostgresTrigger + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: postgres trigger deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/postgres_triggers/get/{path}: + get: + summary: get postgres trigger + operationId: getPostgresTrigger + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: get postgres trigger + content: + application/json: + schema: + $ref: "#/components/schemas/PostgresTrigger" + + /w/{workspace}/postgres_triggers/list: + get: + summary: list postgres triggers + operationId: listPostgresTriggers + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + required: true + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + "200": + description: postgres trigger list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/PostgresTrigger" + + /w/{workspace}/postgres_triggers/exists/{path}: + get: + summary: does postgres trigger exists + operationId: existsPostgresTrigger + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: postgres trigger exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/postgres_triggers/setenabled/{path}: + post: + summary: set enabled postgres trigger + operationId: setPostgresTriggerEnabled + tags: + - postgres_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated postgres trigger enable + required: true + content: + application/json: + schema: + type: object + properties: + enabled: + type: boolean + required: + - enabled + responses: + "200": + description: postgres trigger enabled set + content: + text/plain: + schema: + type: string /groups/list: get: @@ -9094,8 +9486,7 @@ paths: required: true schema: type: string - enum: - [ + enum: [ script, group_, resource, @@ -9109,6 +9500,7 @@ paths: websocket_trigger, kafka_trigger, nats_trigger, + postgres_trigger, ] responses: "200": @@ -9149,6 +9541,7 @@ paths: websocket_trigger, kafka_trigger, nats_trigger, + postgres_trigger, ] requestBody: description: acl to add @@ -9200,6 +9593,7 @@ paths: websocket_trigger, kafka_trigger, nats_trigger, + postgres_trigger, ] requestBody: description: acl to add @@ -10623,6 +11017,12 @@ components: name: token parameters: + Id: + name: id + in: path + required: true + schema: + type: string Key: name: key in: path @@ -10635,6 +11035,12 @@ components: required: true schema: type: string + PublicationName: + name: publication + in: path + required: true + schema: + type: string VersionId: name: version in: path @@ -11057,28 +11463,7 @@ components: lock_error_logs: type: string language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" kind: type: string enum: [script, failure, trigger, command, approval, preprocessor] @@ -11163,28 +11548,7 @@ components: lock: type: string language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" kind: type: string enum: [script, failure, trigger, command, approval, preprocessor] @@ -11391,28 +11755,7 @@ components: is_flow_step: type: boolean language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" email: type: string visible_to_owner: @@ -11515,28 +11858,7 @@ components: is_flow_step: type: boolean language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" is_skipped: type: boolean email: @@ -12062,6 +12384,30 @@ components: - no_main_func - has_preprocessor + ScriptLang: + type: string + enum: + [ + python3, + deno, + go, + bash, + powershell, + postgresql, + mysql, + bigquery, + snowflake, + mssql, + oracledb, + graphql, + nativets, + bun, + php, + rust, + ansible, + csharp + ] + Preview: type: object properties: @@ -12072,28 +12418,7 @@ components: args: $ref: "#/components/schemas/ScriptArgs" language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" tag: type: string kind: @@ -12458,16 +12783,36 @@ components: - is_flow - args - HttpTrigger: + TriggerExtraProperty: type: object properties: - path: + email: + type: string + extra_perms: + type: object + additionalProperties: + type: boolean + workspace_id: type: string edited_by: type: string edited_at: type: string format: date-time + required: + - email + - extra_perms + - workspace_id + - edited_by + - edited_at + + HttpTrigger: + allOf: + - $ref: "#/components/schemas/TriggerExtraProperty" + type: object + properties: + path: + type: string script_path: type: string route_path: @@ -12485,14 +12830,6 @@ components: - s3 is_flow: type: boolean - extra_perms: - type: object - additionalProperties: - type: boolean - email: - type: string - workspace_id: - type: string http_method: type: string enum: @@ -12625,35 +12962,26 @@ components: type: number websocket_count: type: number + postgres_count: + type: number kafka_count: type: number nats_count: type: number WebsocketTrigger: + allOf: + - $ref: "#/components/schemas/TriggerExtraProperty" type: object properties: path: type: string - edited_by: - type: string - edited_at: - type: string - format: date-time script_path: type: string url: type: string is_flow: type: boolean - extra_perms: - type: object - additionalProperties: - type: boolean - email: - type: string - workspace_id: - type: string server_id: type: string last_server_ping: @@ -12793,6 +13121,168 @@ components: - is_flow required: - runnable_result + + Slot: + type: object + properties: + name: + type: string + + SlotList: + type: object + properties: + slot_name: + type: string + active: + type: boolean + + PublicationData: + type: object + properties: + table_to_track: + type: array + items: + $ref: "#/components/schemas/Relations" + transaction_to_track: + type: array + items: + type: string + required: + - transaction_to_track + + TableToTrack: + type: array + items: + type: object + properties: + table_name: + type: string + columns_name: + type: array + items: + type: string + where_clause: + type: string + required: + - table_name + + Relations: + type: object + properties: + schema_name: + type: string + table_to_track: + $ref: "#/components/schemas/TableToTrack" + required: + - schema_name + - table_to_track + + Language: + type: string + enum: + - Typescript + + TemplateScript: + type: object + properties: + postgres_resource_path: + type: string + relations: + type: array + items: + $ref: "#/components/schemas/Relations" + language: + $ref: "#/components/schemas/Language" + required: + - postgres_resource_path + - relations + - language + + PostgresTrigger: + allOf: + - $ref: "#/components/schemas/TriggerExtraProperty" + type: object + properties: + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + postgres_resource_path: + type: string + publication_name: + type: string + server_id: + type: string + replication_slot_name: + type: string + error: + type: string + required: + - path + - script_path + - is_flow + - enabled + - postgres_resource_path + - replication_slot_name + - publication_name + + NewPostgresTrigger: + type: object + properties: + replication_slot_name: + type: string + publication_name: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + postgres_resource_path: + type: string + publication: + $ref: "#/components/schemas/PublicationData" + required: + - path + - script_path + - is_flow + - enabled + - postgres_resource_path + + EditPostgresTrigger: + type: object + properties: + replication_slot_name: + type: string + publication_name: + type: string + path: + type: string + script_path: + type: string + is_flow: + type: boolean + enabled: + type: boolean + postgres_resource_path: + type: string + publication: + $ref: "#/components/schemas/PublicationData" + required: + - path + - script_path + - is_flow + - enabled + - postgres_resource_path + - publication_name + - replication_slot_name KafkaTrigger: type: object @@ -13817,28 +14307,7 @@ components: path: type: string language: - type: string - enum: - [ - python3, - deno, - go, - bash, - powershell, - postgresql, - mysql, - bigquery, - snowflake, - mssql, - oracledb, - graphql, - nativets, - bun, - php, - rust, - ansible, - csharp, - ] + $ref: "#/components/schemas/ScriptLang" required: - raw_code - path diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 234a1fe73a..3b97ba18ce 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -1,5 +1,8 @@ -use crate::db::{ApiAuthed, DB}; -use windmill_common::variables::decrypt; +use crate::{ + db::{ApiAuthed, DB}, + variables::get_variable_or_self, +}; + use anthropic::AnthropicCache; use axum::{ body::Bytes, @@ -17,7 +20,6 @@ use serde::{Deserialize, Deserializer}; use windmill_audit::audit_ee::audit_log; use windmill_audit::ActionKind; use windmill_common::error::{to_anyhow, Result}; -use windmill_common::variables::build_crypt; use windmill_common::error::Error; @@ -344,38 +346,11 @@ lazy_static! { pub static ref AI_KEY_CACHE: Cache = Cache::new(500); } -struct Variable { - value: String, - is_secret: bool, -} - #[derive(Deserialize, Debug)] struct ProxyQueryParams { no_cache: Option, } -async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result { - if !path.starts_with("$var:") { - return Ok(path); - } - let path = path.strip_prefix("$var:").unwrap().to_string(); - let mut variable = sqlx::query_as!( - Variable, - "SELECT value, is_secret - FROM variable - WHERE path = $1 AND workspace_id = $2", - &path, - &w_id - ) - .fetch_one(db) - .await?; - if variable.is_secret { - let mc = build_crypt(db, w_id).await?; - variable.value = decrypt(&mc, variable.value)?; - } - Ok(variable.value) -} - #[derive(Deserialize, Debug)] pub struct AiResource { pub path: String, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4734b3a3ab..006e52f950 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -59,6 +59,8 @@ mod auth; mod capture; mod concurrency_groups; mod configs; +#[cfg(feature = "postgres_trigger")] +mod postgres_triggers; mod db; mod drafts; pub mod ee; @@ -308,6 +310,11 @@ pub async fn run_server( let nats_killpill_rx = rx.resubscribe(); nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx).await; } + #[cfg(feature = "postgres_trigger")] + { + let db_killpill_rx = rx.resubscribe(); + postgres_triggers::start_database(db.clone(), db_killpill_rx).await; + } } // build our application with a route @@ -377,7 +384,16 @@ pub async fn run_server( Router::new() }) .nest("/kafka_triggers", kafka_triggers_service) - .nest("/nats_triggers", nats_triggers_service), + .nest("/nats_triggers", nats_triggers_service) + .nest("/postgres_triggers", { + #[cfg(feature = "postgres_trigger")] + { + postgres_triggers::workspaced_service() + } + + #[cfg(not(feature = "postgres_trigger"))] + Router::new() + }), ) .nest("/workspaces", workspaces::global_service()) .nest( diff --git a/backend/windmill-api/src/postgres_triggers/bool.rs b/backend/windmill-api/src/postgres_triggers/bool.rs new file mode 100644 index 0000000000..9c415780a5 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/bool.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs +* +*/ + +#[derive(Debug, Error)] +pub enum ParseBoolError { + #[error("invalid input value: {0}")] + InvalidInput(String), +} + +pub fn parse_bool(s: &str) -> Result { + match s { + "t" => Ok(true), + "f" => Ok(false), + _ => Err(ParseBoolError::InvalidInput(s.to_string())), + } +} diff --git a/backend/windmill-api/src/postgres_triggers/converter.rs b/backend/windmill-api/src/postgres_triggers/converter.rs new file mode 100644 index 0000000000..f57268c4b8 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/converter.rs @@ -0,0 +1,255 @@ +use core::str; +use std::{ + num::{ParseFloatError, ParseIntError}, + str::FromStr, +}; + +use super::{ + bool::{parse_bool, ParseBoolError}, + hex::{from_bytea_hex, ByteaHexParseError}, +}; +use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Utc}; +use rust_decimal::Decimal; +use rust_postgres::types::Type; +use serde_json::{to_value, Number, Value}; +use thiserror::Error; +use uuid::Uuid; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs +* +*/ + +#[derive(Debug, Error)] +pub enum ConverterError { + #[error("invalid bool value")] + InvalidBool(#[from] ParseBoolError), + + #[error("invalid int value")] + InvalidInt(#[from] ParseIntError), + + #[error("invalid float value")] + InvalidFloat(#[from] ParseFloatError), + + #[error("invalid numeric: {0}")] + InvalidNumeric(#[from] rust_decimal::Error), + + #[error("invalid bytea: {0}")] + InvalidBytea(#[from] ByteaHexParseError), + + #[error("invalid uuid: {0}")] + InvalidUuid(#[from] uuid::Error), + + #[error("invalid json: {0}")] + InvalidJson(#[from] serde_json::Error), + + #[error("invalid timestamp: {0} ")] + InvalidTimestamp(#[from] chrono::ParseError), + + #[error("invalid array: {0}")] + InvalidArray(#[from] ArrayParseError), + + #[error("{0}")] + Custom(String), +} + +fn convert_into(number: T) -> Number +where + T: Sized, + serde_json::Number: From, +{ + serde_json::Number::from(number) +} + +pub struct Converter; + +#[derive(Debug, Error)] +pub enum ArrayParseError { + #[error("input too short")] + InputTooShort, + + #[error("missing braces")] + MissingBraces, +} + +fn f64_to_json_number(raw_val: f64) -> Result { + let temp = serde_json::Number::from_f64(raw_val.into()) + .ok_or(ConverterError::Custom("invalid json-float".to_string()))?; + Ok(Value::Number(temp)) +} + +impl Converter { + pub fn try_from_str(typ: Option, str: &str) -> Result { + let value = match typ.unwrap_or(Type::TEXT) { + Type::BOOL => Value::Bool(parse_bool(str)?), + Type::BOOL_ARRAY => { + Converter::parse_array(str, |str| Ok(Value::Bool(parse_bool(str)?)))? + } + Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => { + Value::String(str.to_string()) + } + Type::CHAR_ARRAY + | Type::BPCHAR_ARRAY + | Type::VARCHAR_ARRAY + | Type::NAME_ARRAY + | Type::TEXT_ARRAY => { + Converter::parse_array(str, |str| Ok(Value::String(str.to_string())))? + } + Type::INT2 => Value::Number(convert_into(str.parse::()?)), + Type::INT2_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::INT4 => Value::Number(convert_into(str.parse::()?)), + Type::INT4_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::INT8 => Value::Number(convert_into(str.parse::()?)), + Type::INT8_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + Type::FLOAT4 => f64_to_json_number(str.parse::()?)?, + Type::FLOAT4_ARRAY => { + Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? + } + Type::FLOAT8 => f64_to_json_number(str.parse::()?)?, + Type::FLOAT8_ARRAY => { + Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? + } + Type::NUMERIC => serde_json::json!(Decimal::from_str(str)?), + Type::NUMERIC_ARRAY => { + Converter::parse_array(str, |str| Ok(serde_json::json!(Decimal::from_str(str)?)))? + } + Type::BYTEA => to_value(from_bytea_hex(str)?).unwrap(), + Type::BYTEA_ARRAY => { + Converter::parse_array(str, |str| Ok(to_value(from_bytea_hex(str)?).unwrap()))? + } + Type::DATE => { + let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; + Value::String(date.to_string()) + } + Type::DATE_ARRAY => Converter::parse_array(str, |str| { + let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; + Ok(Value::String(date.to_string())) + })?, + Type::TIME => { + let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; + Value::String(time.to_string()) + } + Type::TIME_ARRAY => Converter::parse_array(str, |str| { + let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; + Ok(Value::String(time.to_string())) + })?, + Type::TIMESTAMP => { + let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; + Value::String(timestamp.to_string()) + } + Type::TIMESTAMP_ARRAY => Converter::parse_array(str, |str| { + let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; + Ok(Value::String(timestamp.to_string())) + })?, + Type::TIMESTAMPTZ => { + let val = + match DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z") { + Ok(val) => val, + Err(_) => { + DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%:z")? + } + }; + let utc: DateTime = val.into(); + Value::String(utc.to_string()) + } + Type::TIMESTAMPTZ_ARRAY => { + match Converter::parse_array(str, |str| { + let utc: DateTime = + DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z")? + .into(); + Ok(Value::String(utc.to_string())) + }) { + Ok(val) => val, + Err(_) => Converter::parse_array(str, |str| { + let utc: DateTime = DateTime::::parse_from_str( + str, + "%Y-%m-%d %H:%M:%S%.f%#z", + )? + .into(); + Ok(Value::String(utc.to_string())) + })?, + } + } + Type::UUID => Value::String(Uuid::parse_str(str)?.to_string()), + Type::UUID_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::String(Uuid::parse_str(str)?.to_string())) + })?, + Type::JSON | Type::JSONB => serde_json::from_str::(str)?, + Type::JSON_ARRAY | Type::JSONB_ARRAY => Converter::parse_array(str, |str| { + Ok(serde_json::from_str::(str)?) + })?, + Type::OID => Value::Number(convert_into(str.parse::()?)), + Type::OID_ARRAY => Converter::parse_array(str, |str| { + Ok(Value::Number(convert_into(str.parse::()?))) + })?, + _ => Value::String(str.to_string()), + }; + + Ok(value) + } + + fn parse_array

(str: &str, mut parse: P) -> Result + where + P: FnMut(&str) -> Result, + { + if str.len() < 2 { + return Err(ArrayParseError::InputTooShort.into()); + } + + if !str.starts_with('{') || !str.ends_with('}') { + return Err(ArrayParseError::MissingBraces.into()); + } + + let mut res = vec![]; + let str = &str[1..(str.len() - 1)]; + let mut val_str = String::with_capacity(10); + let mut in_quotes = false; + let mut in_escape = false; + let mut chars = str.chars(); + let mut done = str.is_empty(); + + while !done { + loop { + match chars.next() { + Some(c) => match c { + c if in_escape => { + val_str.push(c); + in_escape = false; + } + '"' => in_quotes = !in_quotes, + '\\' => in_escape = true, + ',' if !in_quotes => { + break; + } + c => { + val_str.push(c); + } + }, + None => { + done = true; + break; + } + } + } + let val = if val_str.to_lowercase() == "null" { + Value::Null + } else { + parse(&val_str)? + }; + res.push(val); + val_str.clear(); + } + let arr = Value::Array(res); + Ok(arr) + } +} diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/postgres_triggers/handler.rs new file mode 100644 index 0000000000..fc6adbb6e2 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/handler.rs @@ -0,0 +1,1458 @@ +use std::{ + collections::{ + hash_map::Entry::{Occupied, Vacant}, + HashMap, + }, + str::FromStr, +}; + +use crate::{ + db::{ApiAuthed, DB}, + postgres_triggers::mapper::{Mapper, MappingInfo}, +}; +use axum::{ + extract::{Path, Query}, + Extension, Json, +}; +use chrono::Utc; +use http::StatusCode; +use itertools::Itertools; +use pg_escape::{quote_identifier, quote_literal}; +use quick_cache::sync::Cache; +use rand::Rng; +use rust_postgres::types::Type; +use serde::{Deserialize, Deserializer, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::{ + postgres::{types::Oid, PgConnectOptions, PgSslMode}, + Connection, FromRow, PgConnection, QueryBuilder, +}; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_common::error::Error; +use windmill_common::{ + db::UserDB, + error::{self, JsonResult}, + utils::{not_found_if_none, paginate, Pagination, StripPath}, + worker::CLOUD_HOSTED, +}; + +use super::get_database_resource; +use lazy_static::lazy_static; + +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Database { + pub user: String, + pub password: String, + pub host: String, + pub port: u16, + pub dbname: String, + pub sslmode: String, + pub root_certificate_pem: String, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct TableToTrack { + pub table_name: String, + pub where_clause: Option, + pub columns_name: Vec, +} + +impl TableToTrack { + fn new( + table_name: String, + where_clause: Option, + columns_name: Vec, + ) -> TableToTrack { + TableToTrack { table_name, where_clause, columns_name } + } +} + +lazy_static! { + pub static ref TEMPLATE: Cache = Cache::new(50); +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct Relations { + pub schema_name: String, + pub table_to_track: Vec, +} + +impl Relations { + fn new(schema_name: String, table_to_track: Vec) -> Relations { + Relations { schema_name, table_to_track } + } + + fn add_new_table(&mut self, table_to_track: TableToTrack) { + self.table_to_track.push(table_to_track); + } +} + +#[derive(Deserialize)] +pub struct EditPostgresTrigger { + replication_slot_name: String, + publication_name: String, + path: String, + script_path: String, + is_flow: bool, + postgres_resource_path: String, + publication: Option, +} + +#[derive(Deserialize, Serialize, Debug)] + +pub struct NewPostgresTrigger { + path: String, + script_path: String, + is_flow: bool, + enabled: bool, + postgres_resource_path: String, + replication_slot_name: Option, + publication_name: Option, + publication: Option, +} + +pub async fn get_database_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + let database = get_database_resource(authed, user_db, db, postgres_resource_path, w_id).await?; + + Ok(get_raw_postgres_connection(&database).await?) +} + +pub async fn get_raw_postgres_connection(db: &Database) -> Result { + let options = { + let sslmode = if !db.sslmode.is_empty() { + PgSslMode::from_str(&db.sslmode)? + } else { + PgSslMode::Prefer + }; + let options = PgConnectOptions::new() + .host(&db.host) + .database(&db.dbname) + .port(db.port) + .ssl_mode(sslmode) + .username(&db.user); + + let options = if !db.root_certificate_pem.is_empty() { + options.ssl_root_cert_from_pem(db.root_certificate_pem.as_bytes().to_vec()) + } else { + options + }; + + if !db.password.is_empty() { + options.password(&db.password) + } else { + options + } + }; + + PgConnection::connect_with(&options) + .await + .map_err(Error::SqlErr) +} + +#[derive(Deserialize, Debug)] +pub enum Language { + #[serde(rename = "typescript", alias = "Typescript")] + Typescript, +} + +#[derive(Debug, Deserialize)] +pub struct TemplateScript { + postgres_resource_path: String, + #[serde(deserialize_with = "check_if_not_duplication_relation")] + relations: Option>, + language: Language, +} + +fn check_if_not_duplication_relation<'de, D>( + relations: D, +) -> std::result::Result>, D::Error> +where + D: Deserializer<'de>, +{ + let relations: Option> = Option::deserialize(relations)?; + + match relations { + Some(relations) => { + for relation in relations.iter() { + if relation.schema_name.is_empty() { + return Err(serde::de::Error::custom( + "Schema Name must not be empty".to_string(), + )); + } + + for table_to_track in relation.table_to_track.iter() { + if table_to_track.table_name.trim().is_empty() { + return Err(serde::de::Error::custom( + "Table name must not be empty".to_string(), + )); + } + } + } + + if !relations + .iter() + .map(|relation| relation.schema_name.as_str()) + .all_unique() + { + return Err(serde::de::Error::custom( + "You cannot choose a schema more than one time".to_string(), + )); + } + + Ok(Some(relations)) + } + None => Ok(None), + } +} + +#[derive(FromRow, Deserialize, Serialize, Debug)] +pub struct PostgresTrigger { + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub workspace_id: String, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + pub extra_perms: Option, + pub postgres_resource_path: String, + pub error: Option, + pub server_id: Option, + pub replication_slot_name: String, + pub publication_name: String, + pub last_server_ping: Option>, + pub enabled: bool, +} + +#[derive(Deserialize, Serialize)] +pub struct ListPostgresTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +#[derive(Deserialize)] +pub struct SetEnabled { + pub enabled: bool, +} + +pub async fn create_postgres_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(new_postgres_trigger): Json, +) -> error::Result<(StatusCode, String)> { + let NewPostgresTrigger { + postgres_resource_path, + path, + script_path, + enabled, + is_flow, + publication_name, + replication_slot_name, + publication, + } = new_postgres_trigger; + + if *CLOUD_HOSTED { + return Err(error::Error::BadRequest( + "Postgres triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), + )); + } + + if publication_name.is_none() && publication.is_none() { + return Err(error::Error::BadRequest( + "Publication data is missing".to_string(), + )); + } + + let create_slot = replication_slot_name.is_none(); + let create_publication = publication_name.is_none(); + + let name; + let mut pub_name = publication_name.as_deref().unwrap_or_default(); + let mut slot_name = replication_slot_name.as_deref().unwrap_or_default(); + if create_publication || create_slot { + let generate_random_string = move || { + let timestamp = Utc::now().timestamp_millis().to_string(); + let mut rng = rand::thread_rng(); + let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; + + let random_part = (0..10) + .map(|_| { + charset + .chars() + .nth(rng.gen_range(0..charset.len())) + .unwrap() + }) + .collect::(); + + format!("{}_{}", timestamp, random_part) + }; + + name = format!("windmill_{}", generate_random_string()); + pub_name = &name; + slot_name = &name; + let publication = publication.unwrap(); + + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + new_publication( + &mut connection, + pub_name, + publication.table_to_track.as_deref(), + &publication + .transaction_to_track + .iter() + .map(AsRef::as_ref) + .collect_vec(), + ) + .await?; + + new_slot(&mut connection, slot_name).await?; + } + + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + r#" + INSERT INTO postgres_trigger ( + publication_name, + replication_slot_name, + workspace_id, + path, + script_path, + is_flow, + email, + enabled, + postgres_resource_path, + edited_by + ) + VALUES ( + $1, + $2, + $3, + $4, + $5, + $6, + $7, + $8, + $9, + $10 + )"#, + pub_name, + slot_name, + &w_id, + &path, + script_path, + is_flow, + &authed.email, + enabled, + postgres_resource_path, + &authed.username + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "postgres_triggers.create", + ActionKind::Create, + &w_id, + Some(path.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, path.to_string())) +} + +pub async fn list_postgres_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lst): Query, +) -> error::JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); + let mut sqlb = SqlBuilder::select_from("postgres_trigger") + .fields(&[ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "server_id", + "last_server_ping", + "extra_perms", + "error", + "enabled", + "postgres_resource_path", + "replication_slot_name", + "publication_name", + ]) + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + if let Some(path) = lst.path { + sqlb.and_where_eq("script_path", "?".bind(&path)); + } + if let Some(is_flow) = lst.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + if let Some(path_start) = &lst.path_start { + sqlb.and_where_like_left("path", path_start); + } + let sql = sqlb + .sql() + .map_err(|e| error::Error::InternalErr(e.to_string()))?; + let rows = sqlx::query_as::<_, PostgresTrigger>(&sql) + .fetch_all(&mut *tx) + .await + .map_err(|e| { + tracing::debug!("Error fetching postgres_trigger: {:#?}", e); + windmill_common::error::Error::InternalErr("server error".to_string()) + })?; + tx.commit().await.map_err(|e| { + tracing::debug!("Error commiting postgres_trigger: {:#?}", e); + windmill_common::error::Error::InternalErr("server error".to_string()) + })?; + + Ok(Json(rows)) +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct PublicationData { + #[serde(default, deserialize_with = "check_if_not_duplication_relation")] + table_to_track: Option>, + #[serde(deserialize_with = "check_if_valid_transaction_type")] + transaction_to_track: Vec, +} + +fn check_if_valid_transaction_type<'de, D>( + transaction_type: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; + if transaction_type.len() > 3 { + return Err(serde::de::Error::custom( + "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" + .to_string(), + )); + } + transaction_type.sort_unstable(); + transaction_type.dedup(); + + for transaction in transaction_type.iter() { + match transaction.to_lowercase().as_ref() { + "insert" => {}, + "update" => {}, + "delete" => {}, + _ => { + return Err(serde::de::Error::custom( + "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" + .to_string(), + )) + } + } + } + + Ok(transaction_type) +} + +impl PublicationData { + fn new( + table_to_track: Option>, + transaction_to_track: Vec, + ) -> PublicationData { + PublicationData { table_to_track, transaction_to_track } + } +} + +#[derive(Debug, Serialize)] +pub struct SlotList { + slot_name: Option, + active: Option, +} + +pub async fn list_slot_name( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> error::Result>> { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + let slots = sqlx::query_as!( + SlotList, + r#" + SELECT + slot_name, + active + FROM + pg_replication_slots + WHERE + plugin = 'pgoutput' AND + slot_type = 'logical'; + "# + ) + .fetch_all(&mut connection) + .await?; + + Ok(Json(slots)) +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Slot { + name: String, +} + +async fn new_slot(connection: &mut PgConnection, name: &str) -> error::Result<()> { + let query = format!( + r#" + SELECT + * + FROM + pg_create_logical_replication_slot({}, 'pgoutput');"#, + quote_literal(&name) + ); + + sqlx::query(&query).execute(connection).await?; + + Ok(()) +} + +pub async fn create_slot( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, + Json(Slot { name }): Json, +) -> error::Result { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + new_slot(&mut connection, &name).await?; + + Ok(format!("Slot {} created!", name)) +} + +pub async fn drop_slot_name( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, + Json(Slot { name }): Json, +) -> error::Result { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + let query = format!("SELECT pg_drop_replication_slot({});", quote_literal(&name)); + sqlx::query(&query).execute(&mut connection).await?; + + Ok(format!("Slot name {} deleted!", name)) +} +#[derive(Debug, Serialize)] +struct PublicationName { + publication_name: String, +} + +pub async fn list_database_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> error::Result>> { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + let publication_names = sqlx::query_as!( + PublicationName, + "SELECT pubname AS publication_name FROM pg_publication;" + ) + .fetch_all(&mut connection) + .await?; + + let publications = publication_names + .iter() + .map(|publication| publication.publication_name.to_owned()) + .collect_vec(); + + Ok(Json(publications)) +} + +pub async fn get_publication_info( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, +) -> error::Result> { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + let publication_data = + get_publication_scope_and_transaction(&publication_name, &mut connection).await; + + let (all_table, transaction_to_track) = match publication_data { + Ok(pub_data) => pub_data, + Err(Error::SqlErr(sqlx::Error::RowNotFound)) => { + return Err(Error::NotFound( + "Publication was not found, please create a new publication".to_string(), + )) + } + Err(e) => return Err(e), + }; + + let table_to_track = if !all_table { + Some(get_tracked_relations(&mut connection, &publication_name).await?) + } else { + None + }; + Ok(Json(PublicationData::new( + table_to_track, + transaction_to_track, + ))) +} + +async fn new_publication( + connection: &mut PgConnection, + publication_name: &str, + table_to_track: Option<&[Relations]>, + transaction_to_track: &[&str], +) -> Result<(), Error> { + let mut query = QueryBuilder::new("CREATE PUBLICATION "); + + query.push(quote_identifier(publication_name)); + + match table_to_track { + Some(database_component) if !database_component.is_empty() => { + query.push(" FOR"); + for (i, schema) in database_component.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push(" TABLES IN SCHEMA "); + query.push(quote_identifier(&schema.schema_name)); + } else { + query.push(" TABLE ONLY "); + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push(full_name); + if !table.columns_name.is_empty() { + query.push(" ("); + let columns = table + .columns_name + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push(&columns); + query.push(")"); + } + + if let Some(where_clause) = &table.where_clause { + query.push(" WHERE ("); + query.push(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push(", "); + } + } + } + if i < database_component.len() - 1 { + query.push(", "); + } + } + } + _ => { + query.push(" FOR ALL TABLES "); + } + }; + + if !transaction_to_track.is_empty() { + let transactions = || transaction_to_track.iter().join(", "); + query.push(" WITH (publish = '"); + query.push(transactions()); + query.push("');"); + } + + let query = query.build(); + query.execute(&mut *connection).await?; + + Ok(()) +} + +pub async fn create_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, + Json(publication_data): Json, +) -> error::Result { + let PublicationData { table_to_track, transaction_to_track } = publication_data; + + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + new_publication( + &mut connection, + &publication_name, + table_to_track.as_deref(), + &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(), + ) + .await?; + + Ok(format!( + "Publication {} successfully created!", + publication_name + )) +} + +async fn drop_publication( + publication_name: &str, + connection: &mut PgConnection, +) -> Result<(), Error> { + let mut query = QueryBuilder::new("DROP PUBLICATION IF EXISTS "); + let quoted_publication_name = quote_identifier(publication_name); + query.push(quoted_publication_name); + query.push(";"); + query.build().execute(&mut *connection).await?; + Ok(()) +} + +pub async fn delete_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, +) -> error::Result { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + drop_publication(&publication_name, &mut connection).await?; + + Ok(format!( + "Publication {} successfully deleted!", + publication_name + )) +} + +async fn update_publication( + connection: &mut PgConnection, + publication_name: &str, + PublicationData { table_to_track, transaction_to_track }: PublicationData, +) -> error::Result { + let (all_table, _) = + get_publication_scope_and_transaction(&publication_name, connection).await?; + + let mut query = QueryBuilder::new(""); + let quoted_publication_name = quote_identifier(&publication_name); + + let transaction_to_track_as_str = transaction_to_track.iter().join(","); + + match table_to_track { + Some(ref relations) if !relations.is_empty() => { + if all_table { + drop_publication(&publication_name, connection).await?; + new_publication( + connection, + &publication_name, + table_to_track.as_deref(), + &transaction_to_track.iter().map(AsRef::as_ref).collect_vec(), + ) + .await?; + } else { + query.push("ALTER PUBLICATION "); + query.push("ed_publication_name); + query.push(" SET"); + for (i, schema) in relations.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push(" TABLES IN SCHEMA "); + let quoted_schema = quote_identifier(&schema.schema_name); + query.push("ed_schema); + } else { + query.push(" TABLE ONLY "); + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push(&full_name); + if !table.columns_name.is_empty() { + query.push(" ("); + let columns = table + .columns_name + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push(&columns); + query.push(") "); + } + + if let Some(where_clause) = &table.where_clause { + query.push(" WHERE ("); + query.push(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push(", "); + } + } + } + if i < relations.len() - 1 { + query.push(','); + } + } + query.push(";"); + query.build().execute(&mut *connection).await?; + query.reset(); + query.push("ALTER PUBLICATION "); + query.push("ed_publication_name); + query.push(format!( + " SET (publish = '{}');", + transaction_to_track_as_str + )); + } + } + _ => { + drop_publication(&publication_name, connection).await?; + let to_execute = format!( + r#" + CREATE + PUBLICATION {} FOR ALL TABLES WITH (publish = '{}') + "#, + quoted_publication_name, transaction_to_track_as_str + ); + query.push(&to_execute); + } + }; + + query.build().execute(&mut *connection).await?; + + Ok(format!( + "Publication {} successfully updated!", + publication_name + )) +} + +pub async fn alter_publication( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, + Json(publication_data): Json, +) -> error::Result { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + let message = update_publication(&mut connection, &publication_name, publication_data).await?; + + Ok(message) +} + +async fn get_publication_scope_and_transaction( + publication_name: &str, + connection: &mut PgConnection, +) -> Result<(bool, Vec), Error> { + #[derive(Debug, Deserialize, FromRow)] + struct PublicationTransaction { + all_table: bool, + insert: bool, + update: bool, + delete: bool, + } + + let transaction = sqlx::query_as!( + PublicationTransaction, + r#" + SELECT + puballtables AS all_table, + pubinsert AS insert, + pubupdate AS update, + pubdelete AS delete + FROM + pg_publication + WHERE + pubname = $1 + "#, + publication_name + ) + .fetch_one(&mut *connection) + .await?; + + let mut transaction_to_track = Vec::with_capacity(3); + + if transaction.insert { + transaction_to_track.push("insert".to_string()); + } + if transaction.update { + transaction_to_track.push("update".to_string()); + } + if transaction.delete { + transaction_to_track.push("delete".to_string()); + } + + Ok((transaction.all_table, transaction_to_track)) +} + +async fn get_tracked_relations( + connection: &mut PgConnection, + publication_name: &str, +) -> error::Result> { + #[derive(Debug, Deserialize, FromRow)] + struct PublicationData { + schema_name: Option, + table_name: Option, + columns: Option>, + where_clause: Option, + } + + let publications = sqlx::query_as!( + PublicationData, + r#" + SELECT + schemaname AS schema_name, + tablename AS table_name, + attnames AS columns, + rowfilter AS where_clause + FROM + pg_publication_tables + WHERE + pubname = $1 + "#, + publication_name + ) + .fetch_all(&mut *connection) + .await?; + + let mut table_to_track: HashMap = HashMap::new(); + + for publication in publications { + let schema_name = publication.schema_name.unwrap(); + let entry = table_to_track.entry(schema_name.clone()); + let table_to_track = TableToTrack::new( + publication.table_name.unwrap(), + publication.where_clause, + publication.columns.unwrap(), + ); + match entry { + Occupied(mut occuped) => { + occuped.get_mut().add_new_table(table_to_track); + } + Vacant(vacant) => { + vacant.insert(Relations::new(schema_name, vec![table_to_track])); + } + } + } + Ok(table_to_track.into_values().collect_vec()) +} + +pub async fn get_postgres_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + let trigger = sqlx::query_as!( + PostgresTrigger, + r#" + SELECT + workspace_id, + path, + script_path, + is_flow, + edited_by, + email, + edited_at, + server_id, + last_server_ping, + extra_perms, + error, + enabled, + replication_slot_name, + publication_name, + postgres_resource_path + FROM + postgres_trigger + WHERE + workspace_id = $1 AND + path = $2 + "#, + &w_id, + &path + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let trigger = not_found_if_none(trigger, "Trigger", path)?; + + Ok(Json(trigger)) +} + +pub async fn update_postgres_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(postgres_trigger): Json, +) -> error::Result { + let workspace_path = path.to_path(); + let EditPostgresTrigger { + replication_slot_name, + publication_name, + script_path, + path, + is_flow, + postgres_resource_path, + publication, + } = postgres_trigger; + + if let Some(publication) = publication { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + update_publication(&mut connection, &publication_name, publication).await?; + } + let mut tx = user_db.begin(&authed).await?; + + sqlx::query!( + r#" + UPDATE postgres_trigger + SET + script_path = $1, + path = $2, + is_flow = $3, + edited_by = $4, + email = $5, + postgres_resource_path = $6, + replication_slot_name = $7, + publication_name = $8, + edited_at = now(), + error = NULL, + server_id = NULL + WHERE + workspace_id = $9 AND + path = $10 + "#, + script_path, + path, + is_flow, + &authed.username, + &authed.email, + postgres_resource_path, + replication_slot_name, + publication_name, + w_id, + workspace_path, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "postgres_triggers.update", + ActionKind::Create, + &w_id, + Some(&path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(workspace_path.to_string()) +} + +pub async fn delete_postgres_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::Result { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + r#" + DELETE FROM postgres_trigger + WHERE + workspace_id = $1 AND + path = $2 + "#, + w_id, + path, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "postgres_triggers.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Postgres trigger {path} deleted")) +} + +pub async fn exists_postgres_trigger( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let exists = sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM postgres_trigger + WHERE + path = $1 AND + workspace_id = $2 + )"#, + path, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +pub async fn set_enabled( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> error::Result { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + + // important to set server_id, last_server_ping and error to NULL to stop current postgres listener + let one_o = sqlx::query_scalar!( + r#" + UPDATE postgres_trigger + SET + enabled = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL + WHERE + path = $4 AND + workspace_id = $5 + RETURNING 1 + "#, + payload.enabled, + &authed.email, + &authed.username, + path, + w_id, + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); + + not_found_if_none(one_o, "Postgres trigger", path)?; + + audit_log( + &mut *tx, + &authed, + "postgres_triggers.setenabled", + ActionKind::Update, + &w_id, + Some(path), + Some([("enabled", payload.enabled.to_string().as_ref())].into()), + ) + .await?; + + tx.commit().await?; + + Ok(format!( + "succesfully updated postgres trigger at path {} to status {}", + path, payload.enabled + )) +} + +pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> error::Result { + let template = if let Some((_, template)) = TEMPLATE.remove(&id) { + template + } else { + "".to_string() + }; + Ok(template) +} + +pub async fn create_template_script( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path(w_id): Path, + Json(template_script): Json, +) -> error::Result { + let TemplateScript { postgres_resource_path, relations, language } = template_script; + if relations.is_none() { + return Err(Error::BadRequest( + "You must at least choose schema to fetch table from".to_string(), + )); + } + + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + #[derive(Debug, FromRow, Deserialize)] + struct ColumnInfo { + table_schema: Option, + table_name: Option, + column_name: Option, + oid: Oid, + is_nullable: bool, + } + + let relations = relations.unwrap(); + let mut schema_or_fully_qualified_name = Vec::with_capacity(relations.len()); + let mut columns_list = Vec::new(); + for relation in relations { + if !relation.table_to_track.is_empty() { + for table_to_track in relation.table_to_track { + let fully_qualified_name = + format!("{}.{}", &relation.schema_name, table_to_track.table_name); + schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name)); + + let columns = if !table_to_track.columns_name.is_empty() { + quote_literal(&table_to_track.columns_name.join(",")) + } else { + "''".to_string() + }; + columns_list.push(columns); + } + continue; + } + + schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); + columns_list.push(String::from("''")); + } + + let tables_name = schema_or_fully_qualified_name.join(","); + let columns_list = columns_list.join(","); + + let query = format!( + r#" + WITH table_column_mapping AS ( + SELECT + unnest(ARRAY[{}]) AS table_name, + unnest(ARRAY[{}]) AS column_list + ), + parsed_columns AS ( + SELECT + tcm.table_name, + CASE + WHEN tcm.column_list = '' THEN NULL + ELSE string_to_array(tcm.column_list, ',') + END AS columns + FROM + table_column_mapping tcm + ) + SELECT + ns.nspname AS table_schema, + cls.relname AS table_name, + attr.attname AS column_name, + attr.atttypid AS oid, + attr.attnotnull AS is_nullable + FROM + pg_attribute attr + JOIN + pg_class cls + ON attr.attrelid = cls.oid + JOIN + pg_namespace ns + ON cls.relnamespace = ns.oid + JOIN + parsed_columns pc + ON ns.nspname || '.' || cls.relname = pc.table_name + OR ns.nspname = pc.table_name + WHERE + attr.attnum > 0 -- Exclude system columns + AND NOT attr.attisdropped -- Exclude dropped columns + AND cls.relkind = 'r' -- Restrict to base tables + AND ( + pc.columns IS NULL + OR attr.attname = ANY(pc.columns) + ); + "#, + tables_name, columns_list + ); + + let rows: Vec = sqlx::query_as(&query) + .fetch_all(&mut connection) + .await + .map_err(error::Error::SqlErr)?; + + let mut mapper: HashMap>> = HashMap::new(); + + for row in rows { + let ColumnInfo { table_schema, table_name, column_name, oid, is_nullable } = row; + + let entry = mapper.entry(table_schema.unwrap()); + + let mapped_info = + MappingInfo::new(column_name.unwrap(), Type::from_oid(oid.0), is_nullable); + + match entry { + Occupied(mut occupied) => { + let entry = occupied.get_mut().entry(table_name.unwrap()); + match entry { + Occupied(mut occuped) => { + let mapping_info = occuped.get_mut(); + mapping_info.push(mapped_info); + } + Vacant(vacant) => { + let mut mapping_info = Vec::with_capacity(10); + mapping_info.push(mapped_info); + vacant.insert(mapping_info); + } + } + } + Vacant(vacant) => { + let mut mapping_info = Vec::with_capacity(10); + mapping_info.push(mapped_info); + vacant.insert(HashMap::from([(table_name.unwrap(), mapping_info)])); + } + } + } + + let mapper = Mapper::new(mapper, language); + + let create_template_id = |w_id: &str| -> String { + let uuid = uuid::Uuid::new_v4().to_string(); + let id = format!("{}-{}", &w_id, &uuid); + + id + }; + + let template = mapper.get_template(); + let id = create_template_id(&w_id); + + TEMPLATE.insert(id.clone(), template); + + Ok(id) +} + +pub async fn is_database_in_logical_level( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, postgres_resource_path)): Path<(String, String)>, +) -> error::JsonResult { + let mut connection = get_database_connection( + authed.clone(), + Some(user_db.clone()), + &db, + &postgres_resource_path, + &w_id, + ) + .await?; + + let wal_level = sqlx::query_scalar!("SHOW WAL_LEVEL;") + .fetch_optional(&mut connection) + .await? + .flatten(); + + let is_logical = match wal_level.as_deref() { + Some("logical") => true, + _ => false, + }; + + Ok(Json(is_logical)) +} diff --git a/backend/windmill-api/src/postgres_triggers/hex.rs b/backend/windmill-api/src/postgres_triggers/hex.rs new file mode 100644 index 0000000000..2ada70dd35 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/hex.rs @@ -0,0 +1,45 @@ +use std::num::ParseIntError; + + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs +* +*/ + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum ByteaHexParseError { + #[error("missing prefix '\\x'")] + InvalidPrefix, + + #[error("invalid byte")] + OddNumerOfDigits, + + #[error("parse int result: {0}")] + ParseInt(#[from] ParseIntError), +} + +pub fn from_bytea_hex(s: &str) -> Result, ByteaHexParseError> { + if s.len() < 2 || &s[..2] != "\\x" { + return Err(ByteaHexParseError::InvalidPrefix); + } + + let mut result = Vec::with_capacity((s.len() - 2) / 2); + let s = &s[2..]; + + if s.len() % 2 != 0 { + return Err(ByteaHexParseError::OddNumerOfDigits); + } + + for i in (0..s.len()).step_by(2) { + let val = u8::from_str_radix(&s[i..i + 2], 16)?; + result.push(val); + } + + Ok(result) +} diff --git a/backend/windmill-api/src/postgres_triggers/mapper.rs b/backend/windmill-api/src/postgres_triggers/mapper.rs new file mode 100644 index 0000000000..ec3626c7f7 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/mapper.rs @@ -0,0 +1,134 @@ +use std::collections::HashMap; + +use rust_postgres::types::Type; + +use super::handler::Language; + +fn postgres_to_typescript_type(postgres_type: Option) -> String { + let data_type = match postgres_type { + Some(postgres_type) => match postgres_type { + Type::BOOL => "boolean", + Type::BOOL_ARRAY => "Array", + Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => "string", + Type::CHAR_ARRAY + | Type::BPCHAR_ARRAY + | Type::VARCHAR_ARRAY + | Type::NAME_ARRAY + | Type::TEXT_ARRAY => "Array", + Type::INT2 | Type::INT4 | Type::INT8 | Type::NUMERIC => "number", + Type::INT2_ARRAY | Type::INT4_ARRAY | Type::INT8_ARRAY => "Array", + Type::FLOAT4 | Type::FLOAT8 => "number", + Type::FLOAT8_ARRAY | Type::FLOAT4_ARRAY => "Array", + Type::NUMERIC_ARRAY => "Array", + Type::BYTEA => "Array", + Type::BYTEA_ARRAY => "Array>", + Type::DATE => "string", + Type::DATE_ARRAY => "Array", + Type::TIME => "string", + Type::TIME_ARRAY => "Array", + Type::TIMESTAMPTZ | Type::TIMESTAMP => "Date", + Type::TIMESTAMPTZ_ARRAY | Type::TIMESTAMP_ARRAY => "Array", + Type::UUID => "string", + Type::UUID_ARRAY => "Array", + Type::JSON | Type::JSONB | Type::JSON_ARRAY | Type::JSONB_ARRAY => "unknown", + Type::OID => "number", + Type::OID_ARRAY => "Array", + _ => "string", + }, + None => "string", + }; + + data_type.to_string() +} + +fn into_body_struct(language: Language, mapped_info: Vec) -> String { + let mut block = String::new(); + match language { + Language::Typescript => { + block.push_str("{\r\n"); + for field in mapped_info { + let typescript_type = postgres_to_typescript_type(field.data_type); + let mut key = field.column_name; + if field.is_nullable { + key.push('?'); + } + let full_field = format!("\t\t{}: {},\r\n", key, typescript_type); + block.push_str(&full_field); + } + block.push_str("\t}"); + } + } + block +} + +#[derive(Debug)] +pub struct MappingInfo { + data_type: Option, + is_nullable: bool, + column_name: String, +} + +impl MappingInfo { + pub fn new(column_name: String, data_type: Option, is_nullable: bool) -> Self { + Self { column_name, data_type, is_nullable } + } +} + +pub struct Mapper { + to_template: HashMap>>, + language: Language, +} + +impl Mapper { + pub fn new( + to_template: HashMap>>, + language: Language, + ) -> Self { + Self { to_template, language } + } + + fn into_typescript_template(self) -> Vec { + let mut struct_definitions = Vec::new(); + for (_, mapping_info) in self.to_template { + let last_elem = mapping_info.len() - 1; + for (i, (_, mapped_info)) in mapping_info.into_iter().enumerate() { + let mut struct_body = into_body_struct(Language::Typescript, mapped_info); + let struct_body = if i != last_elem { + struct_body.push_str("\r\n"); + struct_body + } else { + struct_body + }; + struct_definitions.push(struct_body); + } + } + struct_definitions + } + + pub fn get_template(self) -> String { + let struct_definition = match self.language { + Language::Typescript => self.into_typescript_template(), + }; + + let struct_definition = if struct_definition.is_empty() { + "any".to_string() + } else { + struct_definition.join("\t| ") + }; + + format!( + r#" + + +export async function main( + transaction_type: "insert" | "update" | "delete", + schema_name: string, + table_name: string, + row: {} +) {{ +}} + "#, + struct_definition + ) + } +} diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs new file mode 100644 index 0000000000..9aa50f8fec --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -0,0 +1,155 @@ +use crate::{ + db::{ApiAuthed, DB}, + jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, + resources::get_resource_value_interpolated_internal, + users::fetch_api_authed, +}; +use serde_json::value::RawValue; +use std::collections::HashMap; + +use axum::{ + routing::{delete, get, post}, + Router, +}; +use handler::{ + alter_publication, create_postgres_trigger, create_publication, create_slot, + create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name, + exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script, + is_database_in_logical_level, list_database_publication, list_postgres_triggers, + list_slot_name, set_enabled, update_postgres_trigger, Database, PostgresTrigger, +}; +use windmill_common::{db::UserDB, error::Error, utils::StripPath}; +use windmill_queue::PushArgsOwned; + +mod bool; +mod converter; +mod handler; +mod hex; +mod mapper; +mod relation; +mod replication_message; +mod trigger; + +pub use trigger::start_database; + +pub async fn get_database_resource( + authed: ApiAuthed, + user_db: Option, + db: &DB, + database_resource_path: &str, + w_id: &str, +) -> Result { + let resource = get_resource_value_interpolated_internal( + &authed, + user_db, + &db, + &w_id, + &database_resource_path, + None, + "", + ) + .await + .map_err(|_| Error::NotFound("Database resource do not exist".to_string()))?; + + let resource = match resource { + Some(resource) => serde_json::from_value::(resource).map_err(Error::SerdeJson)?, + None => { + return { + Err(Error::NotFound( + "Database resource do not exist".to_string(), + )) + } + } + }; + + Ok(resource) +} + +fn publication_service() -> Router { + Router::new() + .route("/get/:publication_name/*path", get(get_publication_info)) + .route("/create/:publication_name/*path", post(create_publication)) + .route("/update/:publication_name/*path", post(alter_publication)) + .route( + "/delete/:publication_name/*path", + delete(delete_publication), + ) + .route("/list/*path", get(list_database_publication)) +} + +fn slot_service() -> Router { + Router::new() + .route("/list/*path", get(list_slot_name)) + .route("/create/*path", post(create_slot)) + .route("/delete/*path", delete(drop_slot_name)) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/create", post(create_postgres_trigger)) + .route("/list", get(list_postgres_triggers)) + .route("/get/*path", get(get_postgres_trigger)) + .route("/update/*path", post(update_postgres_trigger)) + .route("/delete/*path", delete(delete_postgres_trigger)) + .route("/exists/*path", get(exists_postgres_trigger)) + .route("/setenabled/*path", post(set_enabled)) + .route("/get_template_script/:id", get(get_template_script)) + .route("/create_template_script", post(create_template_script)) + .route( + "/is_valid_postgres_configuration/*path", + get(is_database_in_logical_level), + ) + .nest("/publication", publication_service()) + .nest("/slot", slot_service()) +} + +async fn run_job( + args: Option>>, + extra: Option>>, + db: &DB, + trigger: &PostgresTrigger, +) -> anyhow::Result<()> { + let args = PushArgsOwned { args: args.unwrap_or_default(), extra }; + let label_prefix = Some(format!("db-{}-", trigger.path)); + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + db, + Some("anonymous".to_string()), + ) + .await?; + + let user_db = UserDB::new(db.clone()); + + let run_query = RunJobQuery::default(); + + if trigger.is_flow { + run_flow_by_path_inner( + authed, + db.clone(), + user_db, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await?; + } else { + run_script_by_path_inner( + authed, + db.clone(), + user_db, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await?; + } + + Ok(()) +} diff --git a/backend/windmill-api/src/postgres_triggers/relation.rs b/backend/windmill-api/src/postgres_triggers/relation.rs new file mode 100644 index 0000000000..f893f0ed5b --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/relation.rs @@ -0,0 +1,74 @@ +use core::str; + +use serde_json::{Map, Value}; +use std::{collections::HashMap, str::Utf8Error}; + +use super::{ + converter::{Converter, ConverterError}, + replication_message::{Columns, RelationBody, TupleData}, +}; +use rust_postgres::types::Oid; +#[derive(Debug, thiserror::Error)] +pub enum RelationConversionError { + #[error("Could not find matching table")] + FailToFindMatchingTable, + + #[error("Binary data not supported")] + BinaryFormatNotSupported, + + #[error("decode error: {0}")] + FromBytes(#[from] ConverterError), + + #[error("invalid string value")] + InvalidStr(#[from] Utf8Error), +} + +pub struct RelationConverter(HashMap); + +impl RelationConverter { + pub fn new() -> Self { + Self(HashMap::new()) + } + + pub fn add_relation(&mut self, relation: RelationBody) { + self.0.insert(relation.o_id, relation); + } + + pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> { + self.0 + .get(&o_id) + .map(|relation_body| &relation_body.columns) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> { + self.0 + .get(&o_id) + .ok_or(RelationConversionError::FailToFindMatchingTable) + } + + pub fn body_to_json( + &self, + to_decode: (Oid, Vec), + ) -> Result, RelationConversionError> { + let (o_id, tuple_data) = to_decode; + let mut object: Map = Map::new(); + let columns = self.get_columns(o_id)?; + + for (i, column) in columns.iter().enumerate() { + let value = match &tuple_data[i] { + TupleData::Null | TupleData::UnchangedToast => Value::Null, + TupleData::Binary(_) => { + return Err(RelationConversionError::BinaryFormatNotSupported) + } + TupleData::Text(bytes) => { + let str = str::from_utf8(&bytes[..])?; + Converter::try_from_str(column.type_o_id.clone(), str)? + } + }; + + object.insert(column.name.clone(), value); + } + Ok(object) + } +} diff --git a/backend/windmill-api/src/postgres_triggers/replication_message.rs b/backend/windmill-api/src/postgres_triggers/replication_message.rs new file mode 100644 index 0000000000..e33f4bbf86 --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/replication_message.rs @@ -0,0 +1,510 @@ +#![allow(unused)] + +use core::str; +use std::{ + cmp, + io::{self, Cursor, Read}, + str::Utf8Error, +}; + +use byteorder::{BigEndian, ReadBytesExt}; +use bytes::Bytes; +use rust_postgres::types::{Oid, Type}; +use thiserror::Error; + +use super::trigger::LogicalReplicationSettings; +const PRIMARY_KEEPALIVE_BYTE: u8 = b'k'; +const X_LOG_DATA_BYTE: u8 = b'w'; + +/** +* This implementation is inspired by Postgres replication functionality +* from https://github.com/supabase/pg_replicate +* +* Original implementation: +* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/cdc_event.rs +* +*/ + +#[derive(Debug)] +pub struct PrimaryKeepAliveBody { + pub wal_end: u64, + pub timestamp: i64, + pub reply: bool, +} + +impl PrimaryKeepAliveBody { + pub fn new(wal_end: u64, timestamp: i64, reply: bool) -> PrimaryKeepAliveBody { + PrimaryKeepAliveBody { wal_end, timestamp, reply } + } +} + +const BEGIN_BYTE: u8 = b'B'; +const COMMIT_BYTE: u8 = b'C'; +const ORIGIN_BYTE: u8 = b'O'; +const RELATION_BYTE: u8 = b'R'; +const TYPE_BYTE: u8 = b'Y'; +const INSERT_BYTE: u8 = b'I'; +const UPDATE_BYTE: u8 = b'U'; +const DELETE_BYTE: u8 = b'D'; +const TUPLE_NEW_BYTE: u8 = b'N'; +const TUPLE_KEY_BYTE: u8 = b'K'; +const TUPLE_OLD_BYTE: u8 = b'O'; +const TUPLE_DATA_NULL_BYTE: u8 = b'n'; +const TUPLE_DATA_TOAST_BYTE: u8 = b'u'; +const TUPLE_DATA_TEXT_BYTE: u8 = b't'; +const TUPLE_DATA_BINARY_BYTE: u8 = b'b'; + +const REPLICA_IDENTITY_DEFAULT_BYTE: i8 = 0x64; +const REPLICA_IDENTITY_NOTHING_BYTE: i8 = 0x6E; +const REPLICA_IDENTITY_FULL_BYTE: i8 = 0x66; +const REPLICA_IDENTITY_INDEX_BYTE: i8 = 0x69; + +#[derive(Debug)] +pub enum ReplicaIdentity { + Default, + Nothing, + Full, + Index, +} + +#[derive(Debug)] +pub struct Column { + pub flags: i8, + pub name: String, + pub type_o_id: Option, + pub type_modifier: i32, +} + +impl Column { + pub fn new(flags: i8, name: String, type_o_id: Option, type_modifier: i32) -> Self { + Self { flags, name, type_o_id, type_modifier } + } +} + +pub type Columns = Vec; + +#[derive(Debug)] +pub struct RelationBody { + pub transaction_id: Option, + pub o_id: Oid, + pub namespace: String, + pub name: String, + pub replica_identity: ReplicaIdentity, + pub columns: Columns, +} + +impl RelationBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + namespace: String, + name: String, + replica_identity: ReplicaIdentity, + columns: Columns, + ) -> Self { + Self { transaction_id, o_id, namespace, name, replica_identity, columns } + } +} + +#[derive(Debug)] +pub struct InsertBody { + pub transaction_id: Option, + pub o_id: Oid, + pub tuple: Vec, +} + +impl InsertBody { + pub fn new(transaction_id: Option, o_id: Oid, tuple: Vec) -> Self { + Self { transaction_id, o_id, tuple } + } +} + +#[derive(Debug)] +pub struct UpdateBody { + transaction_id: Option, + pub o_id: Oid, + pub old_tuple: Option>, + pub key_tuple: Option>, + pub new_tuple: Vec, +} + +impl UpdateBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + old_tuple: Option>, + key_tuple: Option>, + new_tuple: Vec, + ) -> Self { + Self { transaction_id, o_id, old_tuple, key_tuple, new_tuple } + } +} + +#[derive(Debug)] +pub struct DeleteBody { + transaction_id: Option, + pub o_id: Oid, + pub old_tuple: Option>, + pub key_tuple: Option>, +} + +impl DeleteBody { + pub fn new( + transaction_id: Option, + o_id: Oid, + old_tuple: Option>, + key_tuple: Option>, + ) -> Self { + Self { transaction_id, o_id, old_tuple, key_tuple } + } +} + +#[derive(Debug)] +pub enum TupleData { + Null, + UnchangedToast, + Text(Bytes), + Binary(Bytes), +} + +impl TupleData { + fn parse(buf: &mut Buffer) -> Result, ConversionError> { + let number_of_columns = buf.read_i16::()?; + let mut tuples = Vec::with_capacity(number_of_columns as usize); + for _ in 0..number_of_columns { + let byte = buf.read_u8()?; + let tuple_data = match byte { + TUPLE_DATA_NULL_BYTE => TupleData::Null, + TUPLE_DATA_TOAST_BYTE => TupleData::UnchangedToast, + TUPLE_DATA_TEXT_BYTE => { + let len = buf.read_i32::()?; + let mut data = vec![0; len as usize]; + buf.read_exact(&mut data)?; + TupleData::Text(data.into()) + } + TUPLE_DATA_BINARY_BYTE => { + let len = buf.read_i32::()?; + let mut data = vec![0; len as usize]; + buf.read_exact(&mut data)?; + TupleData::Binary(data.into()) + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message byte `{}`", byte), + ))); + } + }; + + tuples.push(tuple_data); + } + + Ok(tuples) + } +} + +#[derive(Debug)] +pub enum TransactionBody { + Insert(InsertBody), + Update(UpdateBody), + Delete(DeleteBody), +} + +#[non_exhaustive] +#[derive(Debug)] +pub enum LogicalReplicationMessage { + Begin, + Commit, + Relation(RelationBody), + Type, + Insert(InsertBody), + Update(UpdateBody), + Delete(DeleteBody), +} + +#[derive(Debug)] +pub struct XLogDataBody { + pub wal_start: u64, + pub wal_end: u64, + pub timestamp: i64, + pub data: Bytes, +} + +#[derive(Error, Debug)] +pub enum ConversionError { + #[error("Error: {0}")] + Io(#[from] io::Error), + #[error("Utf8Error conversion: {0}")] + Utf8(#[from] Utf8Error), +} + +struct Buffer { + bytes: Bytes, + idx: usize, +} + +impl Buffer { + pub fn new(bytes: Bytes, idx: usize) -> Buffer { + Buffer { bytes, idx } + } + + fn slice(&self) -> &[u8] { + &self.bytes[self.idx..] + } + + fn read_cstr(&mut self) -> Result { + match self.slice().iter().position(|&x| x == 0) { + Some(pos) => { + let start = self.idx; + let end = start + pos; + let cstr = str::from_utf8(&self.bytes[start..end])?.to_owned(); + self.idx = end + 1; + Ok(cstr) + } + None => Err(ConversionError::Io(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected EOF", + ))), + } + } +} + +impl Read for Buffer { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let len = { + let slice = self.slice(); + let len = cmp::min(slice.len(), buf.len()); + buf[..len].copy_from_slice(&slice[..len]); + len + }; + self.idx += len; + Ok(len) + } +} + +impl XLogDataBody { + pub fn new(wal_start: u64, wal_end: u64, timestamp: i64, data: Bytes) -> XLogDataBody { + XLogDataBody { wal_start, wal_end, timestamp, data } + } + + pub fn parse( + self, + logical_replication_settings: &LogicalReplicationSettings, + ) -> Result { + let mut buf = Buffer::new(self.data.clone(), 0); + let byte = buf.read_u8()?; + + let logical_replication_message = match byte { + BEGIN_BYTE => { + buf.read_i64::()?; + buf.read_i64::()?; + buf.read_i32::()?; + + LogicalReplicationMessage::Begin + } + COMMIT_BYTE => { + buf.read_i8()?; + buf.read_u64::()?; + buf.read_u64::()?; + buf.read_i64::()?; + LogicalReplicationMessage::Commit + } + RELATION_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + + let o_id = buf.read_u32::()?; + let namespace = buf.read_cstr()?; + let name = buf.read_cstr()?; + let replica_identity = match buf.read_i8()? { + REPLICA_IDENTITY_DEFAULT_BYTE => ReplicaIdentity::Default, + REPLICA_IDENTITY_NOTHING_BYTE => ReplicaIdentity::Nothing, + REPLICA_IDENTITY_FULL_BYTE => ReplicaIdentity::Full, + REPLICA_IDENTITY_INDEX_BYTE => ReplicaIdentity::Index, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replica identity byte `{}`", byte), + ))); + } + }; + + let num_of_column = buf.read_i16::()?; + + let mut columns = Vec::with_capacity(num_of_column as usize); + for _ in 0..num_of_column { + let flags = buf.read_i8()?; + let name = buf.read_cstr()?; + let o_id = buf.read_u32::()?; + let type_modifier = buf.read_i32::()?; + let type_o_id = Type::from_oid(o_id); + let column = Column::new(flags, name, type_o_id, type_modifier); + + columns.push(column); + } + + LogicalReplicationMessage::Relation(RelationBody::new( + transaction_id, + o_id, + namespace, + name, + replica_identity, + columns, + )) + } + TYPE_BYTE => { + buf.read_u32::()?; + buf.read_cstr()?; + buf.read_cstr()?; + + LogicalReplicationMessage::Type + } + INSERT_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let byte = buf.read_u8()?; + + let tuple = match byte { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected tuple byte `{}`", byte), + ))); + } + }; + + LogicalReplicationMessage::Insert(InsertBody::new(transaction_id, o_id, tuple)) + } + UPDATE_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let byte = buf.read_u8()?; + let mut key_tuple = None; + let mut old_tuple = None; + + let new_tuple = match byte { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + TUPLE_OLD_BYTE | TUPLE_KEY_BYTE => { + if byte == TUPLE_OLD_BYTE { + old_tuple = Some(TupleData::parse(&mut buf)?); + } else { + key_tuple = Some(TupleData::parse(&mut buf)?); + } + match buf.read_u8()? { + TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected tuple byte `{}`", byte), + ))); + } + } + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown tuple byte `{}`", byte), + ))); + } + }; + + LogicalReplicationMessage::Update(UpdateBody::new( + transaction_id, + o_id, + old_tuple, + key_tuple, + new_tuple, + )) + } + DELETE_BYTE => { + let transaction_id = match logical_replication_settings.streaming { + true => Some(buf.read_i32::()?), + false => None, + }; + let o_id = buf.read_u32::()?; + let tag = buf.read_u8()?; + + let mut key_tuple = None; + let mut old_tuple = None; + + match tag { + TUPLE_OLD_BYTE => old_tuple = Some(TupleData::parse(&mut buf)?), + TUPLE_KEY_BYTE => key_tuple = Some(TupleData::parse(&mut buf)?), + tag => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown tuple tag `{}`", tag), + ))); + } + } + + LogicalReplicationMessage::Delete(DeleteBody::new( + transaction_id, + o_id, + old_tuple, + key_tuple, + )) + } + byte => { + return Err(ConversionError::Io(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message tag `{}`", byte), + ))); + } + }; + + Ok(logical_replication_message) + } +} + +#[non_exhaustive] +#[derive(Debug)] +pub enum ReplicationMessage { + XLogData(XLogDataBody), + PrimaryKeepAlive(PrimaryKeepAliveBody), +} + +impl ReplicationMessage { + pub fn parse(buf: Bytes) -> io::Result { + let (byte, mut message) = buf.split_first().unwrap(); + + let replication_message = match *byte { + X_LOG_DATA_BYTE => { + let len = buf.len(); + let wal_start = message.read_u64::()?; + let wal_end = message.read_u64::()?; + let timestamp = message.read_i64::()?; + let len = len - message.len(); + let data = buf.slice(len..); + ReplicationMessage::XLogData(XLogDataBody::new(wal_start, wal_end, timestamp, data)) + } + PRIMARY_KEEPALIVE_BYTE => { + let wal_end = message.read_u64::()?; + let timestamp = message.read_i64::()?; + let reply = message.read_u8()?; + ReplicationMessage::PrimaryKeepAlive(PrimaryKeepAliveBody::new( + wal_end, + timestamp, + reply == 1, + )) + } + byte => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unknown replication message byte `{}`", byte), + )); + } + }; + + Ok(replication_message) + } +} diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs new file mode 100644 index 0000000000..0ed9891cbc --- /dev/null +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -0,0 +1,523 @@ +use std::{collections::HashMap, pin::Pin}; + +use crate::{ + db::DB, + postgres_triggers::{ + get_database_resource, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, + ReplicationMessage, + }, + run_job, + }, + users::fetch_api_authed, +}; +use bytes::{BufMut, Bytes, BytesMut}; +use chrono::TimeZone; +use futures::{pin_mut, SinkExt, StreamExt}; +use pg_escape::{quote_identifier, quote_literal}; +use rand::seq::SliceRandom; +use rust_postgres::{Client, Config, CopyBothDuplex, NoTls, SimpleQueryMessage}; +use windmill_common::{ + db::UserDB, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME, +}; + +use super::{ + handler::{Database, PostgresTrigger}, + replication_message::PrimaryKeepAliveBody, +}; + +pub struct LogicalReplicationSettings { + pub streaming: bool, +} + +impl LogicalReplicationSettings { + pub fn new(streaming: bool) -> Self { + Self { streaming } + } +} + +#[allow(unused)] +trait RowExist { + fn row_exist(&self) -> bool; +} + +impl RowExist for Vec { + fn row_exist(&self) -> bool { + self.iter() + .find_map(|element| { + if let SimpleQueryMessage::CommandComplete(value) = element { + Some(*value) + } else { + None + } + }) + .is_some_and(|value| value > 0) + } +} + +#[derive(thiserror::Error, Debug)] +enum Error { + #[error("Error from database: {0}")] + Postgres(rust_postgres::Error), + #[error("Error : {0}")] + Common(windmill_common::error::Error), +} + +pub struct PostgresSimpleClient(Client); + +impl PostgresSimpleClient { + async fn new(database: &Database) -> Result { + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .port(database.port) + .user(&database.user) + .replication_mode(rust_postgres::config::ReplicationMode::Logical); + + if !database.password.is_empty() { + config.password(&database.password); + } + + let (client, connection) = config.connect(NoTls).await.map_err(Error::Postgres)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + + Ok(PostgresSimpleClient(client)) + } + + async fn get_logical_replication_stream( + &self, + publication_name: &str, + logical_replication_slot_name: &str, + ) -> Result<(CopyBothDuplex, LogicalReplicationSettings), Error> { + let options = format!( + r#"("proto_version" '2', "publication_names" {})"#, + quote_literal(publication_name), + ); + + let query = format!( + r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, + quote_identifier(logical_replication_slot_name), + options + ); + + Ok(( + self.0 + .copy_both_simple::(query.as_str()) + .await + .map_err(Error::Postgres)?, + LogicalReplicationSettings::new(false), + )) + } + + async fn send_status_update( + primary_keep_alive: PrimaryKeepAliveBody, + copy_both_stream: &mut Pin<&mut CopyBothDuplex>, + ) { + let mut buf = BytesMut::new(); + let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let ts = chrono::Utc::now() + .signed_duration_since(ts) + .num_microseconds() + .unwrap_or(0); + + buf.put_u8(b'r'); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_i64(ts); + buf.put_u8(0); + copy_both_stream.send(buf.freeze()).await.unwrap(); + tracing::debug!("Send update status message"); + } +} + +async fn update_ping( + db: &DB, + postgres_trigger: &PostgresTrigger, + error: Option<&str>, +) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + postgres_trigger + SET + last_server_ping = now(), + error = $1 + WHERE + workspace_id = $2 + AND path = $3 + AND server_id = $4 + AND enabled IS TRUE + RETURNING 1 + "#, + error, + &postgres_trigger.workspace_id, + &postgres_trigger.path, + *INSTANCE_NAME + ) + .fetch_optional(db) + .await; + + match updated { + Ok(updated) => { + if updated.flatten().is_none() { + // allow faster restart of database trigger + sqlx::query!( + r#" + UPDATE + postgres_trigger + SET + last_server_ping = NULL + WHERE + workspace_id = $1 + AND path = $2 + AND server_id IS NULL"#, + &postgres_trigger.workspace_id, + &postgres_trigger.path, + ) + .execute(db) + .await + .ok(); + tracing::info!( + "Postgres trigger {} changed, disabled, or deleted, stopping...", + postgres_trigger.path + ); + return None; + } + } + Err(err) => { + tracing::warn!( + "Error updating ping of postgres trigger {}: {:?}", + postgres_trigger.path, + err + ); + } + }; + + Some(()) +} + +async fn loop_ping(db: &DB, postgres_trigger: &PostgresTrigger, error: Option<&str>) { + loop { + if update_ping(db, postgres_trigger, error).await.is_none() { + return; + } + + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } +} + +async fn disable_with_error(postgres_trigger: &PostgresTrigger, db: &DB, error: String) -> () { + match sqlx::query!( + "UPDATE postgres_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", + error, + postgres_trigger.workspace_id, + postgres_trigger.path, + ) + .execute(db).await { + Ok(_) => { + report_critical_error(format!("Disabling postgres trigger {} because of error: {}", postgres_trigger.path, error), db.clone(), Some(&postgres_trigger.workspace_id), None).await; + }, + Err(disable_err) => { + report_critical_error( + format!("Could not disable postgres trigger {} with err {}, disabling because of error {}", postgres_trigger.path, disable_err, error), + db.clone(), + Some(&postgres_trigger.workspace_id), + None, + ).await; + } + } +} + +async fn listen_to_transactions( + postgres_trigger: &PostgresTrigger, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let start_logical_replication_streaming = async { + let authed = fetch_api_authed( + postgres_trigger.edited_by.clone(), + postgres_trigger.email.clone(), + &postgres_trigger.workspace_id, + &db, + None, + ) + .await + .map_err(Error::Common)?; + + let database = get_database_resource( + authed, + Some(UserDB::new(db.clone())), + &db, + &postgres_trigger.postgres_resource_path, + &postgres_trigger.workspace_id, + ) + .await + .map_err(Error::Common)?; + + let client = PostgresSimpleClient::new(&database).await?; + + let (logical_replication_stream, logical_replication_settings) = client + .get_logical_replication_stream( + &postgres_trigger.publication_name, + &postgres_trigger.replication_slot_name, + ) + .await?; + + Ok::<_, Error>((logical_replication_stream, logical_replication_settings)) + }; + + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = loop_ping(&db, postgres_trigger, Some("Connecting...")) => { + return; + } + result = start_logical_replication_streaming => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = loop_ping(&db, postgres_trigger, None) => { + return; + } + _ = { + async { + match result { + Ok((logical_replication_stream, logical_replication_settings)) => { + pin_mut!(logical_replication_stream); + let mut relations = RelationConverter::new(); + tracing::info!("Starting to listen for postgres trigger {}", postgres_trigger.path); + loop { + let message = logical_replication_stream.next().await; + + let message = match message { + Some(message) => message, + None => { + tracing::info!("Stream for postgres trigger {} is empty, leaving....", postgres_trigger.path); + return; + } + }; + + let message = match message { + Ok(message) => message, + Err(err) => { + let err = format!("Postgres trigger named {} had an error while receiving a message : {}", &postgres_trigger.path, err.to_string()); + disable_with_error(&postgres_trigger, &db, err).await; + return; + } + }; + + let logical_message = match ReplicationMessage::parse(message) { + Ok(logical_message) => logical_message, + Err(err) => { + let err = format!("Postgres trigger named: {} had an error while parsing message: {}", postgres_trigger.path, err.to_string()); + disable_with_error(&postgres_trigger, &db, err).await; + return; + } + }; + + + match logical_message { + ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { + if primary_keep_alive.reply { + PostgresSimpleClient::send_status_update(primary_keep_alive, &mut logical_replication_stream).await; + } + } + ReplicationMessage::XLogData(x_log_data) => { + let logical_replication_message = match x_log_data.parse(&logical_replication_settings) { + Ok(logical_replication_message) => logical_replication_message, + Err(err) => { + tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &postgres_trigger.path, err.to_string()); + continue; + } + }; + + let json = match logical_replication_message { + Relation(relation_body) => { + relations.add_relation(relation_body); + None + } + Begin | Type | Commit => { + None + } + Insert(insert) => { + Some((insert.o_id, relations.body_to_json((insert.o_id, insert.tuple)), "insert")) + } + Update(update) => { + Some((update.o_id, relations.body_to_json((update.o_id, update.new_tuple)), "update")) + } + Delete(delete) => { + let body = delete.old_tuple.unwrap_or(delete.key_tuple.unwrap()); + Some((delete.o_id, relations.body_to_json((delete.o_id, body)), "delete")) + } + }; + if let Some((o_id, Ok(body), transaction_type)) = json { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!("Postgres trigger named: {}, error: {}", &postgres_trigger.path, err.to_string()); + continue; + } + }; + let database_info = HashMap::from([ + ("schema_name".to_string(), to_raw_value(&relation.namespace)), + ("table_name".to_string(), to_raw_value(&relation.name)), + ("transaction_type".to_string(), to_raw_value(&transaction_type)), + ("row".to_string(), to_raw_value(&body)), + ]); + let extra = Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({"kind": "postgres", })), + )])); + let _ = run_job(Some(database_info), extra, &db, postgres_trigger).await; + } + + } + } + } + } + Err(err) => { + tracing::error!("Postgres trigger error while trying to start_logical_replication_streaming: {}", err.to_string()) + } + } + } + } => { + return; + } + } + } + } +} + +async fn try_to_listen_to_database_transactions( + pg_trigger: PostgresTrigger, + db: DB, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let postgres_trigger = sqlx::query_scalar!( + r#" + UPDATE postgres_trigger + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + enabled IS TRUE + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + *INSTANCE_NAME, + pg_trigger.workspace_id, + pg_trigger.path, + ) + .fetch_optional(&db) + .await; + match postgres_trigger { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!("Spawning new task to listen_to_database_transaction"); + tokio::spawn(async move { + listen_to_transactions(&pg_trigger, db.clone(), killpill_rx).await; + }); + } else { + tracing::info!( + "Postgres trigger {} already being listened to", + pg_trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for postgres trigger {}: {:?}", + pg_trigger.path, + err + ); + } + }; +} + +async fn listen_to_unlistened_database_events( + db: &DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let postgres_triggers = sqlx::query_as!( + PostgresTrigger, + r#" + SELECT + workspace_id, + path, + script_path, + replication_slot_name, + publication_name, + is_flow, + edited_by, + email, + edited_at, + server_id, + last_server_ping, + extra_perms, + error, + enabled, + postgres_resource_path + FROM + postgres_trigger + WHERE + enabled IS TRUE + AND (last_server_ping IS NULL OR + last_server_ping < now() - interval '15 seconds' + ) + "# + ) + .fetch_all(db) + .await; + + match postgres_triggers { + Ok(mut triggers) => { + triggers.shuffle(&mut rand::thread_rng()); + for trigger in triggers { + try_to_listen_to_database_transactions( + trigger, + db.clone(), + killpill_rx.resubscribe(), + ) + .await; + } + } + Err(err) => { + tracing::error!("Error fetching postgres triggers: {:?}", err); + } + }; +} + +pub async fn start_database(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { + tokio::spawn(async move { + listen_to_unlistened_database_events(&db, &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_database_events(&db, &killpill_rx).await + } + } + } + }); +} diff --git a/backend/windmill-api/src/slack_approvals.rs b/backend/windmill-api/src/slack_approvals.rs index 1f48bd07dc..9b02f4c73a 100644 --- a/backend/windmill-api/src/slack_approvals.rs +++ b/backend/windmill-api/src/slack_approvals.rs @@ -448,10 +448,16 @@ async fn transform_schemas( let is_required = required.unwrap().contains(key); let default_value = default_args_json.and_then(|json| json.get(key).cloned()); - let dynamic_enums_value = dynamic_enums_json.and_then(|json| json.get(key).cloned()); + let dynamic_enums_value = + dynamic_enums_json.and_then(|json| json.get(key).cloned()); - let input_block = - create_input_block(key, schema, is_required, default_value, dynamic_enums_value); + let input_block = create_input_block( + key, + schema, + is_required, + default_value, + dynamic_enums_value, + ); match input_block { serde_json::Value::Array(arr) => blocks.extend(arr), _ => blocks.push(input_block), @@ -536,7 +542,7 @@ fn create_input_block( // Handle date-time format if let FieldType::String = schema.r#type { - if schema.format.as_deref() == Some("date-time") { + if schema.format.as_deref() == Some("date-time") { tracing::debug!("Date-time type"); let now = chrono::Local::now(); let current_date = now.format("%Y-%m-%d").to_string(); diff --git a/backend/windmill-api/src/triggers.rs b/backend/windmill-api/src/triggers.rs index 0f3edbc87c..7101bdd2a9 100644 --- a/backend/windmill-api/src/triggers.rs +++ b/backend/windmill-api/src/triggers.rs @@ -20,6 +20,7 @@ pub struct TriggersCount { websocket_count: i64, kafka_count: i64, nats_count: i64, + postgres_count: i64, } pub(crate) async fn get_triggers_count_internal( db: &DB, @@ -86,6 +87,16 @@ pub(crate) async fn get_triggers_count_internal( .await? .unwrap_or(0); + let postgres_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM postgres_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + let webhook_count = (if is_flow { sqlx::query_scalar!( "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", @@ -129,6 +140,7 @@ pub(crate) async fn get_triggers_count_internal( websocket_count, kafka_count, nats_count, + postgres_count, })) } diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index aaf3b1dd49..48800621b6 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -676,3 +676,28 @@ pub async fn get_value_internal<'c>( Ok(r) } + +pub async fn get_variable_or_self(path: String, db: &DB, w_id: &str) -> Result { + if !path.starts_with("$var:") { + return Ok(path); + } + let path = path.strip_prefix("$var:").unwrap().to_string(); + + let record = sqlx::query!( + "SELECT value, is_secret + FROM variable + WHERE path = $1 AND workspace_id = $2", + &path, + &w_id + ) + .fetch_one(db) + .await?; + + let mut value = record.value; + if record.is_secret { + let mc = build_crypt(db, w_id).await?; + value = decrypt(&mc, value)?; + } + + Ok(value) +} diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 2f00a1f5a4..d6260691c7 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -380,7 +380,7 @@ async fn exists_websocket_trigger( async fn listen_to_unlistened_websockets( db: &DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) -> () { +) { match sqlx::query_as::<_, WebsocketTrigger>( r#"SELECT * FROM websocket_trigger diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 144d58f3ad..a4adc61977 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -35,7 +35,7 @@ use windmill_audit::ActionKind; use windmill_common::db::UserDB; use windmill_common::s3_helpers::LargeFileStorage; use windmill_common::users::username_to_permissioned_as; -use windmill_common::variables::build_crypt; +use windmill_common::variables::{build_crypt, decrypt, encrypt}; use windmill_common::worker::to_raw_value; #[cfg(feature = "enterprise")] use windmill_common::workspaces::WorkspaceDeploymentUISettings; @@ -52,7 +52,6 @@ use windmill_git_sync::handle_deployment_metadata; #[cfg(feature = "enterprise")] use windmill_common::utils::require_admin_or_devops; -use windmill_common::variables::{decrypt, encrypt}; use hyper::StatusCode; use serde::{Deserialize, Serialize}; use sqlx::{FromRow, Postgres, Transaction}; @@ -1304,6 +1303,7 @@ struct UsedTriggers { pub http_routes_used: bool, pub kafka_used: bool, pub nats_used: bool, + pub postgres_used: bool, } async fn get_used_triggers( @@ -1314,12 +1314,17 @@ async fn get_used_triggers( let mut tx = user_db.begin(&authed).await?; let websocket_used = sqlx::query_as!( UsedTriggers, - r#"SELECT - EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) as "websocket_used!", - EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) as "http_routes_used!", + r#" + SELECT + + EXISTS(SELECT 1 FROM websocket_trigger WHERE workspace_id = $1) AS "websocket_used!", + + EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1) AS "http_routes_used!", EXISTS(SELECT 1 FROM kafka_trigger WHERE workspace_id = $1) as "kafka_used!", - EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!""#, - w_id, + EXISTS(SELECT 1 FROM nats_trigger WHERE workspace_id = $1) as "nats_used!", + EXISTS(SELECT 1 FROM postgres_trigger WHERE workspace_id = $1) AS "postgres_used!" + "#, + w_id ) .fetch_one(&mut *tx) .await?; diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 70fea1ea5a..0a417ff916 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -25,18 +25,16 @@ use axum::{ use http::HeaderName; use itertools::Itertools; -use windmill_common::db::UserDB; -use windmill_common::schedule::Schedule; -use windmill_common::variables::build_crypt; - +use windmill_common::variables::decrypt; use windmill_common::{ + db::UserDB, error::{to_anyhow, Error, Result}, flows::Flow, + schedule::Schedule, scripts::{Schema, Script, ScriptLang}, - variables::ExportableListableVariable, + variables::{build_crypt, ExportableListableVariable}, }; -use windmill_common::variables::decrypt; use hyper::header; use serde::{Deserialize, Serialize}; use serde_json::Value; diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 2c689cecfb..8f1bc299ea 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -61,6 +61,7 @@ async-stream.workspace = true const_format.workspace = true crc.workspace = true windmill-macros.workspace = true + semver.workspace = true croner = "2.0.6" quick_cache.workspace = true diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 4c33fae7b0..e47c4fc87f 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -248,18 +248,16 @@ pub async fn connect_db( Err(_) => { if server_mode { DEFAULT_MAX_CONNECTIONS_SERVER + } else if indexer_mode { + DEFAULT_MAX_CONNECTIONS_INDEXER } else { - if indexer_mode { - DEFAULT_MAX_CONNECTIONS_INDEXER - } else { - DEFAULT_MAX_CONNECTIONS_WORKER - + std::env::var("NUM_WORKERS") - .ok() - .map(|x| x.parse().ok()) - .flatten() - .unwrap_or(1) - - 1 - } + DEFAULT_MAX_CONNECTIONS_WORKER + + std::env::var("NUM_WORKERS") + .ok() + .map(|x| x.parse().ok()) + .flatten() + .unwrap_or(1) + - 1 } } }; diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 7cb96d1927..7f91758c90 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -6,13 +6,13 @@ * LICENSE-AGPL for a copy of the license. */ +use crate::error::Result; +use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; use serde::{Deserialize, Serialize}; use crate::error; -use crate::{worker::WORKER_GROUP, BASE_URL, DB}; - lazy_static::lazy_static! { pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); } @@ -133,7 +133,7 @@ pub async fn get_secret_value_as_admin( let r = if variable.is_secret { let value = variable.value; if !value.is_empty() { - let mc = build_crypt(&db, &w_id).await?; + let mc = build_crypt(db, w_id).await?; decrypt_value_with_mc(value, mc).await? } else { "".to_string() @@ -145,17 +145,14 @@ pub async fn get_secret_value_as_admin( Ok(r) } -pub async fn decrypt_value_with_mc( - value: String, - mc: MagicCrypt256, -) -> Result { - Ok(mc.decrypt_base64_to_string(value).map_err(|e| match e { +pub async fn decrypt_value_with_mc(value: String, mc: MagicCrypt256) -> Result { + mc.decrypt_base64_to_string(value).map_err(|e| match e { MagicCryptError::DecryptError(_) => crate::error::Error::InternalErr( "Could not decrypt value. The value may have been encrypted with a different key." .to_string(), ), _ => crate::error::Error::InternalErr(e.to_string()), - })?) + }) } pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index e69de29bb2..8b13789179 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -0,0 +1 @@ + diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index a89a9d56fa..a90fa6ccf7 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -39,7 +39,6 @@ futures-core.workspace = true futures.workspace = true itertools.workspace = true async-recursion.workspace = true -bigdecimal.workspace = true axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index e21728943e..fee2eaa119 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -1,10 +1,5 @@ #[cfg(unix)] -use std::{ - collections::HashMap, - os::unix::fs::PermissionsExt, - path::PathBuf, - process::Stdio, -}; +use std::{collections::HashMap, os::unix::fs::PermissionsExt, path::PathBuf, process::Stdio}; #[cfg(windows)] use std::{ @@ -29,7 +24,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ bash_executor::BIN_BASH, common::{ - check_executor_binary_exists, get_reserved_variables, read_and_check_result, start_child_process, transform_json, OccupancyMetrics + check_executor_binary_exists, get_reserved_variables, read_and_check_result, + start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, @@ -201,7 +197,11 @@ pub async fn handle_ansible_job( envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - check_executor_binary_exists("ansible-playbook", ANSIBLE_PLAYBOOK_PATH.as_str(), "ansible")?; + check_executor_binary_exists( + "ansible-playbook", + ANSIBLE_PLAYBOOK_PATH.as_str(), + "ansible", + )?; let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?; append_logs(&job.id, &job.workspace_id, logs, db).await; diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index d887f14d81..cc34dd72ab 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -25,6 +25,8 @@ mod job_logger_ee; mod js_eval; #[cfg(feature = "mysql")] mod mysql_executor; +#[cfg(feature = "oracledb")] +mod oracledb_executor; mod pg_executor; #[cfg(feature = "php")] mod php_executor; @@ -36,8 +38,6 @@ mod rust_executor; mod worker; mod worker_flow; mod worker_lockfiles; -#[cfg(feature = "oracledb")] -mod oracledb_executor; pub use worker::*; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index fc5686568e..f6e64a60a1 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -20,7 +20,7 @@ use windmill_parser_sql::{ use windmill_queue::CanceledBy; use crate::{ - common::{check_executor_binary_exists, build_args_map, OccupancyMetrics}, + common::{build_args_map, check_executor_binary_exists, OccupancyMetrics}, handle_child::run_future_with_polling_update_job_poller, AuthedClientBackgroundTask, }; @@ -302,7 +302,11 @@ pub async fn do_oracledb( column_order: &mut Option>, occupancy_metrics: &mut OccupancyMetrics, ) -> windmill_common::error::Result> { - check_executor_binary_exists("the Oracle client lib", ORACLE_LIB_DIR.as_str(), "Oracle Database")?; + check_executor_binary_exists( + "the Oracle client lib", + ORACLE_LIB_DIR.as_str(), + "Oracle Database", + )?; let args = build_args_map(job, client, db).await?.map(Json); let job_args = if args.is_some() { diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index d105881078..7eb0d95429 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -15,7 +15,8 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_main_override, get_reserved_variables, read_result, start_child_process, OccupancyMetrics + check_executor_binary_exists, create_args_and_out_file, get_main_override, + get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 9499dd9eec..0d657fbf16 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -90,12 +90,26 @@ use tokio::{ use rand::Rng; use crate::{ - bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ + bash_executor::{handle_bash_job, handle_powershell_job}, + bun_executor::handle_bun_job, + common::{ build_args_map, cached_result_path, get_cached_resource_value_if_valid, get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics, - }, csharp_executor::handle_csharp_job, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, pg_executor::do_postgresql, result_processor::{process_result, start_background_processor}, worker_flow::{handle_flow, update_flow_status_in_progress}, worker_lockfiles::{ + }, + csharp_executor::handle_csharp_job, + deno_executor::handle_deno_job, + go_executor::handle_go_job, + graphql_executor::do_graphql, + handle_child::SLOW_LOGS, + handle_job_error, + job_logger::NO_LOGS_AT_ALL, + js_eval::{eval_fetch_timeout, transpile_ts}, + pg_executor::do_postgresql, + result_processor::{process_result, start_background_processor}, + worker_flow::{handle_flow, update_flow_status_in_progress}, + worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - } + }, }; #[cfg(feature = "rust")] @@ -331,7 +345,6 @@ const DOTNET_DEFAULT_PATH: &str = "C:\\Program Files\\dotnet\\dotnet.exe"; #[cfg(unix)] const DOTNET_DEFAULT_PATH: &str = "/usr/bin/dotnet"; - lazy_static::lazy_static! { pub static ref JOB_TOKEN: Option = std::env::var("JOB_TOKEN").ok(); diff --git a/frontend/.prettierrc b/frontend/.prettierrc.json similarity index 100% rename from frontend/.prettierrc rename to frontend/.prettierrc.json diff --git a/frontend/src/lib/components/ApiConnectForm.svelte b/frontend/src/lib/components/ApiConnectForm.svelte index 680b54850e..33c3ffb5e6 100644 --- a/frontend/src/lib/components/ApiConnectForm.svelte +++ b/frontend/src/lib/components/ApiConnectForm.svelte @@ -24,8 +24,10 @@ let supabaseWizard = false async function isSupabaseAvailable() { - supabaseWizard = - ((await OauthService.listOauthConnects()) ?? {})['supabase_wizard'] != undefined + try { + supabaseWizard = + ((await OauthService.listOauthConnects()) ?? {})['supabase_wizard'] != undefined + } catch (error) {} } async function loadSchema() { if (!resourceTypeInfo) return diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index ddf6811fb9..c6e4b1983f 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -75,7 +75,6 @@ let scopes: string[] = [] let extra_params: [string, string][] = [] - let path: string let description = '' diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index 35ff13f46e..a0ff007d1b 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -481,7 +481,14 @@ const selectedIdStore = writable(selectedId ?? 'settings-metadata') const selectedTriggerStore = writable< - 'webhooks' | 'emails' | 'schedules' | 'cli' | 'routes' | 'websockets' | 'scheduledPoll' + | 'webhooks' + | 'emails' + | 'schedules' + | 'cli' + | 'routes' + | 'websockets' + | 'postgres' + | 'scheduledPoll' >('webhooks') export function getSelectedId() { @@ -516,6 +523,7 @@ | 'cli' | 'routes' | 'websockets' + | 'postgres' | 'scheduledPoll' ) { selectedTriggerStore.set(selectedTrigger) diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index f042f17b76..c66f29c390 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -16,6 +16,7 @@ VariableService, WebsocketTriggerService, KafkaTriggerService, + PostgresTriggerService, NatsTriggerService } from '$lib/gen' import { superadmin, userStore, workspaceStore } from '$lib/stores' @@ -40,6 +41,7 @@ | 'http_trigger' | 'websocket_trigger' | 'kafka_trigger' + | 'postgres_trigger' | 'nats_trigger' let meta: Meta | undefined = undefined export let fullNamePlaceholder: string | undefined = undefined @@ -234,6 +236,11 @@ workspace: $workspaceStore!, path: path }) + } else if (kind == 'postgres_trigger') { + return await PostgresTriggerService.existsPostgresTrigger({ + workspace: $workspaceStore!, + path: path + }) } else if (kind == 'nats_trigger') { return await NatsTriggerService.existsNatsTrigger({ workspace: $workspaceStore!, diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 9679e3866e..7722cce461 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -6,10 +6,12 @@ type NewScriptWithDraft, ScheduleService, type Script, - type TriggersCount + type TriggersCount, + PostgresTriggerService } from '$lib/gen' import { inferArgs } from '$lib/infer' import { initialCode } from '$lib/script_helpers' + import { page } from '$app/stores' import { defaultScripts, enterpriseLicense, userStore, workspaceStore } from '$lib/stores' import { cleanValueProperties, @@ -230,18 +232,40 @@ $: !disableHistoryChange && replaceStateFn('#' + encodeState({ ...script, primarySchedule: $primaryScheduleStore })) - if (script.content == '') { initContent(script.language, script.kind, template) } - function initContent( + async function isTemplateScript() { + let getInitBlockTemplate = $page.url.searchParams.get('id') + if (getInitBlockTemplate === null) { + return undefined + } + try { + getInitBlockTemplate = await PostgresTriggerService.getTemplateScript({ + workspace: $workspaceStore!, + id: getInitBlockTemplate as string + }) + return getInitBlockTemplate + } catch (error) { + sendUserToast( + 'An error occured when trying to load your template script, please try again later', + true + ) + } + } + + async function initContent( language: SupportedLanguage, kind: Script['kind'] | undefined, template: 'pgsql' | 'mysql' | 'script' | 'docker' | 'powershell' | 'bunnative' ) { scriptEditor?.disableCollaboration() - script.content = initialCode(language, kind, template) + const templateScript = await isTemplateScript() + script.content = initialCode(language, kind, template, templateScript != undefined) + if (templateScript) { + script.content += '\r\n' + templateScript + } scriptEditor?.inferSchema(script.content, language, true) if (script.content != editor?.getCode()) { setCode(script.content) diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 80ecbe5472..c55a56de66 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -48,7 +48,10 @@ })) } else if (itemKind == 'script') { items = ( - await ScriptService.listScripts({ workspace: $workspaceStore!, kinds: kinds.join(',') }) + await ScriptService.listScripts({ + workspace: $workspaceStore!, + kinds: kinds.join(','), + }) ).map((script) => ({ value: script.path, label: `${script.path}${script.summary ? ` | ${truncate(script.summary, 20)}` : ''}` diff --git a/frontend/src/lib/components/Section.svelte b/frontend/src/lib/components/Section.svelte index 5daa694837..4f36a96b0b 100644 --- a/frontend/src/lib/components/Section.svelte +++ b/frontend/src/lib/components/Section.svelte @@ -6,6 +6,7 @@ export let label: string | undefined = undefined export let tooltip: string | undefined = undefined + export let documentationLink: string | undefined = undefined export let eeOnly = false export let small: boolean = false @@ -38,7 +39,7 @@ {#if tooltip} - {tooltip} + {tooltip} {/if} {#if eeOnly} {#if !$enterpriseLicense} diff --git a/frontend/src/lib/components/details/DetailPageDetailPanel.svelte b/frontend/src/lib/components/details/DetailPageDetailPanel.svelte index 09805297c3..1093161d36 100644 --- a/frontend/src/lib/components/details/DetailPageDetailPanel.svelte +++ b/frontend/src/lib/components/details/DetailPageDetailPanel.svelte @@ -12,6 +12,7 @@ | 'cli' | 'routes' | 'websockets' + | 'postgres' | 'scheduledPoll' | 'kafka' | 'nats' = 'webhooks' @@ -54,6 +55,7 @@ + diff --git a/frontend/src/lib/components/details/DetailPageLayout.svelte b/frontend/src/lib/components/details/DetailPageLayout.svelte index bce0bda95a..27c2f46778 100644 --- a/frontend/src/lib/components/details/DetailPageLayout.svelte +++ b/frontend/src/lib/components/details/DetailPageLayout.svelte @@ -26,6 +26,7 @@ | 'cli' | 'routes' | 'websockets' + | 'postgres' | 'scheduledPoll' | 'kafka' | 'nats' @@ -64,6 +65,7 @@ + @@ -110,6 +112,7 @@ + diff --git a/frontend/src/lib/components/details/DetailPageTriggerPanel.svelte b/frontend/src/lib/components/details/DetailPageTriggerPanel.svelte index 9eb1831cb8..c5d4938828 100644 --- a/frontend/src/lib/components/details/DetailPageTriggerPanel.svelte +++ b/frontend/src/lib/components/details/DetailPageTriggerPanel.svelte @@ -24,6 +24,7 @@ | 'routes' | 'websockets' | 'kafka' + | 'postgres' | 'nats' | 'scheduledPoll' = 'webhooks' export let simplfiedPoll: boolean = false @@ -66,6 +67,12 @@ Websockets + + + + Postgres + + @@ -97,6 +104,8 @@ {:else if triggerSelected === 'websockets'} + {:else if triggerSelected === 'postgres'} + {:else if triggerSelected === 'kafka' || triggerSelected === 'nats'}

diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte index a38d3659fe..b3994a707e 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersBadge.svelte @@ -1,5 +1,5 @@ + +{#if open} + +{/if} diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte new file mode 100644 index 0000000000..29bbbf0acd --- /dev/null +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte @@ -0,0 +1,596 @@ + + + + + + {#if !drawerLoading && can_write} + {#if edit} +
+ { + await PostgresTriggerService.setPostgresTriggerEnabled({ + path: initialPath, + workspace: $workspaceStore ?? '', + requestBody: { enabled: e.detail } + }) + sendUserToast( + `${e.detail ? 'enabled' : 'disabled'} postgres trigger ${initialPath}` + ) + }} + /> +
+ {/if} + + {/if} + + {#if drawerLoading} + + {:else} +
+ + {#if edit} + Changes can take up to 30 seconds to take effect. + {:else} + New postgres triggers can take up to 30 seconds to start listening. + {/if} + +
+
+
+ +
+ +
+

+ Pick a database to connect to +

+
+ + {#if postgres_resource_path} + + {#if config.show} + + {#if config.isLogical} + Your database is correctly configured with logical replication enabled. You can + proceed with using the streaming feature + {:else} + Logical replication is not enabled on your database. To use this feature, your + Postgres database must have wal_level configured as 'logical' in your + database configuration. + {/if} + + {/if} + {/if} +
+
+
+

+ Pick a script or flow to be triggered +

+
+ + + {#if script_path === undefined && is_flow === false} +
+ +
+ {/if} +
+
+ {#if postgres_resource_path} +
+
+

+ Choose which table of your database to track as well as what kind of transaction + should fire the script.
+ You must pick a database resource first to make the configuration of your trigger + +

+
+

+ Choose the types of database transactions that should trigger a script or flow. + You can select from Insert, Update, + Delete, or any combination of these operations to define when the + trigger should activate. +

+ +
+
+

+ Select the tables to track. You can choose to track + all tables in your database, + all tables within a specific schema, + specific tables in a schema, or even + specific columns of a table. Additionally, you can apply a + filter to retrieve only rows that do not match the specified criteria. +

+ +
Basic

Choose the relations to track without worrying about the + underlying mechanics of creating a + publication + or slot. This simplified option lets you focus only on + the data you want to monitor.

 +
Advanced

Select a specific publication from your database to + track, and manage it by creating, + updating, or deleting. For + slots, you can create or + delete + them. Both non-active slots and the + currently used slot by the trigger will be retrieved from + your database for management.

 + +
+ + + + +
+
+ { + replication_slot_name = '' + }} + > + + + + {#if selectedSlotAction === 'create'} +
+ + +
+ {:else} + + {/if} +
+
+ +
+
+ { + if (selectedPublicationAction === 'create') { + selectedTable = 'specific' + publication_name = `windmill_publication_${random_adj()}` + relations = [{ schema_name: 'public', table_to_track: [] }] + return + } + + publication_name = '' + relations = [] + transaction_to_track = [] + }} + > + + + + {#if selectedPublicationAction === 'create'} +
+ + +
+ {:else} + + {/if} + +
+
+ +
+ + +
+
+
+ {/if} +
+ {/if} + + diff --git a/frontend/src/lib/components/triggers/postgres/PostgresTriggersPanel.svelte b/frontend/src/lib/components/triggers/postgres/PostgresTriggersPanel.svelte new file mode 100644 index 0000000000..ffc8b4e45e --- /dev/null +++ b/frontend/src/lib/components/triggers/postgres/PostgresTriggersPanel.svelte @@ -0,0 +1,103 @@ + + + { + loadTriggers() + }} + bind:this={postgresTriggerEditor} +/> + +
+ {#if !newItem} + {#if isCloudHosted()} + + Postgres triggers are disabled in the multi-tenant cloud. + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {:else} + + {/if} + {/if} + + {#if databaseTriggers} + {#if databaseTriggers.length == 0} +
No Postgres triggers
+ {:else} +
+ {#each databaseTriggers as databaseTriggers (databaseTriggers.path)} +
+
{databaseTriggers.path}
+
+ +
+
+ {/each} +
+ {/if} + {:else} + + {/if} + + {#if newItem} + + Deploy the {isFlow ? 'flow' : 'script'} to add Postgres triggers. + + {/if} +
diff --git a/frontend/src/lib/components/triggers/postgres/PublicationPicker.svelte b/frontend/src/lib/components/triggers/postgres/PublicationPicker.svelte new file mode 100644 index 0000000000..86f035f691 --- /dev/null +++ b/frontend/src/lib/components/triggers/postgres/PublicationPicker.svelte @@ -0,0 +1,132 @@ + + + + +
+ + + {#each v.table_to_track as table_to_track, j} +
+
+ + + + + +
+
+ {/each} + +
+ +
+ {/each} + {/if} +
+ +
+ {/if} + diff --git a/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte b/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte new file mode 100644 index 0000000000..b0a22b5907 --- /dev/null +++ b/frontend/src/lib/components/triggers/postgres/SlotPicker.svelte @@ -0,0 +1,98 @@ + + + + +
+ +
+
Filter by path of
+ + + + +
+ + +
+ {#if $userStore?.is_super_admin && $userStore.username.includes('@')} + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
+
+ {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} +
No postgres triggers
+ {:else if items?.length} +
+ {#each items.slice(0, nbDisplayed) as { postgres_resource_path, path, edited_by, error, edited_at, script_path, is_flow, extra_perms, canWrite, enabled, server_id } (path)} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + {@const ping = new Date()} + {@const pinging = ping && ping.getTime() > new Date().getTime() - 15 * 1000} + +
+
+ + + postgresTriggerEditor?.openEdit(path, is_flow)} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
+ {path} +
+
+ {postgres_resource_path} +
+
+ runnable: {script_path} +
+ + + + +
+ {#if (enabled && (!pinging || error)) || (!enabled && error) || (enabled && !server_id)} + + + + + +
+ {#if enabled} + {#if !server_id} + Postgres trigger is starting... + {:else} + Could not connect to database{error ? ': ' + error : ''} + {/if} + {:else} + Disabled because of an error: {error} + {/if} +
+ + {:else if enabled} + + + + +
+ Connected to database{!server_id ? ' (shutting down...)' : ''}
+ + {/if} +
+ + { + setTriggerEnabled(path, e.detail) + }} + /> + +
+ + { + goto(href) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite, + action: async () => { + await PostgresTriggerService.deletePostgresTrigger({ + workspace: $workspaceStore ?? '', + path + }) + loadTriggers() + } + }, + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + postgresTriggerEditor?.openEdit(path, is_flow) + } + }, + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: canWrite ? 'Share' : 'See Permissions', + icon: Share, + action: () => { + shareModal.openDrawer(path, 'websocket_trigger') + } + } + ]} + /> +
+
+
+
edited by {edited_by}
the {displayDate(edited_at)}
+
+ {/each} +
+ {:else} + + {/if} + + {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} + + + { + loadTriggers() + }} +/> diff --git a/frontend/src/routes/(root)/(logged)/routes/+page.svelte b/frontend/src/routes/(root)/(logged)/routes/+page.svelte index 2aae39245d..6666df29ec 100644 --- a/frontend/src/routes/(root)/(logged)/routes/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/routes/+page.svelte @@ -6,7 +6,6 @@ import { Button, Skeleton } from '$lib/components/common' import Dropdown from '$lib/components/DropdownV2.svelte' import PageHeader from '$lib/components/PageHeader.svelte' - import RouteEditor from '$lib/components/triggers/RouteEditor.svelte' import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' @@ -21,6 +20,7 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { setQuery } from '$lib/navigation' import { onMount } from 'svelte' + import RouteEditor from '$lib/components/triggers/http/RouteEditor.svelte' type TriggerW = HttpTrigger & { canWrite: boolean } diff --git a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte index 113dda4b99..adb7e73203 100644 --- a/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/scripts/get/[...hash]/+page.svelte @@ -18,7 +18,12 @@ } from '$lib/utils' import Tooltip from '$lib/components/Tooltip.svelte' import ShareModal from '$lib/components/ShareModal.svelte' - import { enterpriseLicense, hubBaseUrlStore, userStore, workspaceStore } from '$lib/stores' + import { + enterpriseLicense, + hubBaseUrlStore, + userStore, + workspaceStore + } from '$lib/stores' import { isDeployable, ALL_DEPLOYABLE } from '$lib/utils_deployable' import { onDestroy } from 'svelte' @@ -42,7 +47,7 @@ import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' import SavedInputs from '$lib/components/SavedInputs.svelte' - import WebhooksPanel from '$lib/components/triggers/WebhooksPanel.svelte' + import WebhooksPanel from '$lib/components/triggers/webhook/WebhooksPanel.svelte' import DetailPageLayout from '$lib/components/details/DetailPageLayout.svelte' import DetailPageHeader from '$lib/components/details/DetailPageHeader.svelte' import CliHelpBox from '$lib/components/CliHelpBox.svelte' @@ -78,14 +83,15 @@ import EmailTriggerPanel from '$lib/components/details/EmailTriggerPanel.svelte' import Star from '$lib/components/Star.svelte' import LogViewer from '$lib/components/LogViewer.svelte' - import RoutesPanel from '$lib/components/triggers/RoutesPanel.svelte' + import RoutesPanel from '$lib/components/triggers/http/RoutesPanel.svelte' import { Highlight } from 'svelte-highlight' import json from 'svelte-highlight/languages/json' import { writable } from 'svelte/store' import TriggersBadge from '$lib/components/graph/renderers/triggers/TriggersBadge.svelte' - import WebsocketTriggersPanel from '$lib/components/triggers/WebsocketTriggersPanel.svelte' - import KafkaTriggersPanel from '$lib/components/triggers/KafkaTriggersPanel.svelte' - import NatsTriggersPanel from '$lib/components/triggers/NatsTriggersPanel.svelte' + import WebsocketTriggersPanel from '$lib/components/triggers/websocket/WebsocketTriggersPanel.svelte' + import KafkaTriggersPanel from '$lib/components/triggers/kafka/KafkaTriggersPanel.svelte' + import NatsTriggersPanel from '$lib/components/triggers/nats/NatsTriggersPanel.svelte' + import PostgresTriggersPanel from '$lib/components/triggers/postgres/PostgresTriggersPanel.svelte' let script: Script | undefined let topHash: string | undefined @@ -728,6 +734,10 @@ + +
+ +
diff --git a/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte index 2009ca3b8f..76d2a34da3 100644 --- a/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/websocket_triggers/+page.svelte @@ -26,9 +26,9 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import { setQuery } from '$lib/navigation' import { onDestroy, onMount } from 'svelte' - import WebsocketTriggerEditor from '$lib/components/triggers/WebsocketTriggerEditor.svelte' import Popover from '$lib/components/Popover.svelte' import { isCloudHosted } from '$lib/cloud' + import WebsocketTriggerEditor from '$lib/components/triggers/websocket/WebsocketTriggerEditor.svelte' type TriggerW = WebsocketTrigger & { canWrite: boolean } @@ -233,7 +233,7 @@
Filter by path of
- +
From 74bb660b9799e79e31c33d96a4f06d77094287d5 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 24 Jan 2025 17:32:29 +0300 Subject: [PATCH 4/6] fix(bun, windows): Fix "A required privilege is not held by the client." (#5126) --- backend/windmill-worker/src/bun_executor.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index dfe9c5b328..77bad32a5d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -906,7 +906,7 @@ pub async fn handle_bun_job( #[cfg(windows)] { target = format!("{job_dir}\\main.js"); - symlink = std::os::windows::fs::symlink_dir(&local_path, &target); + symlink = std::fs::hard_link(&local_path, &target); } symlink.map_err(|e| { From 1eaef8542b6f5f30f85841b253256a67272b6647 Mon Sep 17 00:00:00 2001 From: dieriba Date: Fri, 24 Jan 2025 17:07:48 +0100 Subject: [PATCH 5/6] fix pg trigger display (#5127) --- backend/windmill-api/openapi.yaml | 3 +++ .../routes/(root)/(logged)/postgres_triggers/+page.svelte | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 82bd7a9c0e..537ae2892c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -13221,6 +13221,9 @@ components: type: string error: type: string + last_server_ping: + type: string + format: date-time required: - path - script_path diff --git a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte index 5459fe58eb..c69a447f3b 100644 --- a/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte @@ -55,7 +55,10 @@ if (newTrigger) { triggers[i] = { ...triggers[i], - enabled: newTrigger.enabled + error: newTrigger.error, + last_server_ping: newTrigger.last_server_ping, + enabled: newTrigger.enabled, + server_id: newTrigger.server_id } } } From 1a1ea682166af62133d3df2816490e3ebc92462f Mon Sep 17 00:00:00 2001 From: Alexander Petric Date: Fri, 24 Jan 2025 12:17:00 -0500 Subject: [PATCH 6/6] feat(frontend): operator visibility setting per workspace (#5124) * feat(frontend): operator visibility setting per workspace * section + reactivity * sqlx prep * npm check * more sqlx * fix table bg dark mode * improve db query / reactivity * sqlx prepare * more merge fixing * npm check * npm check --- ...c7ba2f60f41e1abb6ff8332bd2dfa7379d14.json} | 12 +- ...c61296a3ff7489ae12f52a19f9543173ac597.json | 6 + ...dcc40f463cbc52d94ed9315cf9a547d4c89f2.json | 6 + ...5a0652d6a72024a0c17f5271add38168d1ab3.json | 15 ++ ...49bb6cb06581079b5fc5bea5ddd8e7a895b79.json | 23 --- .../20250122211942_operator_settings.down.sql | 1 + .../20250122211942_operator_settings.up.sql | 11 ++ backend/windmill-api/openapi.yaml | 71 ++++++++- backend/windmill-api/src/workspaces.rs | 46 +++++- .../settings/WorkspaceOperatorSettings.svelte | 144 ++++++++++++++++++ .../settings/WorkspaceUserSettings.svelte | 4 +- .../components/sidebar/OperatorMenu.svelte | 38 +++-- frontend/src/lib/stores.ts | 6 +- .../(root)/(logged)/audit_logs/+page.svelte | 9 +- .../(root)/(logged)/folders/+page.svelte | 9 +- .../(root)/(logged)/groups/+page.svelte | 9 +- .../(logged)/kafka_triggers/+page.svelte | 9 +- .../(logged)/nats_triggers/+page.svelte | 9 +- .../(root)/(logged)/resources/+page.svelte | 9 +- .../(root)/(logged)/routes/+page.svelte | 9 +- .../(logged)/runs/[...path]/+page.svelte | 9 +- .../(root)/(logged)/schedules/+page.svelte | 12 +- .../(root)/(logged)/variables/+page.svelte | 9 +- .../(root)/(logged)/workers/+page.svelte | 9 +- 24 files changed, 433 insertions(+), 52 deletions(-) rename backend/.sqlx/{query-6c845f168b6265e6cf92e4d33c5409edfdc1847d0348df2ea55504ddaaa67736.json => query-1452033a8e2b160883a649c986d6c7ba2f60f41e1abb6ff8332bd2dfa7379d14.json} (53%) create mode 100644 backend/.sqlx/query-597b89889c3820fe7986b834c0e5a0652d6a72024a0c17f5271add38168d1ab3.json delete mode 100644 backend/.sqlx/query-64920b845c0ce81fb99497c03b249bb6cb06581079b5fc5bea5ddd8e7a895b79.json create mode 100644 backend/migrations/20250122211942_operator_settings.down.sql create mode 100644 backend/migrations/20250122211942_operator_settings.up.sql create mode 100644 frontend/src/lib/components/settings/WorkspaceOperatorSettings.svelte diff --git a/backend/.sqlx/query-6c845f168b6265e6cf92e4d33c5409edfdc1847d0348df2ea55504ddaaa67736.json b/backend/.sqlx/query-1452033a8e2b160883a649c986d6c7ba2f60f41e1abb6ff8332bd2dfa7379d14.json similarity index 53% rename from backend/.sqlx/query-6c845f168b6265e6cf92e4d33c5409edfdc1847d0348df2ea55504ddaaa67736.json rename to backend/.sqlx/query-1452033a8e2b160883a649c986d6c7ba2f60f41e1abb6ff8332bd2dfa7379d14.json index b4199d4ca1..1bad8b94d0 100644 --- a/backend/.sqlx/query-6c845f168b6265e6cf92e4d33c5409edfdc1847d0348df2ea55504ddaaa67736.json +++ b/backend/.sqlx/query-1452033a8e2b160883a649c986d6c7ba2f60f41e1abb6ff8332bd2dfa7379d14.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", + "query": "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color,\n CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings\n FROM workspace\n JOIN usr ON usr.workspace_id = workspace.id\n JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id\n WHERE usr.email = $1 AND workspace.deleted = false", "describe": { "columns": [ { @@ -22,6 +22,11 @@ "ordinal": 3, "name": "color", "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "operator_settings", + "type_info": "Jsonb" } ], "parameters": { @@ -33,8 +38,9 @@ false, false, false, - true + true, + null ] }, - "hash": "6c845f168b6265e6cf92e4d33c5409edfdc1847d0348df2ea55504ddaaa67736" + "hash": "1452033a8e2b160883a649c986d6c7ba2f60f41e1abb6ff8332bd2dfa7379d14" } diff --git a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json index 64d273ed70..b6fee2c5ff 100644 --- a/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json +++ b/backend/.sqlx/query-1730f39fd1793d45fbb41b21389c61296a3ff7489ae12f52a19f9543173ac597.json @@ -127,6 +127,11 @@ "ordinal": 24, "name": "color", "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" } ], "parameters": { @@ -159,6 +164,7 @@ true, true, true, + true, true ] }, diff --git a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json index 6092b0551a..baf7cb42f5 100644 --- a/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json +++ b/backend/.sqlx/query-55cb03040bc2a8c53dd7fbb42bbdcc40f463cbc52d94ed9315cf9a547d4c89f2.json @@ -127,6 +127,11 @@ "ordinal": 24, "name": "color", "type_info": "Varchar" + }, + { + "ordinal": 25, + "name": "operator_settings", + "type_info": "Jsonb" } ], "parameters": { @@ -159,6 +164,7 @@ true, true, true, + true, true ] }, diff --git a/backend/.sqlx/query-597b89889c3820fe7986b834c0e5a0652d6a72024a0c17f5271add38168d1ab3.json b/backend/.sqlx/query-597b89889c3820fe7986b834c0e5a0652d6a72024a0c17f5271add38168d1ab3.json new file mode 100644 index 0000000000..57989bb6de --- /dev/null +++ b/backend/.sqlx/query-597b89889c3820fe7986b834c0e5a0652d6a72024a0c17f5271add38168d1ab3.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE workspace_settings SET operator_settings = $1 WHERE workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Text" + ] + }, + "nullable": [] + }, + "hash": "597b89889c3820fe7986b834c0e5a0652d6a72024a0c17f5271add38168d1ab3" +} diff --git a/backend/.sqlx/query-64920b845c0ce81fb99497c03b249bb6cb06581079b5fc5bea5ddd8e7a895b79.json b/backend/.sqlx/query-64920b845c0ce81fb99497c03b249bb6cb06581079b5fc5bea5ddd8e7a895b79.json deleted file mode 100644 index d201c1c08e..0000000000 --- a/backend/.sqlx/query-64920b845c0ce81fb99497c03b249bb6cb06581079b5fc5bea5ddd8e7a895b79.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select path from script where hash = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "64920b845c0ce81fb99497c03b249bb6cb06581079b5fc5bea5ddd8e7a895b79" -} diff --git a/backend/migrations/20250122211942_operator_settings.down.sql b/backend/migrations/20250122211942_operator_settings.down.sql new file mode 100644 index 0000000000..4831bcdf10 --- /dev/null +++ b/backend/migrations/20250122211942_operator_settings.down.sql @@ -0,0 +1 @@ +ALTER TABLE workspace_settings DROP COLUMN operator_settings; \ No newline at end of file diff --git a/backend/migrations/20250122211942_operator_settings.up.sql b/backend/migrations/20250122211942_operator_settings.up.sql new file mode 100644 index 0000000000..22bbddb17c --- /dev/null +++ b/backend/migrations/20250122211942_operator_settings.up.sql @@ -0,0 +1,11 @@ +ALTER TABLE workspace_settings ADD COLUMN operator_settings JSONB DEFAULT '{ + "runs": true, + "groups": true, + "folders": true, + "workers": true, + "triggers": true, + "resources": true, + "schedules": true, + "variables": true, + "audit_logs": true +}'; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 537ae2892c..6819bb5553 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1618,6 +1618,29 @@ paths: schema: $ref: "#/components/schemas/User" + /w/{workspace}/workspaces/operator_settings: + post: + operationId: updateOperatorSettings + summary: Update operator settings for a workspace + description: Updates the operator settings for a specific workspace. Requires workspace admin privileges. + tags: + - workspace + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/OperatorSettings" + responses: + '200': + description: Operator settings updated successfully + content: + text/plain: + schema: + type: string + /users/exists/{email}: get: summary: exists email @@ -1745,6 +1768,8 @@ paths: type: boolean color: type: string + operator_settings: + $ref: "#/components/schemas/OperatorSettings" required: - code_completion_enabled - automatic_billing @@ -13643,6 +13668,8 @@ components: type: string color: type: string + operator_settings: + $ref: "#/components/schemas/OperatorSettings" required: - id - name @@ -14501,6 +14528,48 @@ components: required: - trigger_kind + OperatorSettings: + nullable: true + type: object + required: + - runs + - schedules + - resources + - variables + - triggers + - audit_logs + - groups + - folders + - workers + properties: + runs: + type: boolean + description: Whether operators can view runs + schedules: + type: boolean + description: Whether operators can view schedules + resources: + type: boolean + description: Whether operators can view resources + variables: + type: boolean + description: Whether operators can view variables + audit_logs: + type: boolean + description: Whether operators can view audit logs + triggers: + type: boolean + description: Whether operators can view triggers + groups: + type: boolean + description: Whether operators can view groups page + folders: + type: boolean + description: Whether operators can view folders page + workers: + type: boolean + description: Whether operators can view workers page + TeamInfo: type: object required: @@ -14521,7 +14590,7 @@ components: description: List of channels within the team items: $ref: '#/components/schemas/ChannelInfo' - + ChannelInfo: type: object required: diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index a4adc61977..3758f5def4 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -121,7 +121,8 @@ pub fn workspaced_service() -> Router { "/critical_alerts/acknowledge_all", post(acknowledge_all_critical_alerts), ) - .route("/critical_alerts/mute", post(mute_critical_alerts)); + .route("/critical_alerts/mute", post(mute_critical_alerts)) + .route("/operator_settings", post(update_operator_settings)); #[cfg(feature = "stripe")] { @@ -188,6 +189,7 @@ pub struct WorkspaceSettings { pub default_scripts: Option, pub mute_critical_alerts: Option, pub color: Option, + pub operator_settings: Option, } #[derive(FromRow, Serialize, Debug)] @@ -286,6 +288,7 @@ struct UserWorkspace { pub name: String, pub username: String, pub color: Option, + pub operator_settings: Option>, } #[derive(Deserialize)] @@ -1366,7 +1369,8 @@ async fn user_workspaces( let mut tx = db.begin().await?; let workspaces = sqlx::query_as!( UserWorkspace, - "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color + "SELECT workspace.id, workspace.name, usr.username, workspace_settings.color, + CASE WHEN usr.operator THEN workspace_settings.operator_settings ELSE NULL END as operator_settings FROM workspace JOIN usr ON usr.workspace_id = workspace.id JOIN workspace_settings ON workspace_settings.workspace_id = workspace.id @@ -2135,3 +2139,41 @@ async fn mute_critical_alerts( pub async fn mute_critical_alerts() -> Error { Error::NotFound("Critical Alerts require EE".to_string()) } + +#[derive(Deserialize, Serialize)] +struct ChangeOperatorSettings { + runs: bool, + schedules: bool, + resources: bool, + variables: bool, + triggers: bool, + audit_logs: bool, + groups: bool, + folders: bool, + workers: bool, +} + +async fn update_operator_settings( + authed: ApiAuthed, + Path(w_id): Path, + Extension(db): Extension, + Json(settings): Json, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + + let mut tx = db.begin().await?; + + let settings_json = serde_json::json!(settings); + + sqlx::query!( + "UPDATE workspace_settings SET operator_settings = $1 WHERE workspace_id = $2", + settings_json, + &w_id + ) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok("Operator settings updated successfully".to_string()) +} diff --git a/frontend/src/lib/components/settings/WorkspaceOperatorSettings.svelte b/frontend/src/lib/components/settings/WorkspaceOperatorSettings.svelte new file mode 100644 index 0000000000..f12062493f --- /dev/null +++ b/frontend/src/lib/components/settings/WorkspaceOperatorSettings.svelte @@ -0,0 +1,144 @@ + + +
+
+
+
+
+ Configure the operator visibility settings for your workspace. Toggle the settings you + want to enable. +
+
+
+ +
+
+
+ +
+ + + + Section + Description + + + + + + + + + + {#each Object.entries(descriptions) as [key, { title, description }]} + + {title} + {description} + + + + + + + + {/each} + + +
+ +
+ +
+
+
diff --git a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte index 43aa4af06c..11ed4ae402 100644 --- a/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte +++ b/frontend/src/lib/components/settings/WorkspaceUserSettings.svelte @@ -3,7 +3,7 @@ import { Badge, Button, Popup, Skeleton } from '$lib/components/common' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' - + import WorkspaceOperatorSettings from '$lib/components/settings/WorkspaceOperatorSettings.svelte' import InviteUser from '$lib/components/InviteUser.svelte' import PageHeader from '$lib/components/PageHeader.svelte' @@ -441,6 +441,8 @@
+ + {#if showInvites} + link.id === 'home' || + ($userWorkspaces && $workspaceStore && $userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.[link.id] === true) + ) - let secondMenuLinks = [ + $: secondMenuLinks = [ { label: 'Resources', + id: 'resources', href: `${base}/resources` }, { label: 'Variables', + id: 'variables', href: `${base}/variables` }, { label: 'Custom HTTP routes', + id: 'triggers', href: `${base}/routes` }, { label: 'Websocket triggers', + id: 'triggers', href: `${base}/websocket_triggers` }, { label: 'Postgres triggers', - href: `${base}/kafka_triggers` + id: 'triggers', + href: `${base}/postgres_triggers` }, { label: 'Kafka triggers', + id: 'triggers', href: `${base}/kafka_triggers` }, { label: 'NATS triggers', + id: 'triggers', href: `${base}/nats_triggers` }, { label: 'Audit logs', + id: 'audit_logs', href: `${base}/audit_logs` }, { label: 'Groups', + id: 'groups', href: `${base}/groups` }, { label: 'Folders', + id: 'folders', href: `${base}/folders` }, { label: 'Workers', + id: 'workers', href: `${base}/workers` } - ] + ].filter((link) => { + if (!$userWorkspaces || !$workspaceStore) return false; + return $userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.[link.id] === true + }) let moreOpen = false @@ -208,7 +226,7 @@ class="divide-y" role="none" > - {#if moreOpen == false} + {#if moreOpen == false && secondMenuLinks.length > 0}
More...
{:else} {#each secondMenuLinks as menuLink (menuLink.href ?? menuLink.label)} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 353eeeac7c..98886e1bc0 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -1,9 +1,11 @@ import { BROWSER } from 'esm-env' import { derived, type Readable, writable } from 'svelte/store' + import { type WorkspaceDefaultScripts, type TokenResponse, type UserWorkspaceList, + type OperatorSettings } from './gen' import type { IntrospectionQuery } from 'graphql' import { getLocalSetting } from './utils' @@ -60,6 +62,7 @@ export const userWorkspaces: Readable< name: string username: string color: string | null + operator_settings?: OperatorSettings }> > = derived([usersWorkspaceStore, superadmin], ([store, superadmin]) => { const originalWorkspaces = store?.workspaces ?? [] @@ -70,7 +73,8 @@ export const userWorkspaces: Readable< id: 'admins', name: 'Admins', username: 'superadmin', - color: null + color: null, + operator_settings: null } ] } else { diff --git a/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte b/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte index 22f301af5b..498a0d1293 100644 --- a/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/audit_logs/+page.svelte @@ -12,7 +12,7 @@ import SplitPanesWrapper from '$lib/components/splitPanes/SplitPanesWrapper.svelte' import type { AuditLog } from '$lib/gen' - import { enterpriseLicense } from '$lib/stores' + import { enterpriseLicense, userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Splitpanes, Pane } from 'svelte-splitpanes' let username: string = $page.url.searchParams.get('username') ?? 'all' @@ -32,6 +32,12 @@ let auditLogDrawer: Drawer +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.audit_logs} + +{:else}
@@ -118,6 +124,7 @@ />
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/folders/+page.svelte b/frontend/src/routes/(root)/(logged)/folders/+page.svelte index 4bcbc02c8c..090223ebbd 100644 --- a/frontend/src/routes/(root)/(logged)/folders/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/folders/+page.svelte @@ -6,7 +6,7 @@ import Dropdown from '$lib/components/DropdownV2.svelte' import FolderEditor from '$lib/components/FolderEditor.svelte' import PageHeader from '$lib/components/PageHeader.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Button, Drawer, DrawerContent, Popup, Skeleton } from '$lib/components/common' import FolderInfo from '$lib/components/FolderInfo.svelte' import FolderUsageInfo from '$lib/components/FolderUsageInfo.svelte' @@ -78,6 +78,12 @@ +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.folders} + +{:else}
+{/if} diff --git a/frontend/src/routes/(root)/(logged)/groups/+page.svelte b/frontend/src/routes/(root)/(logged)/groups/+page.svelte index d1fc02a826..93fc04d45d 100644 --- a/frontend/src/routes/(root)/(logged)/groups/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/groups/+page.svelte @@ -11,7 +11,7 @@ import PageHeader from '$lib/components/PageHeader.svelte' import SharedBadge from '$lib/components/SharedBadge.svelte' import TableCustom from '$lib/components/TableCustom.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { canWrite } from '$lib/utils' import { Pen, Plus, Trash } from 'lucide-svelte' import DataTable from '$lib/components/table/DataTable.svelte' @@ -73,6 +73,12 @@ +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.groups} + +{:else} {/if} +{/if} diff --git a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte index f66a7beca1..13c9cfd1d9 100644 --- a/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/kafka_triggers/+page.svelte @@ -15,7 +15,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Code, Eye, Pen, Plus, Share, Trash, Circle } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -208,6 +208,12 @@ f={(x) => (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.triggers} + +{:else} {/if} +{/if} (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.triggers} + +{:else} {/if} +{/if} x.path + ' ' + x.resource_type + ' ' + x.description + ' '} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.resources} + +{:else} +{/if} diff --git a/frontend/src/routes/(root)/(logged)/routes/+page.svelte b/frontend/src/routes/(root)/(logged)/routes/+page.svelte index 6666df29ec..bd12e6d5eb 100644 --- a/frontend/src/routes/(root)/(logged)/routes/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/routes/+page.svelte @@ -9,7 +9,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Route, Code, Eye, Pen, Plus, Share, Trash } from 'lucide-svelte' import { goto } from '$lib/navigation' import SearchItems from '$lib/components/SearchItems.svelte' @@ -156,6 +156,12 @@ f={(x) => (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.triggers} + +{:else} {/if} +{/if} +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.runs} + +{:else} {#if innerWidth > 900}
@@ -1371,3 +1377,4 @@
{/if} +{/if} diff --git a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte index f440e54600..cad5a7ee4f 100644 --- a/frontend/src/routes/(root)/(logged)/schedules/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/schedules/+page.svelte @@ -11,7 +11,7 @@ import SharedBadge from '$lib/components/SharedBadge.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import Toggle from '$lib/components/Toggle.svelte' - import { userStore, workspaceStore } from '$lib/stores' + import { userStore, workspaceStore, userWorkspaces } from '$lib/stores' import { Calendar, Circle, @@ -232,9 +232,12 @@ } onMount(() => { + console.log(`on mount: `, $userStore?.operator, $workspaceStore, $userWorkspaces) loadQueryFilters() }) + $: $userWorkspaces && $userStore && $workspaceStore && console.log(`user workspaces: `, $userWorkspaces) + $: updateQueryFilters(selectedFilterKind, filterUserFolders, filterEnabledDisabled) @@ -247,6 +250,12 @@ f={(x) => (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.schedules} + +{:else} {/if} +{/if} x.path + ' ' + x.description} /> +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.variables} + +{:else} {/if} +{/if} +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find(_ => _.id === $workspaceStore)?.operator_settings?.workers} + +{:else} {/if} +{/if}