diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index 5d01baafbd..08a3d179f0 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -60,7 +60,7 @@ use windmill_common::{ global_settings::{ AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, - DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -168,6 +168,14 @@ pub fn global_service() -> Router { "/refresh_custom_instance_user_pwd", post(refresh_custom_instance_user_pwd), ) + .route( + "/external_instance_pg/status", + get(get_external_instance_pg_status), + ) + .route( + "/external_instance_pg/setup", + post(setup_external_instance_pg), + ) .route( "/setup_custom_instance_pg_database/{name}", post(setup_custom_instance_pg_database), @@ -938,6 +946,13 @@ async fn run_setting_pre_write_hook( value: &serde_json::Value, ) -> error::Result<()> { match key { + EXTERNAL_INSTANCE_PG_SETTING => { + windmill_common::external_instance_pg::check_external_instance_pg_write( + db, + Some(value), + ) + .await?; + } // The instance AI config is written as an untyped blob through this generic // endpoint, so it never passes the typed check the workspace handler applies. // Rates that reach a cost total unbounded would make it negative or infinite. @@ -1289,6 +1304,14 @@ async fn set_instance_config( for (key, value) in &settings_diff.upserts { run_setting_pre_write_hook(&db, key, value).await?; } + if settings_diff + .deletes + .iter() + .any(|k| k == EXTERNAL_INSTANCE_PG_SETTING) + { + windmill_common::external_instance_pg::check_external_instance_pg_write(&db, None) + .await?; + } instance_config::apply_settings_diff(&db, &settings_diff) .await @@ -1743,6 +1766,54 @@ async fn refresh_custom_instance_user_pwd( Ok(Json(())) } +async fn get_external_instance_pg_status( + authed: ApiAuthed, + Extension(db): Extension, +) -> JsonResult { + require_super_admin(&db, &authed).await?; + Ok(Json( + windmill_common::external_instance_pg::external_instance_pg_status(&db).await?, + )) +} + +#[derive(Deserialize)] +struct SetupExternalInstancePgBody { + #[serde(default)] + rotate_passwords: bool, +} + +async fn setup_external_instance_pg( + authed: ApiAuthed, + Extension(db): Extension, + Json(body): Json, +) -> JsonResult { + require_super_admin(&db, &authed).await?; + let report = windmill_common::external_instance_pg::setup_external_instance_pg_unchecked( + &db, + body.rotate_passwords, + ) + .await?; + let rotated = body.rotate_passwords.to_string(); + let success = report.success.to_string(); + windmill_audit::audit_oss::audit_log( + &db, + &authed, + "settings.setup_external_instance_pg", + windmill_audit::ActionKind::Update, + "global", + Some(&authed.email), + Some( + [ + ("rotate_passwords", rotated.as_str()), + ("success", success.as_str()), + ] + .into(), + ), + ) + .await?; + Ok(Json(report)) +} + #[derive(Deserialize)] struct SetupCustomInstanceDbBody { tag: Option, diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a3292ecfbc..7358d2b889 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1572,6 +1572,43 @@ paths: schema: type: object + /settings/external_instance_pg/status: + get: + summary: Returns whether the external instance cluster is configured and how its last setup went + operationId: getExternalInstancePgStatus + tags: + - setting + responses: + "200": + description: external instance cluster status + content: + application/json: + schema: + $ref: "#/components/schemas/ExternalInstancePgStatus" + + /settings/external_instance_pg/setup: + post: + summary: Sets up the external instance cluster with its saved admin login, optionally rotating the passwords Windmill manages on it (enterprise edition only) + operationId: setupExternalInstancePg + tags: + - setting + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + rotate_passwords: + type: boolean + responses: + "200": + description: the setup report, also stored as the last setup + content: + application/json: + schema: + $ref: "#/components/schemas/ExternalInstancePgSetupReport" + /settings/list_custom_instance_pg_databases: post: summary: Returns the set-up statuses of custom instance pg databases @@ -33621,6 +33658,44 @@ components: - ducklake - datatable + ExternalInstancePgSetupStep: + type: object + required: [name, status, message] + properties: + name: + type: string + status: + type: string + enum: [ok, warning, error] + message: + type: string + + ExternalInstancePgSetupReport: + type: object + required: [success, finished_at, steps] + properties: + success: + type: boolean + description: no step failed; warnings leave it true + finished_at: + type: string + format: date-time + steps: + type: array + items: + $ref: "#/components/schemas/ExternalInstancePgSetupStep" + + ExternalInstancePgStatus: + type: object + required: [configured, database_count] + properties: + configured: + type: boolean + database_count: + type: integer + last_setup: + $ref: "#/components/schemas/ExternalInstancePgSetupReport" + InstanceDatatableRole: type: object required: [id, name, enabled] diff --git a/backend/windmill-common/src/external_instance_pg.rs b/backend/windmill-common/src/external_instance_pg.rs new file mode 100644 index 0000000000..f7362c6cfa --- /dev/null +++ b/backend/windmill-common/src/external_instance_pg.rs @@ -0,0 +1,171 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The external Postgres cluster behind `external_instance` data tables and Ducklake catalogs. +//! +//! Windmill administers that cluster itself, logged in as the user in +//! [`EXTERNAL_INSTANCE_PG_SETTING`]. It creates `custom_instance_user` and +//! `custom_instance_replication_user` there, with passwords it generates and keeps in +//! [`EXTERNAL_INSTANCE_PG_STATE_SETTING`]. They share their names with the roles on Windmill's own +//! cluster, but they are different roles with different passwords. +//! +//! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not +//! supported: each would keep resetting the passwords the other depends on. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::{ + error::{Error, Result}, + global_settings::{EXTERNAL_INSTANCE_PG_SETTING, EXTERNAL_INSTANCE_PG_STATE_SETTING}, + instance_config::{CustomInstanceDb, ExternalInstancePg}, + DB, +}; + +/// What Windmill keeps about the external cluster. Server-managed and hidden: never part of the +/// instance config, never readable by an agent worker. No `Debug`: it carries live passwords. +#[derive(Serialize, Deserialize, Clone, Default)] +pub struct ExternalInstancePgState { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub user_pwd: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replication_pwd: Option, + /// The databases Windmill created on the cluster. It only ever drops one of these. + #[serde(default)] + pub databases: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_setup: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ExternalInstancePgSetupReport { + /// No step failed. Warnings leave it true. + pub success: bool, + pub finished_at: chrono::DateTime, + pub steps: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct ExternalInstancePgSetupStep { + pub name: String, + pub status: SetupStepStatus, + pub message: String, +} + +#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SetupStepStatus { + Ok, + Warning, + Error, +} + +/// The status the settings page shows without running anything. +#[derive(Serialize, Debug)] +pub struct ExternalInstancePgStatus { + pub configured: bool, + pub database_count: usize, + pub last_setup: Option, +} + +pub async fn read_external_instance_pg_config(db: &DB) -> Result> { + let value = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + EXTERNAL_INSTANCE_PG_SETTING + ) + .fetch_optional(db) + .await?; + value + .map(|v| { + serde_json::from_value(v).map_err(|e| { + Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_SETTING}: {e}")) + }) + }) + .transpose() +} + +pub async fn read_external_instance_pg_state<'c>( + executor: impl sqlx::PgExecutor<'c>, +) -> Result { + let value = sqlx::query_scalar!( + "SELECT value FROM global_settings WHERE name = $1", + EXTERNAL_INSTANCE_PG_STATE_SETTING + ) + .fetch_optional(executor) + .await?; + match value { + None => Ok(ExternalInstancePgState::default()), + Some(v) => serde_json::from_value(v).map_err(|e| { + Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_STATE_SETTING}: {e}")) + }), + } +} + +pub async fn external_instance_pg_status(db: &DB) -> Result { + let configured = read_external_instance_pg_config(db).await?.is_some(); + let state = read_external_instance_pg_state(db).await?; + Ok(ExternalInstancePgStatus { + configured, + database_count: state.databases.len(), + last_setup: state.last_setup, + }) +} + +/// Refuse to unset the cluster while Windmill still has databases on it: every data table and +/// Ducklake catalog there would stop resolving. Allowed on every edition, so a downgraded +/// instance can still clear a setting it no longer uses. +pub async fn ensure_external_instance_pg_removable(db: &DB) -> Result<()> { + let state = read_external_instance_pg_state(db).await?; + if state.databases.is_empty() { + return Ok(()); + } + let names = state + .databases + .keys() + .cloned() + .collect::>() + .join(", "); + Err(Error::BadRequest(format!( + "The external instance cluster still holds databases Windmill created ({names}). Drop \ + them before removing {EXTERNAL_INSTANCE_PG_SETTING}." + ))) +} + +/// Check a write to [`EXTERNAL_INSTANCE_PG_SETTING`] before it happens: `None`, null or an empty +/// string unsets it. Every writer of global settings calls this, the per-key and bulk endpoints +/// as well as the declarative sync. +pub async fn check_external_instance_pg_write( + db: &DB, + value: Option<&serde_json::Value>, +) -> Result<()> { + match value { + None | Some(serde_json::Value::Null) => ensure_external_instance_pg_removable(db).await, + Some(serde_json::Value::String(s)) if s.trim().is_empty() => { + ensure_external_instance_pg_removable(db).await + } + Some(value) => { + crate::external_instance_pg_oss::validate_external_instance_pg_setting(value) + } + } +} + +/// Converge the external cluster on the configured login: check what it can do, create or update +/// Windmill's two roles with the stored passwords, and report anything that would get in the way. +/// With `rotate_passwords`, generate new passwords first. Safe to run again; running it again is +/// how a failed rotation is repaired. +/// +/// Authorization: administers the external cluster with its admin credentials and checks nothing. +/// Callers MUST be superadmin. +pub async fn setup_external_instance_pg_unchecked( + db: &DB, + rotate_passwords: bool, +) -> Result { + crate::external_instance_pg_oss::setup_external_instance_pg_unchecked(db, rotate_passwords) + .await +} diff --git a/backend/windmill-common/src/external_instance_pg_oss.rs b/backend/windmill-common/src/external_instance_pg_oss.rs new file mode 100644 index 0000000000..6dcd417336 --- /dev/null +++ b/backend/windmill-common/src/external_instance_pg_oss.rs @@ -0,0 +1,43 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Where the external instance cluster comes from: the enterprise implementation, or a refusal. +//! `private` alone is not that edition: community builds carry it. + +use crate::error::Error; + +pub fn external_instance_pg_unavailable() -> Error { + Error::BadRequest( + "External instance databases are a Windmill Enterprise Edition feature".to_string(), + ) +} + +#[cfg(all(feature = "private", feature = "enterprise"))] +pub(crate) use crate::external_instance_pg_ee::{ + setup_external_instance_pg_unchecked, validate_external_instance_pg_setting, +}; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +pub(crate) use ce::*; + +#[cfg(not(all(feature = "private", feature = "enterprise")))] +mod ce { + use super::external_instance_pg_unavailable as unavailable; + use crate::{error::Result, external_instance_pg::ExternalInstancePgSetupReport, DB}; + + pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> { + Err(unavailable()) + } + + pub(crate) async fn setup_external_instance_pg_unchecked( + _db: &DB, + _rotate_passwords: bool, + ) -> Result { + Err(unavailable()) + } +} diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index ef63fc2347..7e69c4c866 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -57,6 +57,8 @@ pub const SAML_METADATA_SETTING: &str = "saml_metadata"; pub const SMTP_SETTING: &str = "smtp_settings"; pub const TEAMS_SETTING: &str = "teams"; pub const INDEXER_SETTING: &str = "indexer_settings"; +pub const EXTERNAL_INSTANCE_PG_SETTING: &str = "external_instance_pg"; +pub const EXTERNAL_INSTANCE_PG_STATE_SETTING: &str = "external_instance_pg_state"; pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result"; pub const UNIQUE_ID_SETTING: &str = "uid"; @@ -355,6 +357,9 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ // resolve datatable connections through the dedicated datatable endpoints, never these. "custom_instance_pg_databases", "custom_instance_replication_pwd", + // The external cluster's admin login, and the passwords Windmill generated on it. + EXTERNAL_INSTANCE_PG_SETTING, + EXTERNAL_INSTANCE_PG_STATE_SETTING, ]; /// Whether an agent worker may read the given global setting over HTTP. diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index de3fd7684a..d26b974717 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -350,6 +350,8 @@ pub struct GlobalSettings { pub ducklake_settings: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_instance_pg_databases: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub external_instance_pg: Option, // Opaque settings (EE-private structs or no clear schema) #[serde(skip_serializing_if = "Option::is_none")] @@ -833,6 +835,36 @@ pub struct CustomInstanceDbLogs { pub replication_user_error: Option, } +// --------------------------------------------------------------------------- +// External instance PG cluster +// --------------------------------------------------------------------------- + +/// The external Postgres cluster Windmill manages for `external_instance` data tables and Ducklake +/// catalogs. `user` logs in as the cluster's administrator: it needs `CREATEDB` and `CREATEROLE`. +/// `dbname` is only where that login connects to run cluster-wide statements. +/// +/// Every field defaults rather than being required: this deserializes as part of the whole +/// instance config, and one malformed row must not make every other setting unreadable. The +/// write path and every use reject an incomplete value instead. +#[derive(Deserialize, Serialize, Clone, Debug, Default)] +#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))] +pub struct ExternalInstancePg { + #[serde(default)] + pub host: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub port: Option, + #[serde(default)] + pub user: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub password: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dbname: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sslmode: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub root_certificate_pem: Option, +} + // --------------------------------------------------------------------------- // Autoscaling (worker config) // --------------------------------------------------------------------------- @@ -969,6 +1001,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_settings", "custom_instance_pg_databases", "custom_instance_replication_pwd", + "external_instance_pg_state", "uid", "rsa_keys", "jwt_secret", @@ -994,6 +1027,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ // Server-only (written by setup/refresh via direct SQL), never operator-authored — // hidden so the config machinery can't read, rewrite, or drop it. "custom_instance_replication_pwd", + // Same for the passwords and database registry Windmill keeps for the external cluster. + "external_instance_pg_state", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -1005,6 +1040,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[ "license_key", "ducklake_user_pg_pwd", "custom_instance_replication_pwd", + "external_instance_pg_state", "pip_index_url", "pip_extra_index_url", "npm_config_registry", @@ -1030,6 +1066,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[ &["secret_key", "serviceAccountKey"], ), ("custom_instance_pg_databases", &["user_pwd"]), + ("external_instance_pg", &["password"]), ]; fn redact_json_value(value: &serde_json::Value) -> serde_json::Value { @@ -1359,6 +1396,13 @@ pub async fn sync_global_settings_declarative( .map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?; let diff = diff_global_settings(current, desired, ApplyMode::Replace); + let external_pg_key = crate::global_settings::EXTERNAL_INSTANCE_PG_SETTING; + if diff.deletes.iter().any(|k| k == external_pg_key) { + crate::external_instance_pg::check_external_instance_pg_write(db, None).await?; + } + if let Some(value) = diff.upserts.get(external_pg_key) { + crate::external_instance_pg::check_external_instance_pg_write(db, Some(value)).await?; + } apply_settings_diff(db, &diff).await?; Ok(()) @@ -1491,6 +1535,10 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> { resolve_env_option(&mut pg.user_pwd)?; } + if let Some(pg) = &mut settings.external_instance_pg { + resolve_env_option(&mut pg.password)?; + } + Ok(()) } @@ -2460,39 +2508,33 @@ mod tests { } #[test] - fn custom_instance_replication_pwd_is_isolated_from_config() { - // The replication-role password is server-only: written by setup/refresh via direct - // SQL, never operator-authored. It must stay out of the declarative config surface - // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it. - assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd")); - assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd")); - assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd")); + fn server_generated_db_passwords_are_isolated_from_config() { + // These hold passwords the server generates: written by setup/refresh via direct SQL, + // never operator-authored. They must stay out of the declarative config surface + // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop them. + for key in [ + "custom_instance_replication_pwd", + "external_instance_pg_state", + ] { + assert!(HIDDEN_SETTINGS.contains(&key), "{key}"); + assert!(PROTECTED_SETTINGS.contains(&key), "{key}"); + assert!(SENSITIVE_SETTINGS.contains(&key), "{key}"); - // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted. - let mut desired = BTreeMap::new(); - desired.insert( - "custom_instance_replication_pwd".to_string(), - serde_json::json!("attacker-set"), - ); - let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge); - assert!( - diff.upserts.is_empty(), - "hidden setting must not be upserted" - ); + // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted. + let mut desired = BTreeMap::new(); + desired.insert(key.to_string(), serde_json::json!("attacker-set")); + let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge); + assert!(diff.upserts.is_empty(), "{key} must not be upserted"); - // A current value is never deleted by a Replace that omits it. - let mut current = BTreeMap::new(); - current.insert( - "custom_instance_replication_pwd".to_string(), - serde_json::json!("live"), - ); - let diff = diff_global_settings(¤t, &BTreeMap::new(), ApplyMode::Replace); - assert!( - !diff - .deletes - .contains(&"custom_instance_replication_pwd".to_string()), - "hidden setting must not be deleted" - ); + // A current value is never deleted by a Replace that omits it. + let mut current = BTreeMap::new(); + current.insert(key.to_string(), serde_json::json!("live")); + let diff = diff_global_settings(¤t, &BTreeMap::new(), ApplyMode::Replace); + assert!( + !diff.deletes.contains(&key.to_string()), + "{key} must not be deleted" + ); + } } #[test] diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index a67bb6bfa1..7bc5dc6070 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -58,6 +58,10 @@ pub mod ee_oss; pub mod email_ee; pub mod email_oss; pub mod error; +pub mod external_instance_pg; +#[cfg(all(feature = "private", feature = "enterprise"))] +mod external_instance_pg_ee; +pub mod external_instance_pg_oss; pub mod external_ip; #[cfg(feature = "private")] pub mod feature_usage_ee;