feat(datatables): set up an external instance cluster for data tables

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-09-16 22:35:00 +02:00
co-authored by Claude Opus 5
parent e0360bce5c
commit b3cda2f548
7 changed files with 443 additions and 32 deletions
+72 -1
View File
@@ -60,7 +60,7 @@ use windmill_common::{
global_settings::{
AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -168,6 +168,14 @@ pub fn global_service() -> Router {
"/refresh_custom_instance_user_pwd",
post(refresh_custom_instance_user_pwd),
)
.route(
"/external_instance_pg/status",
get(get_external_instance_pg_status),
)
.route(
"/external_instance_pg/setup",
post(setup_external_instance_pg),
)
.route(
"/setup_custom_instance_pg_database/{name}",
post(setup_custom_instance_pg_database),
@@ -938,6 +946,13 @@ async fn run_setting_pre_write_hook(
value: &serde_json::Value,
) -> error::Result<()> {
match key {
EXTERNAL_INSTANCE_PG_SETTING => {
windmill_common::external_instance_pg::check_external_instance_pg_write(
db,
Some(value),
)
.await?;
}
// The instance AI config is written as an untyped blob through this generic
// endpoint, so it never passes the typed check the workspace handler applies.
// Rates that reach a cost total unbounded would make it negative or infinite.
@@ -1289,6 +1304,14 @@ async fn set_instance_config(
for (key, value) in &settings_diff.upserts {
run_setting_pre_write_hook(&db, key, value).await?;
}
if settings_diff
.deletes
.iter()
.any(|k| k == EXTERNAL_INSTANCE_PG_SETTING)
{
windmill_common::external_instance_pg::check_external_instance_pg_write(&db, None)
.await?;
}
instance_config::apply_settings_diff(&db, &settings_diff)
.await
@@ -1743,6 +1766,54 @@ async fn refresh_custom_instance_user_pwd(
Ok(Json(()))
}
async fn get_external_instance_pg_status(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgStatus> {
require_super_admin(&db, &authed).await?;
Ok(Json(
windmill_common::external_instance_pg::external_instance_pg_status(&db).await?,
))
}
#[derive(Deserialize)]
struct SetupExternalInstancePgBody {
#[serde(default)]
rotate_passwords: bool,
}
async fn setup_external_instance_pg(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Json(body): Json<SetupExternalInstancePgBody>,
) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgSetupReport> {
require_super_admin(&db, &authed).await?;
let report = windmill_common::external_instance_pg::setup_external_instance_pg_unchecked(
&db,
body.rotate_passwords,
)
.await?;
let rotated = body.rotate_passwords.to_string();
let success = report.success.to_string();
windmill_audit::audit_oss::audit_log(
&db,
&authed,
"settings.setup_external_instance_pg",
windmill_audit::ActionKind::Update,
"global",
Some(&authed.email),
Some(
[
("rotate_passwords", rotated.as_str()),
("success", success.as_str()),
]
.into(),
),
)
.await?;
Ok(Json(report))
}
#[derive(Deserialize)]
struct SetupCustomInstanceDbBody {
tag: Option<String>,
+75
View File
@@ -1572,6 +1572,43 @@ paths:
schema:
type: object
/settings/external_instance_pg/status:
get:
summary: Returns whether the external instance cluster is configured and how its last setup went
operationId: getExternalInstancePgStatus
tags:
- setting
responses:
"200":
description: external instance cluster status
content:
application/json:
schema:
$ref: "#/components/schemas/ExternalInstancePgStatus"
/settings/external_instance_pg/setup:
post:
summary: Sets up the external instance cluster with its saved admin login, optionally rotating the passwords Windmill manages on it (enterprise edition only)
operationId: setupExternalInstancePg
tags:
- setting
requestBody:
required: true
content:
application/json:
schema:
type: object
properties:
rotate_passwords:
type: boolean
responses:
"200":
description: the setup report, also stored as the last setup
content:
application/json:
schema:
$ref: "#/components/schemas/ExternalInstancePgSetupReport"
/settings/list_custom_instance_pg_databases:
post:
summary: Returns the set-up statuses of custom instance pg databases
@@ -33621,6 +33658,44 @@ components:
- ducklake
- datatable
ExternalInstancePgSetupStep:
type: object
required: [name, status, message]
properties:
name:
type: string
status:
type: string
enum: [ok, warning, error]
message:
type: string
ExternalInstancePgSetupReport:
type: object
required: [success, finished_at, steps]
properties:
success:
type: boolean
description: no step failed; warnings leave it true
finished_at:
type: string
format: date-time
steps:
type: array
items:
$ref: "#/components/schemas/ExternalInstancePgSetupStep"
ExternalInstancePgStatus:
type: object
required: [configured, database_count]
properties:
configured:
type: boolean
database_count:
type: integer
last_setup:
$ref: "#/components/schemas/ExternalInstancePgSetupReport"
InstanceDatatableRole:
type: object
required: [id, name, enabled]
@@ -0,0 +1,171 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! The external Postgres cluster behind `external_instance` data tables and Ducklake catalogs.
//!
//! Windmill administers that cluster itself, logged in as the user in
//! [`EXTERNAL_INSTANCE_PG_SETTING`]. It creates `custom_instance_user` and
//! `custom_instance_replication_user` there, with passwords it generates and keeps in
//! [`EXTERNAL_INSTANCE_PG_STATE_SETTING`]. They share their names with the roles on Windmill's own
//! cluster, but they are different roles with different passwords.
//!
//! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not
//! supported: each would keep resetting the passwords the other depends on.
use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::{
error::{Error, Result},
global_settings::{EXTERNAL_INSTANCE_PG_SETTING, EXTERNAL_INSTANCE_PG_STATE_SETTING},
instance_config::{CustomInstanceDb, ExternalInstancePg},
DB,
};
/// What Windmill keeps about the external cluster. Server-managed and hidden: never part of the
/// instance config, never readable by an agent worker. No `Debug`: it carries live passwords.
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct ExternalInstancePgState {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub user_pwd: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub replication_pwd: Option<String>,
/// The databases Windmill created on the cluster. It only ever drops one of these.
#[serde(default)]
pub databases: BTreeMap<String, CustomInstanceDb>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_setup: Option<ExternalInstancePgSetupReport>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ExternalInstancePgSetupReport {
/// No step failed. Warnings leave it true.
pub success: bool,
pub finished_at: chrono::DateTime<chrono::Utc>,
pub steps: Vec<ExternalInstancePgSetupStep>,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ExternalInstancePgSetupStep {
pub name: String,
pub status: SetupStepStatus,
pub message: String,
}
#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum SetupStepStatus {
Ok,
Warning,
Error,
}
/// The status the settings page shows without running anything.
#[derive(Serialize, Debug)]
pub struct ExternalInstancePgStatus {
pub configured: bool,
pub database_count: usize,
pub last_setup: Option<ExternalInstancePgSetupReport>,
}
pub async fn read_external_instance_pg_config(db: &DB) -> Result<Option<ExternalInstancePg>> {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
EXTERNAL_INSTANCE_PG_SETTING
)
.fetch_optional(db)
.await?;
value
.map(|v| {
serde_json::from_value(v).map_err(|e| {
Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_SETTING}: {e}"))
})
})
.transpose()
}
pub async fn read_external_instance_pg_state<'c>(
executor: impl sqlx::PgExecutor<'c>,
) -> Result<ExternalInstancePgState> {
let value = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = $1",
EXTERNAL_INSTANCE_PG_STATE_SETTING
)
.fetch_optional(executor)
.await?;
match value {
None => Ok(ExternalInstancePgState::default()),
Some(v) => serde_json::from_value(v).map_err(|e| {
Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_STATE_SETTING}: {e}"))
}),
}
}
pub async fn external_instance_pg_status(db: &DB) -> Result<ExternalInstancePgStatus> {
let configured = read_external_instance_pg_config(db).await?.is_some();
let state = read_external_instance_pg_state(db).await?;
Ok(ExternalInstancePgStatus {
configured,
database_count: state.databases.len(),
last_setup: state.last_setup,
})
}
/// Refuse to unset the cluster while Windmill still has databases on it: every data table and
/// Ducklake catalog there would stop resolving. Allowed on every edition, so a downgraded
/// instance can still clear a setting it no longer uses.
pub async fn ensure_external_instance_pg_removable(db: &DB) -> Result<()> {
let state = read_external_instance_pg_state(db).await?;
if state.databases.is_empty() {
return Ok(());
}
let names = state
.databases
.keys()
.cloned()
.collect::<Vec<_>>()
.join(", ");
Err(Error::BadRequest(format!(
"The external instance cluster still holds databases Windmill created ({names}). Drop \
them before removing {EXTERNAL_INSTANCE_PG_SETTING}."
)))
}
/// Check a write to [`EXTERNAL_INSTANCE_PG_SETTING`] before it happens: `None`, null or an empty
/// string unsets it. Every writer of global settings calls this, the per-key and bulk endpoints
/// as well as the declarative sync.
pub async fn check_external_instance_pg_write(
db: &DB,
value: Option<&serde_json::Value>,
) -> Result<()> {
match value {
None | Some(serde_json::Value::Null) => ensure_external_instance_pg_removable(db).await,
Some(serde_json::Value::String(s)) if s.trim().is_empty() => {
ensure_external_instance_pg_removable(db).await
}
Some(value) => {
crate::external_instance_pg_oss::validate_external_instance_pg_setting(value)
}
}
}
/// Converge the external cluster on the configured login: check what it can do, create or update
/// Windmill's two roles with the stored passwords, and report anything that would get in the way.
/// With `rotate_passwords`, generate new passwords first. Safe to run again; running it again is
/// how a failed rotation is repaired.
///
/// Authorization: administers the external cluster with its admin credentials and checks nothing.
/// Callers MUST be superadmin.
pub async fn setup_external_instance_pg_unchecked(
db: &DB,
rotate_passwords: bool,
) -> Result<ExternalInstancePgSetupReport> {
crate::external_instance_pg_oss::setup_external_instance_pg_unchecked(db, rotate_passwords)
.await
}
@@ -0,0 +1,43 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
//! Where the external instance cluster comes from: the enterprise implementation, or a refusal.
//! `private` alone is not that edition: community builds carry it.
use crate::error::Error;
pub fn external_instance_pg_unavailable() -> Error {
Error::BadRequest(
"External instance databases are a Windmill Enterprise Edition feature".to_string(),
)
}
#[cfg(all(feature = "private", feature = "enterprise"))]
pub(crate) use crate::external_instance_pg_ee::{
setup_external_instance_pg_unchecked, validate_external_instance_pg_setting,
};
#[cfg(not(all(feature = "private", feature = "enterprise")))]
pub(crate) use ce::*;
#[cfg(not(all(feature = "private", feature = "enterprise")))]
mod ce {
use super::external_instance_pg_unavailable as unavailable;
use crate::{error::Result, external_instance_pg::ExternalInstancePgSetupReport, DB};
pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> {
Err(unavailable())
}
pub(crate) async fn setup_external_instance_pg_unchecked(
_db: &DB,
_rotate_passwords: bool,
) -> Result<ExternalInstancePgSetupReport> {
Err(unavailable())
}
}
@@ -57,6 +57,8 @@ pub const SAML_METADATA_SETTING: &str = "saml_metadata";
pub const SMTP_SETTING: &str = "smtp_settings";
pub const TEAMS_SETTING: &str = "teams";
pub const INDEXER_SETTING: &str = "indexer_settings";
pub const EXTERNAL_INSTANCE_PG_SETTING: &str = "external_instance_pg";
pub const EXTERNAL_INSTANCE_PG_STATE_SETTING: &str = "external_instance_pg_state";
pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
pub const UNIQUE_ID_SETTING: &str = "uid";
@@ -355,6 +357,9 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[
// resolve datatable connections through the dedicated datatable endpoints, never these.
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
// The external cluster's admin login, and the passwords Windmill generated on it.
EXTERNAL_INSTANCE_PG_SETTING,
EXTERNAL_INSTANCE_PG_STATE_SETTING,
];
/// Whether an agent worker may read the given global setting over HTTP.
+73 -31
View File
@@ -350,6 +350,8 @@ pub struct GlobalSettings {
pub ducklake_settings: Option<DucklakeSettings>,
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_instance_pg_databases: Option<CustomInstancePgDatabases>,
#[serde(skip_serializing_if = "Option::is_none")]
pub external_instance_pg: Option<ExternalInstancePg>,
// Opaque settings (EE-private structs or no clear schema)
#[serde(skip_serializing_if = "Option::is_none")]
@@ -833,6 +835,36 @@ pub struct CustomInstanceDbLogs {
pub replication_user_error: Option<String>,
}
// ---------------------------------------------------------------------------
// External instance PG cluster
// ---------------------------------------------------------------------------
/// The external Postgres cluster Windmill manages for `external_instance` data tables and Ducklake
/// catalogs. `user` logs in as the cluster's administrator: it needs `CREATEDB` and `CREATEROLE`.
/// `dbname` is only where that login connects to run cluster-wide statements.
///
/// Every field defaults rather than being required: this deserializes as part of the whole
/// instance config, and one malformed row must not make every other setting unreadable. The
/// write path and every use reject an incomplete value instead.
#[derive(Deserialize, Serialize, Clone, Debug, Default)]
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
pub struct ExternalInstancePg {
#[serde(default)]
pub host: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub port: Option<u16>,
#[serde(default)]
pub user: String,
#[serde(skip_serializing_if = "Option::is_none")]
pub password: Option<StringOrSecretRef>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dbname: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub sslmode: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub root_certificate_pem: Option<String>,
}
// ---------------------------------------------------------------------------
// Autoscaling (worker config)
// ---------------------------------------------------------------------------
@@ -969,6 +1001,7 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
"ducklake_settings",
"custom_instance_pg_databases",
"custom_instance_replication_pwd",
"external_instance_pg_state",
"uid",
"rsa_keys",
"jwt_secret",
@@ -994,6 +1027,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[
// Server-only (written by setup/refresh via direct SQL), never operator-authored —
// hidden so the config machinery can't read, rewrite, or drop it.
"custom_instance_replication_pwd",
// Same for the passwords and database registry Windmill keeps for the external cluster.
"external_instance_pg_state",
];
/// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
@@ -1005,6 +1040,7 @@ const SENSITIVE_SETTINGS: &[&str] = &[
"license_key",
"ducklake_user_pg_pwd",
"custom_instance_replication_pwd",
"external_instance_pg_state",
"pip_index_url",
"pip_extra_index_url",
"npm_config_registry",
@@ -1030,6 +1066,7 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
&["secret_key", "serviceAccountKey"],
),
("custom_instance_pg_databases", &["user_pwd"]),
("external_instance_pg", &["password"]),
];
fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
@@ -1359,6 +1396,13 @@ pub async fn sync_global_settings_declarative(
.map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?;
let diff = diff_global_settings(current, desired, ApplyMode::Replace);
let external_pg_key = crate::global_settings::EXTERNAL_INSTANCE_PG_SETTING;
if diff.deletes.iter().any(|k| k == external_pg_key) {
crate::external_instance_pg::check_external_instance_pg_write(db, None).await?;
}
if let Some(value) = diff.upserts.get(external_pg_key) {
crate::external_instance_pg::check_external_instance_pg_write(db, Some(value)).await?;
}
apply_settings_diff(db, &diff).await?;
Ok(())
@@ -1491,6 +1535,10 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> {
resolve_env_option(&mut pg.user_pwd)?;
}
if let Some(pg) = &mut settings.external_instance_pg {
resolve_env_option(&mut pg.password)?;
}
Ok(())
}
@@ -2460,39 +2508,33 @@ mod tests {
}
#[test]
fn custom_instance_replication_pwd_is_isolated_from_config() {
// The replication-role password is server-only: written by setup/refresh via direct
// SQL, never operator-authored. It must stay out of the declarative config surface
// (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it.
assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd"));
assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd"));
fn server_generated_db_passwords_are_isolated_from_config() {
// These hold passwords the server generates: written by setup/refresh via direct SQL,
// never operator-authored. They must stay out of the declarative config surface
// (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop them.
for key in [
"custom_instance_replication_pwd",
"external_instance_pg_state",
] {
assert!(HIDDEN_SETTINGS.contains(&key), "{key}");
assert!(PROTECTED_SETTINGS.contains(&key), "{key}");
assert!(SENSITIVE_SETTINGS.contains(&key), "{key}");
// A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
let mut desired = BTreeMap::new();
desired.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("attacker-set"),
);
let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
assert!(
diff.upserts.is_empty(),
"hidden setting must not be upserted"
);
// A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
let mut desired = BTreeMap::new();
desired.insert(key.to_string(), serde_json::json!("attacker-set"));
let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
assert!(diff.upserts.is_empty(), "{key} must not be upserted");
// A current value is never deleted by a Replace that omits it.
let mut current = BTreeMap::new();
current.insert(
"custom_instance_replication_pwd".to_string(),
serde_json::json!("live"),
);
let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
assert!(
!diff
.deletes
.contains(&"custom_instance_replication_pwd".to_string()),
"hidden setting must not be deleted"
);
// A current value is never deleted by a Replace that omits it.
let mut current = BTreeMap::new();
current.insert(key.to_string(), serde_json::json!("live"));
let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
assert!(
!diff.deletes.contains(&key.to_string()),
"{key} must not be deleted"
);
}
}
#[test]
+4
View File
@@ -58,6 +58,10 @@ pub mod ee_oss;
pub mod email_ee;
pub mod email_oss;
pub mod error;
pub mod external_instance_pg;
#[cfg(all(feature = "private", feature = "enterprise"))]
mod external_instance_pg_ee;
pub mod external_instance_pg_oss;
pub mod external_ip;
#[cfg(feature = "private")]
pub mod feature_usage_ee;