[ee] feat(datatables): external_instance data tables on the external cluster

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/ee-repo-ref.txt

@@ -1 +1 @@
-6e61ff49c01f7b6c3788a0e32b46161572bf0d84
+ff3e87b5e04b35f6a0e24846d7cd48dcca00ab7c

backend/windmill-api-settings/src/lib.rs

@@ -176,6 +176,14 @@ pub fn global_service() -> Router {
             "/external_instance_pg/setup",
             post(setup_external_instance_pg),
         )
+        .route(
+            "/external_instance_pg/databases",
+            get(list_external_instance_pg_databases),
+        )
+        .route(
+            "/external_instance_pg/databases/{name}",
+            post(create_external_instance_pg_database).delete(drop_external_instance_pg_database),
+        )
         .route(
             "/setup_custom_instance_pg_database/{name}",
             post(setup_custom_instance_pg_database),
@@ -1814,6 +1822,95 @@ async fn setup_external_instance_pg(
     Ok(Json(report))
 }
 
+#[derive(Serialize)]
+struct ExternalInstancePgDatabase {
+    #[serde(flatten)]
+    status: windmill_common::instance_config::CustomInstanceDb,
+    used_by_workspaces: Vec<String>,
+}
+
+async fn list_external_instance_pg_databases(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+) -> JsonResult<std::collections::BTreeMap<String, ExternalInstancePgDatabase>> {
+    require_super_admin(&db, &authed).await?;
+    let databases = windmill_common::external_instance_pg::external_instance_databases(&db).await?;
+    let mut usages =
+        windmill_common::external_instance_pg::external_instance_database_usages(&db).await?;
+    Ok(Json(
+        databases
+            .into_iter()
+            .map(|(name, status)| {
+                let used_by_workspaces = usages.remove(&name).unwrap_or_default();
+                (
+                    name,
+                    ExternalInstancePgDatabase {
+                        status,
+                        used_by_workspaces: used_by_workspaces.into_iter().collect(),
+                    },
+                )
+            })
+            .collect(),
+    ))
+}
+
+async fn create_external_instance_pg_database(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(dbname): Path<String>,
+    Json(body): Json<SetupCustomInstanceDbBody>,
+) -> JsonResult<()> {
+    require_super_admin(&db, &authed).await?;
+    let tag = body.tag.as_deref().unwrap_or("datatable");
+    windmill_common::external_instance_pg::create_external_instance_database_unchecked(
+        &db, &dbname, tag,
+    )
+    .await?;
+    windmill_audit::audit_oss::audit_log(
+        &db,
+        &authed,
+        "settings.create_external_instance_pg_database",
+        windmill_audit::ActionKind::Create,
+        "global",
+        Some(&authed.email),
+        Some([("dbname", dbname.as_str()), ("tag", tag)].into()),
+    )
+    .await?;
+    Ok(Json(()))
+}
+
+async fn drop_external_instance_pg_database(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(dbname): Path<String>,
+) -> JsonResult<()> {
+    require_super_admin(&db, &authed).await?;
+    // A data table naming a dropped database fails on every job, far from the drop that caused it.
+    if let Some(workspaces) =
+        windmill_common::external_instance_pg::external_instance_database_usages(&db)
+            .await?
+            .remove(dbname.trim())
+    {
+        return Err(error::Error::BadRequest(format!(
+            "Database '{dbname}' is still used by data tables in {}",
+            workspaces.into_iter().collect::<Vec<_>>().join(", ")
+        )));
+    }
+    windmill_common::external_instance_pg::drop_external_instance_database_unchecked(&db, &dbname)
+        .await?;
+    windmill_audit::audit_oss::audit_log(
+        &db,
+        &authed,
+        "settings.drop_external_instance_pg_database",
+        windmill_audit::ActionKind::Delete,
+        "global",
+        Some(&authed.email),
+        Some([("dbname", dbname.as_str())].into()),
+    )
+    .await?;
+    Ok(Json(()))
+}
+
 #[derive(Deserialize)]
 struct SetupCustomInstanceDbBody {
     tag: Option<String>,

backend/windmill-api-workspaces/src/datatable_migrations.rs

@@ -11,7 +11,7 @@
 //! to keep that file focused on core workspace configuration.
 
 use crate::workspaces::{
-    is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
+    managed_datatable_kind, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
     PgDumpOptions,
 };
 
@@ -1552,7 +1552,9 @@ async fn generate_initial_datatable_migration(
     // without what a replay elsewhere cannot run: the replaying user owns none of this
     // database's objects, and the grants Windmill plants in an instance database (`ALTER
     // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server.
-    let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?;
+    let no_acl = managed_datatable_kind(&db, &w_id, &datatable_name)
+        .await?
+        .is_some();
     let dump_file = pg_dump_database(
         &pg_db,
         PgDumpOptions {

backend/windmill-api-workspaces/src/workspaces.rs

@@ -3084,23 +3084,32 @@ pub(crate) async fn resolve_pg_source_checked(
         .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))
 }
 
-/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL
-/// rather than a user resource.
-pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result<bool> {
+/// The kind of the database backing the data table `name` when Windmill manages it (on its own
+/// cluster or the external one), `None` when it is a user resource.
+pub(crate) async fn managed_datatable_kind(
+    db: &DB,
+    w_id: &str,
+    name: &str,
+) -> Result<Option<DataTableCatalogResourceType>> {
     // Resolved rather than read: a pointer entry owns no database of its own, so only the entry it
-    // lands on can answer. A name that resolves to nothing keeps the historical `false`.
+    // lands on can answer. A name that resolves to nothing keeps the historical `None`.
     Ok(resolve_governing_datatable(db, w_id, name)
         .await
         .ok()
         .and_then(|g| g.datatable.database)
-        .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance))
+        .map(|d| d.resource_type)
+        .filter(|kind| kind.is_windmill_managed()))
 }
 
 /// Same, for the `datatable://<name>` / `$res:<path>` form the import endpoints take.
-async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result<bool> {
+async fn managed_datatable_source_kind(
+    db: &DB,
+    w_id: &str,
+    source: &str,
+) -> Result<Option<DataTableCatalogResourceType>> {
     match source.strip_prefix("datatable://") {
-        Some(name) => is_instance_datatable(db, w_id, name).await,
-        None => Ok(false),
+        Some(name) => managed_datatable_kind(db, w_id, name).await,
+        None => Ok(None),
     }
 }
 
@@ -3199,10 +3208,7 @@ pub(crate) async fn pg_dump_database(
     if let Some(ref password) = pg_db.password {
         cmd.env("PGPASSWORD", password);
     }
-
-    if let Some(ref sslmode) = pg_db.sslmode {
-        cmd.env("PGSSLMODE", sslmode);
-    }
+    let _root_cert = apply_pg_tls_env(&mut cmd, pg_db)?;
 
     let output = cmd
         .output()
@@ -3320,7 +3326,7 @@ async fn comment_out_unsupported_settings(
 
 /// A psql invocation against `pg_db`, carrying the connection settings the CLI reads
 /// from the environment.
-fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command {
+fn psql_command(pg_db: &PgDatabase) -> Result<(tokio::process::Command, Option<DumpFile>)> {
     let mut cmd = tokio::process::Command::new("psql");
     cmd.arg("--host")
         .arg(&pg_db.host)
@@ -3337,10 +3343,41 @@ fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command {
     if let Some(ref password) = pg_db.password {
         cmd.env("PGPASSWORD", password);
     }
+    let root_cert = apply_pg_tls_env(&mut cmd, pg_db)?;
+    Ok((cmd, root_cert))
+}
+
+/// Give libpq the TLS settings `PgDatabase::connect` applies. The returned file holds the root
+/// certificate `PGSSLROOTCERT` names, so it must outlive the command.
+fn apply_pg_tls_env(
+    cmd: &mut tokio::process::Command,
+    pg_db: &PgDatabase,
+) -> Result<Option<DumpFile>> {
     if let Some(ref sslmode) = pg_db.sslmode {
         cmd.env("PGSSLMODE", sslmode);
     }
-    cmd
+    if let Some(pem) = pg_db
+        .root_certificate_pem
+        .as_deref()
+        .filter(|p| !p.is_empty())
+    {
+        let file = DumpFile::new()?;
+        std::fs::write(&file.path, pem)
+            .map_err(|e| Error::internal_err(format!("Failed to write root certificate: {e}")))?;
+        cmd.env("PGSSLROOTCERT", &file.path);
+        return Ok(Some(file));
+    }
+    // Only a connection that asked to be verified against the system trust store. Without a file,
+    // libpq's own default would look for `~/.postgresql/root.crt` and refuse a verify-* mode.
+    if pg_db.accept_invalid_certs == Some(false)
+        && matches!(
+            pg_db.sslmode.as_deref(),
+            Some("verify-full") | Some("verify-ca")
+        )
+    {
+        cmd.env("PGSSLROOTCERT", "system");
+    }
+    Ok(None)
 }
 
 /// GUC names the server backing `pg_db` knows about.
@@ -3350,7 +3387,8 @@ fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command {
 /// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a
 /// plaintext socket and fail before the import ever starts.
 async fn server_setting_names(pg_db: &PgDatabase) -> Result<HashSet<String>> {
-    let output = psql_command(pg_db)
+    let (mut cmd, _root_cert) = psql_command(pg_db)?;
+    let output = cmd
         .arg("--tuples-only")
         .arg("--no-align")
         .arg("--command")
@@ -3384,7 +3422,8 @@ async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<
     let supported_settings = server_setting_names(target_db).await?;
     comment_out_unsupported_settings(dump_file, &supported_settings).await?;
 
-    let output = psql_command(target_db)
+    let (mut cmd, _root_cert) = psql_command(target_db)?;
+    let output = cmd
         .arg("--set")
         .arg("ON_ERROR_STOP=1")
         .arg("--single-transaction")
@@ -3442,7 +3481,15 @@ async fn create_pg_database(
         }
     }
 
-    if is_instance_datatable_source(&db, &w_id, &req.source).await? {
+    let source_kind = managed_datatable_source_kind(&db, &w_id, &req.source).await?;
+    if source_kind == Some(DataTableCatalogResourceType::ExternalInstance) {
+        windmill_common::external_instance_pg::create_external_instance_database_unchecked(
+            &db,
+            &req.target_dbname,
+            "datatable",
+        )
+        .await?;
+    } else if source_kind == Some(DataTableCatalogResourceType::Instance) {
         windmill_common::create_custom_instance_database(&db, &req.target_dbname, "datatable")
             .await?;
     } else {
@@ -3545,12 +3592,12 @@ async fn ensure_datatable_is_clonable(
     }
     // The copy has to name a database of its own. A resource-backed entry reached through a
     // pointer names one this workspace does not own, so there is nothing here to repoint.
-    let is_instance = governing
+    let is_managed = governing
         .datatable
         .database
         .as_ref()
-        .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance);
-    if governing.workspace_id != w_id && !is_instance {
+        .is_some_and(|d| d.resource_type.is_windmill_managed());
+    if governing.workspace_id != w_id && !is_managed {
         return Err(Error::BadRequest(format!(
             "Data table '{name}' points at a resource-backed data table in another workspace \
              and cannot be copied; fork it from the workspace that owns it."
@@ -3607,8 +3654,12 @@ async fn import_pg_database(
     // what it creates it owns. Grants do, except around an instance data table — Windmill
     // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere
     // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access.
-    let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await?
-        || is_instance_datatable_source(&db, &w_id, &req.source).await?;
+    let no_acl = managed_datatable_source_kind(&db, &w_id, &req.target)
+        .await?
+        .is_some()
+        || managed_datatable_source_kind(&db, &w_id, &req.source)
+            .await?
+            .is_some();
 
     let dump_file = pg_dump_database(
         &source_pg,
@@ -3908,6 +3959,7 @@ async fn edit_datatable_config(
                 // so these line up with the `datatable_configured` adoption counts.
                 created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) {
                     Some(DataTableCatalogResourceType::Instance) => "instance",
+                    Some(DataTableCatalogResourceType::ExternalInstance) => "external_instance",
                     Some(DataTableCatalogResourceType::Postgresql) => "postgresql",
                     None => "reference",
                 });
@@ -3963,26 +4015,34 @@ async fn edit_datatable_config(
     // Check that non-superadmins are not abusing Instance databases, which reach a database this
     // workspace does not own. Pointing an entry at another workspace's data table is not checked
     // here because it cannot be requested at all: `reference` is overwritten from the stored entry
-    // above, for every caller.
-    if !is_superadmin {
-        for (name, dt) in new_config.settings.datatables.iter() {
-            let old_dt = old_datatables.get(name);
-            if dt
-                .database
-                .as_ref()
-                .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)
-            {
-                let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| {
-                    o.resource_type == DataTableCatalogResourceType::Instance
-                        && Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path)
-                });
-                if !unchanged {
-                    return Err(Error::BadRequest(
-                        "Only superadmins can create or modify data tables with Instance databases"
-                            .to_string(),
-                    ));
-                }
-            }
+    // above, for every caller. An unchanged entry is left alone either way, so a downgraded
+    // instance can still save settings that already name an external instance database.
+    for (name, dt) in new_config.settings.datatables.iter() {
+        let Some(database) = dt
+            .database
+            .as_ref()
+            .filter(|d| d.resource_type.is_windmill_managed())
+        else {
+            continue;
+        };
+        let unchanged = old_datatables
+            .get(name)
+            .and_then(|o| o.database.as_ref())
+            .is_some_and(|o| {
+                o.resource_type == database.resource_type
+                    && o.resource_path == database.resource_path
+            });
+        if unchanged {
+            continue;
+        }
+        if database.resource_type == DataTableCatalogResourceType::ExternalInstance {
+            windmill_common::external_instance_pg::ensure_external_instance_available()?;
+        }
+        if !is_superadmin {
+            return Err(Error::BadRequest(
+                "Only superadmins can create or modify data tables with Instance databases"
+                    .to_string(),
+            ));
         }
     }
 
@@ -8134,13 +8194,14 @@ async fn point_kept_datatables_at_parent(
         if dt.reference.is_some() {
             continue;
         }
-        // Only instance databases. A resource-backed data table names a resource, and the settings
-        // clone gave the fork its own copy of that resource in its own workspace — pointing at the
-        // parent's entry would silently move the fork onto the parent's resource instead.
+        // Only instance databases, on either cluster. A resource-backed data table names a
+        // resource, and the settings clone gave the fork its own copy of that resource in its own
+        // workspace — pointing at the parent's entry would silently move the fork onto the
+        // parent's resource instead.
         if dt
             .database
             .as_ref()
-            .is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance)
+            .is_none_or(|d| !d.resource_type.is_windmill_managed())
         {
             continue;
         }
@@ -8270,11 +8331,12 @@ async fn apply_forked_datatable(
         })?,
     };
 
-    if database.resource_type == DataTableCatalogResourceType::Instance {
+    if database.resource_type.is_windmill_managed() {
         // The whole `database` object, not just its `resource_path`: a pointer entry has none to
-        // patch. `reference` goes with it — exactly one of the two may be set.
+        // patch. `reference` goes with it — exactly one of the two may be set. The copy was created
+        // on the same cluster as its source, so it keeps the source's kind.
         let new_database = serde_json::json!({
-            "resource_type": "instance",
+            "resource_type": database.resource_type,
             "resource_path": &fdt.new_dbname,
         });
         sqlx::query!(

backend/windmill-api-workspaces/src/workspaces_extra.rs

@@ -1409,9 +1409,7 @@ pub async fn drop_forked_datatable_databases(
             _ => continue,
         };
 
-        if database.resource_type
-            == windmill_common::workspaces::DataTableCatalogResourceType::Instance
-        {
+        if database.resource_type.is_windmill_managed() {
             let db_to_drop = &database.resource_path;
             if !db_to_drop.starts_with("wm_fork_") {
                 errors.push(format!(
@@ -1420,7 +1418,17 @@ pub async fn drop_forked_datatable_databases(
                 ));
                 continue;
             }
-            if let Err(e) = windmill_common::drop_custom_instance_database(&db, db_to_drop).await {
+            let dropped = if database.resource_type
+                == windmill_common::workspaces::DataTableCatalogResourceType::ExternalInstance
+            {
+                windmill_common::external_instance_pg::drop_external_instance_database_unchecked(
+                    &db, db_to_drop,
+                )
+                .await
+            } else {
+                windmill_common::drop_custom_instance_database(&db, db_to_drop).await
+            };
+            if let Err(e) = dropped {
                 errors.push(format!(
                     "Could not drop instance database '{}' for datatable://{}: {}",
                     db_to_drop, dt_name, e

backend/windmill-api/openapi.yaml

@@ -1609,6 +1609,67 @@ paths:
               schema:
                 $ref: "#/components/schemas/ExternalInstancePgSetupReport"
 
+  /settings/external_instance_pg/databases:
+    get:
+      summary: Lists the databases Windmill created on the external instance cluster, with the workspaces whose data tables use each
+      operationId: listExternalInstancePgDatabases
+      tags:
+        - setting
+      responses:
+        "200":
+          description: databases by name
+          content:
+            application/json:
+              schema:
+                type: object
+                additionalProperties:
+                  $ref: "#/components/schemas/CustomInstanceDb"
+
+  /settings/external_instance_pg/databases/{name}:
+    post:
+      summary: Creates a database on the external instance cluster (enterprise edition only)
+      operationId: createExternalInstancePgDatabase
+      tags:
+        - setting
+      parameters:
+        - name: name
+          in: path
+          required: true
+          schema:
+            type: string
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                tag:
+                  $ref: "#/components/schemas/CustomInstanceDbTag"
+      responses:
+        "200":
+          description: database created
+          content:
+            application/json:
+              schema: {}
+    delete:
+      summary: Drops a database Windmill created on the external instance cluster, refused while a data table uses it
+      operationId: dropExternalInstancePgDatabase
+      tags:
+        - setting
+      parameters:
+        - name: name
+          in: path
+          required: true
+          schema:
+            type: string
+      responses:
+        "200":
+          description: database dropped
+          content:
+            application/json:
+              schema: {}
+
   /settings/list_custom_instance_pg_databases:
     post:
       summary: Returns the set-up statuses of custom instance pg databases
@@ -5267,7 +5328,7 @@ paths:
                       type: string
                     resource_type:
                       type: string
-                      enum: [postgres, instance]
+                      enum: [postgres, instance, external_instance]
                     resource_path:
                       type: string
                     governing_workspace_id:
@@ -36065,6 +36126,7 @@ components:
                     enum:
                       - postgresql
                       - instance
+                      - external_instance
                   resource_path:
                     type: string
                 required:

backend/windmill-common/src/external_instance_pg.rs

@@ -17,7 +17,7 @@
 //! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not
 //! supported: each would keep resetting the passwords the other depends on.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 use serde::{Deserialize, Serialize};
 
@@ -124,26 +124,101 @@ pub async fn external_instance_pg_status(db: &DB) -> Result<ExternalInstancePgSt
     })
 }
 
-/// Refuse to unset the cluster while Windmill still has databases on it: every data table and
-/// Ducklake catalog there would stop resolving. Allowed on every edition, so a downgraded
-/// instance can still clear a setting it no longer uses.
+/// The databases Windmill created on the external cluster, without the passwords kept beside them.
+pub async fn external_instance_databases(db: &DB) -> Result<BTreeMap<String, CustomInstanceDb>> {
+    Ok(read_external_instance_pg_state(db).await?.databases)
+}
+
+/// The workspaces whose data tables name each database on the external cluster.
+pub async fn external_instance_database_usages(
+    db: &DB,
+) -> Result<BTreeMap<String, BTreeSet<String>>> {
+    let rows = sqlx::query_as::<_, (String, String)>(
+        "SELECT ws.workspace_id, entry->'database'->>'resource_path'
+         FROM workspace_settings ws
+         CROSS JOIN LATERAL jsonb_each(
+             CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
+                 THEN ws.datatable->'datatables'
+                 ELSE '{}'::jsonb END
+         ) AS dt(k, entry)
+         WHERE entry->'database'->>'resource_type' = 'external_instance'
+           AND entry->'database'->>'resource_path' IS NOT NULL",
+    )
+    .fetch_all(db)
+    .await?;
+    let mut usages: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
+    for (workspace_id, dbname) in rows {
+        usages.entry(dbname).or_default().insert(workspace_id);
+    }
+    Ok(usages)
+}
+
+/// Refuse to unset the cluster while Windmill still has databases on it, or a workspace still
+/// points at one: every data table there would stop resolving. Allowed on every edition, so a
+/// downgraded instance can still clear a setting it no longer uses.
 pub async fn ensure_external_instance_pg_removable(db: &DB) -> Result<()> {
     let state = read_external_instance_pg_state(db).await?;
-    if state.databases.is_empty() {
+    let usages = external_instance_database_usages(db).await?;
+    if state.databases.is_empty() && usages.is_empty() {
         return Ok(());
     }
     let names = state
         .databases
         .keys()
+        .chain(usages.keys())
+        .collect::<BTreeSet<_>>()
+        .into_iter()
         .cloned()
         .collect::<Vec<_>>()
         .join(", ");
     Err(Error::BadRequest(format!(
-        "The external instance cluster still holds databases Windmill created ({names}). Drop \
-         them before removing {EXTERNAL_INSTANCE_PG_SETTING}."
+        "The external instance cluster still holds databases in use ({names}). Drop them and \
+         repoint the data tables using them before removing {EXTERNAL_INSTANCE_PG_SETTING}."
     )))
 }
 
+/// Refuse a workspace setting that newly names an `external_instance` database on an edition
+/// without them.
+pub fn ensure_external_instance_available() -> Result<()> {
+    crate::external_instance_pg_oss::ensure_external_instance_available()
+}
+
+/// The connection an `external_instance` database resolves to: `custom_instance_user`, or the
+/// replication user, on the external cluster.
+///
+/// Authorization: returns live credentials and checks nothing. Callers MUST have authorized access
+/// to the data table that names `dbname`.
+pub async fn external_instance_connection_unchecked(
+    db: &DB,
+    dbname: &str,
+    replication: bool,
+) -> Result<crate::PgDatabase> {
+    crate::external_instance_pg_oss::external_instance_connection_unchecked(db, dbname, replication)
+        .await
+}
+
+/// Create `dbname` on the external cluster and register it. Refuses a name already taken there,
+/// whoever took it.
+///
+/// Authorization: checks nothing. Callers MUST be superadmin, or be cloning a data table they may
+/// fork into a `wm_fork_` database.
+pub async fn create_external_instance_database_unchecked(
+    db: &DB,
+    dbname: &str,
+    tag: &str,
+) -> Result<()> {
+    crate::external_instance_pg_oss::create_external_instance_database_unchecked(db, dbname, tag)
+        .await
+}
+
+/// Drop `dbname` from the external cluster. Only ever a database Windmill registered creating.
+///
+/// Authorization: checks nothing. Callers MUST be superadmin, or be deleting the fork that owns
+/// this `wm_fork_` database.
+pub async fn drop_external_instance_database_unchecked(db: &DB, dbname: &str) -> Result<()> {
+    crate::external_instance_pg_oss::drop_external_instance_database_unchecked(db, dbname).await
+}
+
 /// Check a write to [`EXTERNAL_INSTANCE_PG_SETTING`] before it happens: `None`, null or an empty
 /// string unsets it. Every writer of global settings calls this, the per-key and bulk endpoints
 /// as well as the declarative sync.

backend/windmill-common/src/external_instance_pg_oss.rs

@@ -19,25 +19,61 @@ pub fn external_instance_pg_unavailable() -> Error {
 
 #[cfg(all(feature = "private", feature = "enterprise"))]
 pub(crate) use crate::external_instance_pg_ee::{
-    setup_external_instance_pg_unchecked, validate_external_instance_pg_setting,
+    create_external_instance_database_unchecked, drop_external_instance_database_unchecked,
+    external_instance_connection_unchecked, setup_external_instance_pg_unchecked,
+    validate_external_instance_pg_setting,
 };
 
+#[cfg(all(feature = "private", feature = "enterprise"))]
+pub(crate) fn ensure_external_instance_available() -> crate::error::Result<()> {
+    Ok(())
+}
+
 #[cfg(not(all(feature = "private", feature = "enterprise")))]
 pub(crate) use ce::*;
 
 #[cfg(not(all(feature = "private", feature = "enterprise")))]
 mod ce {
     use super::external_instance_pg_unavailable as unavailable;
-    use crate::{error::Result, external_instance_pg::ExternalInstancePgSetupReport, DB};
+    use crate::{
+        error::Result, external_instance_pg::ExternalInstancePgSetupReport, PgDatabase, DB,
+    };
 
     pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> {
         Err(unavailable())
     }
 
+    pub(crate) fn ensure_external_instance_available() -> Result<()> {
+        Err(unavailable())
+    }
+
     pub(crate) async fn setup_external_instance_pg_unchecked(
         _db: &DB,
         _rotate_passwords: bool,
     ) -> Result<ExternalInstancePgSetupReport> {
         Err(unavailable())
     }
+
+    pub(crate) async fn external_instance_connection_unchecked(
+        _db: &DB,
+        _dbname: &str,
+        _replication: bool,
+    ) -> Result<PgDatabase> {
+        Err(unavailable())
+    }
+
+    pub(crate) async fn create_external_instance_database_unchecked(
+        _db: &DB,
+        _dbname: &str,
+        _tag: &str,
+    ) -> Result<()> {
+        Err(unavailable())
+    }
+
+    pub(crate) async fn drop_external_instance_database_unchecked(
+        _db: &DB,
+        _dbname: &str,
+    ) -> Result<()> {
+        Err(unavailable())
+    }
 }

backend/windmill-common/src/lib.rs

@@ -1086,7 +1086,13 @@ impl PgDatabase {
                 if err_str.contains("password authentication failed for user")
                     && err_str.contains("custom_instance_user")
                 {
-                    if let Some(db) = main_db {
+                    // The external instance cluster has a `custom_instance_user` of its own, whose
+                    // password setup manages. Rotating the local one would break every instance
+                    // data table and fix nothing.
+                    let local = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
+                    let on_local_cluster = local.host == self.host
+                        && local.port.unwrap_or(5432) == self.port.unwrap_or(5432);
+                    if let Some(db) = main_db.filter(|_| on_local_cluster) {
                         tracing::warn!(
                             "custom_instance_user password auth failed, refreshing and retrying..."
                         );

backend/windmill-common/src/workspaces.rs

@@ -1397,6 +1397,18 @@ pub enum DataTableCatalogResourceType {
     #[strum(serialize = "postgres")]
     Postgresql,
     Instance,
+    /// On the external instance cluster ([`crate::external_instance_pg`]). Enterprise Edition.
+    #[serde(rename = "external_instance")]
+    #[strum(serialize = "external_instance")]
+    ExternalInstance,
+}
+
+impl DataTableCatalogResourceType {
+    /// A database Windmill created and administers, on its own cluster or the external one, as
+    /// opposed to one a user brought as a resource.
+    pub fn is_windmill_managed(self) -> bool {
+        matches!(self, Self::Instance | Self::ExternalInstance)
+    }
 }
 
 /// Build a self-teaching error for an unresolved `datatable://<name>` reference.
@@ -1523,7 +1535,8 @@ pub async fn resolve_governing_datatable(
 }
 
 /// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance
-/// database, the user's own resource for a BYO-postgres one.
+/// database, on Windmill's cluster or the external one; the user's own resource for a BYO-postgres
+/// one.
 async fn resolve_datatable_connection_unchecked(
     db: &DB,
     governing: &GoverningDatatable,
@@ -1534,7 +1547,16 @@ async fn resolve_datatable_connection_unchecked(
         .database
         .as_ref()
         .expect("a governing entry owns a database");
-    if database.resource_type == DataTableCatalogResourceType::Instance {
+    if database.resource_type == DataTableCatalogResourceType::ExternalInstance {
+        let pg_creds = crate::external_instance_pg::external_instance_connection_unchecked(
+            db,
+            &database.resource_path,
+            replication,
+        )
+        .await?;
+        serde_json::to_value(&pg_creds)
+            .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))
+    } else if database.resource_type == DataTableCatalogResourceType::Instance {
         let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
         pg_creds.dbname = database.resource_path.clone();
         if replication {