fix(datatables): hold the fork lock across a rename's settings copy

Fork cleanup of the old id could otherwise drop a copy the renamed workspace goes on using. Also
states the authorization contract of the instance database drop helpers.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

backend/Cargo.lock

@@ -15653,7 +15653,6 @@ dependencies = [
  "pin-project-lite",
  "pkcs1",
  "postgres-native-tls 0.5.3",
- "postgres-protocol",
  "prometheus",
  "quick_cache",
  "rand 0.9.0",

backend/Cargo.toml

@@ -624,7 +624,6 @@ wasm-bindgen-test = "^0"
 convert_case = "0.6.0"
 getrandom = "0.2"
 tokio-postgres = {version = "^0.7", features = ["array-impls", "with-serde_json-1", "with-chrono-0_4", "with-uuid-1", "with-bit-vec-0_6"]}
-postgres-protocol = "0.6"
 rust-postgres = { package = "tokio-postgres", git = "https://github.com/MaterializeInc/rust-postgres", rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe"}
 rust-postgres-native-tls = { package = "postgres-native-tls", git = "https://github.com/MaterializeInc/rust-postgres", features = ["runtime"], rev = "78c1222577bb091d69bc22b1bc7ad01c14675abe" }
 bit-vec = "=0.6.3"

backend/ee-repo-ref.txt

@@ -1 +1 @@
-a97b5a5982d67e977dde6222380903f019da39c2
+c2e43f5b5ff753d339b70e232fd17b6ffecb054d

backend/windmill-api-settings/src/lib.rs

@@ -59,7 +59,7 @@ use windmill_common::{
     global_settings::{
         AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
         CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
-        DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
+        DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
         GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
         HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
         INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -167,22 +167,6 @@ pub fn global_service() -> Router {
             "/refresh_custom_instance_user_pwd",
             post(refresh_custom_instance_user_pwd),
         )
-        .route(
-            "/external_instance_pg/status",
-            get(get_external_instance_pg_status),
-        )
-        .route(
-            "/external_instance_pg/setup",
-            post(setup_external_instance_pg),
-        )
-        .route(
-            "/external_instance_pg/databases",
-            get(list_external_instance_pg_databases),
-        )
-        .route(
-            "/external_instance_pg/databases/{name}",
-            post(create_external_instance_pg_database).delete(drop_external_instance_pg_database),
-        )
         .route(
             "/setup_custom_instance_pg_database/{name}",
             post(setup_custom_instance_pg_database),
@@ -892,14 +876,6 @@ pub async fn set_global_setting_internal(
         )));
     }
 
-    if key == EXTERNAL_INSTANCE_PG_SETTING {
-        return windmill_common::external_instance_pg::write_external_instance_pg_setting(
-            db,
-            Some(&value),
-        )
-        .await;
-    }
-
     run_setting_pre_write_hook(db, &key, &value).await?;
 
     match value {
@@ -1281,7 +1257,7 @@ async fn set_instance_config(
     let desired_map = desired.global_settings.to_settings_map();
     if !desired_map.is_empty() {
         let current_map = current.global_settings.to_settings_map();
-        let mut settings_diff =
+        let settings_diff =
             instance_config::diff_global_settings(&current_map, &desired_map, ApplyMode::Merge);
         let ai_config_changed = settings_diff
             .upserts
@@ -1310,15 +1286,8 @@ async fn set_instance_config(
         }
 
         for (key, value) in &settings_diff.upserts {
-            if key != EXTERNAL_INSTANCE_PG_SETTING {
-                run_setting_pre_write_hook(&db, key, value).await?;
-            }
+            run_setting_pre_write_hook(&db, key, value).await?;
         }
-        windmill_common::external_instance_pg::write_external_instance_pg_from_diff(
-            &db,
-            &mut settings_diff,
-        )
-        .await?;
 
         instance_config::apply_settings_diff(&db, &settings_diff)
             .await
@@ -1720,8 +1689,33 @@ async fn list_custom_instance_pg_databases(
         })?;
 
     if !windmill_api_auth::is_super_admin_authed(&db, &authed).await? {
-        // Which workspace reserved a fork copy is nobody else's business: it would enumerate every
-        // pending fork on the instance.
+        // A fork copy's name gives away the workspace it was reserved for, so every pending fork on
+        // the instance would be listed. Kept for members of that workspace, and wherever the
+        // caller's workspaces use it, e.g. the fork it was finalized into.
+        let reserved_visible: BTreeSet<String> = sqlx::query_scalar(
+            r#"SELECT e.k FROM global_settings gs
+               CROSS JOIN LATERAL jsonb_each(gs.value->'databases') AS e(k, v)
+               WHERE gs.name = 'custom_instance_pg_databases' AND e.v->>'workspace_id' IS NOT NULL
+                 AND (EXISTS (SELECT 1 FROM usr WHERE usr.email = $1
+                                AND usr.workspace_id = e.v->>'workspace_id')
+                   OR EXISTS (SELECT 1 FROM usr JOIN workspace_settings ws
+                                ON ws.workspace_id = usr.workspace_id
+                              CROSS JOIN LATERAL jsonb_each(
+                                  CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
+                                      THEN ws.datatable->'datatables' ELSE '{}'::jsonb END) dt
+                              WHERE usr.email = $1
+                                AND dt.value->'database'->>'resource_type' = 'instance'
+                                AND dt.value->'database'->>'resource_path' = e.k))"#,
+        )
+        .bind(&authed.email)
+        .fetch_all(&db)
+        .await?
+        .into_iter()
+        .collect();
+        result.retain(|dbname, entry| {
+            entry.workspace_id.is_none() || reserved_visible.contains(dbname)
+        });
+        // Which workspace reserved a copy is still only for superadmins.
         for entry in result.values_mut() {
             entry.workspace_id = None;
         }
@@ -1783,135 +1777,6 @@ async fn refresh_custom_instance_user_pwd(
     Ok(Json(()))
 }
 
-async fn get_external_instance_pg_status(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgStatus> {
-    require_super_admin(&db, &authed).await?;
-    Ok(Json(
-        windmill_common::external_instance_pg::external_instance_pg_status(&db).await?,
-    ))
-}
-
-#[derive(Deserialize)]
-struct SetupExternalInstancePgBody {
-    #[serde(default)]
-    rotate_passwords: bool,
-}
-
-async fn setup_external_instance_pg(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Json(body): Json<SetupExternalInstancePgBody>,
-) -> JsonResult<windmill_common::external_instance_pg::ExternalInstancePgSetupReport> {
-    require_super_admin(&db, &authed).await?;
-    let report = windmill_common::external_instance_pg::setup_external_instance_pg_unchecked(
-        &db,
-        body.rotate_passwords,
-    )
-    .await?;
-    let rotated = body.rotate_passwords.to_string();
-    let success = report.success.to_string();
-    windmill_audit::audit_oss::audit_log(
-        &db,
-        &authed,
-        "settings.setup_external_instance_pg",
-        windmill_audit::ActionKind::Update,
-        "global",
-        Some(&authed.email),
-        Some(
-            [
-                ("rotate_passwords", rotated.as_str()),
-                ("success", success.as_str()),
-            ]
-            .into(),
-        ),
-    )
-    .await?;
-    Ok(Json(report))
-}
-
-#[derive(Serialize)]
-struct ExternalInstancePgDatabase {
-    #[serde(flatten)]
-    status: windmill_common::instance_config::CustomInstanceDb,
-    used_by_workspaces: Vec<String>,
-}
-
-async fn list_external_instance_pg_databases(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-) -> JsonResult<std::collections::BTreeMap<String, ExternalInstancePgDatabase>> {
-    require_super_admin(&db, &authed).await?;
-    let databases = windmill_common::external_instance_pg::external_instance_databases(&db).await?;
-    let mut usages =
-        windmill_common::external_instance_pg::external_instance_database_usages(&db).await?;
-    Ok(Json(
-        databases
-            .into_iter()
-            .map(|(name, status)| {
-                let used_by_workspaces = usages.remove(&name).unwrap_or_default();
-                (
-                    name,
-                    ExternalInstancePgDatabase {
-                        status,
-                        used_by_workspaces: used_by_workspaces.into_iter().collect(),
-                    },
-                )
-            })
-            .collect(),
-    ))
-}
-
-async fn create_external_instance_pg_database(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(dbname): Path<String>,
-    Json(body): Json<SetupCustomInstanceDbBody>,
-) -> JsonResult<()> {
-    require_super_admin(&db, &authed).await?;
-    let tag = body.tag.as_deref().unwrap_or("datatable");
-    windmill_common::external_instance_pg::create_external_instance_database_unchecked(
-        &db, &dbname, tag, None,
-    )
-    .await?;
-    windmill_audit::audit_oss::audit_log(
-        &db,
-        &authed,
-        "settings.create_external_instance_pg_database",
-        windmill_audit::ActionKind::Create,
-        "global",
-        Some(&authed.email),
-        Some([("dbname", dbname.as_str()), ("tag", tag)].into()),
-    )
-    .await?;
-    Ok(Json(()))
-}
-
-async fn drop_external_instance_pg_database(
-    authed: ApiAuthed,
-    Extension(db): Extension<DB>,
-    Path(dbname): Path<String>,
-) -> JsonResult<()> {
-    require_super_admin(&db, &authed).await?;
-    // A data table naming a dropped database fails on every job, far from the drop that caused it.
-    windmill_common::external_instance_pg::drop_external_instance_database_unchecked(
-        &db, &dbname, None,
-    )
-    .await?;
-    windmill_audit::audit_oss::audit_log(
-        &db,
-        &authed,
-        "settings.drop_external_instance_pg_database",
-        windmill_audit::ActionKind::Delete,
-        "global",
-        Some(&authed.email),
-        Some([("dbname", dbname.as_str())].into()),
-    )
-    .await?;
-    Ok(Json(()))
-}
-
 #[derive(Deserialize)]
 struct SetupCustomInstanceDbBody {
     tag: Option<String>,
@@ -1926,6 +1791,15 @@ async fn setup_custom_instance_pg_database(
     // Before anything is recorded: the status written below replaces the registry entry, and with it
     // the workspace a fork copy is reserved for.
     require_super_admin(&db, &authed).await?;
+    // Fork cleanup checks and drops the database and its entry under this lock. Held from before
+    // the setup creates the database to after its entry is written, neither lands on the other's
+    // half-done state: a dropped database with its entry written back, or the reverse.
+    let mut tx = db.begin().await?;
+    windmill_common::datatable_roles::lock_instance_databases_governance(
+        &mut tx,
+        [dbname.trim()],
+    )
+    .await?;
     let mut logs = CustomInstanceDbLogs::default();
     let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
     let success = result.is_ok();
@@ -1952,8 +1826,9 @@ async fn setup_custom_instance_pg_database(
     )
     .bind(&dbname)
     .bind(&status_json)
-    .fetch_one(&db)
+    .fetch_one(&mut *tx)
     .await?;
+    tx.commit().await?;
     let status: CustomInstanceDb = serde_json::from_value(saved).map_err(to_anyhow)?;
 
     Ok(Json(status))

backend/windmill-api-workspaces/src/datatable_migrations.rs

@@ -11,7 +11,7 @@
 //! to keep that file focused on core workspace configuration.
 
 use crate::workspaces::{
-    managed_datatable_kind, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
+    is_instance_datatable, pg_dump_database, strip_unreplayable_dump_lines, ItemComparison,
     PgDumpOptions,
 };
 
@@ -1556,9 +1556,7 @@ async fn generate_initial_datatable_migration(
     // without what a replay elsewhere cannot run: the replaying user owns none of this
     // database's objects, and the grants Windmill plants in an instance database (`ALTER
     // DEFAULT PRIVILEGES FOR ROLE ...`) fail even replaying onto the same server.
-    let no_acl = managed_datatable_kind(&db, &w_id, &datatable_name)
-        .await?
-        .is_some();
+    let no_acl = is_instance_datatable(&db, &w_id, &datatable_name).await?;
     let dump_file = pg_dump_database(
         &pg_db,
         PgDumpOptions {

backend/windmill-api-workspaces/src/workspaces.rs

@@ -3078,32 +3078,23 @@ pub(crate) async fn resolve_pg_source_checked(
         .map_err(|e| Error::internal_err(format!("Failed to parse database credentials: {}", e)))
 }
 
-/// The kind of the database backing the data table `name` when Windmill manages it (on its own
-/// cluster or the external one), `None` when it is a user resource.
-pub(crate) async fn managed_datatable_kind(
-    db: &DB,
-    w_id: &str,
-    name: &str,
-) -> Result<Option<DataTableCatalogResourceType>> {
+/// Whether the data table `name` is backed by the Windmill instance's own PostgreSQL
+/// rather than a user resource.
+pub(crate) async fn is_instance_datatable(db: &DB, w_id: &str, name: &str) -> Result<bool> {
     // Resolved rather than read: a pointer entry owns no database of its own, so only the entry it
-    // lands on can answer. A name that resolves to nothing keeps the historical `None`.
+    // lands on can answer. A name that resolves to nothing keeps the historical `false`.
     Ok(resolve_governing_datatable(db, w_id, name)
         .await
         .ok()
         .and_then(|g| g.datatable.database)
-        .map(|d| d.resource_type)
-        .filter(|kind| kind.is_windmill_managed()))
+        .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance))
 }
 
 /// Same, for the `datatable://<name>` / `$res:<path>` form the import endpoints take.
-async fn managed_datatable_source_kind(
-    db: &DB,
-    w_id: &str,
-    source: &str,
-) -> Result<Option<DataTableCatalogResourceType>> {
+async fn is_instance_datatable_source(db: &DB, w_id: &str, source: &str) -> Result<bool> {
     match source.strip_prefix("datatable://") {
-        Some(name) => managed_datatable_kind(db, w_id, name).await,
-        None => Ok(None),
+        Some(name) => is_instance_datatable(db, w_id, name).await,
+        None => Ok(false),
     }
 }
 
@@ -3202,7 +3193,10 @@ pub(crate) async fn pg_dump_database(
     if let Some(ref password) = pg_db.password {
         cmd.env("PGPASSWORD", password);
     }
-    let _root_cert = apply_pg_tls_env(&mut cmd, pg_db)?;
+
+    if let Some(ref sslmode) = pg_db.sslmode {
+        cmd.env("PGSSLMODE", sslmode);
+    }
 
     let output = cmd
         .output()
@@ -3320,7 +3314,7 @@ async fn comment_out_unsupported_settings(
 
 /// A psql invocation against `pg_db`, carrying the connection settings the CLI reads
 /// from the environment.
-fn psql_command(pg_db: &PgDatabase) -> Result<(tokio::process::Command, Option<DumpFile>)> {
+fn psql_command(pg_db: &PgDatabase) -> tokio::process::Command {
     let mut cmd = tokio::process::Command::new("psql");
     cmd.arg("--host")
         .arg(&pg_db.host)
@@ -3337,88 +3331,10 @@ fn psql_command(pg_db: &PgDatabase) -> Result<(tokio::process::Command, Option<D
     if let Some(ref password) = pg_db.password {
         cmd.env("PGPASSWORD", password);
     }
-    let root_cert = apply_pg_tls_env(&mut cmd, pg_db)?;
-    Ok((cmd, root_cert))
-}
-
-/// Give libpq the TLS settings `PgDatabase::connect` applies. The returned file holds the root
-/// certificate `PGSSLROOTCERT` names, so it must outlive the command.
-fn apply_pg_tls_env(
-    cmd: &mut tokio::process::Command,
-    pg_db: &PgDatabase,
-) -> Result<Option<DumpFile>> {
     if let Some(ref sslmode) = pg_db.sslmode {
         cmd.env("PGSSLMODE", sslmode);
     }
-    if let Some(pem) = pg_db
-        .root_certificate_pem
-        .as_deref()
-        .filter(|p| !p.is_empty())
-    {
-        let file = DumpFile::new()?;
-        std::fs::write(&file.path, pem)
-            .map_err(|e| Error::internal_err(format!("Failed to write root certificate: {e}")))?;
-        cmd.env("PGSSLROOTCERT", &file.path);
-        return Ok(Some(file));
-    }
-    // Only a connection that asked to be verified against the system trust store. Without a file,
-    // libpq's own default would look for `~/.postgresql/root.crt` and refuse a verify-* mode. libpq
-    // takes the special `system` value with verify-full only, so verify-ca needs the bundle itself.
-    if pg_db.accept_invalid_certs == Some(false) {
-        match pg_db.sslmode.as_deref() {
-            Some("verify-full") => {
-                cmd.env("PGSSLROOTCERT", "system");
-            }
-            Some("verify-ca") => {
-                if let Some(bundle) = windmill_common::system_ca_bundle() {
-                    cmd.env("PGSSLROOTCERT", bundle);
-                }
-            }
-            _ => {}
-        }
-    }
-    Ok(None)
-}
-
-
-#[cfg(test)]
-mod pg_tls_env_tests {
-    use super::apply_pg_tls_env;
-    use windmill_common::PgDatabase;
-
-    fn root_cert_env(sslmode: &str) -> Option<std::ffi::OsString> {
-        let pg_db = PgDatabase {
-            host: "db".to_string(),
-            user: None,
-            password: None,
-            port: None,
-            sslmode: Some(sslmode.to_string()),
-            dbname: "d".to_string(),
-            root_certificate_pem: None,
-            accept_invalid_certs: Some(false),
-            use_iam_auth: None,
-            region: None,
-        };
-        let mut cmd = tokio::process::Command::new("psql");
-        apply_pg_tls_env(&mut cmd, &pg_db).unwrap();
-        cmd.as_std()
-            .get_envs()
-            .find(|(k, _)| *k == "PGSSLROOTCERT")
-            .and_then(|(_, v)| v.map(|v| v.to_os_string()))
-    }
-
-    #[test]
-    fn system_roots_only_through_verify_full() {
-        assert_eq!(
-            root_cert_env("verify-full").as_deref(),
-            Some("system".as_ref())
-        );
-        // libpq refuses `sslrootcert=system` with verify-ca, which would fail every dump and restore.
-        assert_ne!(
-            root_cert_env("verify-ca").as_deref(),
-            Some("system".as_ref())
-        );
-    }
+    cmd
 }
 
 /// GUC names the server backing `pg_db` knows about.
@@ -3428,8 +3344,7 @@ mod pg_tls_env_tests {
 /// and an unset mode, where `PgDatabase::connect` would hand a TLS-only server a
 /// plaintext socket and fail before the import ever starts.
 async fn server_setting_names(pg_db: &PgDatabase) -> Result<HashSet<String>> {
-    let (mut cmd, _root_cert) = psql_command(pg_db)?;
-    let output = cmd
+    let output = psql_command(pg_db)
         .arg("--tuples-only")
         .arg("--no-align")
         .arg("--command")
@@ -3463,8 +3378,7 @@ async fn pg_import_dump(target_db: &PgDatabase, dump_file: &DumpFile) -> Result<
     let supported_settings = server_setting_names(target_db).await?;
     comment_out_unsupported_settings(dump_file, &supported_settings).await?;
 
-    let (mut cmd, _root_cert) = psql_command(target_db)?;
-    let output = cmd
+    let output = psql_command(target_db)
         .arg("--set")
         .arg("ON_ERROR_STOP=1")
         .arg("--single-transaction")
@@ -3522,16 +3436,20 @@ async fn create_pg_database(
         }
     }
 
-    let source_kind = managed_datatable_source_kind(&db, &w_id, &req.source).await?;
-    if source_kind == Some(DataTableCatalogResourceType::ExternalInstance) {
-        windmill_common::external_instance_pg::create_external_instance_database_unchecked(
-            &db,
-            &req.target_dbname,
-            "datatable",
-            Some(&w_id),
-        )
-        .await?;
-    } else if source_kind == Some(DataTableCatalogResourceType::Instance) {
+    if is_instance_datatable_source(&db, &w_id, &req.source).await? {
+        // Held until the copy is registered, as a rename migrates reservations to the new id under
+        // it once the old one is archived: a copy registered after that would be reserved for an
+        // id nothing answers on.
+        let mut tx = db.begin().await?;
+        windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
+        let live = sqlx::query_scalar::<_, bool>("SELECT NOT deleted FROM workspace WHERE id = $1")
+            .bind(&w_id)
+            .fetch_optional(&mut *tx)
+            .await?
+            .unwrap_or(false);
+        if !live {
+            return Err(Error::BadRequest(format!("Workspace '{w_id}' is archived")));
+        }
         windmill_common::create_custom_instance_database(
             &db,
             &req.target_dbname,
@@ -3539,6 +3457,7 @@ async fn create_pg_database(
             Some(&w_id),
         )
         .await?;
+        tx.commit().await?;
     } else {
         let source_pg =
             resolve_pg_source_checked(&db, &user_db, &authed, &w_id, &req.source).await?;
@@ -3639,12 +3558,12 @@ async fn ensure_datatable_is_clonable(
     }
     // The copy has to name a database of its own. A resource-backed entry reached through a
     // pointer names one this workspace does not own, so there is nothing here to repoint.
-    let is_managed = governing
+    let is_instance = governing
         .datatable
         .database
         .as_ref()
-        .is_some_and(|d| d.resource_type.is_windmill_managed());
-    if governing.workspace_id != w_id && !is_managed {
+        .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance);
+    if governing.workspace_id != w_id && !is_instance {
         return Err(Error::BadRequest(format!(
             "Data table '{name}' points at a resource-backed data table in another workspace \
              and cannot be copied; fork it from the workspace that owns it."
@@ -3693,18 +3612,19 @@ async fn import_pg_database(
                         .to_string(),
                 ));
             }
-            if let Some(kind) = managed_datatable_source_kind(&db, &w_id, &req.target).await? {
-                // Held until the restore is done, as fork finalization takes it: a fork must not
-                // commit this database while `psql` is still filling it.
+            if is_instance_datatable_source(&db, &w_id, &req.target).await? {
+                // Held until the restore is done: fork finalization takes the first, and every
+                // save newly naming a database, in any workspace, the second. Nothing may start
+                // using this database while `psql` is still filling it.
                 let mut tx = db.begin().await?;
                 windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
-                windmill_common::ensure_fork_database_available_to(
-                    &db,
-                    kind,
-                    override_dbname,
-                    &w_id,
+                windmill_common::datatable_roles::lock_instance_databases_governance(
+                    &mut tx,
+                    [override_dbname.as_str()],
                 )
                 .await?;
+                windmill_common::ensure_fork_database_available_to(&db, override_dbname, &w_id)
+                    .await?;
                 fork_lock = Some(tx);
             }
         }
@@ -3716,12 +3636,8 @@ async fn import_pg_database(
     // what it creates it owns. Grants do, except around an instance data table — Windmill
     // plants `custom_instance_user` grants in one, which nothing else can replay. Elsewhere
     // the ACLs are user intent (`REVOKE ... FROM PUBLIC`) and dropping them widens access.
-    let no_acl = managed_datatable_source_kind(&db, &w_id, &req.target)
-        .await?
-        .is_some()
-        || managed_datatable_source_kind(&db, &w_id, &req.source)
-            .await?
-            .is_some();
+    let no_acl = is_instance_datatable_source(&db, &w_id, &req.target).await?
+        || is_instance_datatable_source(&db, &w_id, &req.source).await?;
 
     let dump_file = pg_dump_database(
         &source_pg,
@@ -3823,51 +3739,54 @@ async fn edit_ducklake_config(
     )
     .await?;
 
-    let old_ducklakes = sqlx::query_scalar!(
-        r#"
-            SELECT ws.ducklake->'ducklakes' AS ducklake_name
-            FROM workspace_settings ws
-            WHERE ws.workspace_id = $1
-        "#,
-        &w_id
+    // Under the row lock the save writes with, taken before the database locks below as fork
+    // cleanup takes the two.
+    let old_ducklakes = sqlx::query_scalar::<_, Option<serde_json::Value>>(
+        "SELECT ws.ducklake->'ducklakes' FROM workspace_settings ws
+         WHERE ws.workspace_id = $1 FOR UPDATE",
     )
+    .bind(&w_id)
     .fetch_one(&mut *tx)
     .await?
     .unwrap_or(serde_json::Value::Null);
     let old_ducklakes: HashMap<String, Ducklake> =
         serde_json::from_value(old_ducklakes).unwrap_or_default();
 
-    // Check that non-superadmins are not abusing Instance databases. An unchanged catalog is left
-    // alone either way, so a downgraded instance can still save lakes that already name an
-    // external instance database.
-    for (name, dl) in new_config.settings.ducklakes.iter() {
-        let kind = &dl.catalog.resource_type;
-        if !matches!(
-            kind,
-            DucklakeCatalogResourceType::Instance | DucklakeCatalogResourceType::ExternalInstance
-        ) {
-            continue;
-        }
-        let unchanged = old_ducklakes.get(name).is_some_and(|old| {
-            &old.catalog.resource_type == kind
-                && old.catalog.resource_path == dl.catalog.resource_path
-        });
-        if unchanged {
-            continue;
-        }
-        if *kind == DucklakeCatalogResourceType::ExternalInstance {
-            windmill_common::external_instance_pg::ensure_external_instance_available()?;
-            windmill_common::external_instance_pg::ensure_external_instance_database_registered(
-                &mut tx,
-                &dl.catalog.resource_path,
-            )
-            .await?;
-        }
-        if !is_superadmin {
-            return Err(Error::BadRequest(
-                "Only superadmins can create or modify ducklakes with Instance databases"
-                    .to_string(),
-            ));
+    // Fork cleanup decides nothing uses an instance database under this lock, so a catalog newly
+    // put on one must not commit between its check and its drop.
+    windmill_common::datatable_roles::lock_instance_databases_governance(
+        &mut *tx,
+        new_config
+            .settings
+            .ducklakes
+            .iter()
+            .filter(|(name, dl)| {
+                dl.catalog.resource_type == DucklakeCatalogResourceType::Instance
+                    && old_ducklakes.get(name.as_str()).is_none_or(|old| {
+                        old.catalog.resource_type != DucklakeCatalogResourceType::Instance
+                            || old.catalog.resource_path != dl.catalog.resource_path
+                    })
+            })
+            .map(|(_, dl)| dl.catalog.resource_path.as_str()),
+    )
+    .await?;
+
+    // Check that non-superadmins are not abusing Instance databases
+    if !is_superadmin {
+        for (name, dl) in new_config.settings.ducklakes.iter() {
+            if dl.catalog.resource_type == DucklakeCatalogResourceType::Instance {
+                let old_dl = old_ducklakes.get(name);
+                if old_dl.is_none()
+                    || old_dl.unwrap().catalog.resource_type
+                        != DucklakeCatalogResourceType::Instance
+                    || old_dl.unwrap().catalog.resource_path != dl.catalog.resource_path
+                {
+                    return Err(Error::BadRequest(
+                        "Only superadmins can create or modify ducklakes with Instance databases"
+                            .to_string(),
+                    ));
+                }
+            }
         }
     }
 
@@ -4041,7 +3960,6 @@ async fn edit_datatable_config(
                 // so these line up with the `datatable_configured` adoption counts.
                 created_substrates.push(match dt.database.as_ref().map(|d| d.resource_type) {
                     Some(DataTableCatalogResourceType::Instance) => "instance",
-                    Some(DataTableCatalogResourceType::ExternalInstance) => "external_instance",
                     Some(DataTableCatalogResourceType::Postgresql) => "postgresql",
                     None => "reference",
                 });
@@ -4097,41 +4015,26 @@ async fn edit_datatable_config(
     // Check that non-superadmins are not abusing Instance databases, which reach a database this
     // workspace does not own. Pointing an entry at another workspace's data table is not checked
     // here because it cannot be requested at all: `reference` is overwritten from the stored entry
-    // above, for every caller. An unchanged entry is left alone either way, so a downgraded
-    // instance can still save settings that already name an external instance database.
-    for (name, dt) in new_config.settings.datatables.iter() {
-        let Some(database) = dt
-            .database
-            .as_ref()
-            .filter(|d| d.resource_type.is_windmill_managed())
-        else {
-            continue;
-        };
-        let unchanged = old_datatables
-            .get(name)
-            .and_then(|o| o.database.as_ref())
-            .is_some_and(|o| {
-                o.resource_type == database.resource_type
-                    && o.resource_path == database.resource_path
-            });
-        if unchanged {
-            continue;
-        }
-        // Before the registration check, whose refusal would otherwise tell a workspace admin
-        // which databases exist on the cluster.
-        if !is_superadmin {
-            return Err(Error::BadRequest(
-                "Only superadmins can create or modify data tables with Instance databases"
-                    .to_string(),
-            ));
-        }
-        if database.resource_type == DataTableCatalogResourceType::ExternalInstance {
-            windmill_common::external_instance_pg::ensure_external_instance_available()?;
-            windmill_common::external_instance_pg::ensure_external_instance_database_registered(
-                &mut tx,
-                &database.resource_path,
-            )
-            .await?;
+    // above, for every caller.
+    if !is_superadmin {
+        for (name, dt) in new_config.settings.datatables.iter() {
+            let old_dt = old_datatables.get(name);
+            if dt
+                .database
+                .as_ref()
+                .is_some_and(|d| d.resource_type == DataTableCatalogResourceType::Instance)
+            {
+                let unchanged = old_dt.and_then(|o| o.database.as_ref()).is_some_and(|o| {
+                    o.resource_type == DataTableCatalogResourceType::Instance
+                        && Some(&o.resource_path) == dt.database.as_ref().map(|d| &d.resource_path)
+                });
+                if !unchanged {
+                    return Err(Error::BadRequest(
+                        "Only superadmins can create or modify data tables with Instance databases"
+                            .to_string(),
+                    ));
+                }
+            }
         }
     }
 
@@ -8245,14 +8148,13 @@ async fn point_kept_datatables_at_parent(
         if dt.reference.is_some() {
             continue;
         }
-        // Only instance databases, on either cluster. A resource-backed data table names a
-        // resource, and the settings clone gave the fork its own copy of that resource in its own
-        // workspace — pointing at the parent's entry would silently move the fork onto the
-        // parent's resource instead.
+        // Only instance databases. A resource-backed data table names a resource, and the settings
+        // clone gave the fork its own copy of that resource in its own workspace — pointing at the
+        // parent's entry would silently move the fork onto the parent's resource instead.
         if dt
             .database
             .as_ref()
-            .is_none_or(|d| !d.resource_type.is_windmill_managed())
+            .is_none_or(|d| d.resource_type != DataTableCatalogResourceType::Instance)
         {
             continue;
         }
@@ -8382,30 +8284,26 @@ async fn apply_forked_datatable(
         })?,
     };
 
-    if database.resource_type == DataTableCatalogResourceType::ExternalInstance {
-        windmill_common::external_instance_pg::ensure_external_instance_database_registered(
-            tx,
-            &fdt.new_dbname,
+    if database.resource_type == DataTableCatalogResourceType::Instance {
+        // Held until the fork commits, as every save newly naming a database takes it: none may
+        // claim the copy between the check below and this fork's entry landing on it.
+        windmill_common::datatable_roles::lock_instance_databases_governance(
+            &mut **tx,
+            [fdt.new_dbname.as_str()],
         )
         .await?;
     }
-    if database.resource_type.is_windmill_managed()
+    if database.resource_type == DataTableCatalogResourceType::Instance
         && !windmill_api_auth::is_super_admin_authed(db, authed).await?
     {
-        windmill_common::ensure_fork_database_available_to(
-            db,
-            database.resource_type,
-            &fdt.new_dbname,
-            parent_w_id,
-        )
-        .await?;
+        windmill_common::ensure_fork_database_available_to(db, &fdt.new_dbname, parent_w_id)
+            .await?;
     }
-    if database.resource_type.is_windmill_managed() {
+    if database.resource_type == DataTableCatalogResourceType::Instance {
         // The whole `database` object, not just its `resource_path`: a pointer entry has none to
-        // patch. `reference` goes with it — exactly one of the two may be set. The copy was created
-        // on the same cluster as its source, so it keeps the source's kind.
+        // patch. `reference` goes with it — exactly one of the two may be set.
         let new_database = serde_json::json!({
-            "resource_type": database.resource_type,
+            "resource_type": "instance",
             "resource_path": &fdt.new_dbname,
         });
         sqlx::query!(
@@ -8796,8 +8694,6 @@ async fn create_workspace_fork(
     let mut tx: Transaction<'_, Postgres> = db.begin().await?;
     // Before the settings clone reads the parent's data tables: a pointer this fork ends up with
     // must not be written after cleanup of the parent decided that nothing points at its copies.
-    // Also before the external cluster's lifecycle lock, which finalizing an external copy takes:
-    // fork cleanup takes the two in this order.
     windmill_common::workspaces::lock_fork_datatables(&mut tx, &parent_workspace_id).await?;
 
     if nw.is_dev_workspace {

backend/windmill-api-workspaces/src/workspaces_extra.rs

@@ -56,6 +56,11 @@ pub(crate) async fn change_workspace_id(
 
     let mut tx = db.begin().await?;
 
+    // The settings copy below carries every data table entry to the new id, which fork cleanup of
+    // the old id cannot see until this commits: without the lock it could drop a copy the renamed
+    // workspace goes on using. Before the pairing lock, as forking takes the two in that order.
+    windmill_common::workspaces::lock_fork_datatables(&mut tx, &old_id).await?;
+
     // A rename rewrites the workspace's dev flag and reparents its children, so it decides on the
     // same state the pairing handlers do: without this lock a concurrent create/attach could commit
     // an active dev workspace under the shell this rename is about to archive. Both ids, since the
@@ -110,21 +115,6 @@ pub(crate) async fn change_workspace_id(
     .execute(&mut *tx)
     .await?;
 
-    // A fork copy reserved for the old id would otherwise be unreachable: its creator cannot
-    // import into it or finish its fork under the new id, and nothing else would ever drop it.
-    sqlx::query(
-        r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (
-               SELECT COALESCE(jsonb_object_agg(k, CASE WHEN v->>'workspace_id' = $1
-                   THEN jsonb_set(v, '{workspace_id}', to_jsonb($2::text)) ELSE v END), '{}'::jsonb)
-               FROM jsonb_each(COALESCE(value->'databases', '{}'::jsonb)) AS e(k, v)
-           ))
-           WHERE name = 'custom_instance_pg_databases'"#,
-    )
-    .bind(&old_id)
-    .bind(&rw.new_id)
-    .execute(&mut *tx)
-    .await?;
-
     // Duplicate workspace settings (keep copy in old workspace for reference)
     info!("Duplicating workspace_settings table");
     sqlx::query!(
@@ -865,6 +855,10 @@ pub(crate) async fn change_workspace_id(
         }
     }
 
+    // After every workspace_settings write above: fork cleanup locks a settings row before the
+    // registry, so taking the registry first here would deadlock with it.
+    migrate_fork_reservations(&mut tx, &old_id, &rw.new_id).await?;
+
     // Audit log in the same transaction as the workspace changes
     audit_log(
         &mut *tx,
@@ -933,6 +927,14 @@ pub(crate) async fn change_workspace_id(
     let (_schedules_count, canceled_count, _deleted_tokens_count) =
         archive_workspace_impl(&db, &old_id, &authed.username, None).await?;
 
+    // The old id stays live between the commit above and the archive, and a fork copy created for
+    // it in that window registers under it. Creation checks the workspace is live under the fork
+    // lock, so once this has run under it, no copy can be reserved for the old id any more.
+    let mut tx = db.begin().await?;
+    windmill_common::workspaces::lock_fork_datatables(&mut tx, &old_id).await?;
+    migrate_fork_reservations(&mut tx, &old_id, &rw.new_id).await?;
+    tx.commit().await?;
+
     info!(
         "Workspace id change completed: moved {} to {}, archived old workspace",
         old_id, rw.new_id
@@ -944,6 +946,28 @@ pub(crate) async fn change_workspace_id(
     ))
 }
 
+/// A fork copy reserved for the old id would otherwise be unreachable: its creator cannot import
+/// into it or finish its fork under the new id, and nothing else would ever drop it.
+async fn migrate_fork_reservations(
+    tx: &mut Transaction<'_, Postgres>,
+    old_id: &str,
+    new_id: &str,
+) -> Result<()> {
+    sqlx::query(
+        r#"UPDATE global_settings SET value = jsonb_set(value, '{databases}', (
+               SELECT COALESCE(jsonb_object_agg(k, CASE WHEN v->>'workspace_id' = $1
+                   THEN jsonb_set(v, '{workspace_id}', to_jsonb($2::text)) ELSE v END), '{}'::jsonb)
+               FROM jsonb_each(COALESCE(value->'databases', '{}'::jsonb)) AS e(k, v)
+           ))
+           WHERE name = 'custom_instance_pg_databases'"#,
+    )
+    .bind(old_id)
+    .bind(new_id)
+    .execute(&mut **tx)
+    .await?;
+    Ok(())
+}
+
 #[derive(Deserialize)]
 pub(crate) struct DeleteWorkspaceQuery {
     pub(crate) only_delete_forks: Option<bool>,
@@ -1424,7 +1448,9 @@ pub async fn drop_forked_datatable_databases(
             _ => continue,
         };
 
-        if database.resource_type.is_windmill_managed() {
+        if database.resource_type
+            == windmill_common::workspaces::DataTableCatalogResourceType::Instance
+        {
             let db_to_drop = &database.resource_path;
             if !db_to_drop.starts_with("wm_fork_") {
                 errors.push(format!(
@@ -1436,36 +1462,54 @@ pub async fn drop_forked_datatable_databases(
             // The fork's own entry is what is going away; anything else still reaching the copy,
             // a child fork's pointer at this entry included, keeps it. The lock keeps a child fork
             // from gaining such a pointer before the drop.
-            let dropped = async {
-                let mut tx = db.begin().await?;
-                // The three locks a settings save takes, in its order: this workspace's data
-                // tables, its settings row, and the database itself. Without them a save could
-                // rename this entry, or point another one here, either side of the check below.
-                windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
-                sqlx::query("SELECT 1 FROM workspace_settings WHERE workspace_id = $1 FOR UPDATE")
+            // A task of its own, so a client going away cannot stop it between dropping the
+            // database and committing the entry's removal.
+            let dropped = tokio::spawn({
+                let (db, w_id, dt_name, db_to_drop) = (
+                    db.clone(),
+                    w_id.clone(),
+                    dt_name.clone(),
+                    db_to_drop.clone(),
+                );
+                let resource_type = database.resource_type;
+                async move {
+                    let mut tx = db.begin().await?;
+                    // The three locks a settings save takes, in its order: this workspace's data
+                    // tables, its settings row, and the database itself. Without them a save could
+                    // rename this entry, or point another one here, either side of the check below.
+                    windmill_common::workspaces::lock_fork_datatables(&mut tx, &w_id).await?;
+                    // The snapshot above was read unlocked: a save committing since could have
+                    // repointed this entry, and the entry is removed below whatever it names by then.
+                    let current = sqlx::query_scalar::<_, Option<serde_json::Value>>(
+                        "SELECT datatable->'datatables'->$2 FROM workspace_settings
+                     WHERE workspace_id = $1 FOR UPDATE",
+                    )
                     .bind(&w_id)
+                    .bind(&dt_name)
                     .fetch_optional(&mut *tx)
-                    .await?;
-                windmill_common::datatable_roles::lock_instance_databases_governance(
-                    &mut tx,
-                    [db_to_drop.as_str()],
-                )
-                .await?;
-                if database.resource_type
-                    == windmill_common::workspaces::DataTableCatalogResourceType::ExternalInstance
-                {
-                    windmill_common::external_instance_pg::drop_external_instance_database_unchecked(
-                        &db,
-                        db_to_drop,
-                        Some((&w_id, dt_name)),
+                    .await?
+                    .flatten()
+                    .and_then(|v| serde_json::from_value::<DataTable>(v).ok());
+                    if !current.is_some_and(|dt| {
+                        dt.forked_from.is_some()
+                            && dt.database.is_some_and(|d| {
+                                d.resource_type == resource_type && d.resource_path == db_to_drop
+                            })
+                    }) {
+                        return Err(Error::BadRequest(
+                            "the data table changed while it was being cleaned up".to_string(),
+                        ));
+                    }
+                    windmill_common::datatable_roles::lock_instance_databases_governance(
+                        &mut tx,
+                        [db_to_drop.as_str()],
                     )
                     .await?;
-                } else {
                     let uses = windmill_common::workspaces::managed_database_uses(
                         &mut tx,
                         windmill_common::workspaces::DataTableCatalogResourceType::Instance,
-                        db_to_drop,
-                        Some((&w_id, dt_name)),
+                        &db_to_drop,
+                        Some((w_id.as_str(), dt_name.as_str())),
                     )
                     .await?;
                     if !uses.is_empty() {
@@ -1474,22 +1518,31 @@ pub async fn drop_forked_datatable_databases(
                             uses.join(", ")
                         )));
                     }
-                    windmill_common::drop_custom_instance_database(&db, db_to_drop).await?;
-                }
-                // The entry goes with the database: a fork this one is cloned into afterwards must
-                // not inherit a pointer at a data table whose database is gone.
-                sqlx::query(
+                    // The entry goes with the database: a fork this one is cloned into afterwards must
+                    // not inherit a pointer at a data table whose database is gone.
+                    sqlx::query(
                     "UPDATE workspace_settings SET datatable = datatable #- ARRAY['datatables', $2]
                      WHERE workspace_id = $1",
                 )
                 .bind(&w_id)
-                .bind(dt_name)
+                .bind(&dt_name)
                 .execute(&mut *tx)
                 .await?;
-                tx.commit().await?;
-                Ok::<_, Error>(())
-            }
-            .await;
+                    windmill_common::drop_custom_instance_database_keep_entry(&db, &db_to_drop)
+                        .await?;
+                    sqlx::query(
+                        "UPDATE global_settings SET value = value #- ARRAY['databases', $1]
+                     WHERE name = 'custom_instance_pg_databases'",
+                    )
+                    .bind(&db_to_drop)
+                    .execute(&mut *tx)
+                    .await?;
+                    tx.commit().await?;
+                    Ok::<_, Error>(())
+                }
+            })
+            .await
+            .unwrap_or_else(|e| Err(Error::internal_err(format!("cleanup task failed: {e}"))));
             if let Err(e) = dropped {
                 errors.push(format!(
                     "Could not drop instance database '{}' for datatable://{}: {}",
@@ -1857,17 +1910,7 @@ async fn resolve_fork_catalog_pg(
             "ducklake://{ducklake_name}: malformed registry catalog identity `{catalog}`"
         ))
     })?;
-    let catalog_resource = if resource_type == "external_instance" {
-        serde_json::to_value(
-            windmill_common::external_instance_pg::external_instance_connection_unchecked(
-                db,
-                resource_path,
-                false,
-            )
-            .await?,
-        )
-        .map_err(|e| Error::internal_err(format!("serializing pg creds: {e}")))?
-    } else if resource_type == "instance" {
+    let catalog_resource = if resource_type == "instance" {
         let mut pg_creds = windmill_common::PgDatabase::parse_uri(
             &windmill_common::get_database_url().await?.as_str().await,
         )?;

backend/windmill-api/openapi.yaml

@@ -1572,104 +1572,6 @@ paths:
               schema:
                 type: object
 
-  /settings/external_instance_pg/status:
-    get:
-      summary: Returns whether the external instance cluster is configured and how its last setup went
-      operationId: getExternalInstancePgStatus
-      tags:
-        - setting
-      responses:
-        "200":
-          description: external instance cluster status
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ExternalInstancePgStatus"
-
-  /settings/external_instance_pg/setup:
-    post:
-      summary: Sets up the external instance cluster with its saved admin login, optionally rotating the passwords Windmill manages on it (enterprise edition only)
-      operationId: setupExternalInstancePg
-      tags:
-        - setting
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                rotate_passwords:
-                  type: boolean
-      responses:
-        "200":
-          description: the setup report, also stored as the last setup
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ExternalInstancePgSetupReport"
-
-  /settings/external_instance_pg/databases:
-    get:
-      summary: Lists the databases Windmill created on the external instance cluster, with the workspaces whose data tables, Ducklake catalogs or pending fork cleanups use each
-      operationId: listExternalInstancePgDatabases
-      tags:
-        - setting
-      responses:
-        "200":
-          description: databases by name
-          content:
-            application/json:
-              schema:
-                type: object
-                additionalProperties:
-                  $ref: "#/components/schemas/CustomInstanceDb"
-
-  /settings/external_instance_pg/databases/{name}:
-    post:
-      summary: Creates a database on the external instance cluster (enterprise edition only)
-      operationId: createExternalInstancePgDatabase
-      tags:
-        - setting
-      parameters:
-        - name: name
-          in: path
-          required: true
-          schema:
-            type: string
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                tag:
-                  $ref: "#/components/schemas/CustomInstanceDbTag"
-      responses:
-        "200":
-          description: database created
-          content:
-            application/json:
-              schema: {}
-    delete:
-      summary: Drops a database Windmill created on the external instance cluster, refused while a data table, Ducklake catalog or pending fork cleanup uses it
-      operationId: dropExternalInstancePgDatabase
-      tags:
-        - setting
-      parameters:
-        - name: name
-          in: path
-          required: true
-          schema:
-            type: string
-      responses:
-        "200":
-          description: database dropped
-          content:
-            application/json:
-              schema: {}
-
   /settings/list_custom_instance_pg_databases:
     post:
       summary: Returns the set-up statuses of custom instance pg databases
@@ -5328,7 +5230,7 @@ paths:
                       type: string
                     resource_type:
                       type: string
-                      enum: [postgres, instance, external_instance]
+                      enum: [postgres, instance]
                     resource_path:
                       type: string
                     governing_workspace_id:
@@ -33719,44 +33621,6 @@ components:
         - ducklake
         - datatable
 
-    ExternalInstancePgSetupStep:
-      type: object
-      required: [name, status, message]
-      properties:
-        name:
-          type: string
-        status:
-          type: string
-          enum: [ok, warning, error]
-        message:
-          type: string
-
-    ExternalInstancePgSetupReport:
-      type: object
-      required: [success, finished_at, steps]
-      properties:
-        success:
-          type: boolean
-          description: no step failed; warnings leave it true
-        finished_at:
-          type: string
-          format: date-time
-        steps:
-          type: array
-          items:
-            $ref: "#/components/schemas/ExternalInstancePgSetupStep"
-
-    ExternalInstancePgStatus:
-      type: object
-      required: [configured, database_count]
-      properties:
-        configured:
-          type: boolean
-        database_count:
-          type: integer
-        last_setup:
-          $ref: "#/components/schemas/ExternalInstancePgSetupReport"
-
     InstanceDatatableRole:
       type: object
       required: [id, name, enabled]
@@ -34089,7 +33953,7 @@ components:
           type: array
           items:
             type: string
-          description: Workspaces that reference this database through a ducklake catalog or a datatable database of the kind being listed — 'instance' for the instance databases endpoint, 'external_instance' for the external cluster one. Computed at request time, not persisted, and only returned to superadmins.
+          description: Workspaces that reference this database via a ducklake catalog or datatable database with resource_type 'instance'. Computed at request time, not persisted.
         workspace_id:
           type: string
           description: The workspace a member created this database for as a fork copy. Only that workspace can import into it or point a fork at it.
@@ -36066,7 +35930,6 @@ components:
                       - postgresql
                       - mysql
                       - instance
-                      - external_instance
                   resource_path:
                     type: string
                 required:
@@ -36130,7 +35993,6 @@ components:
                     enum:
                       - postgresql
                       - instance
-                      - external_instance
                   resource_path:
                     type: string
                 required:

backend/windmill-common/Cargo.toml

@@ -75,7 +75,6 @@ bitflags.workspace = true
 once_cell.workspace = true
 phf.workspace = true
 tokio-postgres.workspace = true
-postgres-protocol.workspace = true
 postgres-native-tls.workspace = true
 native-tls.workspace = true
 

backend/windmill-common/src/external_instance_pg.rs

@@ -1,429 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-//! The external Postgres cluster behind `external_instance` data tables and Ducklake catalogs.
-//!
-//! Windmill administers that cluster itself, logged in as the user in
-//! [`EXTERNAL_INSTANCE_PG_SETTING`]. It creates `custom_instance_user` and
-//! `custom_instance_replication_user` there, with passwords it generates and keeps in
-//! [`EXTERNAL_INSTANCE_PG_STATE_SETTING`]. They share their names with the roles on Windmill's own
-//! cluster, but they are different roles with different passwords.
-//!
-//! The cluster may hold data Windmill did not create. Two Windmill instances sharing one is not
-//! supported: each would keep resetting the passwords the other depends on.
-
-use std::collections::{BTreeMap, BTreeSet};
-
-use serde::{Deserialize, Serialize};
-
-use crate::{
-    error::{Error, Result},
-    global_settings::{EXTERNAL_INSTANCE_PG_SETTING, EXTERNAL_INSTANCE_PG_STATE_SETTING},
-    instance_config::{CustomInstanceDb, ExternalInstancePg},
-    DB,
-};
-
-/// What Windmill keeps about the external cluster. Server-managed and hidden: never part of the
-/// instance config, never readable by an agent worker. No `Debug`: it carries live passwords.
-#[derive(Serialize, Deserialize, Clone, Default)]
-pub struct ExternalInstancePgState {
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub user_pwd: Option<String>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub replication_pwd: Option<String>,
-    /// The databases Windmill created on the cluster. It only ever drops one of these.
-    #[serde(default)]
-    pub databases: BTreeMap<String, CustomInstanceDb>,
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub last_setup: Option<ExternalInstancePgSetupReport>,
-    /// The cluster ([`external_instance_pg_address`]) the last successful setup converged. Databases
-    /// are only created on a cluster setup succeeded on: the passwords above exist as soon as setup
-    /// first runs, whether or not the cluster accepted them.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub set_up_for: Option<String>,
-}
-
-/// What identifies the cluster a configuration points at. Other fields (admin login, sslmode) can
-/// change without it becoming another cluster.
-pub fn external_instance_pg_address(config: &ExternalInstancePg) -> String {
-    format!(
-        "{}:{}",
-        config.host.trim().to_lowercase(),
-        config.port.unwrap_or(5432)
-    )
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ExternalInstancePgSetupReport {
-    /// No step failed. Warnings leave it true.
-    pub success: bool,
-    pub finished_at: chrono::DateTime<chrono::Utc>,
-    pub steps: Vec<ExternalInstancePgSetupStep>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct ExternalInstancePgSetupStep {
-    pub name: String,
-    pub status: SetupStepStatus,
-    pub message: String,
-}
-
-#[derive(Serialize, Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
-#[serde(rename_all = "lowercase")]
-pub enum SetupStepStatus {
-    Ok,
-    Warning,
-    Error,
-}
-
-/// The status the settings page shows without running anything.
-#[derive(Serialize, Debug)]
-pub struct ExternalInstancePgStatus {
-    pub configured: bool,
-    pub database_count: usize,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub last_setup: Option<ExternalInstancePgSetupReport>,
-}
-
-/// Authorization: returns the cluster's admin password and checks nothing. Callers MUST be
-/// superadmin or an internal server path.
-pub(crate) async fn read_external_instance_pg_config<'c>(
-    executor: impl sqlx::PgExecutor<'c>,
-) -> Result<Option<ExternalInstancePg>> {
-    let value = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        EXTERNAL_INSTANCE_PG_SETTING
-    )
-    .fetch_optional(executor)
-    .await?;
-    value
-        .map(|v| {
-            serde_json::from_value(v).map_err(|e| {
-                Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_SETTING}: {e}"))
-            })
-        })
-        .transpose()
-}
-
-/// Authorization: returns the passwords Windmill generated on the cluster and checks nothing.
-/// Callers MUST be superadmin or an internal server path.
-pub(crate) async fn read_external_instance_pg_state<'c>(
-    executor: impl sqlx::PgExecutor<'c>,
-) -> Result<ExternalInstancePgState> {
-    let value = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        EXTERNAL_INSTANCE_PG_STATE_SETTING
-    )
-    .fetch_optional(executor)
-    .await?;
-    match value {
-        None => Ok(ExternalInstancePgState::default()),
-        Some(v) => serde_json::from_value(v).map_err(|e| {
-            Error::internal_err(format!("reading {EXTERNAL_INSTANCE_PG_STATE_SETTING}: {e}"))
-        }),
-    }
-}
-
-pub async fn external_instance_pg_status(db: &DB) -> Result<ExternalInstancePgStatus> {
-    let configured = read_external_instance_pg_config(db).await?.is_some();
-    let state = read_external_instance_pg_state(db).await?;
-    Ok(ExternalInstancePgStatus {
-        configured,
-        database_count: state.databases.len(),
-        last_setup: state.last_setup,
-    })
-}
-
-/// The databases Windmill created on the external cluster, without the passwords kept beside them.
-///
-/// Authorization: names every database across all workspaces, and the workspace each fork copy is
-/// reserved for, and checks nothing. Callers MUST be superadmin or an internal authorization or
-/// lifecycle path that does not return the names to a workspace caller.
-pub async fn external_instance_databases(db: &DB) -> Result<BTreeMap<String, CustomInstanceDb>> {
-    Ok(read_external_instance_pg_state(db).await?.databases)
-}
-
-/// The workspaces whose data tables or Ducklake catalogs name each database on the external cluster,
-/// and the forks whose Ducklake metadata schemas there are still waiting to be dropped: those rows
-/// outlive a settings change, and cleanup cannot drop a schema in a database that is gone. A row
-/// whose schema is already dropped only waits on object storage, which needs no database.
-///
-/// Authorization: reads every workspace's settings and checks nothing. Callers MUST be superadmin
-/// or an internal lifecycle path.
-pub async fn external_instance_database_usages<'c>(
-    db: impl sqlx::PgExecutor<'c>,
-) -> Result<BTreeMap<String, BTreeSet<String>>> {
-    let rows = sqlx::query_as::<_, (String, String)>(
-        "SELECT ws.workspace_id, entry->'database'->>'resource_path'
-         FROM workspace_settings ws
-         CROSS JOIN LATERAL jsonb_each(
-             CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'
-                 THEN ws.datatable->'datatables'
-                 ELSE '{}'::jsonb END
-         ) AS dt(k, entry)
-         WHERE entry->'database'->>'resource_type' = 'external_instance'
-           AND entry->'database'->>'resource_path' IS NOT NULL
-         UNION ALL
-         SELECT ws.workspace_id, entry->'catalog'->>'resource_path'
-         FROM workspace_settings ws
-         CROSS JOIN LATERAL jsonb_each(
-             CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object'
-                 THEN ws.ducklake->'ducklakes'
-                 ELSE '{}'::jsonb END
-         ) AS dl(k, entry)
-         WHERE entry->'catalog'->>'resource_type' = 'external_instance'
-           AND entry->'catalog'->>'resource_path' IS NOT NULL
-         UNION ALL
-         SELECT workspace_id, substring(catalog FROM length('external_instance:') + 1)
-         FROM fork_ducklake_namespace
-         WHERE catalog LIKE 'external\\_instance:%' AND NOT schema_dropped",
-    )
-    .fetch_all(db)
-    .await?;
-    let mut usages: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
-    for (workspace_id, dbname) in rows {
-        usages.entry(dbname).or_default().insert(workspace_id);
-    }
-    Ok(usages)
-}
-
-/// Refuse to unset the cluster while Windmill still has databases on it, or a workspace still
-/// points at one: every data table there would stop resolving. Allowed on every edition, so a
-/// downgraded instance can still clear a setting it no longer uses.
-pub async fn ensure_external_instance_pg_removable(conn: &mut sqlx::PgConnection) -> Result<()> {
-    let state = read_external_instance_pg_state(&mut *conn).await?;
-    let usages = external_instance_database_usages(&mut *conn).await?;
-    if state.databases.is_empty() && usages.is_empty() {
-        return Ok(());
-    }
-    let names = state
-        .databases
-        .keys()
-        .chain(usages.keys())
-        .collect::<BTreeSet<_>>()
-        .into_iter()
-        .cloned()
-        .collect::<Vec<_>>()
-        .join(", ");
-    Err(Error::BadRequest(format!(
-        "The external instance cluster still holds databases in use ({names}). Drop them and \
-         repoint the data tables and Ducklake catalogs using them before removing {EXTERNAL_INSTANCE_PG_SETTING}."
-    )))
-}
-
-/// Refuse a workspace setting that newly names an `external_instance` database on an edition
-/// without them.
-pub fn ensure_external_instance_available() -> Result<()> {
-    crate::external_instance_pg_oss::ensure_external_instance_available()
-}
-
-/// The connection an `external_instance` database resolves to: `custom_instance_user`, or the
-/// replication user, on the external cluster.
-///
-/// Authorization: returns live credentials and checks nothing. Callers MUST have authorized access
-/// to the data table that names `dbname`.
-pub async fn external_instance_connection_unchecked(
-    db: &DB,
-    dbname: &str,
-    replication: bool,
-) -> Result<crate::PgDatabase> {
-    crate::external_instance_pg_oss::external_instance_connection_unchecked(db, dbname, replication)
-        .await
-}
-
-/// Create `dbname` on the external cluster and register it. Refuses a name already taken there,
-/// whoever took it.
-///
-/// Authorization: checks nothing. Callers MUST be superadmin, or be cloning a data table they may
-/// fork into a `wm_fork_` database.
-pub async fn create_external_instance_database_unchecked(
-    db: &DB,
-    dbname: &str,
-    tag: &str,
-    for_workspace: Option<&str>,
-) -> Result<()> {
-    crate::external_instance_pg_oss::create_external_instance_database_unchecked(
-        db,
-        dbname,
-        tag,
-        for_workspace,
-    )
-    .await
-}
-
-/// Drop `dbname` from the external cluster: only a database Windmill registered creating, and still
-/// carries the mark it set there. Refused while anything uses it
-/// ([`crate::workspaces::managed_database_uses`]), except the `exempt` data table entry: the fork
-/// copy being cleaned up.
-///
-/// Authorization: checks nothing. Callers MUST be superadmin, or be deleting the fork that owns
-/// this `wm_fork_` database.
-pub async fn drop_external_instance_database_unchecked(
-    db: &DB,
-    dbname: &str,
-    exempt: Option<(&str, &str)>,
-) -> Result<()> {
-    crate::external_instance_pg_oss::drop_external_instance_database_unchecked(db, dbname, exempt)
-    .await
-}
-
-/// Serializes everything that changes which databases exist on the external cluster, or which data
-/// tables name them: setup, creates, drops, and data table saves. Held until `tx` ends.
-pub async fn lock_external_instance_pg_state(
-    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
-) -> Result<()> {
-    sqlx::query("SELECT pg_advisory_xact_lock(hashtext($1))")
-        .bind(EXTERNAL_INSTANCE_PG_STATE_SETTING)
-        .execute(&mut **tx)
-        .await?;
-    Ok(())
-}
-
-/// Refuse a data table naming `dbname` unless Windmill created it on the external cluster. Takes
-/// the lock drops take, so none can remove the database before `tx`, which saves the data table,
-/// commits.
-///
-/// Authorization: its refusal says whether Windmill created a database of that name, which is
-/// instance-wide knowledge. Callers MUST have authorized the caller as superadmin first.
-pub async fn ensure_external_instance_database_registered(
-    tx: &mut sqlx::Transaction<'_, sqlx::Postgres>,
-    dbname: &str,
-) -> Result<()> {
-    lock_external_instance_pg_state(tx).await?;
-    if read_external_instance_pg_state(&mut **tx)
-        .await?
-        .databases
-        .contains_key(dbname)
-    {
-        return Ok(());
-    }
-    Err(Error::BadRequest(format!(
-        "Windmill did not create a database named '{dbname}' on the external instance cluster. \
-         Create it from the instance settings first."
-    )))
-}
-
-/// Write [`EXTERNAL_INSTANCE_PG_SETTING`]: `None`, null or an empty string unsets it. Every writer
-/// of global settings goes through this for that key — the per-key and bulk endpoints as well as
-/// the declarative sync — instead of writing the row itself.
-///
-/// The checks and the write share one transaction holding [`lock_external_instance_pg_state`]. A
-/// check taken outside it could pass while a database create still reads the old cluster, which
-/// would then register a database there after the setting names another one.
-///
-/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config
-/// sync, which applies what the operator deployed.
-pub async fn write_external_instance_pg_setting(
-    db: &DB,
-    value: Option<&serde_json::Value>,
-) -> Result<()> {
-    let value = match value {
-        None | Some(serde_json::Value::Null) => None,
-        Some(serde_json::Value::String(s)) if s.trim().is_empty() => None,
-        Some(value) => Some(value),
-    };
-    let mut tx = db.begin().await?;
-    lock_external_instance_pg_state(&mut tx).await?;
-    // Every check runs on this transaction's own connection: it holds the advisory lock, and
-    // taking a second connection from the pool while other writers queue on that lock is how a
-    // small pool deadlocks.
-    match value {
-        None => {
-            ensure_external_instance_pg_removable(&mut tx).await?;
-            sqlx::query("DELETE FROM global_settings WHERE name = $1")
-                .bind(EXTERNAL_INSTANCE_PG_SETTING)
-                .execute(&mut *tx)
-                .await?;
-        }
-        Some(value) => {
-            crate::external_instance_pg_oss::validate_external_instance_pg_setting(value)?;
-            ensure_external_instance_pg_not_repointed(&mut tx, value).await?;
-            sqlx::query(
-                "INSERT INTO global_settings (name, value) VALUES ($1, $2)
-                 ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now()",
-            )
-            .bind(EXTERNAL_INSTANCE_PG_SETTING)
-            .bind(value)
-            .execute(&mut *tx)
-            .await?;
-        }
-    }
-    tx.commit().await?;
-    tracing::info!(
-        "{} global setting {EXTERNAL_INSTANCE_PG_SETTING}",
-        if value.is_some() { "Set" } else { "Unset" }
-    );
-    Ok(())
-}
-
-/// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches
-/// it, and takes it out of the diff so the generic apply does not write it again.
-///
-/// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config
-/// sync, which applies what the operator deployed.
-pub async fn write_external_instance_pg_from_diff(
-    db: &DB,
-    diff: &mut crate::instance_config::SettingsDiff,
-) -> Result<()> {
-    if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) {
-        write_external_instance_pg_setting(db, Some(&value)).await?;
-    }
-    if let Some(i) = diff
-        .deletes
-        .iter()
-        .position(|k| k == EXTERNAL_INSTANCE_PG_SETTING)
-    {
-        diff.deletes.remove(i);
-        write_external_instance_pg_setting(db, None).await?;
-    }
-    Ok(())
-}
-
-/// Refuse pointing the setting at another host or port while databases live on the current one.
-/// Data tables name databases, not clusters, so they would silently resolve to whatever the new
-/// cluster holds under the same names. Other fields (admin login, sslmode) may change freely.
-async fn ensure_external_instance_pg_not_repointed(
-    conn: &mut sqlx::PgConnection,
-    value: &serde_json::Value,
-) -> Result<()> {
-    let Some(current) = read_external_instance_pg_config(&mut *conn).await? else {
-        return Ok(());
-    };
-    let Ok(desired) = serde_json::from_value::<ExternalInstancePg>(value.clone()) else {
-        return Ok(());
-    };
-    if external_instance_pg_address(&current) == external_instance_pg_address(&desired) {
-        return Ok(());
-    }
-    let state = read_external_instance_pg_state(&mut *conn).await?;
-    let usages = external_instance_database_usages(&mut *conn).await?;
-    if state.databases.is_empty() && usages.is_empty() {
-        return Ok(());
-    }
-    Err(Error::BadRequest(format!(
-        "The external instance cluster at {}:{} still holds databases in use. Drop them and repoint \
-         what uses them before pointing {EXTERNAL_INSTANCE_PG_SETTING} at another cluster.",
-        current.host.trim(),
-        current.port.unwrap_or(5432)
-    )))
-}
-
-/// Converge the external cluster on the configured login: check what it can do, create or update
-/// Windmill's two roles with the stored passwords, and report anything that would get in the way.
-/// With `rotate_passwords`, generate new passwords first. Safe to run again; running it again is
-/// how a failed rotation is repaired.
-///
-/// Authorization: administers the external cluster with its admin credentials and checks nothing.
-/// Callers MUST be superadmin.
-pub async fn setup_external_instance_pg_unchecked(
-    db: &DB,
-    rotate_passwords: bool,
-) -> Result<ExternalInstancePgSetupReport> {
-    crate::external_instance_pg_oss::setup_external_instance_pg_unchecked(db, rotate_passwords)
-        .await
-}

backend/windmill-common/src/external_instance_pg_oss.rs

@@ -1,81 +0,0 @@
-/*
- * Author: Ruben Fiszel
- * Copyright: Windmill Labs, Inc 2022
- * This file and its contents are licensed under the AGPLv3 License.
- * Please see the included NOTICE for copyright information and
- * LICENSE-AGPL for a copy of the license.
- */
-
-//! Where the external instance cluster comes from: the enterprise implementation, or a refusal.
-//! `private` alone is not that edition: community builds carry it.
-
-use crate::error::Error;
-
-pub fn external_instance_pg_unavailable() -> Error {
-    Error::BadRequest(
-        "External instance databases are a Windmill Enterprise Edition feature".to_string(),
-    )
-}
-
-#[cfg(all(feature = "private", feature = "enterprise"))]
-pub(crate) use crate::external_instance_pg_ee::{
-    create_external_instance_database_unchecked, drop_external_instance_database_unchecked,
-    external_instance_connection_unchecked, setup_external_instance_pg_unchecked,
-    validate_external_instance_pg_setting,
-};
-
-#[cfg(all(feature = "private", feature = "enterprise"))]
-pub(crate) fn ensure_external_instance_available() -> crate::error::Result<()> {
-    Ok(())
-}
-
-#[cfg(not(all(feature = "private", feature = "enterprise")))]
-pub(crate) use ce::*;
-
-#[cfg(not(all(feature = "private", feature = "enterprise")))]
-mod ce {
-    use super::external_instance_pg_unavailable as unavailable;
-    use crate::{
-        error::Result, external_instance_pg::ExternalInstancePgSetupReport, PgDatabase, DB,
-    };
-
-    pub(crate) fn validate_external_instance_pg_setting(_value: &serde_json::Value) -> Result<()> {
-        Err(unavailable())
-    }
-
-    pub(crate) fn ensure_external_instance_available() -> Result<()> {
-        Err(unavailable())
-    }
-
-    pub(crate) async fn setup_external_instance_pg_unchecked(
-        _db: &DB,
-        _rotate_passwords: bool,
-    ) -> Result<ExternalInstancePgSetupReport> {
-        Err(unavailable())
-    }
-
-    pub(crate) async fn external_instance_connection_unchecked(
-        _db: &DB,
-        _dbname: &str,
-        _replication: bool,
-    ) -> Result<PgDatabase> {
-        Err(unavailable())
-    }
-
-    pub(crate) async fn create_external_instance_database_unchecked(
-        _db: &DB,
-        _dbname: &str,
-        _tag: &str,
-        _for_workspace: Option<&str>,
-    ) -> Result<()> {
-        Err(unavailable())
-    }
-
-    pub(crate) async fn drop_external_instance_database_unchecked(
-        _db: &DB,
-        _dbname: &str,
-        _exempt: Option<(&str, &str)>,
-    ) -> Result<()> {
-        Err(unavailable())
-    }
-}

backend/windmill-common/src/global_settings.rs

@@ -57,8 +57,6 @@ pub const SAML_METADATA_SETTING: &str = "saml_metadata";
 pub const SMTP_SETTING: &str = "smtp_settings";
 pub const TEAMS_SETTING: &str = "teams";
 pub const INDEXER_SETTING: &str = "indexer_settings";
-pub const EXTERNAL_INSTANCE_PG_SETTING: &str = "external_instance_pg";
-pub const EXTERNAL_INSTANCE_PG_STATE_SETTING: &str = "external_instance_pg_state";
 pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
 
 pub const UNIQUE_ID_SETTING: &str = "uid";
@@ -357,9 +355,6 @@ pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[
     // resolve datatable connections through the dedicated datatable endpoints, never these.
     "custom_instance_pg_databases",
     "custom_instance_replication_pwd",
-    // The external cluster's admin login, and the passwords Windmill generated on it.
-    EXTERNAL_INSTANCE_PG_SETTING,
-    EXTERNAL_INSTANCE_PG_STATE_SETTING,
 ];
 
 /// Whether an agent worker may read the given global setting over HTTP.

backend/windmill-common/src/instance_config.rs

@@ -350,8 +350,6 @@ pub struct GlobalSettings {
     pub ducklake_settings: Option<DucklakeSettings>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub custom_instance_pg_databases: Option<CustomInstancePgDatabases>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub external_instance_pg: Option<ExternalInstancePg>,
 
     // Opaque settings (EE-private structs or no clear schema)
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -838,36 +836,6 @@ pub struct CustomInstanceDbLogs {
     pub replication_user_error: Option<String>,
 }
 
-// ---------------------------------------------------------------------------
-// External instance PG cluster
-// ---------------------------------------------------------------------------
-
-/// The external Postgres cluster Windmill manages for `external_instance` data tables and Ducklake
-/// catalogs. `user` logs in as the cluster's administrator: it needs `CREATEDB` and `CREATEROLE`.
-/// `dbname` is only where that login connects to run cluster-wide statements.
-///
-/// Every field defaults rather than being required: this deserializes as part of the whole
-/// instance config, and one malformed row must not make every other setting unreadable. The
-/// write path and every use reject an incomplete value instead.
-#[derive(Deserialize, Serialize, Clone, Debug, Default)]
-#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
-pub struct ExternalInstancePg {
-    #[serde(default)]
-    pub host: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub port: Option<u16>,
-    #[serde(default)]
-    pub user: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub password: Option<StringOrSecretRef>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub dbname: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub sslmode: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub root_certificate_pem: Option<String>,
-}
-
 // ---------------------------------------------------------------------------
 // Autoscaling (worker config)
 // ---------------------------------------------------------------------------
@@ -1004,7 +972,6 @@ pub const PROTECTED_SETTINGS: &[&str] = &[
     "ducklake_settings",
     "custom_instance_pg_databases",
     "custom_instance_replication_pwd",
-    "external_instance_pg_state",
     "uid",
     "rsa_keys",
     "jwt_secret",
@@ -1030,8 +997,6 @@ pub const HIDDEN_SETTINGS: &[&str] = &[
     // Server-only (written by setup/refresh via direct SQL), never operator-authored —
     // hidden so the config machinery can't read, rewrite, or drop it.
     "custom_instance_replication_pwd",
-    // Same for the passwords and database registry Windmill keeps for the external cluster.
-    "external_instance_pg_state",
 ];
 
 /// Top-level settings whose entire value is sensitive and must be fully redacted in logs.
@@ -1043,7 +1008,6 @@ const SENSITIVE_SETTINGS: &[&str] = &[
     "license_key",
     "ducklake_user_pg_pwd",
     "custom_instance_replication_pwd",
-    "external_instance_pg_state",
     "pip_index_url",
     "pip_extra_index_url",
     "npm_config_registry",
@@ -1069,7 +1033,6 @@ const NESTED_SENSITIVE_FIELDS: &[(&str, &[&str])] = &[
         &["secret_key", "serviceAccountKey"],
     ),
     ("custom_instance_pg_databases", &["user_pwd"]),
-    ("external_instance_pg", &["password"]),
 ];
 
 fn redact_json_value(value: &serde_json::Value) -> serde_json::Value {
@@ -1398,8 +1361,7 @@ pub async fn sync_global_settings_declarative(
     crate::global_settings::parse_allowed_origins_setting(desired.get(origins_key))
         .map_err(|e| anyhow::anyhow!("{origins_key}: {e}"))?;
 
-    let mut diff = diff_global_settings(current, desired, ApplyMode::Replace);
-    crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?;
+    let diff = diff_global_settings(current, desired, ApplyMode::Replace);
     apply_settings_diff(db, &diff).await?;
 
     Ok(())
@@ -1532,10 +1494,6 @@ pub fn resolve_env_refs(settings: &mut GlobalSettings) -> Result<(), String> {
         resolve_env_option(&mut pg.user_pwd)?;
     }
 
-    if let Some(pg) = &mut settings.external_instance_pg {
-        resolve_env_option(&mut pg.password)?;
-    }
-
     Ok(())
 }
 
@@ -2505,33 +2463,39 @@ mod tests {
     }
 
     #[test]
-    fn server_generated_db_passwords_are_isolated_from_config() {
-        // These hold passwords the server generates: written by setup/refresh via direct SQL,
-        // never operator-authored. They must stay out of the declarative config surface
-        // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop them.
-        for key in [
-            "custom_instance_replication_pwd",
-            "external_instance_pg_state",
-        ] {
-            assert!(HIDDEN_SETTINGS.contains(&key), "{key}");
-            assert!(PROTECTED_SETTINGS.contains(&key), "{key}");
-            assert!(SENSITIVE_SETTINGS.contains(&key), "{key}");
+    fn custom_instance_replication_pwd_is_isolated_from_config() {
+        // The replication-role password is server-only: written by setup/refresh via direct
+        // SQL, never operator-authored. It must stay out of the declarative config surface
+        // (hidden on read) and be undeletable, so config sync can't read, rewrite, or drop it.
+        assert!(HIDDEN_SETTINGS.contains(&"custom_instance_replication_pwd"));
+        assert!(PROTECTED_SETTINGS.contains(&"custom_instance_replication_pwd"));
+        assert!(SENSITIVE_SETTINGS.contains(&"custom_instance_replication_pwd"));
 
-            // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
-            let mut desired = BTreeMap::new();
-            desired.insert(key.to_string(), serde_json::json!("attacker-set"));
-            let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
-            assert!(diff.upserts.is_empty(), "{key} must not be upserted");
+        // A stray desired value (e.g. flattened into `extra`) is ignored, not upserted.
+        let mut desired = BTreeMap::new();
+        desired.insert(
+            "custom_instance_replication_pwd".to_string(),
+            serde_json::json!("attacker-set"),
+        );
+        let diff = diff_global_settings(&BTreeMap::new(), &desired, ApplyMode::Merge);
+        assert!(
+            diff.upserts.is_empty(),
+            "hidden setting must not be upserted"
+        );
 
-            // A current value is never deleted by a Replace that omits it.
-            let mut current = BTreeMap::new();
-            current.insert(key.to_string(), serde_json::json!("live"));
-            let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
-            assert!(
-                !diff.deletes.contains(&key.to_string()),
-                "{key} must not be deleted"
-            );
-        }
+        // A current value is never deleted by a Replace that omits it.
+        let mut current = BTreeMap::new();
+        current.insert(
+            "custom_instance_replication_pwd".to_string(),
+            serde_json::json!("live"),
+        );
+        let diff = diff_global_settings(&current, &BTreeMap::new(), ApplyMode::Replace);
+        assert!(
+            !diff
+                .deletes
+                .contains(&"custom_instance_replication_pwd".to_string()),
+            "hidden setting must not be deleted"
+        );
     }
 
     #[test]

backend/windmill-common/src/lib.rs

@@ -58,10 +58,6 @@ pub mod ee_oss;
 pub mod email_ee;
 pub mod email_oss;
 pub mod error;
-pub mod external_instance_pg;
-#[cfg(all(feature = "private", feature = "enterprise"))]
-mod external_instance_pg_ee;
-pub mod external_instance_pg_oss;
 pub mod external_ip;
 #[cfg(feature = "private")]
 pub mod feature_usage_ee;
@@ -1086,13 +1082,7 @@ impl PgDatabase {
                 if err_str.contains("password authentication failed for user")
                     && err_str.contains("custom_instance_user")
                 {
-                    // The external instance cluster has a `custom_instance_user` of its own, whose
-                    // password setup manages. Rotating the local one would break every instance
-                    // data table and fix nothing.
-                    let local = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
-                    let on_local_cluster = local.host == self.host
-                        && local.port.unwrap_or(5432) == self.port.unwrap_or(5432);
-                    if let Some(db) = main_db.filter(|_| on_local_cluster) {
+                    if let Some(db) = main_db {
                         tracing::warn!(
                             "custom_instance_user password auth failed, refreshing and retrying..."
                         );
@@ -1485,7 +1475,26 @@ pub fn validate_dbname(dbname: &str) -> error::Result<()> {
 }
 
 /// Drop a custom instance database: validate, terminate connections, DROP DATABASE, remove from global_settings.
+///
+/// Authorization: drops any instance database but Windmill's own and checks nothing. Callers MUST
+/// be superadmin, or have established the caller may drop this one — a fork's owner cleaning up
+/// its own copy that nothing else uses.
 pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Result<()> {
+    drop_custom_instance_database_keep_entry(db, dbname).await?;
+    sqlx::query!(
+        r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#,
+        dbname.trim()
+    )
+    .execute(db)
+    .await?;
+    Ok(())
+}
+
+/// [`drop_custom_instance_database`] leaving its registry entry, for a caller holding row locks in
+/// a transaction: the registry write has to go through that transaction, as waiting on another
+/// connection for a lock the transaction's own peers hold is a deadlock Postgres cannot see. Same
+/// authorization contract.
+pub async fn drop_custom_instance_database_keep_entry(db: &DB, dbname: &str) -> error::Result<()> {
     let dbname = dbname.trim();
     validate_dbname(dbname)?;
 
@@ -1531,14 +1540,6 @@ pub async fn drop_custom_instance_database(db: &DB, dbname: &str) -> error::Resu
         tracing::info!("Database '{}' does not exist, skipping drop", dbname);
     }
 
-    // Always remove from global_settings
-    sqlx::query!(
-        r#"UPDATE global_settings SET value = value #- ARRAY['databases', $1] WHERE name = 'custom_instance_pg_databases'"#,
-        dbname
-    )
-    .execute(db)
-    .await?;
-
     Ok(())
 }
 
@@ -1659,62 +1660,39 @@ pub async fn create_custom_instance_database(
     Ok(())
 }
 
-/// The system's CA bundle file, for libpq clients that cannot take `sslrootcert=system`: that value
-/// needs libpq 16, and verify-full only.
-pub fn system_ca_bundle() -> Option<std::path::PathBuf> {
-    std::env::var_os("SSL_CERT_FILE")
-        .map(std::path::PathBuf::from)
-        .into_iter()
-        .chain(
-            [
-                "/etc/ssl/certs/ca-certificates.crt",
-                "/etc/pki/tls/certs/ca-bundle.crt",
-                "/etc/ssl/cert.pem",
-                "/etc/ssl/ca-bundle.pem",
-            ]
-            .map(std::path::PathBuf::from),
-        )
-        .find(|path| path.is_file())
-}
-
-/// Refuse a workspace member writing a fork copy into, or pointing a fork at, the managed database
-/// `dbname` of `kind`, unless `w_id` created it for that ([`create_custom_instance_database`], or
-/// its external instance counterpart) and nothing uses it yet. The `wm_fork_` prefix is no
-/// authorization: every database of a cluster answers to the same `custom_instance_user`, so a name
-/// is all it takes to reach another workspace's copy.
+/// Refuse a workspace member writing a fork copy into, or pointing a fork at, the instance database
+/// `dbname`, unless `w_id` created it for that ([`create_custom_instance_database`]) and nothing uses
+/// it yet. The `wm_fork_` prefix is no authorization: every instance database answers to the same
+/// `custom_instance_user`, so a name is all it takes to reach another workspace's copy.
 ///
-/// Authorization: reads the global registries and every workspace's settings, and names other
+/// Authorization: reads the global registry and every workspace's settings, and names other
 /// workspaces in its refusal. Callers MUST have authorized `w_id` for the caller first — a member
 /// of it forking or importing there — and MUST NOT call it on a workspace the caller is not in.
 pub async fn ensure_fork_database_available_to(
     db: &DB,
-    kind: workspaces::DataTableCatalogResourceType,
     dbname: &str,
     w_id: &str,
 ) -> error::Result<()> {
-    let created_for = match kind {
-        workspaces::DataTableCatalogResourceType::ExternalInstance => {
-            external_instance_pg::external_instance_databases(db)
-                .await?
-                .remove(dbname)
-                .and_then(|entry| entry.workspace_id)
-        }
-        _ => sqlx::query_scalar::<_, Option<String>>(
-            "SELECT value->'databases'->$1->>'workspace_id' FROM global_settings
-             WHERE name = 'custom_instance_pg_databases'",
-        )
-        .bind(dbname)
-        .fetch_optional(db)
-        .await?
-        .flatten(),
-    };
+    let created_for = sqlx::query_scalar::<_, Option<String>>(
+        "SELECT value->'databases'->$1->>'workspace_id' FROM global_settings
+         WHERE name = 'custom_instance_pg_databases'",
+    )
+    .bind(dbname)
+    .fetch_optional(db)
+    .await?
+    .flatten();
     if created_for.as_deref() != Some(w_id) {
         return Err(Error::BadRequest(format!(
             "Database '{dbname}' was not created for a fork of workspace '{w_id}'"
         )));
     }
-    let uses =
-        workspaces::managed_database_uses(&mut *db.acquire().await?, kind, dbname, None).await?;
+    let uses = workspaces::managed_database_uses(
+        &mut *db.acquire().await?,
+        workspaces::DataTableCatalogResourceType::Instance,
+        dbname,
+        None,
+    )
+    .await?;
     if !uses.is_empty() {
         return Err(Error::BadRequest(format!(
             "Database '{dbname}' is already in use: {}",

backend/windmill-common/src/workspaces.rs

@@ -1397,18 +1397,6 @@ pub enum DataTableCatalogResourceType {
     #[strum(serialize = "postgres")]
     Postgresql,
     Instance,
-    /// On the external instance cluster ([`crate::external_instance_pg`]). Enterprise Edition.
-    #[serde(rename = "external_instance")]
-    #[strum(serialize = "external_instance")]
-    ExternalInstance,
-}
-
-impl DataTableCatalogResourceType {
-    /// A database Windmill created and administers, on its own cluster or the external one, as
-    /// opposed to one a user brought as a resource.
-    pub fn is_windmill_managed(self) -> bool {
-        matches!(self, Self::Instance | Self::ExternalInstance)
-    }
 }
 
 /// Build a self-teaching error for an unresolved `datatable://<name>` reference.
@@ -1692,8 +1680,7 @@ pub async fn resolve_workspace_governing_datatables(
 }
 
 /// Build the `admin` connection for a governing entry: `custom_instance_user` for an instance
-/// database, on Windmill's cluster or the external one; the user's own resource for a BYO-postgres
-/// one.
+/// database, the user's own resource for a BYO-postgres one.
 async fn resolve_datatable_connection_unchecked(
     db: &DB,
     governing: &GoverningDatatable,
@@ -1704,16 +1691,7 @@ async fn resolve_datatable_connection_unchecked(
         .database
         .as_ref()
         .expect("a governing entry owns a database");
-    if database.resource_type == DataTableCatalogResourceType::ExternalInstance {
-        let pg_creds = crate::external_instance_pg::external_instance_connection_unchecked(
-            db,
-            &database.resource_path,
-            replication,
-        )
-        .await?;
-        serde_json::to_value(&pg_creds)
-            .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))
-    } else if database.resource_type == DataTableCatalogResourceType::Instance {
+    if database.resource_type == DataTableCatalogResourceType::Instance {
         let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
         pg_creds.dbname = database.resource_path.clone();
         if replication {
@@ -2257,10 +2235,6 @@ pub enum DucklakeCatalogResourceType {
     Postgresql,
     Mysql,
     Instance,
-    /// On the external instance cluster ([`crate::external_instance_pg`]). Enterprise Edition.
-    #[serde(rename = "external_instance")]
-    #[strum(serialize = "external_instance")]
-    ExternalInstance,
 }
 
 #[derive(Deserialize, Serialize)]
@@ -2788,16 +2762,7 @@ async fn ducklake_conn_data(
     let ducklake = serde_json::from_value::<Ducklake>(ducklake)?;
 
     let catalog_resource =
-        if ducklake.catalog.resource_type == DucklakeCatalogResourceType::ExternalInstance {
-            let pg_creds = crate::external_instance_pg::external_instance_connection_unchecked(
-                db,
-                &ducklake.catalog.resource_path,
-                false,
-            )
-            .await?;
-            serde_json::to_value(&pg_creds)
-                .map_err(|e| Error::internal_err(format!("Error serializing pg creds: {}", e)))?
-        } else if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance {
+        if ducklake.catalog.resource_type == DucklakeCatalogResourceType::Instance {
             let mut pg_creds = PgDatabase::parse_uri(&get_database_url().await?.as_str().await)?;
             pg_creds.dbname = ducklake.catalog.resource_path.clone();
             pg_creds.user = Some("custom_instance_user".to_string());
@@ -3178,14 +3143,6 @@ async fn register_fork_ducklake_namespace(
     {
         return Ok(());
     }
-    let mut tx = db.begin().await?;
-    // A row naming an external database counts as a use of it. Written under the lock a drop takes,
-    // and only while the database is still registered, so a drop cannot slip in between the
-    // settings this attach resolved and the row that protects the database.
-    if let Some(dbname) = catalog.strip_prefix("external_instance:") {
-        crate::external_instance_pg::ensure_external_instance_database_registered(&mut tx, dbname)
-            .await?;
-    }
     sqlx::query!(
         "INSERT INTO fork_ducklake_namespace
            (workspace_id, ducklake_name, metadata_schema, catalog, storage, storage_ref, data_path)
@@ -3200,10 +3157,9 @@ async fn register_fork_ducklake_namespace(
         &storage_ref,
         data_path,
     )
-    .execute(&mut *tx)
+    .execute(db)
     .await
     .map_err(|e| Error::internal_err(format!("registering fork ducklake namespace: {e:#}")))?;
-    tx.commit().await?;
     let mut locations = FORK_DUCKLAKE_REGISTERED
         .get(w_id)
         .filter(|(_, exp)| *exp > now)

backend/windmill-worker/src/duckdb_executor.rs

@@ -1475,7 +1475,6 @@ pub async fn do_duckdb(
                             &job.id,
                             client,
                             &mut hidden_passwords,
-                            job_dir,
                         )
                         .await?,
                     );
@@ -1491,13 +1490,12 @@ pub async fn do_duckdb(
                     &mut hidden_passwords,
                     &job.workspace_id,
                     materialize.as_ref().map(|(_, m)| m.asset_path.as_str()),
-                    job_dir,
                 )
                 .await?
                 {
                     probe_blocks.extend(q);
                 } else if let Some(q) =
-                    transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job, job_dir)
+                    transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job)
                         .await?
                 {
                     probe_blocks.extend(q);
@@ -1554,7 +1552,6 @@ pub async fn do_duckdb(
                             &job.id,
                             client,
                             &mut hidden_passwords,
-                            job_dir,
                         )
                         .await?,
                     );
@@ -1570,13 +1567,12 @@ pub async fn do_duckdb(
                     &mut hidden_passwords,
                     &job.workspace_id,
                     materialize.as_ref().map(|(_, m)| m.asset_path.as_str()),
-                    job_dir,
                 )
                 .await?
                 {
                     v.extend(ducklake_query);
                 } else if let Some(datatable_query) =
-                    transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job, job_dir)
+                    transform_attach_datatable(&query_block, conn, &mut hidden_passwords, job)
                         .await?
                 {
                     v.extend(datatable_query);
@@ -2244,67 +2240,11 @@ fn parse_attach_db_resource<'a>(query: &'a str) -> Option<ParsedAttachDbResource
     None
 }
 
-/// The verification a DuckDB postgres attach keeps, as its libpq `sslmode` and `sslrootcert`.
-///
-/// Attaches have always turned verify-ca and verify-full into `require`, which resources rely on.
-/// A connection that explicitly refuses invalid certificates — the external instance cluster's —
-/// keeps its mode instead: under `require` its shared password would go to whichever server
-/// answers. DuckDB's libpq takes one root file, so it gets the system bundle plus the configured
-/// certificate, written in the job directory: a resource's certificate is workspace-controlled, so
-/// a file per distinct one has to go with the job rather than pile up on the worker.
-fn pg_attach_verification<'a>(
-    res: &'a PgDatabase,
-    job_dir: &str,
-) -> Result<Option<(&'a str, std::path::PathBuf)>> {
-    let mode = match res.sslmode.as_deref() {
-        Some(mode @ ("verify-ca" | "verify-full")) if res.accept_invalid_certs == Some(false) => {
-            mode
-        }
-        _ => return Ok(None),
-    };
-    let bundle = windmill_common::system_ca_bundle()
-        .map(std::fs::read_to_string)
-        .transpose()
-        .map_err(|e| Error::ExecutionErr(format!("Failed to read the system CA bundle: {e}")))?
-        .unwrap_or_default();
-    let pem = res.root_certificate_pem.as_deref().unwrap_or_default();
-    if bundle.is_empty() && pem.is_empty() {
-        return Err(Error::ExecutionErr(format!(
-            "sslmode {mode} needs a root certificate, and this worker has no system CA bundle"
-        )));
-    }
-    let roots = format!("{bundle}\n{pem}\n");
-    use sha2::Digest;
-    let path = std::path::Path::new(job_dir).join(format!(
-        "pg_roots_{}.pem",
-        hex::encode(&sha2::Sha256::digest(roots.as_bytes())[..8])
-    ));
-    if !path.is_file() {
-        std::fs::write(&path, &roots)
-            .map_err(|e| Error::ExecutionErr(format!("Failed to write root certificates: {e}")))?;
-    }
-    Ok(Some((mode, path)))
-}
-
-fn pg_attach_uri(res: &PgDatabase, job_dir: &str) -> Result<String> {
-    let uri = res.to_uri();
-    let Some((mode, roots)) = pg_attach_verification(res, job_dir)? else {
-        return Ok(uri);
-    };
-    let base = uri.strip_suffix("?sslmode=require").ok_or_else(|| {
-        Error::internal_err("unexpected sslmode in a postgres connection URI".to_string())
-    })?;
-    Ok(format!(
-        "{base}?sslmode={mode}&sslrootcert={}",
-        urlencoding::encode(&roots.to_string_lossy())
-    ))
-}
-
-fn format_attach_db_conn_str(db_resource: Value, db_type: &str, job_dir: &str) -> Result<String> {
+fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result<String> {
     let s = match db_type.to_lowercase().as_str() {
         "postgres" | "postgresql" => {
             let res: PgDatabase = serde_json::from_value(db_resource)?;
-            pg_attach_uri(&res, job_dir)?
+            res.to_uri()
         }
         #[cfg(feature = "mysql")]
         "mysql" => {
@@ -2376,7 +2316,6 @@ async fn transform_attach_db_resource_query(
     job_id: &Uuid,
     client: &AuthedClient,
     hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
-    job_dir: &str,
 ) -> Result<Vec<String>> {
     let db_resource: Value = client
         .get_resource_value_interpolated(parsed.resource_path, Some(job_id.to_string()))
@@ -2384,14 +2323,8 @@ async fn transform_attach_db_resource_query(
     if let Some(pwd) = db_resource.get("password").and_then(|p| p.as_str()) {
         hidden_passwords.lock().unwrap().push(pwd.to_string());
     }
-    db_resource_to_attach_statements(
-        db_resource,
-        parsed.name,
-        parsed.db_type,
-        parsed.extra_args,
-        job_dir,
-    )
-    .await
+    db_resource_to_attach_statements(db_resource, parsed.name, parsed.db_type, parsed.extra_args)
+        .await
 }
 
 async fn db_resource_to_attach_statements(
@@ -2399,12 +2332,11 @@ async fn db_resource_to_attach_statements(
     ident_name: &str,
     db_type: &str,
     extra_args: Option<&str>,
-    job_dir: &str,
 ) -> Result<Vec<String>> {
     // Escape single quotes: the connection string is built from resource fields
     // (host/db/user/password) and embedded in a single-quoted DuckDB literal, so an
     // unescaped quote in any field would otherwise break out of the ATTACH statement.
-    let conn_str = format_attach_db_conn_str(db_resource, db_type, job_dir)?.replace('\'', "''");
+    let conn_str = format_attach_db_conn_str(db_resource, db_type)?.replace('\'', "''");
     let attach_str = format!(
         "ATTACH '{}' as {} (TYPE {}{});",
         conn_str,
@@ -2427,7 +2359,6 @@ async fn transform_attach_ducklake(
     hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
     w_id: &str,
     materialize_target: Option<&str>,
-    job_dir: &str,
 ) -> Result<Option<Vec<String>>> {
     lazy_static::lazy_static! {
         static ref RE: regex::Regex = regex::Regex::new(r"(?i)ATTACH\s*'ducklake(://[^':]+)?'\s*AS\s+([^ ;]+)\s*(\([^)]*\))?").unwrap();
@@ -2460,9 +2391,7 @@ async fn transform_attach_ducklake(
         format!(", {}", user_extra_args)
     };
     let db_type = match ducklake.catalog.resource_type {
-        DucklakeCatalogResourceType::Instance | DucklakeCatalogResourceType::ExternalInstance => {
-            "postgres"
-        }
+        DucklakeCatalogResourceType::Instance => "postgres",
         _ => ducklake.catalog.resource_type.as_ref(),
     };
 
@@ -2478,7 +2407,7 @@ async fn transform_attach_ducklake(
     // single-quoted DuckDB literals below, so an unescaped quote in a resource
     // field would break out of the ATTACH statement.
     let db_conn_str =
-        format_attach_db_conn_str(ducklake.catalog_resource, db_type, job_dir)?.replace('\'', "''");
+        format_attach_db_conn_str(ducklake.catalog_resource, db_type)?.replace('\'', "''");
     let storage = ducklake
         .storage
         .storage
@@ -2533,7 +2462,6 @@ async fn transform_attach_ducklake(
             defer,
             materialize_target,
             hidden_passwords,
-            job_dir,
         )?);
     }
     Ok(Some(statements))
@@ -2567,7 +2495,6 @@ fn fork_defer_statements(
     defer: &windmill_common::workspaces::DucklakeForkDefer,
     materialize_target: Option<&str>,
     hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
-    job_dir: &str,
 ) -> Result<Vec<String>> {
     let mut stmts = vec![];
     if defer.ancestors.is_empty() {
@@ -2580,13 +2507,12 @@ fn fork_defer_statements(
             hidden_passwords.lock().unwrap().push(pwd.to_string());
         }
         let db_type = match a.catalog.resource_type {
-            DucklakeCatalogResourceType::Instance
-            | DucklakeCatalogResourceType::ExternalInstance => "postgres",
+            DucklakeCatalogResourceType::Instance => "postgres",
             _ => a.catalog.resource_type.as_ref(),
         };
         stmts.push(get_attach_db_install_str(db_type)?.to_string());
         let conn_str =
-            format_attach_db_conn_str(a.catalog_resource.clone(), db_type, job_dir)?.replace('\'', "''");
+            format_attach_db_conn_str(a.catalog_resource.clone(), db_type)?.replace('\'', "''");
         let storage = a
             .storage
             .storage
@@ -2705,7 +2631,6 @@ async fn transform_attach_datatable(
     conn: &Connection,
     hidden_passwords: &mut Arc<Mutex<Vec<String>>>,
     job: &MiniPulledJob,
-    job_dir: &str,
 ) -> Result<Option<Vec<String>>> {
     let Some(attached) = parse_attach_datatable(query) else {
         return Ok(None);
@@ -2748,7 +2673,6 @@ async fn transform_attach_datatable(
     Ok(Some(pg_secret_attach_statements(
         db_resource,
         attached.alias,
-        job_dir,
     )?))
 }
 
@@ -2769,32 +2693,17 @@ fn datatable_secret_name(alias: &str) -> String {
 
 /// ATTACH a datatable's postgres database through a DuckDB TEMPORARY SECRET holding
 /// the connection parameters; only sslmode rides in the ATTACH string.
-fn pg_secret_attach_statements(
-    db_resource: Value,
-    alias_name: &str,
-    job_dir: &str,
-) -> Result<Vec<String>> {
+fn pg_secret_attach_statements(db_resource: Value, alias_name: &str) -> Result<Vec<String>> {
     let res: PgDatabase = serde_json::from_value(db_resource)?;
     // Escape single quotes: each field is embedded in a single-quoted DuckDB literal,
     // so an unescaped quote would break out of the CREATE SECRET statement.
     let esc = |s: &str| s.replace('\'', "''");
     // The postgres secret type has no sslmode parameter, so it goes in the ATTACH
     // string; only the libpq values PgDatabase::to_uri collapses to are forwarded.
-    let sslmode = match pg_attach_verification(&res, job_dir)? {
-        // A libpq keyword/value string: the path is quoted for libpq, then for the DuckDB literal.
-        Some((mode, roots)) => format!(
-            "{mode} sslrootcert=''{}''",
-            roots
-                .to_string_lossy()
-                .replace('\\', "\\\\")
-                .replace('\'', "\\''")
-        ),
-        None => match res.sslmode.as_deref() {
-            Some("disable") => "disable",
-            Some("require") | Some("verify-ca") | Some("verify-full") => "require",
-            _ => "prefer",
-        }
-        .to_string(),
+    let sslmode = match res.sslmode.as_deref() {
+        Some("disable") => "disable",
+        Some("require") | Some("verify-ca") | Some("verify-full") => "require",
+        _ => "prefer",
     };
     let secret_name = datatable_secret_name(alias_name);
     Ok(vec![
@@ -2887,54 +2796,6 @@ pub struct Arg {
 mod tests {
     use super::*;
 
-    #[test]
-    fn pg_attach_keeps_verification_only_when_required() {
-        let job_dir = std::env::temp_dir().join(format!("wm-test-{}", uuid::Uuid::new_v4()));
-        std::fs::create_dir_all(&job_dir).unwrap();
-        let job_dir = job_dir.to_string_lossy().to_string();
-        let pg = |sslmode: &str, accept_invalid_certs: Option<bool>| PgDatabase {
-            host: "db.internal".to_string(),
-            user: Some("custom_instance_user".to_string()),
-            password: Some("pw".to_string()),
-            port: None,
-            sslmode: Some(sslmode.to_string()),
-            dbname: "dt".to_string(),
-            root_certificate_pem: Some("-----BEGIN CERTIFICATE-----test".to_string()),
-            accept_invalid_certs,
-            use_iam_auth: None,
-            region: None,
-        };
-        let uri = pg_attach_uri(&pg("verify-full", Some(false)), &job_dir).unwrap();
-        assert!(uri.contains("?sslmode=verify-full&sslrootcert="), "{uri}");
-        let root = urlencoding::decode(uri.split("sslrootcert=").nth(1).unwrap()).unwrap();
-        assert!(std::fs::read_to_string(root.as_ref())
-            .unwrap()
-            .contains("-----BEGIN CERTIFICATE-----test"));
-        // Every certificate a job attaches keeps its own file: one attach must not evict another's.
-        for i in 0..40 {
-            let mut other = pg("verify-full", Some(false));
-            other.root_certificate_pem = Some(format!("-----BEGIN CERTIFICATE-----{i}"));
-            let other = pg_attach_uri(&other, &job_dir).unwrap();
-            let path = urlencoding::decode(other.split("sslrootcert=").nth(1).unwrap()).unwrap();
-            assert!(std::path::Path::new(path.as_ref()).is_file(), "{path}");
-        }
-        assert!(std::path::Path::new(root.as_ref()).is_file(), "the first file is still there");
-        let external = serde_json::to_value(pg("verify-full", Some(false))).unwrap();
-        let attach = &pg_secret_attach_statements(external, "dt", &job_dir).unwrap()[3];
-        assert!(
-            attach.starts_with(&format!("ATTACH 'sslmode=verify-full sslrootcert=''{}''", root)),
-            "{attach}"
-        );
-        // A resource that never opted in keeps the historical downgrade.
-        assert!(pg_attach_uri(&pg("verify-full", None), &job_dir)
-            .unwrap()
-            .ends_with("?sslmode=require"));
-        assert!(pg_attach_uri(&pg("require", Some(false)), &job_dir)
-            .unwrap()
-            .ends_with("?sslmode=require"));
-        std::fs::remove_dir_all(&job_dir).unwrap();
-    }
-
     #[test]
     fn attach_datatable_parses_name_and_role() {
         let reference_of = |q: &str| parse_attach_datatable(q).unwrap().reference;
@@ -3086,7 +2947,7 @@ mod tests {
         let mut defer = test_fork_defer(vec![("orders", false)], vec![]);
         defer.ancestors[0].extra_args = Some("ENCRYPTED true".to_string());
         let mut hp = Arc::new(Mutex::new(vec![]));
-        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap();
+        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap();
         let attach = stmts
             .iter()
             .find(|s| s.starts_with("ATTACH IF NOT EXISTS"))
@@ -3110,7 +2971,7 @@ mod tests {
             vec![],
         );
         let mut hp = Arc::new(Mutex::new(vec![]));
-        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap();
+        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap();
         let joined = stmts.join("\n");
         assert!(
             joined.contains(
@@ -3142,7 +3003,7 @@ mod tests {
     fn test_fork_defer_statements_shape() {
         let defer = test_fork_defer(vec![("orders", false), ("dim", true)], vec![]);
         let mut hp = Arc::new(Mutex::new(vec![]));
-        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap();
+        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap();
         let joined = stmts.join("\n");
         // Ancestor attach: read-only, idempotent, never auto-migrating or auto-creating.
         assert!(joined.contains("ATTACH IF NOT EXISTS"), "{joined}");
@@ -3169,7 +3030,7 @@ mod tests {
         let defer = test_fork_defer(vec![("orders", false)], vec!["orders", "orders_current"]);
         let mut hp = Arc::new(Mutex::new(vec![]));
         let stmts =
-            fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp, "/tmp")
+            fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp)
                 .unwrap();
         let joined = stmts.join("\n");
         assert!(!joined.contains("CREATE VIEW"), "{joined}");
@@ -3186,14 +3047,14 @@ mod tests {
         // status can't be trusted) → no DROP VIEW, or the job would wedge on a type mismatch.
         let defer = test_fork_defer(vec![("orders", false)], vec![]);
         let stmts =
-            fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp, "/tmp")
+            fork_defer_statements("lake", "_wm_target", &defer, Some("lake/orders"), &mut hp)
                 .unwrap();
         assert!(!stmts.join("\n").contains("DROP VIEW"), "{stmts:?}");
 
         // Target in a different lake → this lake's defer views are untouched.
         let defer = test_fork_defer(vec![("orders", false)], vec!["orders"]);
         let stmts =
-            fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp, "/tmp").unwrap();
+            fork_defer_statements("lake", "dl", &defer, Some("other/orders"), &mut hp).unwrap();
         let joined = stmts.join("\n");
         assert!(
             joined.contains("CREATE VIEW IF NOT EXISTS dl.\"orders\""),
@@ -3206,7 +3067,7 @@ mod tests {
     fn test_fork_defer_statements_schema_qualified() {
         let defer = test_fork_defer(vec![("staging.raw", false)], vec![]);
         let mut hp = Arc::new(Mutex::new(vec![]));
-        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp, "/tmp").unwrap();
+        let stmts = fork_defer_statements("lake", "dl", &defer, None, &mut hp).unwrap();
         let joined = stmts.join("\n");
         assert!(
             joined.contains("CREATE SCHEMA IF NOT EXISTS dl.\"staging\";"),
@@ -4032,7 +3893,7 @@ mod tests {
             "dbname": "mydb",
             "sslmode": "require"
         });
-        let result = format_attach_db_conn_str(db_resource, "postgres", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "postgres").unwrap();
         // Should be in URI format: postgres://user:password@host:port/dbname?sslmode=require
         assert!(result.starts_with("postgres://"));
         assert!(result.contains("admin:secret123@localhost:5432/mydb"));
@@ -4045,7 +3906,7 @@ mod tests {
             "host": "db.example.com",
             "dbname": "production"
         });
-        let result = format_attach_db_conn_str(db_resource, "postgres", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "postgres").unwrap();
         // Should be in URI format with defaults: postgres://postgres:@host:5432/dbname?sslmode=prefer
         assert!(result.starts_with("postgres://"));
         assert!(result.contains("@db.example.com:5432/production"));
@@ -4058,7 +3919,7 @@ mod tests {
             "host": "localhost",
             "dbname": "test"
         });
-        let result = format_attach_db_conn_str(db_resource, "postgresql", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "postgresql").unwrap();
         // Should be in URI format (postgresql is treated the same as postgres)
         assert!(result.starts_with("postgres://"));
         assert!(result.contains("@localhost:5432/test"));
@@ -4075,7 +3936,7 @@ mod tests {
             "dbname": "wm_datatables",
             "sslmode": "require"
         });
-        let stmts = pg_secret_attach_statements(db_resource, "dt", "/tmp").unwrap();
+        let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap();
         assert_eq!(stmts[0], "INSTALL postgres;");
         assert_eq!(stmts[1], "LOAD postgres;");
         let secret_name = datatable_secret_name("dt");
@@ -4106,7 +3967,7 @@ mod tests {
             if let Some(s) = input {
                 db_resource["sslmode"] = json!(s);
             }
-            let stmts = pg_secret_attach_statements(db_resource, "dt", "/tmp").unwrap();
+            let stmts = pg_secret_attach_statements(db_resource, "dt").unwrap();
             assert!(
                 stmts[3].starts_with(&format!("ATTACH 'sslmode={expected}'")),
                 "sslmode {input:?} → {}",
@@ -4129,7 +3990,7 @@ mod tests {
         let db_resource = json!({
             "project_id": "my-gcp-project"
         });
-        let result = format_attach_db_conn_str(db_resource, "bigquery", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "bigquery").unwrap();
         assert_eq!(result, "project=my-gcp-project");
     }
 
@@ -4138,7 +3999,7 @@ mod tests {
         let db_resource = json!({
             "other_field": "value"
         });
-        let result = format_attach_db_conn_str(db_resource, "bigquery", "/tmp");
+        let result = format_attach_db_conn_str(db_resource, "bigquery");
         assert!(result.is_err());
         assert!(result.unwrap_err().to_string().contains("project_id"));
     }
@@ -4146,7 +4007,7 @@ mod tests {
     #[test]
     fn test_format_attach_db_conn_str_unsupported_type() {
         let db_resource = json!({});
-        let result = format_attach_db_conn_str(db_resource, "oracle", "/tmp");
+        let result = format_attach_db_conn_str(db_resource, "oracle");
         assert!(result.is_err());
         assert!(result
             .unwrap_err()
@@ -4160,7 +4021,7 @@ mod tests {
             "host": "localhost",
             "dbname": "test"
         });
-        let result = format_attach_db_conn_str(db_resource, "POSTGRES", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "POSTGRES").unwrap();
         // Should be in URI format
         assert!(result.starts_with("postgres://"));
         assert!(result.contains("@localhost:5432/test"));
@@ -4177,7 +4038,7 @@ mod tests {
             "database": "app_db",
             "ssl": true
         });
-        let result = format_attach_db_conn_str(db_resource, "mysql", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "mysql").unwrap();
         assert!(result.contains("database=app_db"));
         assert!(result.contains("host=mysql.example.com"));
         assert!(result.contains("ssl_mode=required"));
@@ -4194,7 +4055,7 @@ mod tests {
             "database": "test",
             "ssl": false
         });
-        let result = format_attach_db_conn_str(db_resource, "mysql", "/tmp").unwrap();
+        let result = format_attach_db_conn_str(db_resource, "mysql").unwrap();
         assert!(result.contains("ssl_mode=disabled"));
     }