From fddff1b3ca0bd0aac19b5c3b8976525fae40b325 Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Fri, 21 Nov 2025 16:20:17 +0100 Subject: [PATCH] Migrate ducklake catalogs to more generic custom instance databases --- ...igrate_instance_ducklake_catalogs.down.sql | 10 ++++ ..._migrate_instance_ducklake_catalogs.up.sql | 14 +++++ backend/windmill-api/openapi.yaml | 24 ++++---- backend/windmill-api/src/resources.rs | 10 ++-- backend/windmill-api/src/settings.rs | 57 ++++++++++--------- backend/windmill-common/src/utils.rs | 13 +++++ backend/windmill-common/src/workspaces.rs | 18 +----- backend/windmill-worker/src/common.rs | 7 +-- .../workspaceSettings/DucklakeSettings.svelte | 25 ++++---- 9 files changed, 101 insertions(+), 77 deletions(-) create mode 100644 backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql create mode 100644 backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql diff --git a/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql new file mode 100644 index 0000000000..ae87d1e456 --- /dev/null +++ b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql @@ -0,0 +1,10 @@ +-- Add up migration script here +UPDATE global_settings +SET name = 'ducklake_settings', + value = jsonb_build_object( + 'ducklake_user_pg_pwd', value->'user_pwd', + 'instance_catalog_db_status', value->'status' + ) +WHERE name = 'custom_instance_pg_databases'; + +ALTER ROLE custom_instance_user RENAME TO ducklake_user; \ No newline at end of file diff --git a/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql new file mode 100644 index 0000000000..6458f9daaa --- /dev/null +++ b/backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql @@ -0,0 +1,14 @@ +-- Superadmins have the ability to create databases in the Windmill Postgres instance +-- for use as Ducklake catalogs or Data tables. These databases can be accessed by +-- the 'custom_instance_user'. The setting below stores the password and logs +-- about the creation status of these databases. + +ALTER ROLE ducklake_user RENAME TO custom_instance_user; + +UPDATE global_settings +SET name = 'custom_instance_pg_databases', + value = jsonb_build_object( + 'user_pwd', value->'ducklake_user_pg_pwd', + 'status', value->'instance_catalog_db_status' + ) +WHERE name = 'ducklake_settings'; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 10acdee52f..051f7d985f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -831,26 +831,26 @@ paths: schema: type: boolean - /settings/get_ducklake_instance_catalog_db_status: + /settings/get_custom_instance_pg_databases_status: post: - summary: Returns the set-up statuses of ducklake instance catalog dbs - operationId: getDucklakeInstanceCatalogDbStatus + summary: Returns the set-up statuses of custom instance pg databases + operationId: getCustomInstanceDbStatus tags: - setting responses: "200": - description: Statuses of all ducklake instance catalog dbs + description: Statuses of all custom instance dbs content: application/json: schema: type: object additionalProperties: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" + $ref: "#/components/schemas/CustomInstanceDbStatus" - /settings/setup_ducklake_catalog_db/{name}: + /settings/setup_custom_instance_pg_database/{name}: post: - summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the ducklake_user - operationId: setupDucklakeCatalogDb + summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user + operationId: setupCustomInstanceDb tags: - setting parameters: @@ -866,7 +866,7 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus" + $ref: "#/components/schemas/CustomInstanceDbStatus" /settings/global/{key}: get: @@ -18118,7 +18118,7 @@ components: - SKIP - FAIL - DucklakeInstanceCatalogDbStatusLogs: + CustomInstanceDbStatusLogs: type: object properties: super_admin: @@ -18135,14 +18135,14 @@ components: grant_permissions: $ref: "#/components/schemas/LoggedWizardStatus" - DucklakeInstanceCatalogDbStatus: + CustomInstanceDbStatus: type: object required: - logs - success properties: logs: - $ref: "#/components/schemas/DucklakeInstanceCatalogDbStatusLogs" + $ref: "#/components/schemas/CustomInstanceDbStatusLogs" success: type: boolean description: Whether the operation completed successfully diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 503c60e1f8..14c7054c3a 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -38,10 +38,10 @@ use windmill_common::{ db::{UserDB, UserDbWithAuthed, UserDbWithOptAuthed}, error::{self, Error, JsonResult, Result}, get_database_url, parse_postgres_url, + utils::get_custom_pg_instance_password, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, worker::{CLOUD_HOSTED, TMP_DIR}, - workspaces::get_ducklake_instance_pg_catalog_password, }; pub fn workspaced_service() -> Router { @@ -469,17 +469,17 @@ pub async fn get_resource_value_interpolated_internal( token: &str, allow_cache: bool, ) -> Result> { - // This is a special syntax to help debugging ducklake catalogs stored in the instance - if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") { + // This is a special syntax to help debugging custom instance databases + if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") { require_super_admin(db, &authed.email).await?; let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?; return Ok(Some(serde_json::json!({ "dbname": dbname, "host": pg_creds.host, "port": pg_creds.port, - "user": "ducklake_user", + "user": "custom_instance_user", "sslmode": pg_creds.ssl_mode, - "password": get_ducklake_instance_pg_catalog_password(&db).await?, + "password": get_custom_pg_instance_password(&db).await?, }))); } diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index bff2b282b6..b5b7ab82e8 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -70,12 +70,12 @@ pub fn global_service() -> Router { post(acknowledge_critical_alert), ) .route( - "/get_ducklake_instance_catalog_db_status", - post(get_ducklake_instance_catalog_db_status), + "/get_custom_instance_pg_databases_status", + post(get_custom_instance_pg_databases_status), ) .route( - "/setup_ducklake_catalog_db/:name", - post(setup_ducklake_catalog_db), + "/setup_custom_instance_pg_database/:name", + post(setup_custom_instance_pg_database), ) .route( "/critical_alerts/acknowledge_all", @@ -218,7 +218,12 @@ pub struct Value { } pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> { - if key == "ducklake_user_pg_pwd" || key == "ducklake_settings" { + // ducklake_user_pg_pwd and ducklake_settings were old names stored as standalone global settings. + // Leave them for backward compatibility (CLI will try to delete them if not present in the yaml) + if key == "ducklake_user_pg_pwd" + || key == "ducklake_settings" + || key == "custom_instance_pg_databases" + { tracing::error!("Tried to unset global setting {}, ignored", key); return Ok(()); } @@ -578,15 +583,15 @@ pub async fn acknowledge_all_critical_alerts() -> error::Error { } #[derive(Deserialize, Debug, Serialize)] -struct DucklakeInstanceCatalogDbStatus { - logs: DucklakeInstanceCatalogDbStatusLogs, // (Step, Message)[] +struct CustomInstanceDbStatus { + logs: CustomInstanceDbStatusLogs, // (Step, Message)[] success: bool, error: Option, } #[derive(Deserialize, Debug, Serialize, Default)] #[serde(default)] -struct DucklakeInstanceCatalogDbStatusLogs { +struct CustomInstanceDbStatusLogs { super_admin: String, #[serde(skip_serializing_if = "String::is_empty")] database_credentials: String, @@ -600,50 +605,50 @@ struct DucklakeInstanceCatalogDbStatusLogs { grant_permissions: String, } -async fn get_ducklake_instance_catalog_db_status( +async fn get_custom_instance_pg_databases_status( _authed: ApiAuthed, Extension(db): Extension, -) -> JsonResult> { +) -> JsonResult> { let result = sqlx::query_scalar!( - r#"SELECT value->'instance_catalog_db_status' FROM global_settings WHERE name = 'ducklake_settings'"#, + r#"SELECT value->'status' FROM global_settings WHERE name = 'custom_instance_pg_databases'"#, ) .fetch_one(&db) .await? - .ok_or_else(|| error::Error::ExecutionErr("Couldn't find ducklake_settings".to_string()))?; + .ok_or_else(|| error::Error::ExecutionErr("Couldn't find custom_instance_pg_databases".to_string()))?; let result = serde_json::from_value(result).map_err(|e| { error::Error::ExecutionErr(format!( - "couldn't parse instance_catalog_db_status : {}", + "couldn't parse custom_instance_pg_databases.status : {}", e.to_string() )) })?; return Ok(Json(result)); } -async fn setup_ducklake_catalog_db( +async fn setup_custom_instance_pg_database( authed: ApiAuthed, Extension(db): Extension, Path(dbname): Path, -) -> JsonResult { - let mut logs = DucklakeInstanceCatalogDbStatusLogs::default(); - let result = setup_ducklake_catalog_db_inner(authed, &db, &dbname, &mut logs).await; +) -> JsonResult { + let mut logs = CustomInstanceDbStatusLogs::default(); + let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await; let success = result.is_ok(); let error = result.err().map(|e| e.to_string()); - let status = DucklakeInstanceCatalogDbStatus { logs, success, error }; + let status = CustomInstanceDbStatus { logs, success, error }; let status_json = serde_json::to_value(&status).map_err(to_anyhow)?; // Save that the database was setup successfully sqlx::query!( - r#"UPDATE global_settings SET value = jsonb_set(value, '{instance_catalog_db_status}', (COALESCE(value->'instance_catalog_db_status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'ducklake_settings'"#, + r#"UPDATE global_settings SET value = jsonb_set(value, '{status}', (COALESCE(value->'status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#, json!({ dbname: status_json }) ).execute(&db).await?; Ok(Json(status)) } -async fn setup_ducklake_catalog_db_inner( +async fn setup_custom_instance_pg_database_inner( authed: ApiAuthed, db: &DB, dbname: &str, - logs: &mut DucklakeInstanceCatalogDbStatusLogs, + logs: &mut CustomInstanceDbStatusLogs, ) -> Result<()> { require_super_admin(db, &authed.email).await?; logs.super_admin = "OK".to_string(); @@ -738,16 +743,16 @@ async fn setup_ducklake_catalog_db_inner( client .batch_execute(&format!( - "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user; - GRANT USAGE ON SCHEMA public TO ducklake_user; - GRANT CREATE ON SCHEMA public TO ducklake_user; + "GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user; + GRANT USAGE ON SCHEMA public TO custom_instance_user; + GRANT CREATE ON SCHEMA public TO custom_instance_user; ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;" + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;" )) .await .map_err(|e| { error::Error::ExecutionErr(format!( - "Failed to grant permissions to ducklake_user: {}", + "Failed to grant permissions to custom_instance_user: {}", e.to_string(), )) })?; diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 623486d19c..dc5189eff1 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -953,3 +953,16 @@ impl ExpiringCacheEntry { self.expiry < std::time::Instant::now() } } + +pub async fn get_custom_pg_instance_password(db: &DB) -> Result { + sqlx::query_scalar!( + "SELECT value->>'user_pwd' FROM global_settings WHERE name = 'custom_instance_pg_databases';" + ) + .fetch_optional(db) + .await? + .flatten().ok_or_else(|| + Error::BadRequest(format!( + "Custom instance db password not found, did you run migrations ?" + )) + ) +} diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 0fa8769ffd..20a5d42c71 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -9,6 +9,7 @@ use strum::AsRefStr; use crate::{ error::{to_anyhow, Error, Result}, get_database_url, parse_postgres_url, + utils::get_custom_pg_instance_password, variables::{build_crypt, decrypt}, DB, }; @@ -213,9 +214,9 @@ pub async fn get_ducklake_from_db_unchecked( "dbname": ducklake.catalog.resource_path, "host": pg_creds.host, "port": pg_creds.port, - "user": "ducklake_user", + "user": "custom_instance_user", "sslmode": pg_creds.ssl_mode, - "password": get_ducklake_instance_pg_catalog_password(&db).await?, + "password": get_custom_pg_instance_password(&db).await?, }) } else { transform_json_unchecked( @@ -233,19 +234,6 @@ pub async fn get_ducklake_from_db_unchecked( Ok(ducklake) } -pub async fn get_ducklake_instance_pg_catalog_password(db: &DB) -> Result { - sqlx::query_scalar!( - "SELECT value->>'ducklake_user_pg_pwd' FROM global_settings WHERE name = 'ducklake_settings';" - ) - .fetch_optional(db) - .await? - .flatten().ok_or_else(|| - Error::BadRequest(format!( - "Ducklake instance catalog password not found, did you run migrations ?" - )) - ) -} - // This does not check for any permission. Should never be displayed to a user. #[async_recursion] async fn transform_json_unchecked( diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 2d995ea9bb..06085f5f68 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -237,7 +237,7 @@ pub async fn transform_json_value( Value::String(y) if y.starts_with("$res:") => { let path = y.strip_prefix("$res:").unwrap(); - if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") { + if path.split("/").count() < 2 && !path.starts_with("CUSTOM_INSTANCE_DB/") { return Err(Error::internal_err(format!( "Argument `{name}` is an invalid resource path: {path}", ))); @@ -629,10 +629,7 @@ lazy_static! { static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok(); } -pub fn build_command_with_isolation( - program: &str, - args: &[&str], -) -> Command { +pub fn build_command_with_isolation(program: &str, args: &[&str]) -> Command { use tokio::process::Command; if *crate::ENABLE_UNSHARE_PID { diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index 5b131cf45d..b9714dc167 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -62,7 +62,7 @@ import Select from '../select/Select.svelte' import ResourcePicker from '../ResourcePicker.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' - import { SettingService, WorkspaceService, type DucklakeInstanceCatalogDbStatus } from '$lib/gen' + import { SettingService, WorkspaceService, type CustomInstanceDbStatus } from '$lib/gen' import { type GetSettingsResponse } from '$lib/gen' import { workspaceStore } from '$lib/stores' @@ -126,7 +126,7 @@ ) let instanceCatalogSetupIsRunning = $state(false) - const instanceCatalogStatuses = usePromise(SettingService.getDucklakeInstanceCatalogDbStatus, { + const instanceCatalogStatuses = usePromise(SettingService.getCustomInstanceDbStatus, { clearValueOnRefresh: false }) @@ -396,10 +396,7 @@ -{#snippet instanceCatalogWizard( - status: DucklakeInstanceCatalogDbStatus | undefined, - dbname: string -)} +{#snippet instanceCatalogWizard(status: CustomInstanceDbStatus | undefined, dbname: string)} {@const showManageCatalogButton = status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'} {#if !status} @@ -452,15 +449,15 @@ "Connect to the newly created database with the default admin user (the one in DATABASE_URL, usually 'postgres') to run the next commands" }, { - title: 'Grant permissions to ducklake_user', + title: 'Grant permissions to custom_instance_user', status: status?.logs.grant_permissions, description: - 'Gives ducklake_user the required permissions to use the database as a Ducklake catalog. ducklake_user is already created during a migration and has an auto-generated password stored in global_settings.ducklake_settings.ducklake_user_pg_pwd. These are the commands : \n\n' + - `GRANT CONNECT ON DATABASE "${dbname}" TO ducklake_user;\n` + - 'GRANT USAGE ON SCHEMA public TO ducklake_user;\n' + - 'GRANT CREATE ON SCHEMA public TO ducklake_user;\n' + + 'Gives custom_instance_user the required permissions to use the database as a Ducklake catalog. custom_instance_user is already created during a migration and has an auto-generated password stored in global_settings.custom_instance_pg_databases.user_pwd. These are the commands : \n\n' + + `GRANT CONNECT ON DATABASE "${dbname}" TO custom_instance_user;\n` + + 'GRANT USAGE ON SCHEMA public TO custom_instance_user;\n' + + 'GRANT CREATE ON SCHEMA public TO custom_instance_user;\n' + 'ALTER DEFAULT PRIVILEGES IN SCHEMA public \n' + - ' GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES\n TO ducklake_user;' + ' GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES\n TO custom_instance_user;' } ], status?.error ?? undefined @@ -495,7 +492,7 @@ try { instanceCatalogSetupIsRunning = true - let result = await SettingService.setupDucklakeCatalogDb({ name: dbname }) + let result = await SettingService.setupCustomInstanceDb({ name: dbname }) await instanceCatalogStatuses.refresh() if (result.success) { if (!wasAlreadySuccessful) sendUserToast('Setup successful') @@ -525,7 +522,7 @@ {#if showManageCatalogButton}