starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-11 08:07:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

Migrate ducklake catalogs to more generic custom instance databases

Browse Source
This commit is contained in:
Diego Imbert
2025-11-21 16:20:17 +01:00
parent 042d403983
commit fddff1b3ca
9 changed files with 101 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.down.sql
+10
View File
@@ -0,0 +1,10 @@
-- Add up migration script here
UPDATE global_settings
SET name = 'ducklake_settings',
value = jsonb_build_object(
'ducklake_user_pg_pwd', value->'user_pwd',
'instance_catalog_db_status', value->'status'
)
WHERE name = 'custom_instance_pg_databases';
ALTER ROLE custom_instance_user RENAME TO ducklake_user;
backend/migrations/20251121144314_migrate_instance_ducklake_catalogs.up.sql
+14
View File
@@ -0,0 +1,14 @@
-- Superadmins have the ability to create databases in the Windmill Postgres instance
-- for use as Ducklake catalogs or Data tables. These databases can be accessed by
-- the 'custom_instance_user'. The setting below stores the password and logs
-- about the creation status of these databases.
ALTER ROLE ducklake_user RENAME TO custom_instance_user;
UPDATE global_settings
SET name = 'custom_instance_pg_databases',
value = jsonb_build_object(
'user_pwd', value->'ducklake_user_pg_pwd',
'status', value->'instance_catalog_db_status'
)
WHERE name = 'ducklake_settings';
backend/windmill-api/openapi.yaml
+12 -12
View File
@@ -831,26 +831,26 @@ paths:
schema:
type: boolean
/settings/get_ducklake_instance_catalog_db_status:
/settings/get_custom_instance_pg_databases_status:
post:
summary: Returns the set-up statuses of ducklake instance catalog dbs
operationId: getDucklakeInstanceCatalogDbStatus
summary: Returns the set-up statuses of custom instance pg databases
operationId: getCustomInstanceDbStatus
tags:
- setting
responses:
"200":
description: Statuses of all ducklake instance catalog dbs
description: Statuses of all custom instance dbs
content:
application/json:
schema:
type: object
additionalProperties:
$ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus"
$ref: "#/components/schemas/CustomInstanceDbStatus"
/settings/setup_ducklake_catalog_db/{name}:
/settings/setup_custom_instance_pg_database/{name}:
post:
summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the ducklake_user
operationId: setupDucklakeCatalogDb
summary: Runs CREATE DATABASE on the Windmill Postgres and grants access to the custom_instance_user
operationId: setupCustomInstanceDb
tags:
- setting
parameters:
@@ -866,7 +866,7 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/DucklakeInstanceCatalogDbStatus"
$ref: "#/components/schemas/CustomInstanceDbStatus"
/settings/global/{key}:
get:
@@ -18118,7 +18118,7 @@ components:
- SKIP
- FAIL
DucklakeInstanceCatalogDbStatusLogs:
CustomInstanceDbStatusLogs:
type: object
properties:
super_admin:
@@ -18135,14 +18135,14 @@ components:
grant_permissions:
$ref: "#/components/schemas/LoggedWizardStatus"
DucklakeInstanceCatalogDbStatus:
CustomInstanceDbStatus:
type: object
required:
- logs
- success
properties:
logs:
$ref: "#/components/schemas/DucklakeInstanceCatalogDbStatusLogs"
$ref: "#/components/schemas/CustomInstanceDbStatusLogs"
success:
type: boolean
description: Whether the operation completed successfully
backend/windmill-api/src/resources.rs
+5 -5
View File
@@ -38,10 +38,10 @@ use windmill_common::{
db::{UserDB, UserDbWithAuthed, UserDbWithOptAuthed},
error::{self, Error, JsonResult, Result},
get_database_url, parse_postgres_url,
utils::get_custom_pg_instance_password,
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
variables,
worker::{CLOUD_HOSTED, TMP_DIR},
workspaces::get_ducklake_instance_pg_catalog_password,
};
pub fn workspaced_service() -> Router {
@@ -469,17 +469,17 @@ pub async fn get_resource_value_interpolated_internal(
token: &str,
allow_cache: bool,
) -> Result<Option<serde_json::Value>> {
// This is a special syntax to help debugging ducklake catalogs stored in the instance
if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") {
// This is a special syntax to help debugging custom instance databases
if let Some(dbname) = path.strip_prefix("CUSTOM_INSTANCE_DB/") {
require_super_admin(db, &authed.email).await?;
let pg_creds = parse_postgres_url(&get_database_url().await?.as_str().await)?;
return Ok(Some(serde_json::json!({
"dbname": dbname,
"host": pg_creds.host,
"port": pg_creds.port,
"user": "ducklake_user",
"user": "custom_instance_user",
"sslmode": pg_creds.ssl_mode,
"password": get_ducklake_instance_pg_catalog_password(&db).await?,
"password": get_custom_pg_instance_password(&db).await?,
})));
}
backend/windmill-api/src/settings.rs
+31 -26
View File
@@ -70,12 +70,12 @@ pub fn global_service() -> Router {
post(acknowledge_critical_alert),
)
.route(
"/get_ducklake_instance_catalog_db_status",
post(get_ducklake_instance_catalog_db_status),
"/get_custom_instance_pg_databases_status",
post(get_custom_instance_pg_databases_status),
)
.route(
"/setup_ducklake_catalog_db/:name",
post(setup_ducklake_catalog_db),
"/setup_custom_instance_pg_database/:name",
post(setup_custom_instance_pg_database),
)
.route(
"/critical_alerts/acknowledge_all",
@@ -218,7 +218,12 @@ pub struct Value {
}
pub async fn delete_global_setting(db: &DB, key: &str) -> error::Result<()> {
if key == "ducklake_user_pg_pwd" || key == "ducklake_settings" {
// ducklake_user_pg_pwd and ducklake_settings were old names stored as standalone global settings.
// Leave them for backward compatibility (CLI will try to delete them if not present in the yaml)
if key == "ducklake_user_pg_pwd"
|| key == "ducklake_settings"
|| key == "custom_instance_pg_databases"
{
tracing::error!("Tried to unset global setting {}, ignored", key);
return Ok(());
}
@@ -578,15 +583,15 @@ pub async fn acknowledge_all_critical_alerts() -> error::Error {
}
#[derive(Deserialize, Debug, Serialize)]
struct DucklakeInstanceCatalogDbStatus {
logs: DucklakeInstanceCatalogDbStatusLogs, // (Step, Message)[]
struct CustomInstanceDbStatus {
logs: CustomInstanceDbStatusLogs, // (Step, Message)[]
success: bool,
error: Option<String>,
}
#[derive(Deserialize, Debug, Serialize, Default)]
#[serde(default)]
struct DucklakeInstanceCatalogDbStatusLogs {
struct CustomInstanceDbStatusLogs {
super_admin: String,
#[serde(skip_serializing_if = "String::is_empty")]
database_credentials: String,
@@ -600,50 +605,50 @@ struct DucklakeInstanceCatalogDbStatusLogs {
grant_permissions: String,
}
async fn get_ducklake_instance_catalog_db_status(
async fn get_custom_instance_pg_databases_status(
_authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> JsonResult<HashMap<String, DucklakeInstanceCatalogDbStatus>> {
) -> JsonResult<HashMap<String, CustomInstanceDbStatus>> {
let result = sqlx::query_scalar!(
r#"SELECT value->'instance_catalog_db_status' FROM global_settings WHERE name = 'ducklake_settings'"#,
r#"SELECT value->'status' FROM global_settings WHERE name = 'custom_instance_pg_databases'"#,
)
.fetch_one(&db)
.await?
.ok_or_else(|| error::Error::ExecutionErr("Couldn't find ducklake_settings".to_string()))?;
.ok_or_else(|| error::Error::ExecutionErr("Couldn't find custom_instance_pg_databases".to_string()))?;
let result = serde_json::from_value(result).map_err(|e| {
error::Error::ExecutionErr(format!(
"couldn't parse instance_catalog_db_status : {}",
"couldn't parse custom_instance_pg_databases.status : {}",
e.to_string()
))
})?;
return Ok(Json(result));
}
async fn setup_ducklake_catalog_db(
async fn setup_custom_instance_pg_database(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(dbname): Path<String>,
) -> JsonResult<DucklakeInstanceCatalogDbStatus> {
let mut logs = DucklakeInstanceCatalogDbStatusLogs::default();
let result = setup_ducklake_catalog_db_inner(authed, &db, &dbname, &mut logs).await;
) -> JsonResult<CustomInstanceDbStatus> {
let mut logs = CustomInstanceDbStatusLogs::default();
let result = setup_custom_instance_pg_database_inner(authed, &db, &dbname, &mut logs).await;
let success = result.is_ok();
let error = result.err().map(|e| e.to_string());
let status = DucklakeInstanceCatalogDbStatus { logs, success, error };
let status = CustomInstanceDbStatus { logs, success, error };
let status_json = serde_json::to_value(&status).map_err(to_anyhow)?;
// Save that the database was setup successfully
sqlx::query!(
r#"UPDATE global_settings SET value = jsonb_set(value, '{instance_catalog_db_status}', (COALESCE(value->'instance_catalog_db_status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'ducklake_settings'"#,
r#"UPDATE global_settings SET value = jsonb_set(value, '{status}', (COALESCE(value->'status', '{}'::jsonb) || to_jsonb($1::json))) WHERE name = 'custom_instance_pg_databases'"#,
json!({ dbname: status_json })
).execute(&db).await?;
Ok(Json(status))
}
async fn setup_ducklake_catalog_db_inner(
async fn setup_custom_instance_pg_database_inner(
authed: ApiAuthed,
db: &DB,
dbname: &str,
logs: &mut DucklakeInstanceCatalogDbStatusLogs,
logs: &mut CustomInstanceDbStatusLogs,
) -> Result<()> {
require_super_admin(db, &authed.email).await?;
logs.super_admin = "OK".to_string();
@@ -738,16 +743,16 @@ async fn setup_ducklake_catalog_db_inner(
client
.batch_execute(&format!(
"GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user;
GRANT USAGE ON SCHEMA public TO ducklake_user;
GRANT CREATE ON SCHEMA public TO ducklake_user;
"GRANT CONNECT ON DATABASE \"{dbname}\" TO custom_instance_user;
GRANT USAGE ON SCHEMA public TO custom_instance_user;
GRANT CREATE ON SCHEMA public TO custom_instance_user;
ALTER DEFAULT PRIVILEGES IN SCHEMA public
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;"
GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO custom_instance_user;"
))
.await
.map_err(|e| {
error::Error::ExecutionErr(format!(
"Failed to grant permissions to ducklake_user: {}",
"Failed to grant permissions to custom_instance_user: {}",
e.to_string(),
))
})?;
backend/windmill-common/src/utils.rs
+13
View File
@@ -953,3 +953,16 @@ impl<T> ExpiringCacheEntry<T> {
self.expiry < std::time::Instant::now()
}
}
pub async fn get_custom_pg_instance_password(db: &DB) -> Result<String> {
sqlx::query_scalar!(
"SELECT value->>'user_pwd' FROM global_settings WHERE name = 'custom_instance_pg_databases';"
)
.fetch_optional(db)
.await?
.flatten().ok_or_else(||
Error::BadRequest(format!(
"Custom instance db password not found, did you run migrations ?"
))
)
}
backend/windmill-common/src/workspaces.rs
+3 -15
View File
@@ -9,6 +9,7 @@ use strum::AsRefStr;
use crate::{
error::{to_anyhow, Error, Result},
get_database_url, parse_postgres_url,
utils::get_custom_pg_instance_password,
variables::{build_crypt, decrypt},
DB,
};
@@ -213,9 +214,9 @@ pub async fn get_ducklake_from_db_unchecked(
"dbname": ducklake.catalog.resource_path,
"host": pg_creds.host,
"port": pg_creds.port,
"user": "ducklake_user",
"user": "custom_instance_user",
"sslmode": pg_creds.ssl_mode,
"password": get_ducklake_instance_pg_catalog_password(&db).await?,
"password": get_custom_pg_instance_password(&db).await?,
})
} else {
transform_json_unchecked(
@@ -233,19 +234,6 @@ pub async fn get_ducklake_from_db_unchecked(
Ok(ducklake)
}
pub async fn get_ducklake_instance_pg_catalog_password(db: &DB) -> Result<String> {
sqlx::query_scalar!(
"SELECT value->>'ducklake_user_pg_pwd' FROM global_settings WHERE name = 'ducklake_settings';"
)
.fetch_optional(db)
.await?
.flatten().ok_or_else(||
Error::BadRequest(format!(
"Ducklake instance catalog password not found, did you run migrations ?"
))
)
}
// This does not check for any permission. Should never be displayed to a user.
#[async_recursion]
async fn transform_json_unchecked(
backend/windmill-worker/src/common.rs
+2 -5
View File
@@ -237,7 +237,7 @@ pub async fn transform_json_value(
Value::String(y) if y.starts_with("$res:") => {
let path = y.strip_prefix("$res:").unwrap();
if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") {
if path.split("/").count() < 2 && !path.starts_with("CUSTOM_INSTANCE_DB/") {
return Err(Error::internal_err(format!(
"Argument `{name}` is an invalid resource path: {path}",
)));
@@ -629,10 +629,7 @@ lazy_static! {
static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok();
}
pub fn build_command_with_isolation(
program: &str,
args: &[&str],
) -> Command {
pub fn build_command_with_isolation(program: &str, args: &[&str]) -> Command {
use tokio::process::Command;
if *crate::ENABLE_UNSHARE_PID {
frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte
+11 -14
View File
@@ -62,7 +62,7 @@
import Select from '../select/Select.svelte'
import ResourcePicker from '../ResourcePicker.svelte'
import { usePromise } from '$lib/svelte5Utils.svelte'
import { SettingService, WorkspaceService, type DucklakeInstanceCatalogDbStatus } from '$lib/gen'
import { SettingService, WorkspaceService, type CustomInstanceDbStatus } from '$lib/gen'
import { type GetSettingsResponse } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
@@ -126,7 +126,7 @@
)
let instanceCatalogSetupIsRunning = $state(false)
const instanceCatalogStatuses = usePromise(SettingService.getDucklakeInstanceCatalogDbStatus, {
const instanceCatalogStatuses = usePromise(SettingService.getCustomInstanceDbStatus, {
clearValueOnRefresh: false
})
@@ -396,10 +396,7 @@
<ConfirmationModal {...confirmationModal.props} />
{#snippet instanceCatalogWizard(
status: DucklakeInstanceCatalogDbStatus | undefined,
dbname: string
)}
{#snippet instanceCatalogWizard(status: CustomInstanceDbStatus | undefined, dbname: string)}
{@const showManageCatalogButton =
status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'}
{#if !status}
@@ -452,15 +449,15 @@
"Connect to the newly created database with the default admin user (the one in DATABASE_URL, usually 'postgres') to run the next commands"
},
{
title: 'Grant permissions to ducklake_user',
title: 'Grant permissions to custom_instance_user',
status: status?.logs.grant_permissions,
description:
'Gives ducklake_user the required permissions to use the database as a Ducklake catalog. ducklake_user is already created during a migration and has an auto-generated password stored in global_settings.ducklake_settings.ducklake_user_pg_pwd. These are the commands : \n\n' +
`GRANT CONNECT ON DATABASE "${dbname}" TO ducklake_user;\n` +
'GRANT USAGE ON SCHEMA public TO ducklake_user;\n' +
'GRANT CREATE ON SCHEMA public TO ducklake_user;\n' +
'Gives custom_instance_user the required permissions to use the database as a Ducklake catalog. custom_instance_user is already created during a migration and has an auto-generated password stored in global_settings.custom_instance_pg_databases.user_pwd. These are the commands : \n\n' +
`GRANT CONNECT ON DATABASE "${dbname}" TO custom_instance_user;\n` +
'GRANT USAGE ON SCHEMA public TO custom_instance_user;\n' +
'GRANT CREATE ON SCHEMA public TO custom_instance_user;\n' +
'ALTER DEFAULT PRIVILEGES IN SCHEMA public \n' +
' GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES\n TO ducklake_user;'
' GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES\n TO custom_instance_user;'
}
],
status?.error ?? undefined
@@ -495,7 +492,7 @@
try {
instanceCatalogSetupIsRunning = true
let result = await SettingService.setupDucklakeCatalogDb({ name: dbname })
let result = await SettingService.setupCustomInstanceDb({ name: dbname })
await instanceCatalogStatuses.refresh()
if (result.success) {
if (!wasAlreadySuccessful) sendUserToast('Setup successful')
@@ -525,7 +522,7 @@
{#if showManageCatalogButton}
<ExploreAssetButton
class="flex-1"
asset={{ kind: 'resource', path: 'INSTANCE_DUCKLAKE_CATALOG/' + dbname }}
asset={{ kind: 'resource', path: 'CUSTOM_INSTANCE_DB/' + dbname }}
_resourceMetadata={{ resource_type: 'postgresql' }}
{dbManagerDrawer}
disabled={!$isCustomInstanceDbEnabled}