feat: Database manager for Ducklake instance catalogs (#6785)

* OVERRIDE_DATA_PATH

* DB Manager for Ducklake instance catalog debugging

backend/windmill-api/src/resources.rs

@@ -11,7 +11,7 @@ use std::collections::HashMap;
 use crate::{
     db::{ApiAuthed, DB},
     users::{maybe_refresh_folders, require_owner_of_path, Tokened},
-    utils::{check_scopes, BulkDeleteRequest},
+    utils::{check_scopes, require_super_admin, BulkDeleteRequest},
     var_resource_cache::{cache_resource, get_cached_resource},
     webhook_util::{WebhookMessage, WebhookShared},
 };
@@ -34,9 +34,11 @@ use windmill_audit::ActionKind;
 use windmill_common::{
     db::{UserDB, UserDbWithOptAuthed},
     error::{Error, JsonResult, Result},
+    get_database_url, parse_postgres_url,
     utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
     variables,
     worker::CLOUD_HOSTED,
+    workspaces::get_ducklake_instance_pg_catalog_password,
 };
 
 pub fn workspaced_service() -> Router {
@@ -462,6 +464,20 @@ pub async fn get_resource_value_interpolated_internal(
     token: &str,
     allow_cache: bool,
 ) -> Result<Option<serde_json::Value>> {
+    // This is a special syntax to help debugging ducklake catalogs stored in the instance
+    if let Some(dbname) = path.strip_prefix("INSTANCE_DUCKLAKE_CATALOG/") {
+        require_super_admin(db, &authed.email).await?;
+        let pg_creds = parse_postgres_url(&get_database_url().await?)?;
+        return Ok(Some(serde_json::json!({
+            "dbname": dbname,
+            "host": pg_creds.host,
+            "port": pg_creds.port,
+            "user": "ducklake_user",
+            "sslmode": pg_creds.ssl_mode,
+            "password": get_ducklake_instance_pg_catalog_password(&db).await?,
+        })));
+    }
+
     if allow_cache {
         if let Some(cached_value) = get_cached_resource(&workspace, &path) {
             return Ok(Some(cached_value));

backend/windmill-worker/src/common.rs

@@ -236,7 +236,8 @@ pub async fn transform_json_value(
         }
         Value::String(y) if y.starts_with("$res:") => {
             let path = y.strip_prefix("$res:").unwrap();
-            if path.split("/").count() < 2 {
+
+            if path.split("/").count() < 2 && !path.starts_with("INSTANCE_DUCKLAKE_CATALOG/") {
                 return Err(Error::internal_err(format!(
                     "Argument `{name}` is an invalid resource path: {path}",
                 )));

backend/windmill-worker/src/duckdb_executor.rs

@@ -488,6 +488,15 @@ async fn transform_attach_ducklake(
     let storage = ducklake.storage.storage.as_deref().unwrap_or("_default_");
     let data_path = ducklake.storage.path;
 
+    // Ducklake 0.3 only requires DATA_PATH at creation and then stores it internally in the catalog
+    // But it will fail if DATA_PATH changes afterwards which is annoying for us
+    // So we always enable override
+    let extra_args = if extra_args.contains("OVERRIDE_DATA_PATH") {
+        extra_args
+    } else {
+        format!(", OVERRIDE_DATA_PATH TRUE{extra_args}")
+    };
+
     let attach_str = format!(
         "ATTACH 'ducklake:{db_type}:{db_conn_str}' AS {alias_name} (DATA_PATH 's3://{storage}/{data_path}'{extra_args});",
     );

frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte

@@ -176,6 +176,7 @@
 	}
 
 	let dbManagerDrawer: DbManagerDrawer | undefined = $state()
+	let instanceCatalogPopover: Popover | undefined = $state()
 	let confirmationModal = createAsyncConfirmationModal()
 </script>
 
@@ -291,7 +292,7 @@
 									contentClasses="py-5 px-6 w-[34rem] bg-surface-secondary -translate-y-2"
 									closeOnOtherPopoverOpen
 									closeOnOutsideClick
-									{...confirmationModal.props.open ? { isOpen: false } : {}}
+									bind:this={instanceCatalogPopover}
 								>
 									<svelte:fragment slot="trigger">
 										<Button spacingSize="xs2" variant="border" color="light" btnClasses="h-6">
@@ -397,6 +398,8 @@
 	status: DucklakeInstanceCatalogDbStatus | undefined,
 	dbname: string
 )}
+	{@const showManageCatalogButton =
+		status?.logs.created_database === 'OK' || status?.logs.created_database === 'SKIP'}
 	{#if !status}
 		<div class="mb-4 text-secondary text-sm">
 			{dbname} needs to be configured in the Windmill postgres instance
@@ -461,51 +464,71 @@
 			status?.error ?? undefined
 		)}
 	/>
+	{#if showManageCatalogButton}
+		<div class="text-tertiary text-xs mt-6">
+			Note: the 'Manage catalog' button below is different from the Manage Ducklake button. This
+			will show you the content of the PostgreSQL database used as a catalog, while the other button
+			shows you the actual content of the ducklake (the parquet files).
+		</div>
+	{/if}
+	<div class="flex gap-2 mt-2">
+		<Button
+			wrapperClasses="flex-1"
+			size="sm"
+			disabled={!isInstanceCatalogEnabled}
+			onClick={async () => {
+				if (instanceCatalogSetupIsRunning) return
 
-	<Button
-		wrapperClasses="mt-6"
-		size="sm"
-		disabled={!isInstanceCatalogEnabled}
-		onClick={async () => {
-			if (instanceCatalogSetupIsRunning) return
-
-			let wasAlreadySuccessful = status?.success ?? false
-			if (status?.logs.created_database != 'OK' && status?.logs.created_database != 'SKIP') {
-				let confirm = await confirmationModal.ask({
-					title: 'Confirm setup',
-					children: `This will create a new database ${dbname} in the Windmill PostgreSQL instance`,
-					confirmationText: 'Setup catalog'
-				})
-				if (!confirm) return
-			}
-
-			try {
-				instanceCatalogSetupIsRunning = true
-				let result = await SettingService.setupDucklakeCatalogDb({ name: dbname })
-				await instanceCatalogStatuses.refresh()
-				if (result.success) {
-					if (!wasAlreadySuccessful) sendUserToast('Setup successful')
-					else sendUserToast('Everything OK')
-				} else {
-					sendUserToast(result.error ?? 'An error occured', true)
+				let wasAlreadySuccessful = status?.success ?? false
+				if (status?.logs.created_database != 'OK' && status?.logs.created_database != 'SKIP') {
+					instanceCatalogPopover?.close()
+					let confirm = await confirmationModal.ask({
+						title: 'Confirm setup',
+						children: `This will create a new database ${dbname} in the Windmill PostgreSQL instance`,
+						confirmationText: 'Setup catalog'
+					})
+					instanceCatalogPopover?.open()
+					if (!confirm) return
 				}
-			} catch (e) {
-				sendUserToast('Unexpected error, check console for details', true)
-				console.error('Error setting up ducklake instance catalog', e)
-			} finally {
-				instanceCatalogSetupIsRunning = false
-			}
-		}}
-		loading={instanceCatalogSetupIsRunning}
-	>
-		{#if !isInstanceCatalogEnabled}
-			Only superadmins can setup instance catalogs
-		{:else if status?.success}
-			Check again
-		{:else if status?.error}
-			Try again
-		{:else}
-			Setup {dbname}
+
+				try {
+					instanceCatalogSetupIsRunning = true
+					let result = await SettingService.setupDucklakeCatalogDb({ name: dbname })
+					await instanceCatalogStatuses.refresh()
+					if (result.success) {
+						if (!wasAlreadySuccessful) sendUserToast('Setup successful')
+						else sendUserToast('Everything OK')
+					} else {
+						sendUserToast(result.error ?? 'An error occured', true)
+					}
+				} catch (e) {
+					sendUserToast('Unexpected error, check console for details', true)
+					console.error('Error setting up ducklake instance catalog', e)
+				} finally {
+					instanceCatalogSetupIsRunning = false
+				}
+			}}
+			loading={instanceCatalogSetupIsRunning}
+		>
+			{#if !isInstanceCatalogEnabled}
+				Only superadmins can setup instance catalogs
+			{:else if status?.success}
+				Check again
+			{:else if status?.error}
+				Try again
+			{:else}
+				Setup {dbname}
+			{/if}
+		</Button>
+		{#if showManageCatalogButton}
+			<ExploreAssetButton
+				class="flex-1"
+				asset={{ kind: 'resource', path: 'INSTANCE_DUCKLAKE_CATALOG/' + dbname }}
+				_resourceMetadata={{ resource_type: 'postgresql' }}
+				{dbManagerDrawer}
+				disabled={!isInstanceCatalogEnabled}
+				onClick={() => instanceCatalogPopover?.close()}
+			/>
 		{/if}
-	</Button>
+	</div>
 {/snippet}