data tables settings

backend/migrations/20251121144144_datatables.down.sql

@@ -0,0 +1,4 @@
+-- Add down migration script here
+
+ALTER TABLE workspace_settings
+DROP COLUMN datatable;

backend/migrations/20251121144144_datatables.up.sql

@@ -0,0 +1,4 @@
+-- Add up migration script here
+
+ALTER TABLE workspace_settings
+ADD COLUMN datatable JSONB;

backend/windmill-api/openapi.yaml

@@ -2077,6 +2077,8 @@ paths:
                     $ref: "#/components/schemas/LargeFileStorage"
                   ducklake:
                     $ref: "#/components/schemas/DucklakeSettings"
+                  datatable:
+                    $ref: "#/components/schemas/DataTableSettings"
                   git_sync:
                     $ref: "#/components/schemas/WorkspaceGitSyncSettings"
                   deploy_ui:
@@ -2789,6 +2791,32 @@ paths:
             application/json:
               schema: {}
 
+  /w/{workspace}/workspaces/edit_datatable_config:
+    post:
+      summary: edit datatable settings
+      operationId: editDataTableConfig
+      tags:
+        - workspace
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      requestBody:
+        description: DataTable settings
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [settings]
+              properties:
+                settings:
+                  $ref: "#/components/schemas/DataTableSettings"
+      responses:
+        "200":
+          description: status
+          content:
+            application/json:
+              schema: {}
+
   /w/{workspace}/workspaces/edit_git_sync_config:
     post:
       summary: edit workspace git sync settings
@@ -19428,6 +19456,29 @@ components:
                 required:
                   - path
 
+    DataTableSettings:
+      type: object
+      required: [datatables]
+      properties:
+        datatables:
+          type: object
+          additionalProperties:
+            type: object
+            required: [database]
+            properties:
+              database:
+                type: object
+                properties:
+                  resource_type:
+                    type: string
+                    enum:
+                      - postgresql
+                      - instance
+                  resource_path:
+                    type: string
+                required:
+                  - resource_type
+
     DynamicInputData:
       type: object
       properties:

backend/windmill-api/src/workspaces.rs

@@ -42,7 +42,9 @@ use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
 use windmill_common::workspaces::GitRepositorySettings;
 #[cfg(feature = "enterprise")]
 use windmill_common::workspaces::WorkspaceDeploymentUISettings;
-use windmill_common::workspaces::WorkspaceGitSyncSettings;
+use windmill_common::workspaces::{
+    DataTable, DataTableCatalogResourceType, WorkspaceGitSyncSettings,
+};
 use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType};
 use windmill_common::{
     error::{Error, JsonResult, Result},
@@ -124,6 +126,7 @@ pub fn workspaced_service() -> Router {
         )
         .route("/edit_ducklake_config", post(edit_ducklake_config))
         .route("/list_ducklakes", get(list_ducklakes))
+        .route("/edit_datatable_config", post(edit_datatable_config))
         .route("/edit_git_sync_config", post(edit_git_sync_config))
         .route("/edit_git_sync_repository", post(edit_git_sync_repository))
         .route(
@@ -327,12 +330,6 @@ struct LargeFileStorageWithSecondary {
     #[serde(default)]
     secondary_storage: HashMap<String, LargeFileStorage>,
 }
-
-#[derive(Deserialize, Serialize, Debug)]
-pub struct DucklakeSettings {
-    pub ducklakes: HashMap<String, Ducklake>,
-}
-
 #[derive(Deserialize, Debug)]
 struct EditLargeFileStorageConfig {
     large_file_storage: Option<LargeFileStorageWithSecondary>,
@@ -343,6 +340,21 @@ struct EditDucklakeConfig {
     settings: DucklakeSettings,
 }
 
+#[derive(Deserialize, Serialize, Debug)]
+pub struct DucklakeSettings {
+    pub ducklakes: HashMap<String, Ducklake>,
+}
+
+#[derive(Deserialize, Debug)]
+struct EditDataTableConfig {
+    settings: DataTableSettings,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct DataTableSettings {
+    pub datatables: HashMap<String, DataTable>,
+}
+
 #[derive(Deserialize)]
 struct CreateWorkspace {
     id: String,
@@ -1247,6 +1259,101 @@ async fn edit_ducklake_config(
     Ok(format!("Edit ducklake config for workspace {}", &w_id))
 }
 
+async fn edit_datatable_config(
+    authed: ApiAuthed,
+    Extension(db): Extension<DB>,
+    Path(w_id): Path<String>,
+    ApiAuthed { is_admin, username, email, .. }: ApiAuthed,
+    Json(new_config): Json<EditDataTableConfig>,
+) -> Result<String> {
+    require_admin(is_admin, &username)?;
+    let is_superadmin = require_super_admin(&db, &email).await.is_ok();
+
+    let mut tx = db.begin().await?;
+
+    let args_for_audit = format!("{:?}", new_config.settings);
+    audit_log(
+        &mut *tx,
+        &authed,
+        "workspaces.edit_datatable_config",
+        ActionKind::Update,
+        &w_id,
+        Some(&authed.email),
+        Some([("datatable", args_for_audit.as_str())].into()),
+    )
+    .await?;
+
+    // Check that all database resources exist to prevent
+    // exploiting the shared property to see any resource
+    for dl in new_config.settings.datatables.values() {
+        if dl.database.resource_type == DataTableCatalogResourceType::Instance {
+            continue;
+        }
+        let database_res = sqlx::query_scalar!(
+            "SELECT 1 FROM resource WHERE workspace_id = $1 AND path = $2",
+            &w_id,
+            &dl.database.resource_path
+        )
+        .fetch_optional(&mut *tx)
+        .await?
+        .flatten();
+
+        if database_res.is_none() {
+            return Err(Error::BadRequest(format!(
+                "Data table database resource {} not found in workspace {}",
+                dl.database.resource_path, &w_id
+            )));
+        }
+    }
+
+    // Check that non-superadmins are not abusing Instance catalogs
+    if !is_superadmin {
+        let old_datatables = sqlx::query_scalar!(
+            r#"
+                SELECT ws.datatable->'datatables' AS datatable_name
+                FROM workspace_settings ws
+                WHERE ws.workspace_id = $1
+            "#,
+            &w_id
+        )
+        .fetch_one(&db)
+        .await?
+        .unwrap_or(serde_json::Value::Null);
+        let old_datatables: HashMap<String, DataTable> =
+            serde_json::from_value(old_datatables).unwrap_or_default();
+        for (name, dt) in new_config.settings.datatables.iter() {
+            if dt.database.resource_type == DataTableCatalogResourceType::Instance {
+                let old_dt = old_datatables.get(name);
+                if old_dt.is_none()
+                    || old_dt.unwrap().database.resource_type
+                        != DataTableCatalogResourceType::Instance
+                    || old_dt.unwrap().database.resource_path != dt.database.resource_path
+                {
+                    return Err(Error::BadRequest(
+                        "Only superadmins can create or modify data tables with Instance catalogs"
+                            .to_string(),
+                    ));
+                }
+            }
+        }
+    }
+
+    let config: serde_json::Value = serde_json::to_value(new_config.settings)
+        .map_err(|err| Error::internal_err(err.to_string()))?;
+
+    sqlx::query!(
+        "UPDATE workspace_settings SET datatable = $1 WHERE workspace_id = $2",
+        config,
+        &w_id
+    )
+    .execute(&mut *tx)
+    .await?;
+
+    tx.commit().await?;
+
+    Ok(format!("Edit datatable config for workspace {}", &w_id))
+}
+
 #[derive(Deserialize)]
 pub struct EditGitSyncConfig {
     #[cfg(feature = "enterprise")]

backend/windmill-common/src/workspaces.rs

@@ -149,6 +149,27 @@ pub async fn get_team_plan_status(_db: &crate::DB, _w_id: &str) -> Result<TeamPl
     Ok(team_plan_info)
 }
 
+#[derive(Deserialize, Serialize, Debug)]
+pub struct DataTable {
+    pub database: DataTableDatabase,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct DataTableDatabase {
+    pub resource_type: DataTableCatalogResourceType,
+    pub resource_path: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, PartialEq)]
+#[serde(rename_all = "lowercase")]
+#[derive(AsRefStr)]
+#[strum(serialize_all = "lowercase")]
+pub enum DataTableCatalogResourceType {
+    #[strum(serialize = "postgres")]
+    Postgresql,
+    Instance,
+}
+
 #[derive(Deserialize, Serialize, Debug)]
 pub struct Ducklake {
     pub catalog: DucklakeCatalog,

frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte

@@ -1,14 +1,44 @@
 <script lang="ts" module>
-	export type DataTablesSettingsStruct = {
+	export type DataTableSettingsType = {
 		dataTables: {
 			name: string
 			database: {
-				resource_type: 'postgresql' | 'mysql' | 'instance'
-				resource_path: string | undefined
+				resource_type: 'postgresql' | 'instance'
+				resource_path?: string | undefined
 			}
 		}[]
 	}
 
+	export function convertDataTableSettingsFromBackend(
+		settings: GetSettingsResponse['datatable']
+	): DataTableSettingsType {
+		const s: DataTableSettingsType = { dataTables: [] }
+		if (settings?.datatables) {
+			for (const [name, rest] of Object.entries(settings.datatables)) {
+				s.dataTables.push({ name, ...rest })
+			}
+		}
+		return s
+	}
+	export function convertDataTableSettingsToBackend(
+		settings: DataTableSettingsType
+	): NonNullable<GetSettingsResponse['datatable']> {
+		const s: GetSettingsResponse['datatable'] = { datatables: {} }
+		for (const dataTable of settings.dataTables) {
+			const database = dataTable.database
+			if (dataTable.name in s.datatables)
+				throw 'Settings contain duplicate dataTable name: ' + dataTable.name
+			if (!database.resource_path) throw 'No resource selected for ' + dataTable.name
+			if (database.resource_type === 'instance' && database.resource_path === 'windmill')
+				throw dataTable.name + ' database cannot be called "windmill"'
+
+			s.datatables[dataTable.name] = {
+				database: dataTable.database
+			}
+		}
+		return s
+	}
+
 	let DEFAULT_DATATABLE_DB_NAME = 'datatable_db'
 </script>
 
@@ -31,6 +61,17 @@
 	import { isCustomInstanceDbEnabled } from './utils.svelte'
 	import { random_adj } from '../random_positive_adjetive'
 	import { sendUserToast } from '$lib/toast'
+	import { SettingService, WorkspaceService, type GetSettingsResponse } from '$lib/gen'
+	import { workspaceStore } from '$lib/stores'
+	import { createAsyncConfirmationModal } from '../common/confirmationModal/asyncConfirmationModal.svelte'
+	import ConfirmationModal from '../common/confirmationModal/ConfirmationModal.svelte'
+	import { resource } from 'runed'
+
+	type Props = {
+		dataTableSettings: DataTableSettingsType
+	}
+
+	let { dataTableSettings = $bindable() }: Props = $props()
 
 	let tableHeadNames = ['Name', 'Database', '', ''] as const
 	let tableHeadTooltips: Partial<Record<(typeof tableHeadNames)[number], string | undefined>> = {
@@ -38,9 +79,7 @@
 		Database: 'The database where the data is stored.'
 	}
 
-	let tempSettings: DataTablesSettingsStruct = $state({
-		dataTables: []
-	})
+	let tempSettings: DataTableSettingsType = $derived(dataTableSettings)
 
 	function removeDataTable(index: number) {
 		tempSettings.dataTables.splice(index, 1)
@@ -58,15 +97,38 @@
 			}
 		})
 	}
+	const instanceCatalogStatuses = resource([], SettingService.getCustomInstanceDbStatus)
 
 	async function onSave() {
 		try {
+			if (
+				$isCustomInstanceDbEnabled &&
+				dataTableSettings.dataTables.some(
+					(d) =>
+						d.database.resource_type === 'instance' &&
+						!instanceCatalogStatuses.current?.[d.database.resource_path ?? '']?.success
+				)
+			) {
+				let confirm = await confirmationModal.ask({
+					title: 'Some instance catalogs are not setup',
+					children: 'Are you sure you want to save without setting them up ?',
+					confirmationText: 'Save anyway'
+				})
+				if (!confirm) return
+			}
+			const settings = convertDataTableSettingsToBackend(tempSettings)
+			await WorkspaceService.editDataTableConfig({
+				workspace: $workspaceStore!,
+				requestBody: { settings }
+			})
+			dataTableSettings = tempSettings
 			sendUserToast('Data table settings saved successfully')
 		} catch (e) {
 			sendUserToast(e, true)
 			console.error('Error saving data table settings', e)
 		}
 	}
+	let confirmationModal = createAsyncConfirmationModal()
 </script>
 
 <div class="flex flex-col gap-4 my-8">
@@ -74,7 +136,7 @@
 		<div class="text-sm font-semibold text-emphasis">Data tables</div>
 		<Description link="https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables">
 			Store relational data out of the box. Interact with a fully managed PostgreSQL database
-			directly from the windmill SDK.
+			directly from the Windmill SDK.
 		</Description>
 	</div>
 </div>
@@ -179,3 +241,5 @@
 </DataTable>
 
 <Button wrapperClasses="mt-4 mb-16 max-w-fit" on:click={onSave}>Save</Button>
+
+<ConfirmationModal {...confirmationModal.props} />

frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte

@@ -59,7 +59,10 @@
 	import UnsavedConfirmationModal from '$lib/components/common/confirmationModal/UnsavedConfirmationModal.svelte'
 	import TextInput from '$lib/components/text_input/TextInput.svelte'
 	import CollapseLink from '$lib/components/CollapseLink.svelte'
-	import DataTableSettings from '$lib/components/workspaceSettings/DataTableSettings.svelte'
+	import DataTableSettings, {
+		convertDataTableSettingsFromBackend,
+		type DataTableSettingsType
+	} from '$lib/components/workspaceSettings/DataTableSettings.svelte'
 
 	let slackInitialPath: string = $state('')
 	let slackScriptPath: string = $state('')
@@ -111,9 +114,9 @@
 		secondaryStorage: undefined
 	})
 
-	let ducklakeSettings: DucklakeSettingsType = $state({
-		ducklakes: []
-	})
+	let dataTableSettings: DataTableSettingsType = $state({ dataTables: [] })
+
+	let ducklakeSettings: DucklakeSettingsType = $state({ ducklakes: [] })
 	let ducklakeSavedSettings: DucklakeSettingsType = $state(untrack(() => ducklakeSettings))
 
 	let workspaceDefaultAppPath: string | undefined = $state(undefined)
@@ -307,6 +310,7 @@
 			!!$enterpriseLicense
 		)
 		initialS3ResourceSettings = clone(s3ResourceSettings)
+		dataTableSettings = convertDataTableSettingsFromBackend(settings.datatable)
 		ducklakeSettings = convertDucklakeSettingsFromBackend(settings.ducklake)
 		ducklakeSavedSettings = clone(ducklakeSettings)
 
@@ -1114,7 +1118,7 @@
 				}}
 			/>
 		{:else if tab == 'windmill_data_tables'}
-			<DataTableSettings />
+			<DataTableSettings bind:dataTableSettings />
 		{:else if tab == 'windmill_lfs'}
 			<StorageSettings
 				bind:s3ResourceSettings