Data table assetexplore

backend/migrations/20251121144144_datatables.up.sql

@@ -1,4 +1,4 @@
--- Add up migration script here
+ALTER TYPE asset_kind ADD VALUE 'datatable';
 
 ALTER TABLE workspace_settings
-ADD COLUMN datatable JSONB;
+ADD COLUMN datatable JSONB;

backend/parsers/windmill-parser-ts/src/asset_parser.rs

@@ -74,27 +74,36 @@ impl AssetsFinder {
             Some(Expr::Member(MemberExpr { prop: MemberProp::Ident(i), .. })) => i.sym.as_str(),
             _ => return Err(()),
         };
-        let (kind, access_type, arg_pos) = match ident {
-            "loadS3File" => (AssetKind::S3Object, Some(R), 0),
-            "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0),
-            "writeS3File" => (AssetKind::S3Object, Some(W), 0),
-            "getResource" => (AssetKind::Resource, None, 0),
-            "setResource" => (AssetKind::Resource, Some(W), 1),
-            "databaseUrlFromResource" => (AssetKind::Resource, None, 0),
-            "denoS3LightClientSettings" => (AssetKind::Resource, None, 0),
-            "duckdbConnectionSettings" => (AssetKind::Resource, None, 0),
-            "polarsConnectionSettings" => (AssetKind::Resource, None, 0),
+        let (kind, access_type, arg_pos, default) = match ident {
+            "loadS3File" => (AssetKind::S3Object, Some(R), 0, None),
+            "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0, None),
+            "writeS3File" => (AssetKind::S3Object, Some(W), 0, None),
+            "getResource" => (AssetKind::Resource, None, 0, None),
+            "setResource" => (AssetKind::Resource, Some(W), 1, None),
+            "databaseUrlFromResource" => (AssetKind::Resource, None, 0, None),
+            "denoS3LightClientSettings" => (AssetKind::Resource, None, 0, None),
+            "duckdbConnectionSettings" => (AssetKind::Resource, None, 0, None),
+            "polarsConnectionSettings" => (AssetKind::Resource, None, 0, None),
+            "datatable" => (AssetKind::DataTable, None, 0, Some("main")),
+            "ducklake" => (AssetKind::DataTable, None, 0, Some("main")),
             _ => return Err(()),
         };
 
         let arg_value = node.args.get(arg_pos);
 
-        match arg_value.map(|e| e.expr.as_ref()) {
-            Some(Expr::Lit(Lit::Str(Str { value, .. }))) => {
+        match (arg_value.map(|e| e.expr.as_ref()), default) {
+            (Some(Expr::Lit(Lit::Str(Str { value, .. }))), _) => {
                 let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value);
                 self.assets
                     .push(ParseAssetsResult { kind, path: path.to_string(), access_type });
             }
+            (None, Some(default)) => {
+                self.assets.push(ParseAssetsResult {
+                    kind,
+                    path: default.to_string(),
+                    access_type,
+                });
+            }
             _ => return Err(()),
         }
         Ok(())

backend/parsers/windmill-parser/src/asset_parser.rs

@@ -16,6 +16,7 @@ pub enum AssetKind {
     S3Object,
     Resource,
     Ducklake,
+    DataTable,
 }
 
 #[derive(Serialize)]
@@ -66,6 +67,8 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> {
         Some((AssetKind::Resource, &s[5..]))
     } else if s.starts_with("ducklake://") {
         Some((AssetKind::Ducklake, &s[11..]))
+    } else if s.starts_with("datatable://") {
+        Some((AssetKind::DataTable, &s[11..]))
     } else {
         None
     }

backend/windmill-api/openapi.yaml

@@ -20170,6 +20170,7 @@ components:
         - s3object
         - resource
         - ducklake
+        - datatable
     Asset:
       type: object
       properties:

frontend/src/lib/components/DBManagerDrawer.svelte

@@ -17,6 +17,7 @@
 	import {
 		dbDeleteTableActionWithPreviewScript,
 		dbTableOpsWithPreviewScripts,
+		getDatabaseArg,
 		getDucklakeSchema,
 		wrapDucklakeQuery,
 		type DbInput
@@ -202,8 +203,7 @@
 							dbType,
 							previewSql: (values) => makeCreateTableQuery(values, dbType, selectedSchemaKey),
 							async onConfirm(values) {
-								const dbArg =
-									input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+								const dbArg = getDatabaseArg(input)
 								const language = getLanguageByResourceType(dbType)
 								let query = makeCreateTableQuery(values, dbType, selectedSchemaKey)
 								if (input?.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)

frontend/src/lib/components/ExploreAssetButton.svelte

@@ -65,12 +65,18 @@
 			s3FilePicker?.open(assetUri)
 		} else if (asset.kind === 'ducklake') {
 			dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake: asset.path })
+		} else if (asset.kind === 'datatable') {
+			dbManagerDrawer?.openDrawer({
+				type: 'database',
+				resourceType: 'postgresql',
+				resourcePath: `datatable://${asset.path}`
+			})
 		}
 		onClick?.()
 	}}
 	endIcon={asset.kind === 's3object'
 		? { icon: File }
-		: asset.kind === 'resource'
+		: asset.kind === 'resource' || asset.kind === 'datatable'
 			? { icon: Database }
 			: asset.kind === 'ducklake'
 				? { icon: DucklakeIcon }
@@ -78,9 +84,7 @@
 >
 	{#if asset.kind === 's3object'}
 		<span class:hidden={noText}>Explore</span>
-	{:else if asset.kind === 'resource'}
-		<span class:hidden={noText}>Manage</span>
-	{:else if asset.kind === 'ducklake'}
+	{:else if asset.kind === 'resource' || asset.kind === 'ducklake' || asset.kind === 'datatable'}
 		<span class:hidden={noText}>Manage</span>
 	{/if}
 </Button>

frontend/src/lib/components/SqlRepl.svelte

@@ -55,7 +55,7 @@
 	import { getLanguageByResourceType } from './apps/components/display/dbtable/utils'
 	import StepHistory, { type StepHistoryData } from './flows/propPicker/StepHistory.svelte'
 	import { Pane, Splitpanes } from 'svelte-splitpanes'
-	import { getDbType, wrapDucklakeQuery, type DbInput } from './dbOps'
+	import { getDatabaseArg, getDbType, wrapDucklakeQuery, type DbInput } from './dbOps'
 
 	type Props = {
 		input: DbInput
@@ -101,7 +101,7 @@
 					})
 					.join(';')
 			}
-			const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+			const dbArg = getDatabaseArg(input)
 
 			if (input?.type === 'ducklake') {
 				transformedCode = wrapDucklakeQuery(transformedCode, input.ducklake)

frontend/src/lib/components/apps/components/display/dbtable/utils.ts

@@ -9,7 +9,7 @@ import {
 import { tryEvery } from '$lib/utils'
 import { stringifySchema } from '$lib/components/copilot/lib'
 import { runScriptAndPollResult } from '$lib/components/jobs/utils'
-import type { DbInput } from '$lib/components/dbOps'
+import { getDatabaseArg, type DbInput } from '$lib/components/dbOps'
 
 export enum ColumnIdentity {
 	ByDefault = 'By Default',
@@ -69,7 +69,7 @@ export async function loadTableMetaData(
 		requestBody: {
 			language,
 			content,
-			args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+			args: getDatabaseArg(input)
 		}
 	})
 
@@ -118,7 +118,7 @@ export async function loadAllTablesMetaData(
 			requestBody: {
 				language,
 				content: await makeLoadTableMetaDataQuery(input, workspace, undefined),
-				args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+				args: getDatabaseArg(input)
 			}
 		})) as ({ table_name: string; schema_name?: string } & object)[]
 		if (input.type === 'database' && input.resourceType === 'ms_sql_server') {
@@ -562,7 +562,9 @@ export async function getDbSchemas(
 				language: scripts[resourceType].lang as Preview['language'],
 				content: scripts[resourceType].code,
 				args: {
-					[scripts[resourceType].argName]: '$res:' + resourcePath
+					[scripts[resourceType].argName]: resourcePath.startsWith('datatable://')
+						? resourcePath
+						: '$res:' + resourcePath
 				}
 			}
 		})

frontend/src/lib/components/dbOps.ts

@@ -16,11 +16,7 @@ import { stringifySchema } from './copilot/lib'
 import { assert } from '$lib/utils'
 
 export type DbInput =
-	| {
-			type: 'database'
-			resourceType: DbType
-			resourcePath: string
-	  }
+	| { type: 'database'; resourceType: DbType; resourcePath: string }
 	| { type: 'ducklake'; ducklake: string }
 
 export type IDbTableOps = {
@@ -58,7 +54,7 @@ export function dbTableOpsWithPreviewScripts({
 }): IDbTableOps {
 	const dbType = getDbType(input)
 	const language = getLanguageByResourceType(dbType)
-	const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+	const dbArg = getDatabaseArg(input)
 	return {
 		dbType,
 		tableKey,
@@ -139,7 +135,7 @@ export function dbDeleteTableActionWithPreviewScript({
 	workspace: string
 	input: DbInput
 }): DbTableActionFactory {
-	const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {}
+	const dbArg = getDatabaseArg(input)
 
 	return ({ tableKey, refresh }) => ({
 		confirmTitle: `Are you sure you want to delete '${tableKey}' ? This action is irreversible`,
@@ -224,3 +220,14 @@ export function wrapDucklakeQuery(query: string, ducklake: string): string {
 	let attach = `ATTACH 'ducklake://${ducklake}' AS dl;USE dl;\n`
 	return query.replace(/^(--.*\n)*/, (match) => match + attach)
 }
+
+export function getDatabaseArg(input: DbInput | undefined) {
+	if (input?.type === 'database') {
+		if (input.resourcePath.startsWith('datatable://')) {
+			return { database: 'datatable://' + input.resourcePath }
+		} else {
+			return { database: '$res:' + input.resourcePath }
+		}
+	}
+	return {}
+}

frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte

@@ -68,6 +68,9 @@
 	import { resource } from 'runed'
 	import CustomInstanceDbSelect from './CustomInstanceDbSelect.svelte'
 	import DBManagerDrawer from '../DBManagerDrawer.svelte'
+	import { Popover } from '../meltComponents'
+	import ExploreAssetButton from '../ExploreAssetButton.svelte'
+	import { deepEqual } from 'fast-equals'
 
 	type Props = {
 		dataTableSettings: DataTableSettingsType
@@ -134,6 +137,16 @@
 
 	let confirmationModal = createAsyncConfirmationModal()
 	let dbManagerDrawer: DBManagerDrawer | undefined = $state()
+	let dirtyMap = $derived.by(() => {
+		const map: Record<string, boolean> = {}
+		for (let i = 0; i < tempSettings.dataTables.length; i++) {
+			let temp = tempSettings.dataTables[i]
+			let dt = dataTableSettings.dataTables.find((d) => d.name === temp.name)
+			console.log('Comparing', temp, dt, deepEqual(dt, temp))
+			map[temp.name] = !deepEqual(dt, temp)
+		}
+		return map
+	})
 </script>
 
 <div class="flex flex-col gap-4 my-8">
@@ -224,7 +237,31 @@
 						</div>
 					</div>
 				</Cell>
-				<Cell class="w-12"></Cell>
+				<Cell class="w-12">
+					{#if dirtyMap[dataTable.name]}
+						<Popover
+							openOnHover
+							contentClasses="p-2 text-sm text-secondary italic"
+							class="cursor-not-allowed"
+						>
+							<svelte:fragment slot="trigger">
+								<ExploreAssetButton
+									class="h-9"
+									asset={{ kind: 'datatable', path: dataTable.name }}
+									{dbManagerDrawer}
+									disabled
+								/>
+							</svelte:fragment>
+							<svelte:fragment slot="content">Please save settings first</svelte:fragment>
+						</Popover>
+					{:else}
+						<ExploreAssetButton
+							class="h-9"
+							asset={{ kind: 'datatable', path: dataTable.name }}
+							{dbManagerDrawer}
+						/>
+					{/if}
+				</Cell>
 				<Cell class="w-12">
 					<CloseButton small on:close={() => removeDataTable(dataTableIndex)} />
 				</Cell>