From 1dd002e70ca3de5b8c068e8d410bbaa43e601b9b Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Wed, 26 Nov 2025 18:18:08 +0100 Subject: [PATCH] Data table assetexplore --- .../20251121144144_datatables.up.sql | 4 +- .../windmill-parser-ts/src/asset_parser.rs | 33 ++++++++++------ .../windmill-parser/src/asset_parser.rs | 3 ++ backend/windmill-api/openapi.yaml | 1 + .../src/lib/components/DBManagerDrawer.svelte | 4 +- .../lib/components/ExploreAssetButton.svelte | 12 ++++-- frontend/src/lib/components/SqlRepl.svelte | 4 +- .../apps/components/display/dbtable/utils.ts | 10 +++-- frontend/src/lib/components/dbOps.ts | 21 ++++++---- .../DataTableSettings.svelte | 39 ++++++++++++++++++- 10 files changed, 97 insertions(+), 34 deletions(-) diff --git a/backend/migrations/20251121144144_datatables.up.sql b/backend/migrations/20251121144144_datatables.up.sql index 6feea09429..7c58c263c6 100644 --- a/backend/migrations/20251121144144_datatables.up.sql +++ b/backend/migrations/20251121144144_datatables.up.sql @@ -1,4 +1,4 @@ --- Add up migration script here +ALTER TYPE asset_kind ADD VALUE 'datatable'; ALTER TABLE workspace_settings -ADD COLUMN datatable JSONB; +ADD COLUMN datatable JSONB; \ No newline at end of file diff --git a/backend/parsers/windmill-parser-ts/src/asset_parser.rs b/backend/parsers/windmill-parser-ts/src/asset_parser.rs index 6b81272ec1..cf370813a1 100644 --- a/backend/parsers/windmill-parser-ts/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-ts/src/asset_parser.rs @@ -74,27 +74,36 @@ impl AssetsFinder { Some(Expr::Member(MemberExpr { prop: MemberProp::Ident(i), .. })) => i.sym.as_str(), _ => return Err(()), }; - let (kind, access_type, arg_pos) = match ident { - "loadS3File" => (AssetKind::S3Object, Some(R), 0), - "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0), - "writeS3File" => (AssetKind::S3Object, Some(W), 0), - "getResource" => (AssetKind::Resource, None, 0), - "setResource" => (AssetKind::Resource, Some(W), 1), - "databaseUrlFromResource" => (AssetKind::Resource, None, 0), - "denoS3LightClientSettings" => (AssetKind::Resource, None, 0), - "duckdbConnectionSettings" => (AssetKind::Resource, None, 0), - "polarsConnectionSettings" => (AssetKind::Resource, None, 0), + let (kind, access_type, arg_pos, default) = match ident { + "loadS3File" => (AssetKind::S3Object, Some(R), 0, None), + "loadS3FileStream" => (AssetKind::S3Object, Some(R), 0, None), + "writeS3File" => (AssetKind::S3Object, Some(W), 0, None), + "getResource" => (AssetKind::Resource, None, 0, None), + "setResource" => (AssetKind::Resource, Some(W), 1, None), + "databaseUrlFromResource" => (AssetKind::Resource, None, 0, None), + "denoS3LightClientSettings" => (AssetKind::Resource, None, 0, None), + "duckdbConnectionSettings" => (AssetKind::Resource, None, 0, None), + "polarsConnectionSettings" => (AssetKind::Resource, None, 0, None), + "datatable" => (AssetKind::DataTable, None, 0, Some("main")), + "ducklake" => (AssetKind::DataTable, None, 0, Some("main")), _ => return Err(()), }; let arg_value = node.args.get(arg_pos); - match arg_value.map(|e| e.expr.as_ref()) { - Some(Expr::Lit(Lit::Str(Str { value, .. }))) => { + match (arg_value.map(|e| e.expr.as_ref()), default) { + (Some(Expr::Lit(Lit::Str(Str { value, .. }))), _) => { let path = parse_asset_syntax(&value).map(|(_, p)| p).unwrap_or(&value); self.assets .push(ParseAssetsResult { kind, path: path.to_string(), access_type }); } + (None, Some(default)) => { + self.assets.push(ParseAssetsResult { + kind, + path: default.to_string(), + access_type, + }); + } _ => return Err(()), } Ok(()) diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index a721590ae9..6914f55f1f 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -16,6 +16,7 @@ pub enum AssetKind { S3Object, Resource, Ducklake, + DataTable, } #[derive(Serialize)] @@ -66,6 +67,8 @@ pub fn parse_asset_syntax(s: &str) -> Option<(AssetKind, &str)> { Some((AssetKind::Resource, &s[5..])) } else if s.starts_with("ducklake://") { Some((AssetKind::Ducklake, &s[11..])) + } else if s.starts_with("datatable://") { + Some((AssetKind::DataTable, &s[11..])) } else { None } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1212b8b652..dbb61e7722 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -20170,6 +20170,7 @@ components: - s3object - resource - ducklake + - datatable Asset: type: object properties: diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index 583f7f30a6..f54a211c7a 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -17,6 +17,7 @@ import { dbDeleteTableActionWithPreviewScript, dbTableOpsWithPreviewScripts, + getDatabaseArg, getDucklakeSchema, wrapDucklakeQuery, type DbInput @@ -202,8 +203,7 @@ dbType, previewSql: (values) => makeCreateTableQuery(values, dbType, selectedSchemaKey), async onConfirm(values) { - const dbArg = - input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) const language = getLanguageByResourceType(dbType) let query = makeCreateTableQuery(values, dbType, selectedSchemaKey) if (input?.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake) diff --git a/frontend/src/lib/components/ExploreAssetButton.svelte b/frontend/src/lib/components/ExploreAssetButton.svelte index 18cdf58e3b..16608ff0fd 100644 --- a/frontend/src/lib/components/ExploreAssetButton.svelte +++ b/frontend/src/lib/components/ExploreAssetButton.svelte @@ -65,12 +65,18 @@ s3FilePicker?.open(assetUri) } else if (asset.kind === 'ducklake') { dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake: asset.path }) + } else if (asset.kind === 'datatable') { + dbManagerDrawer?.openDrawer({ + type: 'database', + resourceType: 'postgresql', + resourcePath: `datatable://${asset.path}` + }) } onClick?.() }} endIcon={asset.kind === 's3object' ? { icon: File } - : asset.kind === 'resource' + : asset.kind === 'resource' || asset.kind === 'datatable' ? { icon: Database } : asset.kind === 'ducklake' ? { icon: DucklakeIcon } @@ -78,9 +84,7 @@ > {#if asset.kind === 's3object'} Explore - {:else if asset.kind === 'resource'} - Manage - {:else if asset.kind === 'ducklake'} + {:else if asset.kind === 'resource' || asset.kind === 'ducklake' || asset.kind === 'datatable'} Manage {/if} diff --git a/frontend/src/lib/components/SqlRepl.svelte b/frontend/src/lib/components/SqlRepl.svelte index a5b50c2b0b..92af361073 100644 --- a/frontend/src/lib/components/SqlRepl.svelte +++ b/frontend/src/lib/components/SqlRepl.svelte @@ -55,7 +55,7 @@ import { getLanguageByResourceType } from './apps/components/display/dbtable/utils' import StepHistory, { type StepHistoryData } from './flows/propPicker/StepHistory.svelte' import { Pane, Splitpanes } from 'svelte-splitpanes' - import { getDbType, wrapDucklakeQuery, type DbInput } from './dbOps' + import { getDatabaseArg, getDbType, wrapDucklakeQuery, type DbInput } from './dbOps' type Props = { input: DbInput @@ -101,7 +101,7 @@ }) .join(';') } - const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) if (input?.type === 'ducklake') { transformedCode = wrapDucklakeQuery(transformedCode, input.ducklake) diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index 24b9fd1058..76f5f016b8 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -9,7 +9,7 @@ import { import { tryEvery } from '$lib/utils' import { stringifySchema } from '$lib/components/copilot/lib' import { runScriptAndPollResult } from '$lib/components/jobs/utils' -import type { DbInput } from '$lib/components/dbOps' +import { getDatabaseArg, type DbInput } from '$lib/components/dbOps' export enum ColumnIdentity { ByDefault = 'By Default', @@ -69,7 +69,7 @@ export async function loadTableMetaData( requestBody: { language, content, - args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + args: getDatabaseArg(input) } }) @@ -118,7 +118,7 @@ export async function loadAllTablesMetaData( requestBody: { language, content: await makeLoadTableMetaDataQuery(input, workspace, undefined), - args: input.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + args: getDatabaseArg(input) } })) as ({ table_name: string; schema_name?: string } & object)[] if (input.type === 'database' && input.resourceType === 'ms_sql_server') { @@ -562,7 +562,9 @@ export async function getDbSchemas( language: scripts[resourceType].lang as Preview['language'], content: scripts[resourceType].code, args: { - [scripts[resourceType].argName]: '$res:' + resourcePath + [scripts[resourceType].argName]: resourcePath.startsWith('datatable://') + ? resourcePath + : '$res:' + resourcePath } } }) diff --git a/frontend/src/lib/components/dbOps.ts b/frontend/src/lib/components/dbOps.ts index bef53e06f5..6132e2b25b 100644 --- a/frontend/src/lib/components/dbOps.ts +++ b/frontend/src/lib/components/dbOps.ts @@ -16,11 +16,7 @@ import { stringifySchema } from './copilot/lib' import { assert } from '$lib/utils' export type DbInput = - | { - type: 'database' - resourceType: DbType - resourcePath: string - } + | { type: 'database'; resourceType: DbType; resourcePath: string } | { type: 'ducklake'; ducklake: string } export type IDbTableOps = { @@ -58,7 +54,7 @@ export function dbTableOpsWithPreviewScripts({ }): IDbTableOps { const dbType = getDbType(input) const language = getLanguageByResourceType(dbType) - const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) return { dbType, tableKey, @@ -139,7 +135,7 @@ export function dbDeleteTableActionWithPreviewScript({ workspace: string input: DbInput }): DbTableActionFactory { - const dbArg = input?.type === 'database' ? { database: '$res:' + input.resourcePath } : {} + const dbArg = getDatabaseArg(input) return ({ tableKey, refresh }) => ({ confirmTitle: `Are you sure you want to delete '${tableKey}' ? This action is irreversible`, @@ -224,3 +220,14 @@ export function wrapDucklakeQuery(query: string, ducklake: string): string { let attach = `ATTACH 'ducklake://${ducklake}' AS dl;USE dl;\n` return query.replace(/^(--.*\n)*/, (match) => match + attach) } + +export function getDatabaseArg(input: DbInput | undefined) { + if (input?.type === 'database') { + if (input.resourcePath.startsWith('datatable://')) { + return { database: 'datatable://' + input.resourcePath } + } else { + return { database: '$res:' + input.resourcePath } + } + } + return {} +} diff --git a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte index 5a42c7b898..eedb566a80 100644 --- a/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DataTableSettings.svelte @@ -68,6 +68,9 @@ import { resource } from 'runed' import CustomInstanceDbSelect from './CustomInstanceDbSelect.svelte' import DBManagerDrawer from '../DBManagerDrawer.svelte' + import { Popover } from '../meltComponents' + import ExploreAssetButton from '../ExploreAssetButton.svelte' + import { deepEqual } from 'fast-equals' type Props = { dataTableSettings: DataTableSettingsType @@ -134,6 +137,16 @@ let confirmationModal = createAsyncConfirmationModal() let dbManagerDrawer: DBManagerDrawer | undefined = $state() + let dirtyMap = $derived.by(() => { + const map: Record = {} + for (let i = 0; i < tempSettings.dataTables.length; i++) { + let temp = tempSettings.dataTables[i] + let dt = dataTableSettings.dataTables.find((d) => d.name === temp.name) + console.log('Comparing', temp, dt, deepEqual(dt, temp)) + map[temp.name] = !deepEqual(dt, temp) + } + return map + })
@@ -224,7 +237,31 @@
- + + {#if dirtyMap[dataTable.name]} + + + + + Please save settings first + + {:else} + + {/if} + removeDataTable(dataTableIndex)} />