Improve alter table query speed + duckdb nits (#7538)

* Fetch alter table metadata much faster

* Upgrade duckdb to 1.4.3

* Disable transactional DDL for Ducklake (bug on their side)

backend/windmill-duckdb-ffi-internal/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2024"
 
 [dependencies]
 chrono = "0.4.41"
-duckdb = { rev = "fe0702529de6ec5a568337726bba9355503157d2", git = "https://github.com/windmill-labs/duckdb-rs.git", features = ["bundled"] }
+duckdb = { version = "1.4.3", features = ["bundled"] }
 rust_decimal = "1.37.2"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = { version = "^1", features = ["preserve_order", "raw_value"] }

backend/windmill-duckdb-ffi-internal/src/lib.rs

@@ -267,11 +267,7 @@ fn do_duckdb_inner(
                             (0..stmt.column_count())
                                 .map(|i| {
                                     let logical_type = stmt.column_logical_type(i);
-                                    if logical_type.is_invalid() {
-                                        None
-                                    } else {
-                                        logical_type.get_alias()
-                                    }
+                                    logical_type.get_alias()
                                 })
                                 .collect::<Vec<_>>(),
                         );

frontend/src/lib/components/apps/components/display/dbtable/queries/alterTable.ts

@@ -476,5 +476,9 @@ function normalizeNewFkToOldColNames(
 }
 
 export function dbSupportsTransactionalDdl(dbType: DbType): boolean {
-	return dbType === 'postgresql' || dbType === 'ms_sql_server' || dbType === 'duckdb'
+	return dbType === 'postgresql' || dbType === 'ms_sql_server'
+	// Commented out temporarily because Ducklake transactional DDL sometimes put the
+	// ducklake in an unusable state.
+	// See : https://github.com/duckdb/ducklake/issues/683
+	// || dbType === 'duckdb'
 }

frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts

@@ -3,6 +3,7 @@ import { wrapDucklakeQuery } from '$lib/components/ducklake'
 import { runScriptAndPollResult } from '$lib/components/jobs/utils'
 import type { ScriptLang } from '$lib/gen'
 import type { TableEditorForeignKey } from '../tableEditor'
+import type { TableMetadata } from '../utils'
 
 /**
  * Raw foreign key result from database queries
@@ -402,57 +403,74 @@ export async function fetchTableRelationalKeys(
 	schema: string | undefined,
 	workspace: string,
 	dbArg: Record<string, any>,
-	language: ScriptLang
-): Promise<{ foreignKeys: TableEditorForeignKey[]; pk_constraint_name?: string }> {
-	let foreignKeys: TableEditorForeignKey[] = []
-	let pk_constraint_name: string | undefined = undefined
-	try {
-		if (dbType !== 'bigquery') {
-			let fkQuery = makeForeignKeysQuery(dbType, table, schema)
-			if (input.type === 'ducklake') fkQuery = wrapDucklakeQuery(fkQuery, input.ducklake)
+	language: ScriptLang,
+	getColDefs: () => Promise<TableMetadata>
+): Promise<{
+	foreignKeys: TableEditorForeignKey[]
+	pk_constraint_name?: string
+	colDefs: TableMetadata
+}> {
+	let fkPromise = async () => {
+		try {
+			if (dbType !== 'bigquery') {
+				let fkQuery = makeForeignKeysQuery(dbType, table, schema)
+				if (input.type === 'ducklake') fkQuery = wrapDucklakeQuery(fkQuery, input.ducklake)
 
-			const fkResult = await runScriptAndPollResult({
-				workspace,
-				requestBody: { args: dbArg, content: fkQuery, language }
-			})
-
-			let rawForeignKeys = fkResult as RawForeignKey[]
-
-			if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
-				// Lowercase keys for consistency
-				rawForeignKeys = rawForeignKeys.map((fk) => {
-					const lowerFk: any = {}
-					Object.keys(fk).forEach((key) => {
-						lowerFk[key.toLowerCase()] = fk[key]
-					})
-					return lowerFk
+				const fkResult = await runScriptAndPollResult({
+					workspace,
+					requestBody: { args: dbArg, content: fkQuery, language }
 				})
-				foreignKeys = transformForeignKeys(rawForeignKeys)
+
+				let rawForeignKeys = fkResult as RawForeignKey[]
+
+				if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
+					// Lowercase keys for consistency
+					rawForeignKeys = rawForeignKeys.map((fk) => {
+						const lowerFk: any = {}
+						Object.keys(fk).forEach((key) => {
+							lowerFk[key.toLowerCase()] = fk[key]
+						})
+						return lowerFk
+					})
+					return transformForeignKeys(rawForeignKeys)
+				}
 			}
+		} catch (e) {
+			console.warn('Failed to fetch foreign keys:', e)
 		}
-	} catch (e) {
-		console.warn('Failed to fetch foreign keys:', e)
+		return []
 	}
 
-	try {
-		if (dbType !== 'bigquery' && dbType !== 'mysql') {
-			let pkQuery = makePrimaryKeyConstraintQuery(dbType, table, schema)
-			if (input.type === 'ducklake') pkQuery = wrapDucklakeQuery(pkQuery, input.ducklake)
+	let pkPromise = async () => {
+		try {
+			if (dbType !== 'bigquery' && dbType !== 'mysql') {
+				let pkQuery = makePrimaryKeyConstraintQuery(dbType, table, schema)
+				if (input.type === 'ducklake') pkQuery = wrapDucklakeQuery(pkQuery, input.ducklake)
 
-			const pkResult = await runScriptAndPollResult({
-				workspace,
-				requestBody: { args: dbArg, content: pkQuery, language }
-			})
+				const pkResult = await runScriptAndPollResult({
+					workspace,
+					requestBody: { args: dbArg, content: pkQuery, language }
+				})
 
-			let rawPkResult = pkResult as RawPrimaryKeyConstraint[]
+				let rawPkResult = pkResult as RawPrimaryKeyConstraint[]
 
-			if (rawPkResult && Array.isArray(rawPkResult) && rawPkResult.length > 0) {
-				const pkRecord: any = rawPkResult[0]
-				pk_constraint_name = pkRecord?.constraint_name || pkRecord?.CONSTRAINT_NAME || ''
+				if (rawPkResult && Array.isArray(rawPkResult) && rawPkResult.length > 0) {
+					const pkRecord: any = rawPkResult[0]
+					const pk_constraint_name: string =
+						pkRecord?.constraint_name || pkRecord?.CONSTRAINT_NAME || ''
+					return pk_constraint_name
+				}
 			}
+		} catch (e) {
+			console.warn('Failed to fetch primary key constraint:', e)
 		}
-	} catch (e) {
-		console.warn('Failed to fetch primary key constraint:', e)
 	}
-	return { foreignKeys, pk_constraint_name }
+
+	const [foreignKeys, pk_constraint_name, colDefs] = await Promise.all([
+		fkPromise(),
+		pkPromise(),
+		getColDefs()
+	])
+
+	return { foreignKeys, pk_constraint_name, colDefs }
 }

frontend/src/lib/components/dbOps.ts

@@ -191,15 +191,15 @@ export function dbSchemaOpsWithPreviewScripts({
 			})
 		},
 		onFetchTableEditorDefinition: async ({ table, schema, getColDefs }) => {
-			let colDefs = await getColDefs()
-			let { foreignKeys, pk_constraint_name } = await fetchTableRelationalKeys(
+			let { foreignKeys, pk_constraint_name, colDefs } = await fetchTableRelationalKeys(
 				input,
 				dbType,
 				table,
 				schema,
 				workspace,
 				dbArg,
-				language
+				language,
+				getColDefs
 			)
 
 			return buildTableEditorValues({