diff --git a/frontend/src/lib/components/DBManagerContent.svelte b/frontend/src/lib/components/DBManagerContent.svelte index 456098da0a..ffdc1fbc7d 100644 --- a/frontend/src/lib/components/DBManagerContent.svelte +++ b/frontend/src/lib/components/DBManagerContent.svelte @@ -91,7 +91,6 @@ if (!input) return await Promise.all([ (async () => { - if (input?.type == 'database' && input.resourceType === 'bigquery') return cachedColDefs = (await loadAllTablesMetaData($workspaceStore, input)) ?? cachedColDefs })(), (async () => { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts index e7de7a594e..b6af07e0a6 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts @@ -1,4 +1,4 @@ -import { JobService, ResourceService } from '$lib/gen' +import { JobService, ResourceService, type ScriptLang } from '$lib/gen' import { runScriptAndPollResult } from '$lib/components/jobs/utils' import type { DbInput } from '$lib/components/dbTypes' @@ -24,16 +24,11 @@ export async function loadTableMetaData( ): Promise { if (!input || !table || !workspace) return undefined - let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType) - let content = await makeLoadTableMetaDataQuery(input, workspace, table) + let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, table) const job = await JobService.runScriptPreview({ workspace: workspace, - requestBody: { - language, - content, - args: getDatabaseArg(input) - } + requestBody: { language, content: query, args: getDatabaseArg(input) } }) const maxRetries = 8 @@ -69,25 +64,18 @@ export async function loadAllTablesMetaData( ): Promise | undefined> { if (!input || !workspace) return undefined - let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType) - try { + let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, undefined) let result = (await runScriptAndPollResult({ workspace: workspace, - requestBody: { - language, - content: await makeLoadTableMetaDataQuery(input, workspace, undefined), - args: getDatabaseArg(input) - } + requestBody: { language, content: query, args: getDatabaseArg(input) } })) as ({ table_name: string; schema_name?: string } & object)[] const map: Record = {} for (const _col of result) { const col = lowercaseKeys(_col) const tableKey = col.schema_name ? `${col.schema_name}.${col.table_name}` : col.table_name - if (!(tableKey in map)) { - map[tableKey] = [] - } + map[tableKey] ??= [] map[tableKey].push(col) } @@ -103,9 +91,9 @@ async function makeLoadTableMetaDataQuery( input: DbInput, workspace: string, table: string | undefined -): Promise { +): Promise<{ query: string; language: ScriptLang }> { if (input.type === 'ducklake') { - return `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__; + const query = `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__; SELECT COLUMN_NAME as field, DATA_TYPE as DataType, @@ -117,12 +105,13 @@ async function makeLoadTableMetaDataQuery( TABLE_NAME as table_name FROM information_schema.columns c WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()` + return { query, language: 'duckdb' } } else if (input.resourceType === 'mysql') { const resourceObj = (await ResourceService.getResourceValue({ workspace, path: input.resourcePath })) as any - return ` + const query = ` SELECT COLUMN_NAME as field, COLUMN_TYPE as DataType, @@ -152,8 +141,9 @@ async function makeLoadTableMetaDataQuery( TABLE_NAME, ORDINAL_POSITION; ` + return { query, language: 'mysql' } } else if (input.resourceType === 'postgresql') { - return ` + const query = ` SELECT a.attname as field, pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType, @@ -199,8 +189,9 @@ async function makeLoadTableMetaDataQuery( ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'}; ` + return { query, language: 'postgresql' } } else if (input.resourceType === 'ms_sql_server') { - return ` + const query = ` SELECT c.COLUMN_NAME as field, c.DATA_TYPE as DataType, @@ -244,11 +235,12 @@ WHERE ORDER BY c.ORDINAL_POSITION; ` + return { query, language: 'mssql' } } else if ( input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth' ) { - return ` + const query = ` select COLUMN_NAME as field, DATA_TYPE as DataType, COLUMN_DEFAULT as DefaultValue, @@ -272,10 +264,10 @@ ORDER BY } order by ORDINAL_POSITION; ` + return { query, language: 'snowflake' } } else if (input.resourceType === 'bigquery') { - // TODO: find a solution for this (query uses hardcoded dataset name) - if (!table) throw new Error('Table name is required for BigQuery') - return `SELECT + if (table) { + const query = `SELECT c.COLUMN_NAME as field, DATA_TYPE as DataType, CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue, @@ -291,6 +283,40 @@ FROM WHERE c.TABLE_NAME = '${table.split('.')[1]}' order by c.ORDINAL_POSITION;` + return { query, language: 'bigquery' } + } else { + const query = `import { BigQuery } from '@google-cloud/bigquery@7.5.0'; +export async function main(database: bigquery) { +const bq = new BigQuery({ + credentials: database +}) +const [datasets] = await bq.getDatasets(); +if (!datasets) return {} +const schema = {} as any +let queries = datasets.map(dataset => \` + (SELECT + c.COLUMN_NAME as field, + '\${dataset.id}' as schema_name, + c.TABLE_NAME as table_name, + DATA_TYPE as DataType, + CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue, + CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey, + 'No' as IsIdentity, + IS_NULLABLE as IsNullable, + false as IsEnum +FROM + \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS c + LEFT JOIN + \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p + on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME +ORDER BY c.ORDINAL_POSITION)\` +) +let query = queries.join('\\nUNION ALL \\n') +const [rows] = await bq.query(query) +return rows +}` + return { query, language: 'bun' } + } } else { throw new Error('Unsupported database type:' + input.resourceType) }