starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-19 00:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Move database manager SQL queries to backend (#8306)

Browse Source 
* SQL Query builders in Rust

* Remove frontend sql scripts and substitute at execution

* fix null value bug

* Handle WM_INTERNAL_DB marker for apps deployed prior

* Revert policy handling

* Fix database studio empty string as where clause

* check policy

* Revert "check policy"

This reverts commit 3ea7899979.

* Revert "Fix database studio empty string as where clause"

This reverts commit 432fc87915.

* Revert

* legacy comments

* Move DDL queries to backend

* tests

* move bigquery bun scripts to backend

* expand markers + other nits

* fix: escape sql literals in query builders and async preview sql

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: quote all user-supplied identifiers in query builders to prevent SQL injection

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: suppress dead_code warnings for deserialization-only fields and test-only helpers

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: correct DDL test assertions and drop_table schema handling for non-schema DBs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* MySQL fix

* Fix 0/1 bool

* MySQL fix Yes/No casing

* Better error toasts

* Fix ms sql ntext cast

* fix: quote table name in Snowflake SHOW PRIMARY KEYS query

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: quote schema and table in Snowflake SHOW IMPORTED KEYS query

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: quote BigQuery dataset name in metadata query

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* fix: remove invalid + separator in MSSQL CONCAT for count query

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-03-24 10:45:37 +01:00
committed by GitHub
parent 37ebaf4d0a
commit aa30fd252d
20 changed files with 4828 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/src/apps.rs
+1
View File
@@ -2053,6 +2053,7 @@ async fn execute_component(
.triggerables_v2
.as_ref()
.ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?;
let policy_triggerables = triggerables_v2
.get(path) // start with `path` in case we can avoid the next` format!`.
.or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path)))
backend/windmill-api/src/internal_db.rs
+37
View File
@@ -0,0 +1,37 @@
use crate::db::ApiAuthed;
use axum::{extract::Path, routing::post, Json, Router};
use serde::{Deserialize, Serialize};
use windmill_common::{
error::{Error, Result},
query_builders::try_expand_internal_db_query,
scripts::ScriptLang,
};
pub fn workspaced_service() -> Router {
Router::new().route("/expand_marker", post(expand_marker))
}
#[derive(Deserialize)]
struct ExpandMarkerRequest {
language: ScriptLang,
content: String,
}
#[derive(Serialize)]
struct ExpandMarkerResponse {
code: String,
}
async fn expand_marker(
_authed: ApiAuthed,
Path(_w_id): Path<String>,
Json(req): Json<ExpandMarkerRequest>,
) -> Result<Json<ExpandMarkerResponse>> {
match try_expand_internal_db_query(&req.content, &req.language) {
Some(Ok(expanded)) => Ok(Json(ExpandMarkerResponse { code: expanded.code })),
Some(Err(msg)) => Err(Error::BadRequest(msg)),
None => Err(Error::BadRequest(
"Content is not a WM_INTERNAL_DB marker".to_string(),
)),
}
}
backend/windmill-api/src/lib.rs
+2
View File
@@ -98,6 +98,7 @@ mod indexer_oss;
mod inkeep_ee;
mod inkeep_oss;
mod integration;
mod internal_db;
mod live_migrations;
#[cfg(all(feature = "private", feature = "parquet"))]
pub mod s3_proxy_ee;
@@ -554,6 +555,7 @@ pub async fn run_server(
.nest("/groups", groups::workspaced_service())
.nest("/groups_history", group_history::workspaced_service())
.nest("/inputs", windmill_api_inputs::workspaced_service())
.nest("/internal_db", internal_db::workspaced_service())
.nest("/job_metrics", job_metrics::workspaced_service())
.nest("/job_helpers", job_helpers_service)
.nest("/jobs", jobs::workspaced_service())
backend/windmill-common/src/lib.rs
+1
View File
@@ -77,6 +77,7 @@ pub mod oidc_oss;
#[cfg(feature = "private")]
pub mod otel_ee;
pub mod otel_oss;
pub mod query_builders;
pub mod queue;
pub mod result_stream;
pub mod runnable_settings;
backend/windmill-common/src/query_builders.rs
+4465
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/worker.rs
+23
View File
@@ -4209,6 +4209,29 @@ pub async fn run_language_executor(
modules: &Option<std::collections::HashMap<String, ScriptModule>>,
run_inline: bool,
) -> error::Result<Box<RawValue>> {
// Expand WM_INTERNAL_DB markers into real SQL before dispatching
let expanded_code: String;
let mut language = language;
let code = if let Some(ref lang) = language {
match windmill_common::query_builders::try_expand_internal_db_query(code, lang) {
Some(Ok(expanded)) => {
if let Some(lang_override) = expanded.language_override {
language = Some(lang_override);
}
expanded_code = expanded.code;
&expanded_code
}
Some(Err(e)) => {
return Err(Error::ExecutionErr(format!(
"Failed to expand WM_INTERNAL_DB marker: {}",
e
)));
}
None => code, // Not a marker, use original code
}
} else {
code
};
if let Some(modules) = modules {
#[cfg(feature = "python")]
let base_dir = if language == Some(ScriptLang::Python3) {
frontend/src/lib/components/DBManager.svelte
+7 -13
View File
@@ -572,36 +572,30 @@
dbTableEditorState = { open: false }
}}
{dbType}
computePreview={({ values }) => {
computePreview={async ({ values }) => {
if (dbTableEditorState.alterTableKey && dbTableEditorAlterTableData.current) {
let diff = diffTableEditorValues(dbTableEditorAlterTableData.current, values)
let queries = dbSchemaOps.previewAlterSql({
values: diff,
schema: selected.schemaKey
})
let sql = await dbSchemaOps.previewAlterSql({ values: diff, schema: selected.schemaKey })
let alert = !dbSupportsTransactionalDdl(dbType)
? {
title: capitalize(dbType) + ' does not support transactional DDL',
body: 'Any of these statements failing may leave your database in an intermediate state.'
}
: undefined
return { sql: queries.join('\n'), ...(alert ? { alert } : {}) }
return { sql, ...(alert ? { alert } : {}) }
} else {
return { sql: dbSchemaOps.previewCreateSql({ values, schema: selected.schemaKey }) }
let sql = await dbSchemaOps.previewCreateSql({ values, schema: selected.schemaKey })
return { sql }
}
}}
computeBtnProps={({ values }) => {
if (dbTableEditorState.alterTableKey && dbTableEditorAlterTableData.current) {
let diff = diffTableEditorValues(dbTableEditorAlterTableData.current, values)
let queries = dbSchemaOps.previewAlterSql({
values: diff,
schema: selected.schemaKey
})
if (!queries.length) {
if (!diff.operations.length) {
return { text: 'No changes detected', disabled: true }
}
return {
text: `Alter table (${pluralize(queries.length, 'change')} detected)`
text: `Alter table (${pluralize(diff.operations.length, 'change')} detected)`
}
} else {
return { text: 'Create table' }
frontend/src/lib/components/DBTable.svelte
+2 -2
View File
@@ -115,8 +115,8 @@
refresh?.()
sendUserToast('Row deleted')
})
.catch(() => {
sendUserToast('Error deleting row', true)
.catch((e) => {
sendUserToast(`Error deleting row: ${e?.message ?? e}`, true)
})
}
})
frontend/src/lib/components/DBTableEditor.svelte
+41 -32
View File
@@ -42,6 +42,7 @@
<script lang="ts">
import { ArrowRight, ClipboardCopy, Plus, Settings, X } from 'lucide-svelte'
import { copyToClipboard } from '$lib/utils'
import { Button } from './common'
import { Cell } from './table'
import DataTable from './table/DataTable.svelte'
@@ -51,7 +52,6 @@
import Popover from './meltComponents/Popover.svelte'
import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte'
import { sendUserToast } from '$lib/toast'
import { copyToClipboard } from '$lib/utils'
import { getFlatTableNamesFromSchema, type DBSchema } from '$lib/stores'
import { twMerge } from 'tailwind-merge'
import DarkModeObserver from './DarkModeObserver.svelte'
@@ -78,7 +78,7 @@
onConfirm: (params: { values: TableEditorValues }) => void | Promise<void>
computePreview: (params: {
values: TableEditorValues
}) => { sql: string; alert?: { title: string; body?: string } }
}) => Promise<{ sql: string; alert?: { title: string; body?: string } }>
computeBtnProps: (params: { values: TableEditorValues }) => { text: string; disabled?: boolean }
}
@@ -137,6 +137,8 @@
})
| undefined = $state()
let previewLoading = $state(false)
let darkMode = $state(false)
let btnProps = new Debounced(() => computeBtnProps({ values }), 500)
@@ -440,25 +442,34 @@
</div>
<Button
disabled={!!errors || btnProps.current.disabled}
loading={btnProps.pending}
on:click={() => {
let preview = computePreview?.({ values })
askingForConfirmation = {
onConfirm: async () => {
try {
askingForConfirmation && (askingForConfirmation.loading = true)
await onConfirm({ values })
} catch (e) {
let msg: string | undefined = (e as Error)?.message
if (typeof msg !== 'string') msg = e ? JSON.stringify(e) : 'An error occurred'
sendUserToast(msg, true)
}
askingForConfirmation = undefined
},
title: 'Confirm running the following:',
confirmationText: btnProps.current.text,
open: true,
...(preview && { codeContent: preview.sql, alert: preview.alert })
loading={btnProps.pending || previewLoading}
on:click={async () => {
previewLoading = true
try {
let preview = await computePreview({ values })
askingForConfirmation = {
onConfirm: async () => {
try {
askingForConfirmation && (askingForConfirmation.loading = true)
await onConfirm({ values })
} catch (e) {
let msg: string | undefined = (e as Error)?.message
if (typeof msg !== 'string') msg = e ? JSON.stringify(e) : 'An error occurred'
sendUserToast(msg, true)
}
askingForConfirmation = undefined
},
title: 'Confirm running the following:',
confirmationText: btnProps.current.text,
open: true,
...(preview && { codeContent: preview.sql, alert: preview.alert })
}
} catch (e) {
let msg: string | undefined = (e as Error)?.message
if (typeof msg !== 'string') msg = e ? JSON.stringify(e) : 'An error occurred'
sendUserToast(msg, true)
} finally {
previewLoading = false
}
}}
>
@@ -479,17 +490,15 @@
</Alert>
{/if}
{#if askingForConfirmation?.codeContent}
<div class="bg-surface-secondary border border-surface-selected rounded-md p-2 relative">
<code class="whitespace-pre-wrap">
{askingForConfirmation.codeContent}
</code>
<Button
on:click={() => copyToClipboard(askingForConfirmation?.codeContent)}
size="xs"
startIcon={{ icon: ClipboardCopy }}
color="none"
wrapperClasses="absolute z-10 top-0 right-0"
></Button>
<div class="bg-surface-secondary border border-surface-selected rounded-md p-2 relative group">
<button
class="absolute top-2 right-2 opacity-0 group-hover:opacity-100 transition-opacity p-1 rounded hover:bg-surface-hover"
onclick={() => copyToClipboard(askingForConfirmation?.codeContent)}
title="Copy to clipboard"
>
<ClipboardCopy size={14} />
</button>
<pre class="whitespace-pre-wrap text-sm"><code>{askingForConfirmation.codeContent}</code></pre>
</div>
{/if}
</ConfirmationModal>
frontend/src/lib/components/apps/components/display/dbtable/DeleteRow.svelte
+2 -2
View File
@@ -62,8 +62,8 @@
onCancel: () => {
sendUserToast('Error deleting row', true)
},
onError: () => {
sendUserToast('Error updating row', true)
onError: (e) => {
sendUserToast(`Error deleting row: ${e?.message ?? e}`, true)
}
}
)
frontend/src/lib/components/apps/components/display/dbtable/InsertRowRunnable.svelte
+2 -2
View File
@@ -59,8 +59,8 @@
onCancel: () => {
sendUserToast('Error inserting row', true)
},
onError: () => {
sendUserToast('Error inserting row', true)
onError: (e) => {
sendUserToast(`Error inserting row: ${e?.message ?? e}`, true)
}
})
}
frontend/src/lib/components/apps/components/display/dbtable/UpdateCell.svelte
+2 -2
View File
@@ -64,8 +64,8 @@
onCancel: () => {
sendUserToast('Error updating value', true)
},
onError: () => {
sendUserToast('Error updating value', true)
onError: (e) => {
sendUserToast(`Error updating value: ${e?.message ?? e}`, true)
}
}
)
frontend/src/lib/components/apps/components/display/dbtable/metadata.ts
+57 -245
View File
@@ -1,4 +1,4 @@
import { JobService, ResourceService, type ScriptLang } from '$lib/gen'
import { JobService, ResourceService } from '$lib/gen'
import { runScriptAndPollResult } from '$lib/components/jobs/utils'
import type { DbInput } from '$lib/components/dbTypes'
@@ -15,9 +15,18 @@ import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores'
import { stringifyGraphqlSchema, stringifySchema } from '$lib/components/copilot/lib'
import type { DbType } from '$lib/components/dbTypes'
import { getDatabaseArg } from '$lib/components/dbOps'
import { getDatabaseArg, getDbType } from '$lib/components/dbOps'
import { sendUserToast } from '$lib/toast'
function makeMetadataMarker(
op: string,
payload: Record<string, unknown>,
ducklake: string | undefined
): string {
if (ducklake) payload.ducklake = ducklake
return `-- WM_INTERNAL_DB_${op} ${JSON.stringify(payload)}`
}
export async function loadTableMetaData(
input: DbInput,
workspace: string | undefined,
@@ -25,11 +34,26 @@ export async function loadTableMetaData(
): Promise<TableMetadata | undefined> {
if (!input || !table || !workspace) return undefined
let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, table)
const dbType = getDbType(input)
const language = getLanguageByResourceType(dbType)
const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
const dbArg = getDatabaseArg(input)
// MySQL needs the database name for metadata queries
let databaseName: string | undefined
if (input.type === 'database' && input.resourceType === 'mysql') {
const resourceObj = (await ResourceService.getResourceValue({
workspace,
path: input.resourcePath
})) as any
databaseName = resourceObj?.database
}
const content = makeMetadataMarker('LOAD_TABLE_METADATA', { table, databaseName }, ducklake)
const job = await JobService.runScriptPreview({
workspace: workspace,
requestBody: { language, content: query, args: getDatabaseArg(input) }
workspace,
requestBody: { language, content, args: dbArg }
})
const maxRetries = 8
@@ -39,7 +63,7 @@ export async function loadTableMetaData(
await new Promise((resolve) => setTimeout(resolve, 1000 * (attempts || 0.6)))
const testResult = (await JobService.getCompletedJob({
workspace: workspace,
workspace,
id: job
})) as any
@@ -78,10 +102,30 @@ export async function loadAllTablesMetaData(
if (!input || !workspace) return undefined
try {
let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, undefined)
const dbType = getDbType(input)
const dbArg = getDatabaseArg(input)
const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
// MySQL needs the database name for metadata queries
let databaseName: string | undefined
if (input.type === 'database' && input.resourceType === 'mysql') {
const resourceObj = (await ResourceService.getResourceValue({
workspace,
path: input.resourcePath
})) as any
databaseName = resourceObj?.database
}
const language = getLanguageByResourceType(dbType)
const content = makeMetadataMarker(
'LOAD_TABLE_METADATA',
{ table: undefined, databaseName },
ducklake
)
let result = (await runScriptAndPollResult({
workspace: workspace,
requestBody: { language, content: query, args: getDatabaseArg(input) }
workspace,
requestBody: { language, content, args: dbArg }
})) as ({ table_name: string; schema_name?: string } & object)[]
const map: Record<string, TableMetadata> = {}
@@ -101,241 +145,6 @@ export async function loadAllTablesMetaData(
}
}
async function makeLoadTableMetaDataQuery(
input: DbInput,
workspace: string,
table: string | undefined
): Promise<{ query: string; language: ScriptLang }> {
if (input.type === 'ducklake') {
const query = `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
SELECT
COLUMN_NAME as field,
DATA_TYPE as DataType,
COLUMN_DEFAULT as DefaultValue,
false as IsPrimaryKey,
false as IsIdentity,
IS_NULLABLE as IsNullable,
false as IsEnum,
TABLE_NAME as table_name
FROM information_schema.columns c
WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()`
return { query, language: 'duckdb' }
} else if (input.resourceType === 'mysql') {
const resourceObj = (await ResourceService.getResourceValue({
workspace,
path: input.resourcePath
})) as any
const query = `
SELECT
COLUMN_NAME as field,
COLUMN_TYPE as DataType,
COLUMN_DEFAULT as DefaultValue,
CASE WHEN COLUMN_KEY = 'PRI' THEN 1 ELSE 0 END as IsPrimaryKey,
CASE WHEN EXTRA like '%auto_increment%' THEN 'YES' ELSE 'NO' END as IsIdentity,
CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
CASE WHEN DATA_TYPE = 'enum' THEN true ELSE false END as IsEnum${
table
? ''
: `,
TABLE_NAME as table_name`
}
FROM
INFORMATION_SCHEMA.COLUMNS${
table
? `
WHERE
TABLE_NAME = '${table.split('.').reverse()[0]}' AND TABLE_SCHEMA = '${
table.split('.').reverse()[1] ?? resourceObj?.database ?? ''
}'`
: `
WHERE
TABLE_SCHEMA NOT IN ('mysql', 'performance_schema', 'information_schema', 'sys')`
}
ORDER BY
TABLE_NAME,
ORDINAL_POSITION;
`
return { query, language: 'mysql' }
} else if (input.resourceType === 'postgresql') {
const query = `
SELECT
a.attname as field,
pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
(SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid, true) for 128)
FROM pg_catalog.pg_attrdef d
WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) as DefaultValue,
(SELECT CASE WHEN i.indisprimary THEN true ELSE 'NO' END
FROM pg_catalog.pg_class tbl, pg_catalog.pg_class idx, pg_catalog.pg_index i, pg_catalog.pg_attribute att
WHERE tbl.oid = a.attrelid AND idx.oid = i.indexrelid AND att.attrelid = tbl.oid
AND i.indrelid = tbl.oid AND att.attnum = any(i.indkey) AND att.attname = a.attname LIMIT 1) as IsPrimaryKey,
CASE a.attidentity
WHEN 'd' THEN 'By Default'
WHEN 'a' THEN 'Always'
ELSE 'No'
END as IsIdentity,
CASE a.attnotnull
WHEN false THEN 'YES'
ELSE 'NO'
END as IsNullable,
(SELECT true
FROM pg_catalog.pg_enum e
WHERE e.enumtypid = a.atttypid FETCH FIRST ROW ONLY) as IsEnum${
table
? ''
: `,
ns.nspname AS schema_name,
c.relname AS table_name`
}
FROM pg_catalog.pg_attribute a${
table
? `
WHERE a.attrelid = (SELECT c.oid FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid WHERE relname = '${
table.split('.').reverse()[0]
}' AND ns.nspname = '${table.split('.').reverse()[1] ?? 'public'}')
AND a.attnum > 0 AND NOT a.attisdropped
`
: `
JOIN pg_catalog.pg_class c ON a.attrelid = c.oid
JOIN pg_catalog.pg_namespace ns ON c.relnamespace = ns.oid
WHERE c.relkind = 'r' AND a.attnum > 0 AND NOT a.attisdropped
AND ns.nspname != 'pg_catalog' AND ns.nspname != 'information_schema'`
}
ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'};
`
return { query, language: 'postgresql' }
} else if (input.resourceType === 'ms_sql_server') {
const query = `
SELECT
c.COLUMN_NAME as field,
c.DATA_TYPE as DataType,
c.COLUMN_DEFAULT as DefaultValue,
CASE WHEN COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'IsIdentity') = 1 THEN 'By Default' ELSE 'No' END as IsIdentity,
CASE WHEN pk.COLUMN_NAME IS NOT NULL THEN 1 ELSE 0 END as IsPrimaryKey,
CASE WHEN c.IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
CASE WHEN c.DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum,
dc.name as default_constraint_name${
table
? ''
: `,
c.TABLE_NAME as table_name`
}
FROM
INFORMATION_SCHEMA.COLUMNS c
LEFT JOIN (
SELECT
ku.TABLE_SCHEMA,
ku.TABLE_NAME,
ku.COLUMN_NAME
FROM
INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc
INNER JOIN INFORMATION_SCHEMA.KEY_COLUMN_USAGE ku
ON tc.CONSTRAINT_TYPE = 'PRIMARY KEY'
AND tc.CONSTRAINT_NAME = ku.CONSTRAINT_NAME
AND tc.TABLE_SCHEMA = ku.TABLE_SCHEMA
AND tc.TABLE_NAME = ku.TABLE_NAME
) pk ON c.TABLE_SCHEMA = pk.TABLE_SCHEMA
AND c.TABLE_NAME = pk.TABLE_NAME
AND c.COLUMN_NAME = pk.COLUMN_NAME
LEFT JOIN sys.default_constraints dc
ON dc.parent_object_id = OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME)
AND dc.parent_column_id = COLUMNPROPERTY(OBJECT_ID(c.TABLE_SCHEMA + '.' + c.TABLE_NAME), c.COLUMN_NAME, 'ColumnId')${
table
? `
WHERE
c.TABLE_NAME = '${table}'`
: ''
}
ORDER BY
c.ORDINAL_POSITION;
`
return { query, language: 'mssql' }
} else if (
input.resourceType === 'snowflake' ||
(input.resourceType as any) === 'snowflake_oauth'
) {
const query = `
select COLUMN_NAME as field,
DATA_TYPE as DataType,
COLUMN_DEFAULT as DefaultValue,
CASE WHEN COLUMN_DEFAULT like 'AUTOINCREMENT%' THEN 'By Default' ELSE 'No' END as IsIdentity,
0 as IsPrimaryKey, -- a one-query solution is not trivial, we will use SHOW PRIMARY KEYS separately
CASE WHEN IS_NULLABLE = 'YES' THEN 'YES' ELSE 'NO' END as IsNullable,
CASE WHEN DATA_TYPE = 'enum' THEN 1 ELSE 0 END as IsEnum${
table
? ''
: `,
table_name as table_name,
table_schema as schema_name`
}
from information_schema.columns${
table
? `
where table_name = '${table.split('.').reverse()[0]}' and table_schema = '${
table.split('.').reverse()[1] ?? 'PUBLIC'
}'`
: "\nwhere table_schema <> 'INFORMATION_SCHEMA'\n"
}
order by ORDINAL_POSITION;
`
return { query, language: 'snowflake' }
} else if (input.resourceType === 'bigquery') {
if (table) {
const query = `SELECT
c.COLUMN_NAME as field,
DATA_TYPE as DataType,
CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
'No' as IsIdentity,
IS_NULLABLE as IsNullable,
false as IsEnum
FROM
${table.split('.')[0]}.INFORMATION_SCHEMA.COLUMNS c
LEFT JOIN
${table.split('.')[0]}.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
WHERE
c.TABLE_NAME = '${table.split('.')[1]}'
order by c.ORDINAL_POSITION;`
return { query, language: 'bigquery' }
} else {
const query = `import { BigQuery } from '@google-cloud/bigquery@7.5.0';
export async function main(database: bigquery) {
const bq = new BigQuery({
credentials: database
})
const [datasets] = await bq.getDatasets();
if (!datasets) return {}
const schema = {} as any
let queries = datasets.map(dataset => \`
(SELECT
c.COLUMN_NAME as field,
'\${dataset.id}' as schema_name,
c.TABLE_NAME as table_name,
DATA_TYPE as DataType,
CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
'No' as IsIdentity,
IS_NULLABLE as IsNullable,
false as IsEnum
FROM
\\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS c
LEFT JOIN
\\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
ORDER BY c.ORDINAL_POSITION)\`
)
let query = queries.join('\\nUNION ALL \\n')
const [rows] = await bq.query(query)
return rows
}`
return { query, language: 'bun' }
}
} else {
throw new Error('Unsupported database type:' + input.resourceType)
}
}
type SnowflakeShowPrimaryKeysResult = {
column_name: string
database_name: string
@@ -379,12 +188,15 @@ async function fetchSnowflakePrimaryKeys(
dbArg: any,
tableKey?: string
): Promise<SnowflakeShowPrimaryKeysResult[]> {
const payload: Record<string, unknown> = {}
if (tableKey) payload.table = tableKey
const content = makeMetadataMarker('SNOWFLAKE_PRIMARY_KEYS', payload, undefined)
return (await JobService.runScriptPreviewAndWaitResult({
workspace,
requestBody: {
language: 'snowflake',
args: dbArg,
content: tableKey ? `SHOW PRIMARY KEYS IN TABLE ${tableKey}` : 'SHOW PRIMARY KEYS IN ACCOUNT'
content
}
})) as SnowflakeShowPrimaryKeysResult[]
}
frontend/src/lib/components/apps/components/display/dbtable/queries/count.ts
+9
View File
@@ -1,3 +1,12 @@
/**
* LEGACY: These query builders generate full SQL on the frontend.
* They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps
* whose policies were generated with expanded SQL digests.
*
* New code (Database Manager) should use WM_INTERNAL_DB markers instead,
* which are expanded server-side by the Rust query_builders module.
* See: dbOps.ts → dbTableOpsWithPreviewScripts()
*/
import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
import { wrapDucklakeQuery } from '../../../../../ducklake'
import type { DbType, DbInput } from '$lib/components/dbTypes'
frontend/src/lib/components/apps/components/display/dbtable/queries/delete.ts
+9
View File
@@ -1,3 +1,12 @@
/**
* LEGACY: These query builders generate full SQL on the frontend.
* They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps
* whose policies were generated with expanded SQL digests.
*
* New code (Database Manager) should use WM_INTERNAL_DB markers instead,
* which are expanded server-side by the Rust query_builders module.
* See: dbOps.ts → dbTableOpsWithPreviewScripts()
*/
import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
import type { DbType, DbInput } from '$lib/components/dbTypes'
import { wrapDucklakeQuery } from '../../../../../ducklake'
frontend/src/lib/components/apps/components/display/dbtable/queries/insert.ts
+9
View File
@@ -1,3 +1,12 @@
/**
* LEGACY: These query builders generate full SQL on the frontend.
* They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps
* whose policies were generated with expanded SQL digests.
*
* New code (Database Manager) should use WM_INTERNAL_DB markers instead,
* which are expanded server-side by the Rust query_builders module.
* See: dbOps.ts → dbTableOpsWithPreviewScripts()
*/
import type { AppInput } from '$lib/components/apps/inputType'
import { wrapDucklakeQuery } from '../../../../../ducklake'
import type { DbType, DbInput } from '$lib/components/dbTypes'
frontend/src/lib/components/apps/components/display/dbtable/queries/relationalKeys.ts
+1 -1
View File
@@ -275,7 +275,7 @@ function makeSnowflakeForeignKeysQuery(tableName: string, schemaName: string): s
* pk_database_name, pk_schema_name, pk_table_name, pk_column_name, key_sequence,
* update_rule, delete_rule, fk_name, pk_name, deferrability
*/
function transformSnowflakeForeignKeys(snowflakeResults: any[]): RawForeignKey[] {
export function transformSnowflakeForeignKeys(snowflakeResults: any[]): RawForeignKey[] {
if (!snowflakeResults || !Array.isArray(snowflakeResults)) {
return []
}
frontend/src/lib/components/apps/components/display/dbtable/queries/select.ts
+9
View File
@@ -1,3 +1,12 @@
/**
* LEGACY: These query builders generate full SQL on the frontend.
* They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps
* whose policies were generated with expanded SQL digests.
*
* New code (Database Manager) should use WM_INTERNAL_DB markers instead,
* which are expanded server-side by the Rust query_builders module.
* See: dbOps.ts → dbTableOpsWithPreviewScripts()
*/
import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
import { wrapDucklakeQuery } from '../../../../../ducklake'
import type { DbType, DbInput } from '$lib/components/dbTypes'
frontend/src/lib/components/apps/components/display/dbtable/queries/update.ts
+9
View File
@@ -1,3 +1,12 @@
/**
* LEGACY: These query builders generate full SQL on the frontend.
* They exist only for backwards compatibility with Database Studio (dbexplorercomponent) apps
* whose policies were generated with expanded SQL digests.
*
* New code (Database Manager) should use WM_INTERNAL_DB markers instead,
* which are expanded server-side by the Rust query_builders module.
* See: dbOps.ts → dbTableOpsWithPreviewScripts()
*/
import type { AppInput, RunnableByName } from '$lib/components/apps/inputType'
import { wrapDucklakeQuery } from '../../../../../ducklake'
import type { DbInput, DbType } from '$lib/components/dbTypes'
frontend/src/lib/components/dbOps.ts
+140 -58
View File
@@ -3,29 +3,21 @@ import {
type ColumnDef,
type TableMetadata
} from './apps/components/display/dbtable/utils'
import { makeSelectQuery } from './apps/components/display/dbtable/queries/select'
import { runScriptAndPollResult } from './jobs/utils'
import { makeCountQuery } from './apps/components/display/dbtable/queries/count'
import { makeUpdateQuery } from './apps/components/display/dbtable/queries/update'
import { makeDeleteQuery } from './apps/components/display/dbtable/queries/delete'
import { makeInsertQuery } from './apps/components/display/dbtable/queries/insert'
import { makeDeleteTableQuery } from './apps/components/display/dbtable/queries/deleteTable'
import type { DBSchema, SQLSchema } from '$lib/stores'
import { stringifySchema } from './copilot/lib'
import type { DbInput, DbType } from './dbTypes'
import { wrapDucklakeQuery } from './ducklake'
import { assert } from '$lib/utils'
import {
buildTableEditorValues,
type TableEditorValues
} from './apps/components/display/dbtable/tableEditor'
import { type AlterTableValues } from './apps/components/display/dbtable/queries/alterTable'
import {
makeAlterTableQueries,
makeAlterTableQuery,
type AlterTableValues
} from './apps/components/display/dbtable/queries/alterTable'
import { makeCreateTableQuery } from './apps/components/display/dbtable/queries/createTable'
import { fetchTableRelationalKeys } from './apps/components/display/dbtable/queries/relationalKeys'
transformForeignKeys,
transformSnowflakeForeignKeys,
type RawForeignKey
} from './apps/components/display/dbtable/queries/relationalKeys'
export type IDbTableOps = {
dbType: DbType
@@ -63,28 +55,35 @@ export function dbTableOpsWithPreviewScripts({
const dbType = getDbType(input)
const language = getLanguageByResourceType(dbType)
const dbArg = getDatabaseArg(input)
const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
function makeMarker(op: string, payload: Record<string, unknown>): string {
if (ducklake) payload.ducklake = ducklake
return `-- WM_INTERNAL_DB_${op} ${JSON.stringify(payload)}`
}
return {
dbType,
tableKey,
colDefs,
getCount: async ({ quicksearch }) => {
let countQuery = makeCountQuery(dbType, tableKey, undefined, colDefs)
if (input.type === 'ducklake') countQuery = wrapDucklakeQuery(countQuery, input.ducklake)
const content = makeMarker('COUNT', { table: tableKey, columnDefs: colDefs })
const result = await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, quicksearch }, language, content: countQuery }
requestBody: { args: { ...dbArg, quicksearch }, language, content }
})
const count = result?.[0].count as number
return count
},
getRows: async (params) => {
let query = makeSelectQuery(tableKey, colDefs, undefined, dbType, undefined, {
const content = makeMarker('SELECT', {
table: tableKey,
columnDefs: colDefs,
fixPgIntTypes: true
})
if (input.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
let items = (await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...params }, language, content: query }
requestBody: { args: { ...dbArg, ...params }, language, content }
})) as unknown[]
if (!items || !Array.isArray(items)) {
throw 'items is not an array'
@@ -92,31 +91,32 @@ export function dbTableOpsWithPreviewScripts({
return items
},
onUpdate: async ({ values }, colDef, newValue) => {
let updateQuery = makeUpdateQuery(tableKey, colDef, colDefs, dbType)
if (input.type === 'ducklake') updateQuery = wrapDucklakeQuery(updateQuery, input.ducklake)
const content = makeMarker('UPDATE', {
table: tableKey,
column: colDef,
columns: colDefs
})
await runScriptAndPollResult({
workspace,
requestBody: {
args: { ...dbArg, value_to_update: newValue, ...values },
language,
content: updateQuery
content
}
})
},
onDelete: async ({ values }) => {
let deleteQuery = makeDeleteQuery(tableKey, colDefs, dbType)
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
const content = makeMarker('DELETE', { table: tableKey, columns: colDefs })
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...values }, language, content: deleteQuery }
requestBody: { args: { ...dbArg, ...values }, language, content }
})
},
onInsert: async ({ values }) => {
let insertQuery = makeInsertQuery(tableKey, colDefs, dbType)
if (input.type === 'ducklake') insertQuery = wrapDucklakeQuery(insertQuery, input.ducklake)
const content = makeMarker('INSERT', { table: tableKey, columns: colDefs })
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg, ...values }, language, content: insertQuery }
requestBody: { args: { ...dbArg, ...values }, language, content }
})
}
}
@@ -125,9 +125,9 @@ export function dbTableOpsWithPreviewScripts({
export type IDbSchemaOps = {
onDelete: (params: { tableKey: string; schema?: string }) => Promise<void>
onCreate: (params: { values: TableEditorValues; schema?: string }) => Promise<void>
previewCreateSql: (params: { values: TableEditorValues; schema?: string }) => string
previewCreateSql: (params: { values: TableEditorValues; schema?: string }) => Promise<string>
onAlter: (params: { values: AlterTableValues; schema?: string }) => Promise<void>
previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => string[]
previewAlterSql: (params: { values: AlterTableValues; schema?: string }) => Promise<string>
onCreateSchema: (params: { schema: string }) => Promise<void>
onDeleteSchema: (params: { schema: string }) => Promise<void>
onFetchTableEditorDefinition: (params: {
@@ -147,61 +147,130 @@ export function dbSchemaOpsWithPreviewScripts({
const dbType = getDbType(input)
const dbArg = getDatabaseArg(input)
const language = getLanguageByResourceType(dbType)
const ducklake = input.type === 'ducklake' ? input.ducklake : undefined
function makeMarker(op: string, payload: Record<string, unknown>): string {
if (ducklake) payload.ducklake = ducklake
return `-- WM_INTERNAL_DB_${op} ${JSON.stringify(payload)}`
}
return {
onDelete: async ({ tableKey, schema }) => {
let deleteQuery = makeDeleteTableQuery(tableKey, dbType, schema)
if (input.type === 'ducklake') deleteQuery = wrapDucklakeQuery(deleteQuery, input.ducklake)
const content = makeMarker('DROP_TABLE', { table: tableKey, schema })
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg }, language, content: deleteQuery }
requestBody: { args: { ...dbArg }, language, content }
})
},
onCreate: async ({ values, schema }) => {
let query = makeCreateTableQuery(values, dbType, schema)
if (input?.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
const content = makeMarker('CREATE_TABLE', {
name: values.name,
columns: values.columns,
foreignKeys: values.foreignKeys,
schema
})
await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: query, language }
requestBody: { args: dbArg, content, language }
})
},
previewCreateSql: ({ values, schema }) => makeCreateTableQuery(values, dbType, schema),
previewCreateSql: async ({ values, schema }) => {
const content = makeMarker('CREATE_TABLE', {
name: values.name,
columns: values.columns,
foreignKeys: values.foreignKeys,
schema
})
return expandMarker(workspace, language, content)
},
onAlter: async ({ values, schema }) => {
let query = makeAlterTableQuery(values, dbType, schema)
if (input.type === 'ducklake') query = wrapDucklakeQuery(query, input.ducklake)
const content = makeMarker('ALTER_TABLE', {
name: values.name,
operations: values.operations,
schema
})
await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: query, language }
requestBody: { args: dbArg, content, language }
})
},
previewAlterSql: ({ values, schema }) => makeAlterTableQueries(values, dbType, schema),
previewAlterSql: async ({ values, schema }) => {
const content = makeMarker('ALTER_TABLE', {
name: values.name,
operations: values.operations,
schema
})
return expandMarker(workspace, language, content)
},
onCreateSchema: async ({ schema }) => {
let createSchemaQuery = `CREATE SCHEMA ${schema};`
if (input.type === 'ducklake')
createSchemaQuery = wrapDucklakeQuery(createSchemaQuery, input.ducklake)
const content = makeMarker('CREATE_SCHEMA', { schema })
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg }, language, content: createSchemaQuery }
requestBody: { args: { ...dbArg }, language, content }
})
},
onDeleteSchema: async ({ schema }) => {
let dropSchemaQuery = `DROP SCHEMA ${schema} CASCADE;`
if (input.type === 'ducklake')
dropSchemaQuery = wrapDucklakeQuery(dropSchemaQuery, input.ducklake)
const content = makeMarker('DROP_SCHEMA', { schema })
await runScriptAndPollResult({
workspace,
requestBody: { args: { ...dbArg }, language, content: dropSchemaQuery }
requestBody: { args: { ...dbArg }, language, content }
})
},
onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => {
let { foreignKeys, pk_constraint_name } = await fetchTableRelationalKeys(
input,
dbType,
table,
schema,
workspace,
dbArg,
language
)
let foreignKeys: import('./apps/components/display/dbtable/tableEditor').TableEditorForeignKey[] =
[]
let pk_constraint_name: string | undefined
// Fetch foreign keys (not supported for BigQuery)
if (dbType !== 'bigquery') {
try {
const fkContent = makeMarker('FOREIGN_KEYS', { table, schema })
const fkResult = await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: fkContent, language }
})
let rawForeignKeys: RawForeignKey[]
if (dbType === 'snowflake') {
rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[])
} else {
rawForeignKeys = fkResult as RawForeignKey[]
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
rawForeignKeys = rawForeignKeys.map((fk) => {
const lowerFk: any = {}
Object.keys(fk).forEach((key) => {
lowerFk[key.toLowerCase()] = fk[key]
})
return lowerFk
})
}
}
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
foreignKeys = transformForeignKeys(rawForeignKeys)
}
} catch (e) {
console.warn('Failed to fetch foreign keys:', e)
}
}
// Fetch primary key constraint name (not supported for BigQuery/MySQL)
if (dbType !== 'bigquery' && dbType !== 'mysql') {
try {
const pkContent = makeMarker('PRIMARY_KEY_CONSTRAINT', { table, schema })
const pkResult = (await runScriptAndPollResult({
workspace,
requestBody: { args: dbArg, content: pkContent, language }
})) as { constraint_name?: string; CONSTRAINT_NAME?: string }[]
if (pkResult && Array.isArray(pkResult) && pkResult.length > 0) {
const pkRecord: any = pkResult[0]
pk_constraint_name = pkRecord?.constraint_name || pkRecord?.CONSTRAINT_NAME || ''
}
} catch (e) {
console.warn('Failed to fetch primary key constraint:', e)
}
}
return buildTableEditorValues({
tableName: table,
@@ -278,3 +347,16 @@ export function getDatabaseArg(input: DbInput | undefined) {
}
return {}
}
async function expandMarker(workspace: string, language: string, content: string): Promise<string> {
const response = await fetch(`/api/w/${workspace}/internal_db/expand_marker`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ language, content })
})
if (!response.ok) {
throw new Error(await response.text())
}
const result = (await response.json()) as { code: string }
return result.code
}