Playwright E2E (#7520)

* clean plate

* npm i

* log in e2e

* global setup login

* set license key

* Revert "set license key"

This reverts commit 86d5db2c48.

* create datatable test

* fix wrong pg_creds

* data table + db manager e2e test

* DbManagerPage class

* small refactor

* create resource test + improvements

* text db manager in resources

* Factor test logic in classes

* refactoring

* refacto

* alter table test

* alter table e2e test

* set schema in test

* nits

* fix wrong schema var

* Correct setup and parallelization

* reducedMotion

* tests passing headless !

* bigger timeout

* start e2e docker compose

* e2e runs on all databases

* nit test uid fix

* refactp

* stash

* Better Workspace Storage settings

* minio setup

* nit

* nit

* super nit

* Permission settings in modal

* badge indicator

* Fetch alter table metadata much faster

* Upgrade duckdb to 1.4.3

* Ducklake tests

* Disable transactional DDL for Ducklake (bug on their side)

* git ignore env

* bigquery tests passes

* getJsonEnv

* load coldef in parallel

* Make Bigquery schema fetching much faster

* makeLoadTableMetaDataQuery for entire db in bigquery

* refactor getDbSchemas to avoid assignment side effect

* fix col def

* Better loading state mgmt

* snowflake

* fix snowflake primary keys

* Test CI

* fix setTimeout type

* remove type node

* test e2e ci

* Revert "test e2e ci"

This reverts commit bf98a755dc.

* remove ci

* fix snowflake pk query in alternate schemas

* nit wait for coldefs

* nit snowflake

* Snowflake fk fix

* UNPROCESSABLE_ENTITY instead of INTERNAL_ERROR

* nits

* fix alter pk in snowflake

* yet other fixes

* snowflake tests pass

* nits
This commit is contained in:
Diego Imbert
2026-01-22 16:20:01 +00:00
committed by GitHub
parent 7385726741
commit a384b4c23d
58 changed files with 1694 additions and 784 deletions
@@ -145,6 +145,7 @@
options={{
right: 'As JSON'
}}
class="as-json-toggle"
/>
<TestConnection {resourceType} {args} />
{#if resourceType == 'postgresql'}
@@ -46,6 +46,7 @@
>
<DrawerContent
title="Add a resource"
id="add-resource-drawer"
on:close={drawer.closeDrawer}
tooltip="Resources represent connections to third party systems. Learn more on how to integrate external APIs."
documentationLink="https://www.windmill.dev/docs/integrations/integrations_on_windmill"
@@ -561,6 +561,7 @@
placeholder="Search resource type"
bind:value={filter}
class="text-2xl grow"
id="search-resource-type"
/>
</div>
+35 -44
View File
@@ -1,18 +1,10 @@
<script lang="ts">
import { type DBSchema } from '$lib/stores'
import {
ChevronDownIcon,
EditIcon,
Loader2,
MoreVertical,
Plus,
Table2,
Trash2Icon
} from 'lucide-svelte'
import { ChevronDownIcon, EditIcon, Loader2, Plus, Table2, Trash2Icon } from 'lucide-svelte'
import { Pane, Splitpanes } from 'svelte-splitpanes'
import { ClearableInput, Drawer, DrawerContent } from './common'
import { sendUserToast } from '$lib/toast'
import { type ColumnDef, type DbFeatures } from './apps/components/display/dbtable/utils'
import { type ColumnDef } from './apps/components/display/dbtable/utils'
import DBTable from './DBTable.svelte'
import type { IDbSchemaOps, IDbTableOps } from './dbOps'
import DropdownV2 from './DropdownV2.svelte'
@@ -29,7 +21,8 @@
diffTableEditorValues
} from './apps/components/display/dbtable/queries/alterTable'
import { resource } from 'runed'
import { capitalize, pluralize } from '$lib/utils'
import { capitalize, onlyAlphaNumAndUnderscore, pluralize } from '$lib/utils'
import type { DbFeatures } from './apps/components/display/dbtable/dbFeatures'
/** Represents a selected table with its schema */
export interface SelectedTable {
@@ -41,7 +34,7 @@
dbType: DbType
dbSchema: DBSchema
dbSupportsSchemas: boolean
getColDefs: (tableKey: string) => Promise<ColumnDef[]>
colDefs: Record<string, ColumnDef[]> | undefined
dbTableOpsFactory: (params: { colDefs: ColumnDef[]; tableKey: string }) => IDbTableOps
dbSchemaOps: IDbSchemaOps
refresh?: () => void
@@ -63,8 +56,8 @@
dbSchema,
dbTableOpsFactory,
dbSchemaOps,
getColDefs,
dbSupportsSchemas,
colDefs,
refresh,
initialSchemaKey,
initialTableKey,
@@ -209,15 +202,16 @@
open: false
})
let dbTableEditorAlterTableData = resource(
() => dbTableEditorState.alterTableKey,
async (table) => {
[() => dbTableEditorState.alterTableKey, () => colDefs],
async ([table]) => {
if (!table) return
let tableKey2 =
dbSupportsSchemas && selected.schemaKey ? `${selected.schemaKey}.${table}` : table
if (!colDefs?.[tableKey2]) return
return await dbSchemaOps.onFetchTableEditorDefinition({
table: table,
schema: selected.schemaKey,
getColDefs: () => getColDefs(tableKey2)
colDefs: colDefs[tableKey2]
})
}
)
@@ -226,14 +220,14 @@
let newSchemaName = $state('')
// Check if the sanitized schema name already exists
const sanitizedNewSchemaName = $derived(
newSchemaName
.trim()
.toLowerCase()
.replace(/[^a-zA-Z0-9_]/g, '')
)
const sanitizedNewSchemaName = $derived.by(() => {
let s = newSchemaName.trim().replace(/[^a-zA-Z0-9_]/g, '')
if (dbType === 'snowflake') s = s.toUpperCase()
return s
})
const schemaAlreadyExists = $derived(
sanitizedNewSchemaName !== '' && schemaKeys.includes(sanitizedNewSchemaName)
sanitizedNewSchemaName !== '' &&
schemaKeys.map((s) => s.toLowerCase()).includes(sanitizedNewSchemaName.toLowerCase())
)
</script>
@@ -247,20 +241,20 @@
<Select
bind:value={selected.schemaKey}
items={safeSelectItems(schemaKeys)}
id="db-schema-select"
transformInputSelectedText={(s) => `Schema: ${s}`}
RightIcon={ChevronDownIcon}
placeholder="Search or create schema..."
showPlaceholderOnOpen
onCreateItem={(schema) => {
schema = schema
.trim()
.toLowerCase()
.replace(/[^a-zA-Z0-9_]/g, '')
schema = schema.trim().replace(/[^a-zA-Z0-9_]/g, '')
if (dbType === 'snowflake') schema = schema.toUpperCase()
askingForConfirmation = {
confirmationText: `Create ${schema}`,
type: 'reload',
title: `This will run 'CREATE SCHEMA ${schema}' on your database. Are you sure ?`,
open: true,
id: 'db-create-schema-confirmation-modal',
onConfirm: async () => {
askingForConfirmation && (askingForConfirmation.loading = true)
try {
@@ -451,7 +445,10 @@
onclick={() => (selected.tableKey = tableKey)}
>
<Table2 class="text-primary shrink-0" size={16} />
<p class="truncate text-ellipsis grow text-left text-emphasis text-xs">{tableKey}</p>
<p
class="db-manager-table-key truncate text-ellipsis grow text-left text-emphasis text-xs"
>{tableKey}</p
>
<DropdownV2
items={() => [
{
@@ -462,6 +459,7 @@
title: `Are you sure you want to delete ${tableKey} ? This action is irreversible`,
confirmationText: 'Delete permanently',
open: true,
id: 'db-manager-delete-table-confirmation-modal',
onConfirm: async () => {
askingForConfirmation && (askingForConfirmation.loading = true)
try {
@@ -489,14 +487,8 @@
}
]}
class="w-fit"
>
<svelte:fragment slot="buttonReplacement">
<MoreVertical
size={8}
class="w-8 h-8 p-2 hover:bg-surface-hover cursor-pointer rounded-md"
/>
</svelte:fragment>
</DropdownV2>
btnId={'db-manager-table-actions-' + onlyAlphaNumAndUnderscore(tableKey)}
/>
</button>
{/each}
{/if}
@@ -513,13 +505,9 @@
{/if}
</Pane>
<Pane class="p-3 pt-1">
{#if tableKey}
{#await getColDefs(tableKey) then colDefs}
{#if colDefs && colDefs?.length}
{@const dbTableOps = dbTableOpsFactory({ colDefs, tableKey })}
<DBTable {dbTableOps} />
{/if}
{/await}
{#if tableKey && colDefs?.[tableKey]?.length}
{@const dbTableOps = dbTableOpsFactory({ colDefs: colDefs[tableKey], tableKey })}
<DBTable {dbTableOps} />
{/if}
</Pane>
</Splitpanes>
@@ -538,6 +526,7 @@
on:close={() => (dbTableEditorState = { open: false })}
>
<DrawerContent
id="db-table-editor-drawer"
on:close={() => (dbTableEditorState = { open: false })}
title={dbTableEditorState.alterTableKey
? `Alter ${dbTableEditorState.alterTableKey}`
@@ -602,7 +591,7 @@
}
}}
/>
{:else if dbTableEditorAlterTableData.loading}
{:else if dbTableEditorAlterTableData.loading || !colDefs}
<Loader2 class="animate-spin" size={32} />
{:else}
<p class="text-sm text-tertiary">Failed to load table definition.</p>
@@ -643,6 +632,7 @@
type: 'reload',
title: `This will run 'CREATE SCHEMA ${sanitizedNewSchemaName}' on your database. Are you sure?`,
open: true,
id: 'db-create-schema-confirmation-modal',
onConfirm: async () => {
askingForConfirmation && (askingForConfirmation.loading = true)
try {
@@ -680,6 +670,7 @@
type: 'reload',
title: `This will run 'CREATE SCHEMA ${sanitizedNewSchemaName}' on your database. Are you sure?`,
open: true,
id: 'db-create-schema-confirmation-modal',
onConfirm: async () => {
askingForConfirmation && (askingForConfirmation.loading = true)
try {
@@ -2,11 +2,7 @@
import { dbSchemas, workspaceStore, type DBSchema } from '$lib/stores'
import { sendUserToast, sortArray } from '$lib/utils'
import { Loader2 } from 'lucide-svelte'
import {
dbSupportsSchemas,
getDbFeatures,
type TableMetadata
} from './apps/components/display/dbtable/utils'
import { dbSupportsSchemas } from './apps/components/display/dbtable/utils'
import DbManager from './DBManager.svelte'
import {
dbSchemaOpsWithPreviewScripts,
@@ -17,21 +13,18 @@
import { Pane, Splitpanes } from 'svelte-splitpanes'
import SqlRepl from './SqlRepl.svelte'
import SimpleAgTable from './SimpleAgTable.svelte'
import { untrack, type Snippet } from 'svelte'
import { type Snippet } from 'svelte'
import type { DbInput } from './dbTypes'
import {
getDbSchemas,
loadAllTablesMetaData,
loadTableMetaData
} from './apps/components/display/dbtable/metadata'
import { getDbSchemas, loadAllTablesMetaData } from './apps/components/display/dbtable/metadata'
import type { SelectedTable } from './DBManager.svelte'
import { getDbFeatures } from './apps/components/display/dbtable/dbFeatures'
import { resource } from 'runed'
interface Props {
input?: DbInput
showRepl?: boolean
hasReplResult?: boolean
isRefreshing?: boolean
selectedSchemaKey?: string | undefined
selectedTableKey?: string | undefined
dbSelector?: Snippet<[]>
@@ -47,7 +40,6 @@
input,
showRepl = true,
hasReplResult = $bindable(false),
isRefreshing = $bindable(false),
selectedSchemaKey = $bindable(undefined),
selectedTableKey = $bindable(undefined),
dbSelector,
@@ -67,56 +59,39 @@
}
}
// `refreshCount` is a derived state. `refreshing` is the source of truth
let refreshCount = $state(0)
$effect(() => {
if (refreshing) untrack(() => (refreshCount += 1))
})
let refreshing = $state(false)
$effect(() => {
if (refreshing) getSchema()
})
// Sync refreshing state with bindable prop
$effect(() => {
isRefreshing = refreshing
})
export const refresh = () => !refreshing && (refreshing = true)
// Initial schema load
$effect(() => {
if (input) {
untrack(() => getSchema())
let colDefs = resource(
() => [input],
async () => {
if (!input) return
return await loadAllTablesMetaData($workspaceStore, input)
}
})
async function getSchema() {
if (!input) return
const dbSchemasPath = getDbSchemasPath(input)
if ($dbSchemas[dbSchemasPath] && !refreshing) return
const oldDbSchema = $dbSchemas[dbSchemasPath]
if (input.type == 'database') {
await getDbSchemas(
input.resourceType,
input.resourcePath,
$workspaceStore,
$dbSchemas,
(message: string) => {
sendUserToast(message, true)
}
)
} else if (input.type == 'ducklake') {
$dbSchemas[dbSchemasPath] = await getDucklakeSchema({
workspace: $workspaceStore!,
ducklake: input.ducklake
})
)
let dbSchemasPromise = resource(
() => [input],
async () => {
if (!input) return
const dbSchemasPath = getDbSchemasPath(input)
if (input.type == 'database') {
$dbSchemas[dbSchemasPath] = await getDbSchemas(
input.resourceType,
input.resourcePath,
$workspaceStore,
(message: string) => sendUserToast(message, true)
)
} else if (input.type == 'ducklake') {
$dbSchemas[dbSchemasPath] = await getDucklakeSchema({
workspace: $workspaceStore!,
ducklake: input.ducklake
})
}
}
// avoid infinite loop on error due to the way getDbSchemas is implemented
// and relying on an assignement side effect
if (oldDbSchema !== $dbSchemas[dbSchemasPath]) $dbSchemas = $dbSchemas
refreshing = false
)
export const refresh = () => {
colDefs.refetch()
dbSchemasPromise.refetch()
}
export function isLoading() {
return colDefs.loading || dbSchemasPromise.loading
}
let replPanelSize = $state(36)
@@ -129,30 +104,6 @@
hasReplResult = !!replResultData
})
let cachedColDefs: Record<string, TableMetadata> = {}
let cachedLastRefreshCount = 0
async function getColDefs(tableKey: string): Promise<TableMetadata> {
if (cachedLastRefreshCount !== refreshCount) cachedColDefs = {}
cachedLastRefreshCount = refreshCount
if (cachedColDefs[tableKey]) return cachedColDefs[tableKey]
if (!input) return []
try {
cachedColDefs = (await loadAllTablesMetaData($workspaceStore, input)) ?? cachedColDefs
return cachedColDefs[tableKey]
} catch (e) {
if (input?.type == 'ducklake')
throw 'Impossible that loadAllTablesMetaData fails for Ducklake'
// Query is not implemented for all dbs, need a fallback
const result = await loadTableMetaData(input, $workspaceStore, tableKey)
if (result) cachedColDefs[tableKey] = result
return result ?? []
}
}
// Export for parent components
export function clearReplResult() {
replResultData = undefined
@@ -197,7 +148,7 @@
<DbManager
dbSupportsSchemas={input?.type == 'database' && dbSupportsSchemas(input.resourceType)}
{dbSchema}
{getColDefs}
colDefs={colDefs.current}
dbTableOpsFactory={({ colDefs, tableKey }) =>
dbTableOpsWithPreviewScripts({
colDefs,
@@ -85,7 +85,6 @@
let dbManagerContent: DBManagerContent | undefined = $state()
let hasReplResult = $state(false)
let isRefreshing = $state(false)
</script>
<svelte:window bind:innerWidth={windowWidth} />
@@ -108,15 +107,11 @@
}}
CloseIcon={hasReplResult ? ArrowLeft : undefined}
noPadding
id="db-manager-drawer"
>
{#if effectiveInput && $workspaceStore}
{#key selectedDatatable}
<DBManagerContent
bind:this={dbManagerContent}
input={effectiveInput}
bind:hasReplResult
bind:isRefreshing
>
<DBManagerContent bind:this={dbManagerContent} input={effectiveInput} bind:hasReplResult>
{#snippet dbSelector()}
{#if isDatatableInput}
{#if datatables.loading}
@@ -140,7 +135,7 @@
{/if}
{#snippet actions()}
<Button
loading={isRefreshing}
loading={dbManagerContent?.isLoading() ?? false}
on:click={() => dbManagerContent?.refresh()}
startIcon={{ icon: RefreshCcw }}
size="xs"
@@ -39,20 +39,15 @@
</script>
<script lang="ts">
import { ArrowRight, ClipboardCopy, Info, Plus, Settings, X } from 'lucide-svelte'
import { ArrowRight, ClipboardCopy, Plus, Settings, X } from 'lucide-svelte'
import { Button } from './common'
import { Cell } from './table'
import DataTable from './table/DataTable.svelte'
import Head from './table/Head.svelte'
import {
datatypeHasLength,
dbSupportsSchemas,
type DbFeatures
} from './apps/components/display/dbtable/utils'
import { datatypeHasLength } from './apps/components/display/dbtable/utils'
import { DB_TYPES } from '$lib/consts'
import Popover from './meltComponents/Popover.svelte'
import Tooltip from './meltComponents/Tooltip.svelte'
import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte'
import { sendUserToast } from '$lib/toast'
import { copyToClipboard } from '$lib/utils'
@@ -70,6 +65,8 @@
datatypeDefaultLength
} from './apps/components/display/dbtable/tableEditor'
import Alert from './common/alert/Alert.svelte'
import type { DbFeatures } from './apps/components/display/dbtable/dbFeatures'
import Label from './Label.svelte'
type Props = {
dbType: DbType
@@ -122,8 +119,7 @@
...(datatypeHasLength(defaultColumnType) && {
datatype_length: datatypeDefaultLength(defaultColumnType)
}),
...(primaryKey && { primaryKey }),
...(!features?.defaultToNotNull && { nullable: true })
...(primaryKey && { primaryKey })
})
}
if (!initialValues) {
@@ -158,7 +154,7 @@
/>
</label>
<div class="flex flex-col">
<div class="flex flex-col" id="columns-section">
<!-- svelte-ignore a11y_label_has_associated_control -->
<label>Columns</label>
<DataTable>
@@ -222,11 +218,19 @@
</Cell>
<Cell last class="flex items-center mt-1.5">
{#if features?.primaryKeys}
<input type="checkbox" class="!w-4 !h-4" bind:checked={column.primaryKey} />
<input
type="checkbox"
class="!w-4 !h-4 primary-key-checkbox"
bind:checked={column.primaryKey}
/>
{/if}
<Popover class="ml-8" contentClasses="py-3 px-5 flex flex-col gap-6">
<Popover
class="ml-8"
contentClasses="py-3 px-5 flex flex-col gap-6"
enableFlyTransition
>
{#snippet trigger()}
<Settings size={18} />
<Settings size={18} class="settings-menu-btn" />
{/snippet}
{#snippet content()}
{#if datatypeHasLength(column.datatype)}
@@ -236,27 +240,26 @@
</label>
{/if}
{#if features?.defaultValues}
<label class="text-xs">
<span class="flex gap-1 mb-1">
Default value
<Tooltip>
<Info size={14} />
{#snippet text()}
Surround your expressions with curly brackets:
<code>
{'{NOW()}'}
</code>.
<br />
By default, it will be parsed as a literal
{/snippet}
</Tooltip>
</span>
<input type="text" placeholder="NULL" bind:value={column.defaultValue} />
</label>
<Label
class="flex gap-1 mb-1"
label="Default Value"
tooltip="Parsed as literal by default. Use curly brackets for expressions (e.g. {'{NOW()}'} )."
>
<input
class="default-value"
type="text"
placeholder="NULL"
bind:value={column.defaultValue}
/>
</Label>
{/if}
{#if !column.primaryKey}
<label class="flex gap-2 items-center text-xs">
<input type="checkbox" class="!w-4 !h-4" bind:checked={column.nullable} />
<input
type="checkbox"
class="nullable-checkbox !w-4 !h-4"
bind:checked={column.nullable}
/>
Nullable
</label>
{/if}
@@ -266,7 +269,7 @@
color="light"
startIcon={{ icon: X }}
wrapperClasses="w-fit ml-2"
btnClasses="p-0"
btnClasses="delete-column-btn p-0"
on:click={() => values.columns.splice(i, 1)}
/>
</Cell>
@@ -288,7 +291,7 @@
</DataTable>
</div>
{#if features?.foreignKeys}
<div class="flex flex-col">
<div class="flex flex-col" id="foreign-keys-section">
<!-- svelte-ignore a11y_label_has_associated_control -->
<label>Foreign Keys</label>
<DataTable>
@@ -304,14 +307,15 @@
<tr>
<Cell first class="flex">
<Select
inputClass={twMerge('!w-48')}
inputClass={twMerge('fk-table-select !w-48')}
error={fkErrors?.emptyTarget}
placeholder=""
bind:value={foreignKey.targetTable}
items={getFlatTableNamesFromSchema(dbSchema).map((o) => ({
value: o,
label:
(currentSchema && o.startsWith(currentSchema)) || !dbSupportsSchemas(dbType)
!features?.schemas ||
(currentSchema && o.toLowerCase().startsWith(currentSchema.toLowerCase()))
? o.split('.')[1]
: o
}))}
@@ -326,7 +330,7 @@
from x-overflowing -->
<div class="grow h-[2rem] relative">
<Select
class="!absolute inset-0"
class="fk-source-col-select !absolute inset-0"
error={fkErrors?.nonExistingSourceColumns.includes(
column.sourceColumn
)}
@@ -341,7 +345,7 @@
<ArrowRight size={16} class="h-fit shrink-0" />
<div class="grow h-[2rem] relative">
<Select
class="!absolute inset-0"
class="fk-target-col-select !absolute inset-0"
error={fkErrors?.nonExistingTargetColumns.includes(
column.targetColumn
)}
@@ -357,26 +361,35 @@
</div>
</div>
<div class="ml-auto flex">
{#if columnIndex === 0}
<Popover contentClasses="py-3 px-5 w-52 flex flex-col gap-6">
{#if columnIndex === 0 && features.enforcedForeignKeys}
<Popover
contentClasses="py-3 px-5 w-52 flex flex-col gap-4"
enableFlyTransition
>
{#snippet trigger()}
<Settings size={18} />
<Settings class="fk-settings-btn" size={18} />
{/snippet}
{#snippet content()}
<span>
ON DELETE <select bind:value={foreignKey.onDelete}>
<option value="NO ACTION" selected>NO ACTION</option>
<option value="CASCADE" selected>CASCADE</option>
<option value="SET NULL" selected>SET NULL</option>
<Label label="On delete">
<select
class="fk-on-delete-select"
bind:value={foreignKey.onDelete}
>
<option value="NO ACTION" selected>No action</option>
<option value="CASCADE" selected>Cascade</option>
<option value="SET NULL" selected>Set null</option>
</select>
</span>
<span>
ON UPDATE <select bind:value={foreignKey.onUpdate}>
<option value="NO ACTION" selected>NO ACTION</option>
<option value="CASCADE" selected>CASCADE</option>
<option value="SET NULL" selected>SET NULL</option>
</Label>
<Label label="On update">
<select
class="fk-on-update-select"
bind:value={foreignKey.onUpdate}
>
<option value="NO ACTION" selected>No action</option>
<option value="CASCADE" selected>Cascade</option>
<option value="SET NULL" selected>Set null</option>
</select>
</span>
</Label>
{/snippet}
</Popover>
{/if}
@@ -384,7 +397,7 @@
color="light"
startIcon={{ icon: X }}
wrapperClasses="w-fit ml-2"
btnClasses="p-0"
btnClasses="fk-delete-btn p-0"
on:click={foreignKey.columns.length > 1
? () => foreignKey.columns.splice(columnIndex, 1)
: () => values.foreignKeys.splice(foreignKeyIndex, 1)}
@@ -454,6 +467,7 @@
<Portal>
<ConfirmationModal
id="db-table-editor-confirmation-modal"
{...askingForConfirmation ?? { confirmationText: '', title: '' }}
on:canceled={() => (askingForConfirmation = undefined)}
on:confirmed={askingForConfirmation?.onConfirm ?? (() => {})}
@@ -35,6 +35,7 @@
customWidth?: number | undefined
customMenu?: boolean
class?: string | undefined
btnId?: string | undefined
enableFlyTransition?: boolean
size?: ButtonType.UnifiedSize
btnText?: string
@@ -60,6 +61,7 @@
enableFlyTransition = false,
size = 'md',
btnText = '',
btnId = undefined,
buttonReplacement,
menu,
maxHeight = undefined
@@ -142,6 +144,7 @@
}
}
}}
id={btnId}
data-menu
>
{#if buttonReplacement}
+6 -9
View File
@@ -591,15 +591,12 @@
const resourcePath = newSchemaRes.replace('$res:', '')
dbSchema = $dbSchemas[resourcePath]
if (dbSchema === undefined) {
if (lang === 'graphql') {
await getDbSchemas('graphql', resourcePath, $workspaceStore, $dbSchemas, (e) => {
console.error('error getting graphql db schema', e)
})
} else if (lang === 'sql') {
await getDbSchemas(scriptLang ?? '', resourcePath, $workspaceStore, $dbSchemas, (e) => {
console.error(`error getting SQL (${scriptLang}) db schema`, e)
})
}
$dbSchemas[resourcePath] = await getDbSchemas(
lang === 'graphql' ? 'graphql' : (scriptLang ?? ''),
resourcePath,
$workspaceStore,
(e) => console.error(`error getting ${lang} (${scriptLang}) db schema`, e)
)
}
dbSchema = $dbSchemas[resourcePath]
} else {
@@ -210,6 +210,7 @@
inputClass={selectInputClass}
placeholder={placeholder ?? `${resourceType ?? 'any'} resource`}
itemLabelWrapperClasses="flex-1"
id="resource-picker-select"
>
{#snippet endSnippet({ item, close })}
<Button
@@ -270,6 +270,7 @@
<div class="flex gap-2 items-start flex-wrap justify-between mt-2 md:mt-6">
<div class="flex-row-reverse flex-wrap flex w-full gap-4">
<Button
id="run-form-run-button"
{loading}
variant="accent"
unifiedSize="md"
+5 -1
View File
@@ -45,6 +45,7 @@
import { type ToastAction } from '$lib/toast'
import { processMessage } from './toast'
import { onDestroy, untrack } from 'svelte'
import { twMerge } from 'tailwind-merge'
interface Props {
message: string
@@ -90,7 +91,10 @@
<!-- svelte-ignore a11y_no_static_element_interactions -->
<div
class="pointer-events-auto w-full max-w-sm overflow-hidden bg-surface-tertiary drop-shadow-base shadow-lg ring-1 ring-black ring-opacity-5 border rounded-md"
class={twMerge(
'pointer-events-auto w-full max-w-sm overflow-hidden bg-surface-tertiary drop-shadow-base shadow-lg ring-1 ring-black ring-opacity-5 border rounded-md',
error ? 'toast-error' : 'toast-success'
)}
onmouseenter={() => state && (state.hover = true)}
onmouseleave={() => state && (state.hover = false)}
>
@@ -45,7 +45,7 @@
</Button>
<Drawer bind:this={insertDrawer} size="800px">
<DrawerContent title="Insert row" on:close={insertDrawer.closeDrawer}>
<DrawerContent title="Insert row" on:close={insertDrawer.closeDrawer} id="insert-row-drawer">
{#snippet actions()}
<Button
variant="accent"
@@ -233,11 +233,11 @@
ducklake: dbPath.split('ducklake://')[1]
})
} else {
await getDbSchemas(
const resourcePath = dbPath.split('$res:')[1]
dbSchemas[resourcePath] = await getDbSchemas(
resolvedConfig?.type?.selected,
dbPath.split('$res:')[1],
resourcePath,
$workspaceStore,
dbSchemas,
() => {},
{ useLegacyScripts: true }
)
@@ -0,0 +1,47 @@
// Separate file to import in both frontend and e2e
import type { DbInput } from '../../../../dbTypes'
import { dbSupportsSchemas } from './utils'
export type DbFeatures = {
foreignKeys?: boolean
enforcedForeignKeys?: boolean
primaryKeys?: boolean
defaultValues?: boolean
schemas?: boolean
}
export function getDbFeatures(dbInput: DbInput): Required<DbFeatures> {
const def: Required<DbFeatures> = {
foreignKeys: true,
primaryKeys: true,
defaultValues: true,
enforcedForeignKeys: true,
schemas: dbInput.type !== 'ducklake' && dbSupportsSchemas(dbInput.resourceType)
}
if (dbInput.type == 'ducklake')
return {
...def,
foreignKeys: false,
enforcedForeignKeys: false,
primaryKeys: false
}
if (dbInput.resourceType == 'bigquery')
return {
...def,
foreignKeys: false,
primaryKeys: true,
defaultValues: false,
enforcedForeignKeys: false
}
if (dbInput.resourceType == 'snowflake')
return {
...def,
enforcedForeignKeys: false
}
return { ...def }
}
@@ -1,4 +1,4 @@
import { JobService, ResourceService } from '$lib/gen'
import { JobService, ResourceService, type ScriptLang } from '$lib/gen'
import { runScriptAndPollResult } from '$lib/components/jobs/utils'
import type { DbInput } from '$lib/components/dbTypes'
@@ -11,9 +11,8 @@ import {
} from './utils'
import { type Preview } from '$lib/gen'
import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores'
import type { DBSchema, GraphqlSchema, SQLSchema } from '$lib/stores'
import { tryEvery } from '$lib/utils'
import { stringifySchema } from '$lib/components/copilot/lib'
import type { DbType } from '$lib/components/dbTypes'
import { getDatabaseArg } from '$lib/components/dbOps'
@@ -25,16 +24,11 @@ export async function loadTableMetaData(
): Promise<TableMetadata | undefined> {
if (!input || !table || !workspace) return undefined
let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
let content = await makeLoadTableMetaDataQuery(input, workspace, table)
let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, table)
const job = await JobService.runScriptPreview({
workspace: workspace,
requestBody: {
language,
content,
args: getDatabaseArg(input)
}
requestBody: { language, content: query, args: getDatabaseArg(input) }
})
const maxRetries = 8
@@ -51,7 +45,19 @@ export async function loadTableMetaData(
if (testResult.success) {
attempts = maxRetries
return testResult.result.map(lowercaseKeys)
const result = testResult.result.map(lowercaseKeys)
// For Snowflake, fetch primary keys separately
if (
input.type === 'database' &&
(input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth')
) {
const map: Record<string, TableMetadata> = { [table]: result }
await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace, table)
return map[table]
}
return result
} else {
attempts++
}
@@ -70,29 +76,22 @@ export async function loadAllTablesMetaData(
): Promise<Record<string, TableMetadata> | undefined> {
if (!input || !workspace) return undefined
let language = input.type == 'ducklake' ? 'duckdb' : getLanguageByResourceType(input.resourceType)
try {
let { language, query } = await makeLoadTableMetaDataQuery(input, workspace, undefined)
let result = (await runScriptAndPollResult({
workspace: workspace,
requestBody: {
language,
content: await makeLoadTableMetaDataQuery(input, workspace, undefined),
args: getDatabaseArg(input)
}
requestBody: { language, content: query, args: getDatabaseArg(input) }
})) as ({ table_name: string; schema_name?: string } & object)[]
const map: Record<string, TableMetadata> = {}
for (const _col of result) {
const col = lowercaseKeys(_col)
const tableKey = col.schema_name ? `${col.schema_name}.${col.table_name}` : col.table_name
if (!(tableKey in map)) {
map[tableKey] = []
}
map[tableKey] ??= []
map[tableKey].push(col)
}
fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace)
await fetchAndAddSnowflakePrimaryKeysInMap(map, input, workspace)
return map
} catch (e) {
@@ -104,9 +103,9 @@ async function makeLoadTableMetaDataQuery(
input: DbInput,
workspace: string,
table: string | undefined
): Promise<string> {
): Promise<{ query: string; language: ScriptLang }> {
if (input.type === 'ducklake') {
return `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
const query = `ATTACH 'ducklake://${input.ducklake}' AS __ducklake__;
SELECT
COLUMN_NAME as field,
DATA_TYPE as DataType,
@@ -118,12 +117,13 @@ async function makeLoadTableMetaDataQuery(
TABLE_NAME as table_name
FROM information_schema.columns c
WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()`
return { query, language: 'duckdb' }
} else if (input.resourceType === 'mysql') {
const resourceObj = (await ResourceService.getResourceValue({
workspace,
path: input.resourcePath
})) as any
return `
const query = `
SELECT
COLUMN_NAME as field,
COLUMN_TYPE as DataType,
@@ -153,8 +153,9 @@ async function makeLoadTableMetaDataQuery(
TABLE_NAME,
ORDINAL_POSITION;
`
return { query, language: 'mysql' }
} else if (input.resourceType === 'postgresql') {
return `
const query = `
SELECT
a.attname as field,
pg_catalog.format_type(a.atttypid, a.atttypmod) as DataType,
@@ -200,8 +201,9 @@ async function makeLoadTableMetaDataQuery(
ORDER BY ${table ? 'a.attnum' : 'ns.nspname, c.relname, a.attnum'};
`
return { query, language: 'postgresql' }
} else if (input.resourceType === 'ms_sql_server') {
return `
const query = `
SELECT
c.COLUMN_NAME as field,
c.DATA_TYPE as DataType,
@@ -245,11 +247,12 @@ WHERE
ORDER BY
c.ORDINAL_POSITION;
`
return { query, language: 'mssql' }
} else if (
input.resourceType === 'snowflake' ||
(input.resourceType as any) === 'snowflake_oauth'
) {
return `
const query = `
select COLUMN_NAME as field,
DATA_TYPE as DataType,
COLUMN_DEFAULT as DefaultValue,
@@ -273,10 +276,10 @@ ORDER BY
}
order by ORDINAL_POSITION;
`
return { query, language: 'snowflake' }
} else if (input.resourceType === 'bigquery') {
// TODO: find a solution for this (query uses hardcoded dataset name)
if (!table) throw new Error('Table name is required for BigQuery')
return `SELECT
if (table) {
const query = `SELECT
c.COLUMN_NAME as field,
DATA_TYPE as DataType,
CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
@@ -292,6 +295,40 @@ FROM
WHERE
c.TABLE_NAME = '${table.split('.')[1]}'
order by c.ORDINAL_POSITION;`
return { query, language: 'bigquery' }
} else {
const query = `import { BigQuery } from '@google-cloud/bigquery@7.5.0';
export async function main(database: bigquery) {
const bq = new BigQuery({
credentials: database
})
const [datasets] = await bq.getDatasets();
if (!datasets) return {}
const schema = {} as any
let queries = datasets.map(dataset => \`
(SELECT
c.COLUMN_NAME as field,
'\${dataset.id}' as schema_name,
c.TABLE_NAME as table_name,
DATA_TYPE as DataType,
CASE WHEN COLUMN_DEFAULT = 'NULL' THEN '' ELSE COLUMN_DEFAULT END as DefaultValue,
CASE WHEN constraint_name is not null THEN true ELSE false END as IsPrimaryKey,
'No' as IsIdentity,
IS_NULLABLE as IsNullable,
false as IsEnum
FROM
\\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS c
LEFT JOIN
\\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.KEY_COLUMN_USAGE p
on c.table_name = p.table_name AND c.column_name = p.COLUMN_NAME
ORDER BY c.ORDINAL_POSITION)\`
)
let query = queries.join('\\nUNION ALL \\n')
const [rows] = await bq.query(query)
return rows
}`
return { query, language: 'bun' }
}
} else {
throw new Error('Unsupported database type:' + input.resourceType)
}
@@ -311,14 +348,23 @@ async function fetchAndAddSnowflakePrimaryKeysInMap(
workspace: string,
tableKey?: string
) {
if (input.type == 'database' && input.resourceType === 'snowflake') {
if (
input.type == 'database' &&
(input.resourceType === 'snowflake' || (input.resourceType as any) === 'snowflake_oauth')
) {
let pkResult = await fetchSnowflakePrimaryKeys(workspace, getDatabaseArg(input), tableKey)
for (const pk of pkResult) {
const tableKey = `${pk.schema_name}.${pk.table_name}`.toUpperCase()
if (tableKey in map) {
for (const col of map[tableKey]) {
if (col.field.toLowerCase() === pk.column_name.toLowerCase()) {
col.isprimarykey = true
const pkTableKey = `${pk.schema_name}.${pk.table_name}`.toUpperCase()
// Also check the original casing and the provided tableKey
const keysToCheck = [pkTableKey, `${pk.schema_name}.${pk.table_name}`, tableKey].filter(
Boolean
) as string[]
for (const key of keysToCheck) {
if (key in map) {
for (const col of map[key]) {
if (col.field.toLowerCase() === pk.column_name.toLowerCase()) {
col.isprimarykey = true
}
}
}
}
@@ -336,7 +382,7 @@ async function fetchSnowflakePrimaryKeys(
requestBody: {
language: 'snowflake',
args: dbArg,
content: tableKey ? `SHOW PRIMARY KEYS IN TABLE ${tableKey}` : 'SHOW PRIMARY KEYS'
content: tableKey ? `SHOW PRIMARY KEYS IN TABLE ${tableKey}` : 'SHOW PRIMARY KEYS IN ACCOUNT'
}
})) as SnowflakeShowPrimaryKeysResult[]
}
@@ -353,25 +399,20 @@ export async function getDbSchemas(
resourceType: string,
resourcePath: string,
workspace: string | undefined,
dbSchemas: DBSchemas,
errorCallback: (message: string) => void,
options: {
useLegacyScripts?: boolean // To avoid breaking app policies
} = {}
): Promise<void> {
): Promise<DBSchema | undefined> {
let scripts = options.useLegacyScripts ? legacyScripts : scriptsV2
let sqlScript = scripts[getLanguageByResourceType(resourceType)]
if (!sqlScript) return
if (!resourceType || !resourcePath || !workspace || !sqlScript) return
return new Promise(async (resolve, reject) => {
if (!resourceType || !resourcePath || !workspace) {
resolve()
return
}
const job = await JobService.runScriptPreview({
workspace: workspace,
let result: unknown
try {
result = await JobService.runScriptPreviewAndWaitResult({
workspace,
requestBody: {
language: sqlScript.lang as Preview['language'],
content: sqlScript.code,
@@ -382,91 +423,43 @@ export async function getDbSchemas(
}
}
})
} catch (e) {
console.error(e)
errorCallback('Error fetching schema: ' + ((e as Error)?.message || e))
return
}
tryEvery({
tryCode: async () => {
if (resourcePath) {
const testResult = await JobService.getCompletedJob({
workspace,
id: job
})
if (!testResult.success) {
console.error(testResult.result?.['error']?.['message'])
} else {
if (testResult.result === 'WINDMILL_TOO_BIG') {
console.info('Result is too big, fetching result separately')
const data = await JobService.getCompletedJobResult({
workspace,
id: job
})
testResult.result = data
}
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = sqlScript
let schema: any
try {
schema =
processingFn !== undefined ? processingFn(testResult.result) : testResult.result
} catch (e) {
console.error(e)
errorCallback('Error processing schema')
resolve()
return
}
const dbSchema = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
dbSchemas[resourcePath] = {
...dbSchema,
stringified: stringifySchema(dbSchema)
}
} else {
if (
typeof testResult.result !== 'object' ||
!('__schema' in (testResult?.result ?? {}))
) {
console.error('Invalid GraphQL schema')
errorCallback('Invalid GraphQL schema')
} else {
const dbSchema = {
lang: 'graphql' as GraphqlSchema['lang'],
schema: testResult.result
}
dbSchemas[resourcePath] = {
...(dbSchema as any),
stringified: stringifySchema(dbSchema as any)
}
}
}
}
}
resolve()
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = sqlScript
let schema: any
try {
schema = processingFn !== undefined ? processingFn(result) : result
} catch (e) {
console.error(e)
errorCallback('Error processing schema')
return
}
const dbSchema = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
return { ...dbSchema, stringified: stringifySchema(dbSchema) }
} else {
if (typeof result !== 'object' || !('__schema' in (result ?? {}))) {
console.error('Invalid GraphQL schema')
errorCallback('Invalid GraphQL schema')
return
} else {
const dbSchema = {
lang: 'graphql' as GraphqlSchema['lang'],
schema: result
}
},
timeoutCode: async () => {
console.error('Could not query schema within 5s')
errorCallback('Could not query schema within 5s')
try {
await JobService.cancelQueuedJob({
workspace,
id: job,
requestBody: {
reason: 'Could not query schema within 5s'
}
})
} catch (err) {
console.error(err)
}
reject()
},
interval: 500,
timeout: 5000
})
})
return { ...(dbSchema as any), stringified: stringifySchema(dbSchema as any) }
}
}
}
}
export async function getTablesByResource(
@@ -48,7 +48,7 @@ describe('makeAlterTableQueries', () => {
'postgresql'
)
expect(normalize(queries[0])).toBe(normalize('ALTER TABLE users DROP COLUMN age;'))
expect(normalize(queries[0])).toBe(normalize('ALTER TABLE users DROP COLUMN "age";'))
})
it('renames a column', () => {
@@ -123,14 +123,23 @@ export function makeAlterTableQueries(
break
case 'renameTable':
queries.push(`ALTER TABLE ${tableRef} RENAME TO ${op.to};`)
const tableNewRef = dbType === 'snowflake' && schema ? `${schema.trim()}.${op.to}` : op.to
queries.push(`ALTER TABLE ${tableRef} RENAME TO ${tableNewRef};`)
break
case 'addPrimaryKey':
const notEnforced = dbType === 'bigquery' ? ' NOT ENFORCED' : ''
queries.push(
`ALTER TABLE ${tableRef} ADD PRIMARY KEY (${op.columns.join(', ')})${notEnforced};`
)
if (dbType === 'snowflake') {
// Snowflake requires CONSTRAINT syntax with explicit name
const constraintName = `${values.name}_pk`
queries.push(
`ALTER TABLE ${tableRef} ADD CONSTRAINT ${constraintName} PRIMARY KEY (${op.columns.join(', ')});`
)
} else {
const notEnforced = dbType === 'bigquery' ? ' NOT ENFORCED' : ''
queries.push(
`ALTER TABLE ${tableRef} ADD PRIMARY KEY (${op.columns.join(', ')})${notEnforced};`
)
}
break
case 'dropPrimaryKey':
@@ -218,7 +227,7 @@ function renderDropDefaultValue(
case 'bigquery':
return `ALTER TABLE ${tableRef} ALTER COLUMN ${columnName} DROP DEFAULT;`
case 'ms_sql_server':
return `ALTER TABLE ${tableRef} DROP CONSTRAINT ${defaultConstraintName};`
return `ALTER TABLE ${tableRef} DROP CONSTRAINT ${renderDbQuotedIdentifier(defaultConstraintName ?? '', dbType)};`
default:
throw new Error(`Unsupported database type: ${dbType}`)
}
@@ -295,7 +304,7 @@ function renderDropForeignKey(
): string {
if (dbType === 'mysql')
return `ALTER TABLE ${tableRef} DROP FOREIGN KEY ${renderDbQuotedIdentifier(fk_constraint_name, dbType)};`
return `ALTER TABLE ${tableRef} DROP CONSTRAINT ${fk_constraint_name};`
return `ALTER TABLE ${tableRef} DROP CONSTRAINT ${renderDbQuotedIdentifier(fk_constraint_name, dbType)};`
}
function renderDropPrimaryKey(
@@ -6,6 +6,8 @@ import type { TableEditorValues, TableEditorValuesColumn } from '../tableEditor'
export function makeCreateTableQuery(values: TableEditorValues, dbType: DbType, schema?: string) {
const pkCount = values.columns.reduce((p, c) => p + (c.primaryKey ? 1 : 0), 0)
if (dbType === 'snowflake') schema ??= 'PUBLIC'
function transformColumn(c: TableEditorValuesColumn): string {
const datatype = c.datatype_length ? `${c.datatype}(${c.datatype_length})` : c.datatype
const defValue = c.defaultValue && formatDefaultValue(c.defaultValue, datatype, dbType)
@@ -3,7 +3,6 @@ import { wrapDucklakeQuery } from '$lib/components/ducklake'
import { runScriptAndPollResult } from '$lib/components/jobs/utils'
import type { ScriptLang } from '$lib/gen'
import type { TableEditorForeignKey } from '../tableEditor'
import type { TableMetadata } from '../utils'
/**
* Raw foreign key result from database queries
@@ -267,32 +266,34 @@ WHERE
// Snowflake queries
function makeSnowflakeForeignKeysQuery(tableName: string, schemaName: string): string {
return `
SELECT
tc.constraint_name as fk_constraint_name,
kcu.column_name as source_column,
rc.unique_constraint_schema || '.' || ccu.table_name as target_table,
ccu.column_name as target_column,
COALESCE(rc.delete_rule, 'NO ACTION') as on_delete,
COALESCE(rc.update_rule, 'NO ACTION') as on_update
FROM
information_schema.table_constraints tc
JOIN information_schema.referential_constraints rc
ON tc.constraint_name = rc.constraint_name
AND tc.constraint_schema = rc.constraint_schema
JOIN information_schema.key_column_usage kcu
ON tc.constraint_name = kcu.constraint_name
AND tc.constraint_schema = kcu.constraint_schema
JOIN information_schema.constraint_column_usage ccu
ON rc.unique_constraint_name = ccu.constraint_name
AND rc.unique_constraint_schema = ccu.constraint_schema
WHERE
tc.constraint_type = 'FOREIGN KEY'
AND tc.table_name = '${tableName}'
AND tc.table_schema = '${schemaName}'
ORDER BY
tc.constraint_name, kcu.ordinal_position;
`.trim()
return `SHOW IMPORTED KEYS IN TABLE ${schemaName}.${tableName}`
}
/**
* Transforms Snowflake SHOW IMPORTED KEYS result to RawForeignKey format
* Snowflake returns columns: fk_database_name, fk_schema_name, fk_table_name, fk_column_name,
* pk_database_name, pk_schema_name, pk_table_name, pk_column_name, key_sequence,
* update_rule, delete_rule, fk_name, pk_name, deferrability
*/
function transformSnowflakeForeignKeys(snowflakeResults: any[]): RawForeignKey[] {
if (!snowflakeResults || !Array.isArray(snowflakeResults)) {
return []
}
return snowflakeResults.map((row) => ({
fk_constraint_name: row.fk_name || row.FK_NAME || '',
source_column: row.fk_column_name || row.FK_COLUMN_NAME || '',
target_table: `${row.pk_schema_name || row.PK_SCHEMA_NAME || ''}.${row.pk_table_name || row.PK_TABLE_NAME || ''}`,
target_column: row.pk_column_name || row.PK_COLUMN_NAME || '',
on_delete: (row.delete_rule || row.DELETE_RULE || 'NO ACTION').toUpperCase() as
| 'CASCADE'
| 'SET NULL'
| 'NO ACTION',
on_update: (row.update_rule || row.UPDATE_RULE || 'NO ACTION').toUpperCase() as
| 'CASCADE'
| 'SET NULL'
| 'NO ACTION'
}))
}
function makeSnowflakePrimaryKeyQuery(tableName: string, schemaName: string): string {
@@ -403,12 +404,10 @@ export async function fetchTableRelationalKeys(
schema: string | undefined,
workspace: string,
dbArg: Record<string, any>,
language: ScriptLang,
getColDefs: () => Promise<TableMetadata>
language: ScriptLang
): Promise<{
foreignKeys: TableEditorForeignKey[]
pk_constraint_name?: string
colDefs: TableMetadata
}> {
let fkPromise = async () => {
try {
@@ -421,17 +420,27 @@ export async function fetchTableRelationalKeys(
requestBody: { args: dbArg, content: fkQuery, language }
})
let rawForeignKeys = fkResult as RawForeignKey[]
let rawForeignKeys: RawForeignKey[]
// Snowflake returns a different format from SHOW IMPORTED KEYS
if (dbType === 'snowflake') {
rawForeignKeys = transformSnowflakeForeignKeys(fkResult as any[])
} else {
rawForeignKeys = fkResult as RawForeignKey[]
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
// Lowercase keys for consistency
rawForeignKeys = rawForeignKeys.map((fk) => {
const lowerFk: any = {}
Object.keys(fk).forEach((key) => {
lowerFk[key.toLowerCase()] = fk[key]
})
return lowerFk
})
}
}
if (rawForeignKeys && Array.isArray(rawForeignKeys)) {
// Lowercase keys for consistency
rawForeignKeys = rawForeignKeys.map((fk) => {
const lowerFk: any = {}
Object.keys(fk).forEach((key) => {
lowerFk[key.toLowerCase()] = fk[key]
})
return lowerFk
})
return transformForeignKeys(rawForeignKeys)
}
}
@@ -466,11 +475,7 @@ export async function fetchTableRelationalKeys(
}
}
const [foreignKeys, pk_constraint_name, colDefs] = await Promise.all([
fkPromise(),
pkPromise(),
getColDefs()
])
const [foreignKeys, pk_constraint_name] = await Promise.all([fkPromise(), pkPromise()])
return { foreignKeys, pk_constraint_name, colDefs }
return { foreignKeys, pk_constraint_name }
}
@@ -75,6 +75,7 @@ export function columnDefToTableEditorValuesColumn(
} else {
datatype = colDef.datatype?.replace(/\s+/g, ' ').toUpperCase() || 'UNKNOWN'
}
datatype = normalizeDatatypeAlias(datatype)
const defaultValue = colDef.defaultvalue
? colDef.defaultvalue.startsWith("'") &&
@@ -95,3 +96,15 @@ export function columnDefToTableEditorValuesColumn(
default_constraint_name: colDef.default_constraint_name
}
}
function normalizeDatatypeAlias(datatype: string): string {
// Normalize some common datatype variations
const dt = datatype.toUpperCase().trim()
if (dt === 'INTEGER') return 'INT'
if (dt === 'DOUBLE PRECISION') return 'DOUBLE'
if (dt === 'TIMESTAMP WITHOUT TIME ZONE') return 'TIMESTAMP'
if (dt === 'TIMESTAMP WITH TIME ZONE') return 'TIMESTAMPTZ'
if (dt === 'CHARACTER VARYING') return 'VARCHAR'
if (dt === 'CHARACTER') return 'CHAR'
return datatype
}
@@ -7,7 +7,7 @@ import {
type IntrospectionQuery
} from 'graphql'
import type { DbInput, DbType } from '$lib/components/dbTypes'
import type { DbType } from '$lib/components/dbTypes'
export enum ColumnIdentity {
ByDefault = 'By Default',
@@ -157,23 +157,34 @@ const bq = new BigQuery({
credentials: args
})
const [datasets] = await bq.getDatasets();
const schema = {}
for (const dataset of datasets) {
schema[dataset.id] = {}
const query = "SELECT table_name, ARRAY_AGG(STRUCT(if(is_nullable = 'YES', true, false) AS required, column_name AS name, data_type AS type, if(column_default = 'NULL', null, column_default) AS \`default\`) ORDER BY ordinal_position) AS schema \
FROM \`{dataset.id}\`.INFORMATION_SCHEMA.COLUMNS \
GROUP BY table_name".replace('{dataset.id}', dataset.id)
const [rows] = await bq.query(query)
for (const row of rows) {
schema[dataset.id][row.table_name] = {}
for (const col of row.schema) {
const colName = col.name
delete col.name
if (col.default === null) {
delete col.default
}
schema[dataset.id][row.table_name][colName] = col
if (!datasets) return {}
const schema = {} as any
let queries = datasets.map(dataset => \`
SELECT
table_name,
'\${dataset.id}' as dataset,
ARRAY_AGG(STRUCT(
if(is_nullable = 'YES', true, false) AS required,
column_name AS name,
data_type AS type,
if(column_default = 'NULL', null, column_default) AS \\\`default\\\`
) ORDER BY ordinal_position)
AS schema
FROM \\\`\${dataset.id}\\\`.INFORMATION_SCHEMA.COLUMNS
GROUP BY table_name\`
)
let query = queries.join('\\nUNION ALL \\n')
const [rows] = await bq.query(query)
for (const row of rows) {
schema[row.dataset] ??= {}
schema[row.dataset][row.table_name] = {}
for (const col of row.schema) {
const colName = col.name
delete col.name
if (col.default === null) {
delete col.default
}
schema[row.dataset][row.table_name][colName] = col
}
}
return schema
@@ -428,31 +439,3 @@ export function sqlDataTypeToJsTypeHeuristic(datatype: string): string {
return 'any'
}
}
export type DbFeatures = {
foreignKeys?: boolean
primaryKeys?: boolean
defaultValues?: boolean
defaultToNotNull?: boolean
}
export function getDbFeatures(dbInput: DbInput): Required<DbFeatures> {
const def: Required<DbFeatures> = {
foreignKeys: true,
primaryKeys: true,
defaultValues: true,
defaultToNotNull: true
}
if (dbInput.type == 'ducklake') return { ...def, foreignKeys: false, primaryKeys: false }
if (dbInput.resourceType == 'bigquery')
return {
foreignKeys: false,
primaryKeys: true,
defaultValues: false,
defaultToNotNull: false
}
return { ...def }
}
@@ -9,10 +9,11 @@
small?: boolean
Icon?: any | undefined
class?: string
id?: string | undefined
onClick?: () => void | undefined | any
}
let { noBg = false, small = false, Icon, class: className, onClick }: Props = $props()
let { noBg = false, small = false, Icon, class: className, id, onClick }: Props = $props()
const dispatch = createEventDispatcher()
</script>
@@ -20,6 +21,7 @@
<Button
on:click={() => (dispatch('close'), onClick?.())}
on:pointerdown={(e) => e.stopPropagation()}
{id}
startIcon={{ icon: Icon ?? X }}
iconOnly
unifiedSize="sm"
@@ -100,7 +100,7 @@
}
let {
id = '',
id,
aiId = undefined,
aiDescription = undefined,
size = 'md',
@@ -14,6 +14,7 @@
open?: boolean
type?: 'danger' | 'reload'
showIcon?: boolean
id?: string
children?: Snippet
onConfirmed?: () => void | Promise<void>
onCanceled?: () => void
@@ -27,6 +28,7 @@
open = false,
type: _type,
showIcon = true,
id,
children,
onConfirmed,
onCanceled
@@ -84,6 +86,7 @@
transition:fadeFast|local
class={'fixed top-0 bottom-0 left-0 right-0 z-[5000]'}
role="dialog"
{id}
>
<div
class={classNames(
@@ -4,6 +4,7 @@
import Disposable from './Disposable.svelte'
import ConditionalPortal from './ConditionalPortal.svelte'
import { chatState } from '$lib/components/copilot/chat/sharedChatState.svelte'
import { useReducedMotion } from '$lib/svelte5Utils.svelte'
interface Props {
open?: boolean
@@ -23,7 +24,7 @@
let {
open = $bindable(undefined),
duration = 0.3,
duration: _duration = 0.3,
placement = 'right',
size = '600px',
alwaysOpen = false,
@@ -43,6 +44,8 @@
let disposable: Disposable | undefined = $state(undefined)
let reducedMotion = useReducedMotion()
let duration = $derived(reducedMotion.val ? 0 : _duration)
let durationMs = $derived(duration * 1000)
export function toggleDrawer() {
@@ -19,6 +19,7 @@
CloseIcon?: any | undefined
fullScreen?: boolean
eeOnly?: boolean
id?: string | undefined
actions?: import('svelte').Snippet
titleExtra?: import('svelte').Snippet
children?: import('svelte').Snippet
@@ -36,6 +37,7 @@
CloseIcon = undefined,
fullScreen = true,
eeOnly = false,
id,
actions,
titleExtra,
children
@@ -44,7 +46,10 @@
const dispatch = createEventDispatcher()
</script>
<div class={classNames('flex flex-col divide-y', fullScreen ? 'h-screen max-h-screen' : 'h-full')}>
<div
class={classNames('flex flex-col divide-y', fullScreen ? 'h-screen max-h-screen' : 'h-full')}
{id}
>
<div class="flex justify-between w-full items-center pl-2 pr-4 py-2 gap-2">
<div class="flex items-center gap-2 w-full truncate">
<div
@@ -56,7 +61,7 @@
}
}}
>
<CloseButton on:close Icon={CloseIcon} />
<CloseButton on:close Icon={CloseIcon} id="{id}-close-btn" />
</div>
<span class="font-semibold text-emphasis truncate text-lg max-w-sm"
>{title ?? ''}
@@ -91,6 +91,7 @@
<slot name="header-right" />
<div class="w-8">
<button
id="modal-close-button"
on:click={() => {
close()
}}
@@ -444,16 +444,16 @@ export function createDbSchemaTool<T>(): Tool<T> {
workspace: workspace,
path: args.resourcePath
})
const newDbSchemas = {}
await getDbSchemas(
resource.resource_type,
args.resourcePath,
workspace,
newDbSchemas,
(error) => {
console.error(error)
}
)
const newDbSchemas = {
[args.resourcePath]: await getDbSchemas(
resource.resource_type,
args.resourcePath,
workspace,
(error) => {
console.error(error)
}
)
}
dbSchemas.update((schemas) => ({ ...schemas, ...newDbSchemas }))
const dbs = get(dbSchemas)
const db = dbs[args.resourcePath]
+4 -5
View File
@@ -133,7 +133,7 @@ export type IDbSchemaOps = {
onFetchTableEditorDefinition: (params: {
table: string
schema?: string
getColDefs: () => Promise<TableMetadata>
colDefs: TableMetadata
}) => Promise<TableEditorValues>
}
@@ -192,16 +192,15 @@ export function dbSchemaOpsWithPreviewScripts({
requestBody: { args: { ...dbArg }, language, content: dropSchemaQuery }
})
},
onFetchTableEditorDefinition: async ({ table, schema, getColDefs }) => {
let { foreignKeys, pk_constraint_name, colDefs } = await fetchTableRelationalKeys(
onFetchTableEditorDefinition: async ({ table, schema, colDefs }) => {
let { foreignKeys, pk_constraint_name } = await fetchTableRelationalKeys(
input,
dbType,
table,
schema,
workspace,
dbArg,
language,
getColDefs
language
)
return buildTableEditorValues({
@@ -410,7 +410,8 @@
<TextInput
inputProps={{
autofocus: true,
placeholder: HOME_SEARCH_PLACEHOLDER
placeholder: HOME_SEARCH_PLACEHOLDER,
id: 'home-search-input'
}}
size="md"
bind:value={filter}
@@ -28,7 +28,6 @@
// For DB manager
let dbManagerContent: DBManagerContent | undefined = $state()
let hasReplResult = $state(false)
let isRefreshing = $state(false)
let windowWidth = $state(window.innerWidth)
let expand = $state(false)
@@ -168,7 +167,6 @@
bind:this={dbManagerContent}
input={dbInput}
bind:hasReplResult
bind:isRefreshing
bind:selectedSchemaKey
bind:selectedTableKey
multiSelectMode={true}
@@ -216,7 +214,7 @@
</Button>
<Button
loading={isRefreshing}
loading={dbManagerContent?.isLoading() ?? false}
on:click={() => dbManagerContent?.refresh()}
startIcon={{ icon: RefreshCcw }}
size="xs"
@@ -37,7 +37,7 @@ export function createSchemasResource(getDatatable: () => string | undefined) {
if (!dbSchema) {
try {
await getDbSchemas('postgresql', resourcePath, workspace, schemas, (msg) =>
schemas[resourcePath] = await getDbSchemas('postgresql', resourcePath, workspace, (msg) =>
console.error('Schema error:', msg)
)
dbSchema = get(dbSchemas)[resourcePath]
@@ -5,6 +5,8 @@
import type { ProcessedItem } from './utils.svelte'
import { twMerge } from 'tailwind-merge'
import { PlusIcon } from 'lucide-svelte'
import { useReducedMotion } from '$lib/svelte5Utils.svelte'
import { watch } from 'runed'
let {
processedItems: _processedItems,
@@ -58,6 +60,7 @@
let listEl: HTMLDivElement | undefined = $state()
let dropdownPos = $state(computeDropdownPos())
let keyArrowPos = $state<number | undefined>()
let reducedMotion = useReducedMotion()
function computeDropdownPos(): {
width: number
@@ -100,32 +103,42 @@
// Also CSS transitions are smoother because they do not rely on JS / animation frames
let uiState = $state({ domExists: open, visible: open, timeout: null as number | null })
let initial = true
$effect(() => {
let isOpen = open && !disabled
untrack(() => {
if (initial) {
initial = false
return
}
if (uiState.timeout) clearTimeout(uiState.timeout)
uiState = {
domExists: true,
visible: !isOpen,
timeout: setTimeout(() => {
if (isOpen) {
uiState.visible = true
uiState.timeout = null
} else if (!isOpen) {
uiState.visible = false
uiState.timeout = setTimeout(() => {
uiState.domExists = false
uiState.timeout = null
}, 500) // leave time for transition to finish
watch(
() => open && !disabled,
(isOpen) => {
untrack(() => {
if (initial) {
initial = false
return
}
if (reducedMotion.val) {
uiState = {
domExists: open && !disabled,
visible: open && !disabled,
timeout: null
}
}, 0) // We need the height to be 0 then change immediately for the transition to play
}
})
})
return
}
if (uiState.timeout) clearTimeout(uiState.timeout)
uiState = {
domExists: true,
visible: !isOpen,
timeout: setTimeout(() => {
if (isOpen) {
uiState.visible = true
uiState.timeout = null
} else if (!isOpen) {
uiState.visible = false
uiState.timeout = setTimeout(() => {
uiState.domExists = false
uiState.timeout = null
}, 500) // leave time for transition to finish
}
}, 0) // We need the height to be 0 then change immediately for the transition to play
}
})
}
)
</script>
<svelte:window
@@ -153,6 +166,7 @@
{#if uiState.domExists}
<div
class={twMerge(
open ? 'select-dropdown-open' : 'select-dropdown-closed',
disablePortal ? 'absolute z-[5002]' : 'fixed z-[10000]',
'text-primary text-sm select-none',
dropdownPos.isBelow ? '' : 'flex flex-col justify-end',
@@ -165,7 +179,8 @@
>
<div
class={twMerge(
'overflow-clip rounded-md drop-shadow-base transition-height',
'overflow-clip rounded-md drop-shadow-base',
!reducedMotion.val ? 'transition-height' : '',
dropdownPos.isBelow ? '' : 'flex flex-col justify-end'
)}
style="height: {uiState.visible ? dropdownPos.height : 0}px;"
@@ -52,6 +52,7 @@
placeholder="Search or create..."
showPlaceholderOnOpen
items={onlySelectedTags}
id="custom-instance-db-select"
disabled={!$isCustomInstanceDbEnabled}
>
{#snippet endSnippet({ item })}
@@ -161,6 +161,7 @@
{/if}
<Button
size="sm"
id="run-custom-instance-db-setup-button"
variant={!status?.success ? 'accent' : 'default'}
endIcon={status?.success ? undefined : { icon: ArrowRight }}
disabled={!$isCustomInstanceDbEnabled}
@@ -192,7 +192,7 @@
{#each tempSettings.dataTables as dataTable, dataTableIndex}
<Row>
<Cell first class="w-48 relative">
<TextInput bind:value={dataTable.name} inputProps={{ placeholder: 'Name' }} />
<TextInput bind:value={dataTable.name} inputProps={{ placeholder: 'Name', id: 'name' }} />
</Cell>
<Cell>
<div class="flex gap-2">
@@ -221,6 +221,7 @@
}
}
}
id="database-type-select"
class="w-28"
/>
</div>
@@ -202,7 +202,7 @@
</div>
{/if}
<DataTable>
<DataTable containerClass="ducklake-settings-table">
<Head>
<tr>
{#each tableHeadNames as name, i}
@@ -235,7 +235,11 @@
<code class="px-1 py-0.5 border rounded-md">ATTACH 'ducklake' AS dl;</code> shorthand
</Tooltip>
{/if}
<TextInput bind:value={ducklake.name} inputProps={{ placeholder: 'Name' }} />
<TextInput
bind:value={ducklake.name}
inputProps={{ placeholder: 'Name' }}
class="ducklake-name"
/>
</Cell>
<Cell>
<div class="flex gap-2">
@@ -309,11 +313,12 @@
else delete ducklake.storage.storage
}
}
class="w-48"
class="ducklake-workspace-storage-select w-48"
inputClass="!placeholder-secondary"
/>
<TextInput
inputProps={{ placeholder: 'Data path (defaults to /)' }}
class="ducklake-storage-data-path"
bind:value={ducklake.storage.path}
/>
</div>
@@ -132,7 +132,7 @@
</Alert>
{/if}
{#if s3ResourceSettings}
<DataTable containerClass="mt-4">
<DataTable containerClass="storage-settings-table mt-4">
<Head>
<tr>
{#each tableHeadNames as name, i}
@@ -152,7 +152,11 @@
{#if tableRow[0] === null}
<TextInput inputProps={{ placeholder: 'Primary storage', disabled: true }} />
{:else}
<TextInput bind:value={tableRow[0]} inputProps={{ placeholder: 'Name' }} />
<TextInput
bind:value={tableRow[0]}
inputProps={{ placeholder: 'Name' }}
class="secondary-storage-name-input"
/>
{/if}
</Cell>
<Cell>
@@ -167,6 +171,7 @@
{ value: 'gcloud_storage', label: 'Google Cloud Storage' }
]}
bind:value={tableRow[1].resourceType}
id="storage-resource-type-select"
class="w-40"
/>
</div>
-2
View File
@@ -156,8 +156,6 @@ export const BIGQUERY_TYPES = [
'integer[]',
'int64',
'int64[]',
'float',
'float[]',
'float64',
'float64[]',
'numeric',
+19
View File
@@ -188,6 +188,25 @@ export class ChangeOnDeepInequality<T> {
}
}
export function useReducedMotion(): { val: boolean } {
if (typeof window === 'undefined') return { val: false }
const query = window.matchMedia('(prefers-reduced-motion: reduce)')
let s = $state(query.matches)
$effect(() => {
const handler = (event: MediaQueryListEvent) => {
s = event.matches
}
query.addEventListener('change', handler)
return () => query.removeEventListener('change', handler)
})
return {
get val() {
return s
}
}
}
// Prevents flickering when data is unloaded (undefined) then reloaded quickly
// But still becomes undefined if data is not reloaded within the timeout
// so the user has feedback that the data is not available anymore.
+4
View File
@@ -1957,6 +1957,10 @@ export function countChars(str: string, char: string): number {
return count
}
export function onlyAlphaNumAndUnderscore(str: string): string {
return str.replace(/[^a-zA-Z0-9_]/g, '')
}
export function buildReactiveObj<T extends object>(fields: {
[name in keyof T]: [() => T[name], (v: T[name]) => void]
}): T {