starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 08:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

bigquery tests passes

Browse Source
This commit is contained in:
Diego Imbert
2026-01-15 21:40:19 +01:00
parent 4c3d6687cb
commit 819efda9a7
8 changed files with 112 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/e2e/DbManagerPage.ts
+18 -10
View File
@@ -121,12 +121,15 @@ export async function runDbManagerAlterTableTest(page: Page, dbType: _DbType) {
await tableEditor.setTableName(postsTableName)
await idCol.delete()
await friendCol.setName('person_id')
await friendCol.setType(getDbDatatype(dbType, 'BIGINT'))
if (dbType !== 'bigquery' && dbType !== 'snowflake') {
await friendCol.setType(getDbDatatype(dbType, 'BIGINT'))
}
await friendCol.setSettings({
defaultValue: dbFeatures.defaultValues ? '123' : undefined,
nullable: false
})
if (dbFeatures.primaryKeys) {
if (dbFeatures.primaryKeys && dbType !== 'bigquery') {
// Bigquery cannot rename a table with primary keys
await friendCol.setPrimaryKey(true)
await createdAtCol.setPrimaryKey(true)
await contentColumn.setPrimaryKey(true)
@@ -143,13 +146,15 @@ export async function runDbManagerAlterTableTest(page: Page, dbType: _DbType) {
tableEditor = new TableEditorDrawer(page)
await idCol.checkNotExists()
await friendCol.checkNameIs('person_id')
await friendCol.checkTypeIs(getDbDatatype(dbType, 'BIGINT'))
if (dbType !== 'bigquery' && dbType !== 'snowflake') {
await friendCol.checkTypeIs(getDbDatatype(dbType, 'BIGINT'))
}
if (dbFeatures.defaultValues) {
await friendCol.checkSettingsIs({ defaultValue: /123/ })
}
await createdAtCol.checkTypeIs(getDbDatatype(dbType, 'TIMESTAMP'))
await createdAtCol.checkNameIs('created_at')
if (dbFeatures.primaryKeys) {
if (dbFeatures.primaryKeys && dbType !== 'bigquery') {
await friendCol.checkPrimaryKeyIs(true)
await createdAtCol.checkPrimaryKeyIs(true)
await contentColumn.checkPrimaryKeyIs(true)
@@ -188,7 +193,7 @@ export class DbManagerPage {
async selectTable(tableName: string) {
const tableKey = this.page.locator('.db-manager-table-key', { hasText: tableName })
await expect(tableKey).toBeVisible({ timeout: 10000 })
await expect(tableKey).toBeVisible()
await tableKey.click()
}
@@ -203,7 +208,7 @@ export class DbManagerPage {
async openActionsMenu(tableName: string): Promise<TableActionsMenu> {
const actionsBtn = this.dbManager().locator(`#db-manager-table-actions-${tableName}`)
await expect(actionsBtn).toBeVisible({ timeout: 10000 })
await expect(actionsBtn).toBeVisible()
await actionsBtn.click()
return new TableActionsMenu(this.page)
}
@@ -298,7 +303,7 @@ class TableEditorDrawer {
async expectNoChangesDetected() {
const btn = this.tableEditor().locator(`button:has-text("No changes detected")`)
return await expect(btn).toBeVisible({ timeout: 10000 })
return await expect(btn).toBeVisible()
}
getColumn(columnName: string): Column {
@@ -355,7 +360,7 @@ class Column {
async checkTypeIs(columnType: string) {
const newColTypeSelect = (await this.row()).locator('td').nth(1).locator('input')
await expect(newColTypeSelect).toHaveValue(columnType)
await expect(newColTypeSelect).toHaveValue(new RegExp(`^${columnType}$`, 'i'))
}
async delete() {
@@ -430,7 +435,7 @@ class InsertRowDrawer {
drawer = () => this.page.locator('#insert-row-drawer')
async fillField(fieldName: string, value: string) {
await expect(this.drawer()).toBeVisible({ timeout: 10000 })
await expect(this.drawer()).toBeVisible()
// For now, assumes single field - could be enhanced to handle multiple fields
await this.drawer().locator('textarea').fill(value, { force: true })
}
@@ -451,7 +456,7 @@ class DataGrid {
async expectCellValue(value: string) {
const cell = this.dbManager.locator('.ag-cell-value', { hasText: value })
await expect(cell).toBeVisible({ timeout: 10000 })
await expect(cell).toBeVisible()
}
async editCell(oldValue: string, newValue: string) {
@@ -497,6 +502,9 @@ function getDbInput(dbType: _DbType): DbInput {
// Ensure exact casing of datatype as per DB_TYPES
function getDbDatatype(dbType: _DbType, datatype: string): string {
if (dbType === 'ms_sql_server' && datatype.toLowerCase() === 'timestamp') datatype = 'datetime2'
if (dbType === 'bigquery' && datatype.toLowerCase() === 'text') datatype = 'string'
if (dbType === 'bigquery' && datatype.toLowerCase() === 'int') datatype = 'int64'
if (dbType === 'snowflake' && datatype.toLowerCase() === 'text') datatype = 'varchar'
const allDataTypes = DB_TYPES[dbType == 'ducklake' ? 'duckdb' : dbType] || []
return allDataTypes.find((dt) => dt.toLowerCase() === datatype.toLowerCase()) || datatype
}
frontend/e2e/db-manager.spec.ts
+46 -36
View File
@@ -3,9 +3,50 @@ import { runDbManagerAlterTableTest, runDbManagerSimpleCRUDTest } from './DbMana
import { DbType } from '../src/lib/components/dbTypes'
import { Toast, prettify } from './utils'
const resourceByDbType = {
postgresql: {
host: 'postgres_e2e',
port: 5432,
dbname: 'test_db',
user: 'test_user',
password: 'postgres_password',
sslmode: 'disable'
},
mysql: {
host: 'mysql_e2e',
port: 3306,
user: 'test_user',
database: 'test_db',
password: 'test_password',
ssl: false
},
oracle: {
user: 'test_user',
password: 'test_password',
database: 'oracle_e2e:1521/test_db'
},
ms_sql_server: {
host: 'mssql_e2e',
user: 'sa',
password: 'MsSql_Pass123!',
port: 1433,
dbname: 'master',
instance_name: '',
trust_cert: true,
ca_cert: '',
encrypt: true
},
bigquery: getBigQueryResource(),
snowflake: undefined // TODO
} as const
test.describe('Database resources', () => {
for (const dbType of ['postgresql', 'mysql', 'bigquery', 'ms_sql_server', 'snowflake'] as const) {
test.describe(prettify(dbType), () => {
test.skip(
resourceByDbType[dbType] === undefined,
`No resource config for ${dbType}, set ${dbType.toUpperCase()}_RESOURCE`
)
test(`simple CRUD with DB Manager`, async ({ page }) => {
await setupNewResourceAndOpenDbManager(page, dbType)
await runDbManagerSimpleCRUDTest(page, dbType)
@@ -163,42 +204,10 @@ async function setupCustomInstanceDb(row: Locator, page: Page, name: string) {
await closeModalBtn.click()
}
const resourceByDbType = {
postgresql: {
host: 'postgres_e2e',
port: 5432,
dbname: 'test_db',
user: 'test_user',
password: 'postgres_password',
sslmode: 'disable'
},
mysql: {
host: 'mysql_e2e',
port: 3306,
user: 'test_user',
database: 'test_db',
password: 'test_password',
ssl: false
},
oracle: {
user: 'test_user',
password: 'test_password',
database: 'oracle_e2e:1521/test_db'
},
ms_sql_server: {
host: 'mssql_e2e',
user: 'sa',
password: 'MsSql_Pass123!',
port: 1433,
dbname: 'master',
instance_name: '',
trust_cert: true,
ca_cert: '',
encrypt: true
},
bigquery: {}, // TODO
snowflake: {} // TODO
} as const
function getBigQueryResource(): object | undefined {
const bigqueryResource = process.env.BIGQUERY_RESOURCE
if (bigqueryResource) return JSON.parse(bigqueryResource)
}
const wsStorageResources = {
s3: {
@@ -301,6 +310,7 @@ async function setupNewResource(
const jsonEditor = page.locator('.simple-editor .view-lines')
await expect(jsonEditor).toBeVisible()
await jsonEditor.click({ clickCount: 4 }) // Select all existing text
console.log('Pasting resource config:', resourceObj)
await page.evaluate((c) => navigator.clipboard.writeText(c), JSON.stringify(resourceObj))
await page.keyboard.press('ControlOrMeta+V')
frontend/package-lock.json
Generated
-1
View File
@@ -6493,7 +6493,6 @@
"resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz",
"integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==",
"license": "MIT",
"peer": true,
"engines": {
"node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0"
}
frontend/playwright.config.ts
+4 -1
View File
@@ -25,8 +25,11 @@ export default defineConfig({
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html',
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
timeout: 60 * 1000,
timeout: 90 * 1000,
expect: { timeout: 25 * 1000 },
use: {
actionTimeout: 25 * 1000,
/* Base URL to use in actions like `await page.goto('')`. */
baseURL: process.env.BASE_URL || 'http://localhost:3000',
frontend/src/lib/components/DBTableEditor.svelte
+1 -2
View File
@@ -119,8 +119,7 @@
...(datatypeHasLength(defaultColumnType) && {
datatype_length: datatypeDefaultLength(defaultColumnType)
}),
...(primaryKey && { primaryKey }),
...(!features?.defaultToNotNull && { nullable: true })
...(primaryKey && { primaryKey })
})
}
if (!initialValues) {
frontend/src/lib/components/apps/components/display/dbtable/dbFeatures.ts
+1 -4
View File
@@ -7,7 +7,6 @@ export type DbFeatures = {
foreignKeys?: boolean
primaryKeys?: boolean
defaultValues?: boolean
defaultToNotNull?: boolean
schemas?: boolean
}
@@ -16,7 +15,6 @@ export function getDbFeatures(dbInput: DbInput): Required<DbFeatures> {
foreignKeys: true,
primaryKeys: true,
defaultValues: true,
defaultToNotNull: true,
schemas: dbInput.type !== 'ducklake' && dbSupportsSchemas(dbInput.resourceType)
}
@@ -27,8 +25,7 @@ export function getDbFeatures(dbInput: DbInput): Required<DbFeatures> {
...def,
foreignKeys: false,
primaryKeys: true,
defaultValues: false,
defaultToNotNull: false
defaultValues: false
}
return { ...def }
frontend/src/lib/components/apps/components/display/dbtable/metadata.ts
+42 -92
View File
@@ -13,7 +13,6 @@ import {
import { type Preview } from '$lib/gen'
import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores'
import { tryEvery } from '$lib/utils'
import { stringifySchema } from '$lib/components/copilot/lib'
import type { DbType } from '$lib/components/dbTypes'
import { getDatabaseArg } from '$lib/components/dbOps'
@@ -362,16 +361,12 @@ export async function getDbSchemas(
let scripts = options.useLegacyScripts ? legacyScripts : scriptsV2
let sqlScript = scripts[getLanguageByResourceType(resourceType)]
if (!sqlScript) return
if (!resourceType || !resourcePath || !workspace || !sqlScript) return
return new Promise(async (resolve, reject) => {
if (!resourceType || !resourcePath || !workspace) {
resolve()
return
}
const job = await JobService.runScriptPreview({
workspace: workspace,
let result: unknown
try {
result = await JobService.runScriptPreviewAndWaitResult({
workspace,
requestBody: {
language: sqlScript.lang as Preview['language'],
content: sqlScript.code,
@@ -382,91 +377,46 @@ export async function getDbSchemas(
}
}
})
} catch (e) {
console.error(e)
return errorCallback('Error fetching schema: ' + ((e as Error)?.message || e))
}
tryEvery({
tryCode: async () => {
if (resourcePath) {
const testResult = await JobService.getCompletedJob({
workspace,
id: job
})
if (!testResult.success) {
console.error(testResult.result?.['error']?.['message'])
} else {
if (testResult.result === 'WINDMILL_TOO_BIG') {
console.info('Result is too big, fetching result separately')
const data = await JobService.getCompletedJobResult({
workspace,
id: job
})
testResult.result = data
}
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = sqlScript
let schema: any
try {
schema =
processingFn !== undefined ? processingFn(testResult.result) : testResult.result
} catch (e) {
console.error(e)
errorCallback('Error processing schema')
resolve()
return
}
const dbSchema = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
dbSchemas[resourcePath] = {
...dbSchema,
stringified: stringifySchema(dbSchema)
}
} else {
if (
typeof testResult.result !== 'object' ||
!('__schema' in (testResult?.result ?? {}))
) {
console.error('Invalid GraphQL schema')
errorCallback('Invalid GraphQL schema')
} else {
const dbSchema = {
lang: 'graphql' as GraphqlSchema['lang'],
schema: testResult.result
}
dbSchemas[resourcePath] = {
...(dbSchema as any),
stringified: stringifySchema(dbSchema as any)
}
}
}
}
}
resolve()
if (resourceType !== undefined) {
if (resourceType !== 'graphql') {
const { processingFn } = sqlScript
let schema: any
try {
schema = processingFn !== undefined ? processingFn(result) : result
} catch (e) {
console.error(e)
return errorCallback('Error processing schema')
}
const dbSchema = {
lang: resourceTypeToLang(resourceType) as SQLSchema['lang'],
schema,
publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo
}
dbSchemas[resourcePath] = {
...dbSchema,
stringified: stringifySchema(dbSchema)
}
} else {
if (typeof result !== 'object' || !('__schema' in (result ?? {}))) {
console.error('Invalid GraphQL schema')
return errorCallback('Invalid GraphQL schema')
} else {
const dbSchema = {
lang: 'graphql' as GraphqlSchema['lang'],
schema: result
}
},
timeoutCode: async () => {
console.error('Could not query schema within 5s')
errorCallback('Could not query schema within 5s')
try {
await JobService.cancelQueuedJob({
workspace,
id: job,
requestBody: {
reason: 'Could not query schema within 5s'
}
})
} catch (err) {
console.error(err)
dbSchemas[resourcePath] = {
...(dbSchema as any),
stringified: stringifySchema(dbSchema as any)
}
reject()
},
interval: 500,
timeout: 5000
})
})
}
}
}
}
export async function getTablesByResource(
frontend/src/lib/consts.ts
-2
View File
@@ -156,8 +156,6 @@ export const BIGQUERY_TYPES = [
'integer[]',
'int64',
'int64[]',
'float',
'float[]',
'float64',
'float64[]',
'numeric',