From 819efda9a7b040bc2c18de10ae41f6d2552296ff Mon Sep 17 00:00:00 2001 From: Diego Imbert Date: Thu, 15 Jan 2026 21:40:19 +0100 Subject: [PATCH] bigquery tests passes --- frontend/e2e/DbManagerPage.ts | 28 ++-- frontend/e2e/db-manager.spec.ts | 82 ++++++----- frontend/package-lock.json | 1 - frontend/playwright.config.ts | 5 +- .../src/lib/components/DBTableEditor.svelte | 3 +- .../components/display/dbtable/dbFeatures.ts | 5 +- .../components/display/dbtable/metadata.ts | 134 ++++++------------ frontend/src/lib/consts.ts | 2 - 8 files changed, 112 insertions(+), 148 deletions(-) diff --git a/frontend/e2e/DbManagerPage.ts b/frontend/e2e/DbManagerPage.ts index d183e6a3da..19dbfbf95e 100644 --- a/frontend/e2e/DbManagerPage.ts +++ b/frontend/e2e/DbManagerPage.ts @@ -121,12 +121,15 @@ export async function runDbManagerAlterTableTest(page: Page, dbType: _DbType) { await tableEditor.setTableName(postsTableName) await idCol.delete() await friendCol.setName('person_id') - await friendCol.setType(getDbDatatype(dbType, 'BIGINT')) + if (dbType !== 'bigquery' && dbType !== 'snowflake') { + await friendCol.setType(getDbDatatype(dbType, 'BIGINT')) + } await friendCol.setSettings({ defaultValue: dbFeatures.defaultValues ? '123' : undefined, nullable: false }) - if (dbFeatures.primaryKeys) { + if (dbFeatures.primaryKeys && dbType !== 'bigquery') { + // Bigquery cannot rename a table with primary keys await friendCol.setPrimaryKey(true) await createdAtCol.setPrimaryKey(true) await contentColumn.setPrimaryKey(true) @@ -143,13 +146,15 @@ export async function runDbManagerAlterTableTest(page: Page, dbType: _DbType) { tableEditor = new TableEditorDrawer(page) await idCol.checkNotExists() await friendCol.checkNameIs('person_id') - await friendCol.checkTypeIs(getDbDatatype(dbType, 'BIGINT')) + if (dbType !== 'bigquery' && dbType !== 'snowflake') { + await friendCol.checkTypeIs(getDbDatatype(dbType, 'BIGINT')) + } if (dbFeatures.defaultValues) { await friendCol.checkSettingsIs({ defaultValue: /123/ }) } await createdAtCol.checkTypeIs(getDbDatatype(dbType, 'TIMESTAMP')) await createdAtCol.checkNameIs('created_at') - if (dbFeatures.primaryKeys) { + if (dbFeatures.primaryKeys && dbType !== 'bigquery') { await friendCol.checkPrimaryKeyIs(true) await createdAtCol.checkPrimaryKeyIs(true) await contentColumn.checkPrimaryKeyIs(true) @@ -188,7 +193,7 @@ export class DbManagerPage { async selectTable(tableName: string) { const tableKey = this.page.locator('.db-manager-table-key', { hasText: tableName }) - await expect(tableKey).toBeVisible({ timeout: 10000 }) + await expect(tableKey).toBeVisible() await tableKey.click() } @@ -203,7 +208,7 @@ export class DbManagerPage { async openActionsMenu(tableName: string): Promise { const actionsBtn = this.dbManager().locator(`#db-manager-table-actions-${tableName}`) - await expect(actionsBtn).toBeVisible({ timeout: 10000 }) + await expect(actionsBtn).toBeVisible() await actionsBtn.click() return new TableActionsMenu(this.page) } @@ -298,7 +303,7 @@ class TableEditorDrawer { async expectNoChangesDetected() { const btn = this.tableEditor().locator(`button:has-text("No changes detected")`) - return await expect(btn).toBeVisible({ timeout: 10000 }) + return await expect(btn).toBeVisible() } getColumn(columnName: string): Column { @@ -355,7 +360,7 @@ class Column { async checkTypeIs(columnType: string) { const newColTypeSelect = (await this.row()).locator('td').nth(1).locator('input') - await expect(newColTypeSelect).toHaveValue(columnType) + await expect(newColTypeSelect).toHaveValue(new RegExp(`^${columnType}$`, 'i')) } async delete() { @@ -430,7 +435,7 @@ class InsertRowDrawer { drawer = () => this.page.locator('#insert-row-drawer') async fillField(fieldName: string, value: string) { - await expect(this.drawer()).toBeVisible({ timeout: 10000 }) + await expect(this.drawer()).toBeVisible() // For now, assumes single field - could be enhanced to handle multiple fields await this.drawer().locator('textarea').fill(value, { force: true }) } @@ -451,7 +456,7 @@ class DataGrid { async expectCellValue(value: string) { const cell = this.dbManager.locator('.ag-cell-value', { hasText: value }) - await expect(cell).toBeVisible({ timeout: 10000 }) + await expect(cell).toBeVisible() } async editCell(oldValue: string, newValue: string) { @@ -497,6 +502,9 @@ function getDbInput(dbType: _DbType): DbInput { // Ensure exact casing of datatype as per DB_TYPES function getDbDatatype(dbType: _DbType, datatype: string): string { if (dbType === 'ms_sql_server' && datatype.toLowerCase() === 'timestamp') datatype = 'datetime2' + if (dbType === 'bigquery' && datatype.toLowerCase() === 'text') datatype = 'string' + if (dbType === 'bigquery' && datatype.toLowerCase() === 'int') datatype = 'int64' + if (dbType === 'snowflake' && datatype.toLowerCase() === 'text') datatype = 'varchar' const allDataTypes = DB_TYPES[dbType == 'ducklake' ? 'duckdb' : dbType] || [] return allDataTypes.find((dt) => dt.toLowerCase() === datatype.toLowerCase()) || datatype } diff --git a/frontend/e2e/db-manager.spec.ts b/frontend/e2e/db-manager.spec.ts index efd8e7a29a..31797b69f7 100644 --- a/frontend/e2e/db-manager.spec.ts +++ b/frontend/e2e/db-manager.spec.ts @@ -3,9 +3,50 @@ import { runDbManagerAlterTableTest, runDbManagerSimpleCRUDTest } from './DbMana import { DbType } from '../src/lib/components/dbTypes' import { Toast, prettify } from './utils' +const resourceByDbType = { + postgresql: { + host: 'postgres_e2e', + port: 5432, + dbname: 'test_db', + user: 'test_user', + password: 'postgres_password', + sslmode: 'disable' + }, + mysql: { + host: 'mysql_e2e', + port: 3306, + user: 'test_user', + database: 'test_db', + password: 'test_password', + ssl: false + }, + oracle: { + user: 'test_user', + password: 'test_password', + database: 'oracle_e2e:1521/test_db' + }, + ms_sql_server: { + host: 'mssql_e2e', + user: 'sa', + password: 'MsSql_Pass123!', + port: 1433, + dbname: 'master', + instance_name: '', + trust_cert: true, + ca_cert: '', + encrypt: true + }, + bigquery: getBigQueryResource(), + snowflake: undefined // TODO +} as const + test.describe('Database resources', () => { for (const dbType of ['postgresql', 'mysql', 'bigquery', 'ms_sql_server', 'snowflake'] as const) { test.describe(prettify(dbType), () => { + test.skip( + resourceByDbType[dbType] === undefined, + `No resource config for ${dbType}, set ${dbType.toUpperCase()}_RESOURCE` + ) test(`simple CRUD with DB Manager`, async ({ page }) => { await setupNewResourceAndOpenDbManager(page, dbType) await runDbManagerSimpleCRUDTest(page, dbType) @@ -163,42 +204,10 @@ async function setupCustomInstanceDb(row: Locator, page: Page, name: string) { await closeModalBtn.click() } -const resourceByDbType = { - postgresql: { - host: 'postgres_e2e', - port: 5432, - dbname: 'test_db', - user: 'test_user', - password: 'postgres_password', - sslmode: 'disable' - }, - mysql: { - host: 'mysql_e2e', - port: 3306, - user: 'test_user', - database: 'test_db', - password: 'test_password', - ssl: false - }, - oracle: { - user: 'test_user', - password: 'test_password', - database: 'oracle_e2e:1521/test_db' - }, - ms_sql_server: { - host: 'mssql_e2e', - user: 'sa', - password: 'MsSql_Pass123!', - port: 1433, - dbname: 'master', - instance_name: '', - trust_cert: true, - ca_cert: '', - encrypt: true - }, - bigquery: {}, // TODO - snowflake: {} // TODO -} as const +function getBigQueryResource(): object | undefined { + const bigqueryResource = process.env.BIGQUERY_RESOURCE + if (bigqueryResource) return JSON.parse(bigqueryResource) +} const wsStorageResources = { s3: { @@ -301,6 +310,7 @@ async function setupNewResource( const jsonEditor = page.locator('.simple-editor .view-lines') await expect(jsonEditor).toBeVisible() await jsonEditor.click({ clickCount: 4 }) // Select all existing text + console.log('Pasting resource config:', resourceObj) await page.evaluate((c) => navigator.clipboard.writeText(c), JSON.stringify(resourceObj)) await page.keyboard.press('ControlOrMeta+V') diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1f4645bbff..8fb8db99f1 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -6493,7 +6493,6 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.11.0.tgz", "integrity": "sha512-mS1lbMsxgQj6hge1XZ6p7GPhbrtFwUFYi3wRzXAC/FmYnyXMTvvI3td3rjmQ2u8ewXueaSvRPWaEcgVVOT9Jnw==", "license": "MIT", - "peer": true, "engines": { "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts index 6aaec234c4..796feb82e9 100644 --- a/frontend/playwright.config.ts +++ b/frontend/playwright.config.ts @@ -25,8 +25,11 @@ export default defineConfig({ /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: 'html', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - timeout: 60 * 1000, + timeout: 90 * 1000, + expect: { timeout: 25 * 1000 }, use: { + actionTimeout: 25 * 1000, + /* Base URL to use in actions like `await page.goto('')`. */ baseURL: process.env.BASE_URL || 'http://localhost:3000', diff --git a/frontend/src/lib/components/DBTableEditor.svelte b/frontend/src/lib/components/DBTableEditor.svelte index ee5bdf7972..69818a0dd9 100644 --- a/frontend/src/lib/components/DBTableEditor.svelte +++ b/frontend/src/lib/components/DBTableEditor.svelte @@ -119,8 +119,7 @@ ...(datatypeHasLength(defaultColumnType) && { datatype_length: datatypeDefaultLength(defaultColumnType) }), - ...(primaryKey && { primaryKey }), - ...(!features?.defaultToNotNull && { nullable: true }) + ...(primaryKey && { primaryKey }) }) } if (!initialValues) { diff --git a/frontend/src/lib/components/apps/components/display/dbtable/dbFeatures.ts b/frontend/src/lib/components/apps/components/display/dbtable/dbFeatures.ts index b71db8170b..5584ac0e11 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/dbFeatures.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/dbFeatures.ts @@ -7,7 +7,6 @@ export type DbFeatures = { foreignKeys?: boolean primaryKeys?: boolean defaultValues?: boolean - defaultToNotNull?: boolean schemas?: boolean } @@ -16,7 +15,6 @@ export function getDbFeatures(dbInput: DbInput): Required { foreignKeys: true, primaryKeys: true, defaultValues: true, - defaultToNotNull: true, schemas: dbInput.type !== 'ducklake' && dbSupportsSchemas(dbInput.resourceType) } @@ -27,8 +25,7 @@ export function getDbFeatures(dbInput: DbInput): Required { ...def, foreignKeys: false, primaryKeys: true, - defaultValues: false, - defaultToNotNull: false + defaultValues: false } return { ...def } diff --git a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts index 50ea65e39b..e7de7a594e 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/metadata.ts @@ -13,7 +13,6 @@ import { import { type Preview } from '$lib/gen' import type { DBSchema, DBSchemas, GraphqlSchema, SQLSchema } from '$lib/stores' -import { tryEvery } from '$lib/utils' import { stringifySchema } from '$lib/components/copilot/lib' import type { DbType } from '$lib/components/dbTypes' import { getDatabaseArg } from '$lib/components/dbOps' @@ -362,16 +361,12 @@ export async function getDbSchemas( let scripts = options.useLegacyScripts ? legacyScripts : scriptsV2 let sqlScript = scripts[getLanguageByResourceType(resourceType)] - if (!sqlScript) return + if (!resourceType || !resourcePath || !workspace || !sqlScript) return - return new Promise(async (resolve, reject) => { - if (!resourceType || !resourcePath || !workspace) { - resolve() - return - } - - const job = await JobService.runScriptPreview({ - workspace: workspace, + let result: unknown + try { + result = await JobService.runScriptPreviewAndWaitResult({ + workspace, requestBody: { language: sqlScript.lang as Preview['language'], content: sqlScript.code, @@ -382,91 +377,46 @@ export async function getDbSchemas( } } }) + } catch (e) { + console.error(e) + return errorCallback('Error fetching schema: ' + ((e as Error)?.message || e)) + } - tryEvery({ - tryCode: async () => { - if (resourcePath) { - const testResult = await JobService.getCompletedJob({ - workspace, - id: job - }) - if (!testResult.success) { - console.error(testResult.result?.['error']?.['message']) - } else { - if (testResult.result === 'WINDMILL_TOO_BIG') { - console.info('Result is too big, fetching result separately') - const data = await JobService.getCompletedJobResult({ - workspace, - id: job - }) - testResult.result = data - } - if (resourceType !== undefined) { - if (resourceType !== 'graphql') { - const { processingFn } = sqlScript - let schema: any - try { - schema = - processingFn !== undefined ? processingFn(testResult.result) : testResult.result - } catch (e) { - console.error(e) - errorCallback('Error processing schema') - resolve() - return - } - const dbSchema = { - lang: resourceTypeToLang(resourceType) as SQLSchema['lang'], - schema, - publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo - } - dbSchemas[resourcePath] = { - ...dbSchema, - stringified: stringifySchema(dbSchema) - } - } else { - if ( - typeof testResult.result !== 'object' || - !('__schema' in (testResult?.result ?? {})) - ) { - console.error('Invalid GraphQL schema') - - errorCallback('Invalid GraphQL schema') - } else { - const dbSchema = { - lang: 'graphql' as GraphqlSchema['lang'], - schema: testResult.result - } - dbSchemas[resourcePath] = { - ...(dbSchema as any), - stringified: stringifySchema(dbSchema as any) - } - } - } - } - } - resolve() + if (resourceType !== undefined) { + if (resourceType !== 'graphql') { + const { processingFn } = sqlScript + let schema: any + try { + schema = processingFn !== undefined ? processingFn(result) : result + } catch (e) { + console.error(e) + return errorCallback('Error processing schema') + } + const dbSchema = { + lang: resourceTypeToLang(resourceType) as SQLSchema['lang'], + schema, + publicOnly: !!schema.public || !!schema.PUBLIC || !!schema.dbo + } + dbSchemas[resourcePath] = { + ...dbSchema, + stringified: stringifySchema(dbSchema) + } + } else { + if (typeof result !== 'object' || !('__schema' in (result ?? {}))) { + console.error('Invalid GraphQL schema') + return errorCallback('Invalid GraphQL schema') + } else { + const dbSchema = { + lang: 'graphql' as GraphqlSchema['lang'], + schema: result } - }, - timeoutCode: async () => { - console.error('Could not query schema within 5s') - errorCallback('Could not query schema within 5s') - try { - await JobService.cancelQueuedJob({ - workspace, - id: job, - requestBody: { - reason: 'Could not query schema within 5s' - } - }) - } catch (err) { - console.error(err) + dbSchemas[resourcePath] = { + ...(dbSchema as any), + stringified: stringifySchema(dbSchema as any) } - reject() - }, - interval: 500, - timeout: 5000 - }) - }) + } + } + } } export async function getTablesByResource( diff --git a/frontend/src/lib/consts.ts b/frontend/src/lib/consts.ts index 702791b74e..677ffa8ad4 100644 --- a/frontend/src/lib/consts.ts +++ b/frontend/src/lib/consts.ts @@ -156,8 +156,6 @@ export const BIGQUERY_TYPES = [ 'integer[]', 'int64', 'int64[]', - 'float', - 'float[]', 'float64', 'float64[]', 'numeric',