From c08eb0abc8444f09959e6e5cc229b546b01a4cc5 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 11 Apr 2024 01:12:41 +0200 Subject: [PATCH] fix: remove requirement on full wasm parser for row insert of db studio --- backend/Cargo.lock | 11 +++ backend/Cargo.toml | 1 + .../parsers/windmill-parser-sql/src/lib.rs | 2 +- frontend/package-lock.json | 6 ++ frontend/package.json | 1 + .../lib/components/LightweightArgInput.svelte | 2 +- .../display/dbtable/InsertRow.svelte | 59 +++++++----- frontend/src/lib/infer.ts | 93 +------------------ frontend/src/lib/inferArgSig.ts | 91 ++++++++++++++++++ 9 files changed, 152 insertions(+), 114 deletions(-) create mode 100644 frontend/src/lib/inferArgSig.ts diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 1d8ff834a6..bb6b6761e7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10051,6 +10051,17 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-sql-datatype-parser-wasm" +version = "1.305.0" +dependencies = [ + "serde", + "wasm-bindgen", + "wasm-bindgen-test", + "windmill-parser", + "windmill-parser-sql", +] + [[package]] name = "windmill-worker" version = "1.305.0" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 30bc919c62..5af686c328 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -20,6 +20,7 @@ members = [ "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", + "./parsers/windmill-sql-datatype-parser-wasm", ] [workspace.package] diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index c64ed7f846..cdca933d90 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -5,7 +5,7 @@ use regex::Regex; use serde_json::json; use std::collections::HashMap; -use windmill_parser::{Arg, MainArgSignature, Typ}; +pub use windmill_parser::{Arg, MainArgSignature, Typ}; pub fn parse_mysql_sig(code: &str) -> anyhow::Result { let parsed = parse_mysql_file(&code)?; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 23343882fd..3c0f582302 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -52,6 +52,7 @@ "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.1.0", "windmill-parser-wasm": "^1.286.2", + "windmill-sql-datatype-parser-wasm": "1.305.0", "y-monaco": "^0.1.4", "y-websocket": "^1.5.0", "yaml": "^2.3.4", @@ -10095,6 +10096,11 @@ "resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.286.2.tgz", "integrity": "sha512-xwmIRy/QJoT/p5MpZw5O1Ed4T25gOjHvR4VpkJTxeMhhzRzhXA52gBRABRGL1lZImo74kWTHdMVQ/UdKB7yO0g==" }, + "node_modules/windmill-sql-datatype-parser-wasm": { + "version": "1.305.0", + "resolved": "https://registry.npmjs.org/windmill-sql-datatype-parser-wasm/-/windmill-sql-datatype-parser-wasm-1.305.0.tgz", + "integrity": "sha512-tn0yVn61g6EQp4t7zVnH6ZeL1CbjSSPinpMB8PwbxCTcv+dj5ibWRzYmNu+bqzcLjwdMTUA6VRHsbYJ3Ne4HpQ==" + }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index 8d223fd5ae..df3fc4ee70 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -134,6 +134,7 @@ "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.1.0", "windmill-parser-wasm": "^1.286.2", + "windmill-sql-datatype-parser-wasm": "1.305.0", "y-monaco": "^0.1.4", "y-websocket": "^1.5.0", "yaml": "^2.3.4", diff --git a/frontend/src/lib/components/LightweightArgInput.svelte b/frontend/src/lib/components/LightweightArgInput.svelte index cc13cd9356..4feaf1df4f 100644 --- a/frontend/src/lib/components/LightweightArgInput.svelte +++ b/frontend/src/lib/components/LightweightArgInput.svelte @@ -305,7 +305,7 @@ {/each} - {:else} + {:else if value != undefined} List is not an array {/if} diff --git a/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte b/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte index 91e32a8e30..ca678ad3d9 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte +++ b/frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte @@ -9,19 +9,20 @@ type ColumnDef } from './utils' - import wasmUrl from 'windmill-parser-wasm/windmill_parser_wasm_bg.wasm?url' import init, { parse_sql, parse_mysql, parse_bigquery, parse_snowflake, parse_mssql - } from 'windmill-parser-wasm' - import type { MainArgSignature } from '$lib/gen' - import { makeInsertQuery } from './queries/insert' - import { argSigToJsonSchemaType } from '$lib/infer' + } from 'windmill-sql-datatype-parser-wasm' + import wasmUrl from 'windmill-sql-datatype-parser-wasm/windmill_sql_datatype_parser_wasm_bg.wasm?url' + init(wasmUrl) + import { argSigToJsonSchemaType } from '$lib/inferArgSig' + import fi from 'date-fns/locale/fi' + export let args: Record = {} export let dbType: DbType = 'postgresql' @@ -63,50 +64,66 @@ } }) as FieldMetadata[] | undefined - async function parseSQLArgs(code: string, dbType: DbType) { - await init(wasmUrl) - - let rawSchema = '' + function parseSQLArgs(field: string, dbType: DbType): string { + let rawType = '' switch (dbType) { case 'mysql': - rawSchema = parse_mysql(code) + rawType = parse_mysql(field) break case 'postgresql': - rawSchema = parse_sql(code) + rawType = parse_sql(field) break case 'bigquery': - rawSchema = parse_bigquery(code) + rawType = parse_bigquery(field) break case 'snowflake': - rawSchema = parse_snowflake(code) + rawType = parse_snowflake(field) break case 'ms_sql_server': - rawSchema = parse_mssql(code) + rawType = parse_mssql(field) break default: throw new Error('Language not supported') } - const args: MainArgSignature = JSON.parse(rawSchema) - - return args + return rawType } + function rawTypeToSchemaType(typ: string) { + if (typ.startsWith('list-')) { + return { + list: rawTypeToSchemaType(typ.replace('list-', '')) + } + } else if (typ == 'str') { + return { + str: undefined + } + } else { + return typ + } + } async function builtSchema(fields: FieldMetadata[], dbType: DbType) { const properties: { [name: string]: SchemaProperty } = {} const required: string[] = [] - const insertCode = makeInsertQuery('ignoredtable', columnDefs, dbType) - const args = await parseSQLArgs(insertCode, dbType) + await init(wasmUrl) fields.forEach((field) => { const schemaProperty: SchemaProperty = { type: 'string' } - const parsedArg = args.args.find((arg) => arg.name === field.name) + const parsedArg = columnDefs.find((arg) => arg.field === field.name) if (parsedArg) { - argSigToJsonSchemaType(parsedArg.typ, schemaProperty) + let typ: any = rawTypeToSchemaType(parseSQLArgs(parsedArg.datatype, dbType)) + argSigToJsonSchemaType(typ, schemaProperty) + console.log( + 'schemaProperty', + schemaProperty, + field.name, + typ, + parseSQLArgs(parsedArg.datatype, dbType) + ) } if (field.defaultValue) { diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 9c592dd7ab..6be7f39146 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -1,6 +1,6 @@ import { ScriptService, type MainArgSignature, FlowService, Script } from '$lib/gen' import { get, writable } from 'svelte/store' -import type { Schema, SchemaProperty, SupportedLanguage } from './common.js' +import type { Schema, SupportedLanguage } from './common.js' import { emptySchema, sortObject } from './utils.js' import { tick } from 'svelte' import init, { @@ -21,6 +21,7 @@ import init, { } from 'windmill-parser-wasm' import wasmUrl from 'windmill-parser-wasm/windmill_parser_wasm_bg.wasm?url' import { workspaceStore } from './stores.js' +import { argSigToJsonSchemaType } from './inferArgSig.js' init(wasmUrl) @@ -151,96 +152,6 @@ export async function inferArgs( await tick() } -export function argSigToJsonSchemaType( - t: - | string - | { resource: string | null } - | { list: string | { str: any } | { object: { key: string; typ: any }[] } | null } - | { str: string[] | null } - | { object: { key: string; typ: any }[] }, - oldS: SchemaProperty -): void { - const newS: SchemaProperty = { type: '' } - if (t === 'int') { - newS.type = 'integer' - } else if (t === 'float') { - newS.type = 'number' - } else if (t === 'bool') { - newS.type = 'boolean' - } else if (t === 'email') { - newS.type = 'string' - newS.format = 'email' - } else if (t === 'sql') { - newS.type = 'string' - newS.format = 'sql' - } else if (t === 'yaml') { - newS.type = 'string' - newS.format = 'yaml' - } else if (t === 'bytes') { - newS.type = 'string' - newS.contentEncoding = 'base64' - } else if (t === 'datetime') { - newS.type = 'string' - newS.format = 'date-time' - } else if (typeof t !== 'string' && `object` in t) { - newS.type = 'object' - if (t.object) { - const properties = {} - for (const prop of t.object) { - properties[prop.key] = {} - argSigToJsonSchemaType(prop.typ, properties[prop.key]) - } - newS.properties = properties - } - } else if (typeof t !== 'string' && `str` in t) { - newS.type = 'string' - if (t.str) { - newS.enum = t.str - } - } else if (typeof t !== 'string' && `resource` in t) { - newS.type = 'object' - newS.format = `resource-${t.resource}` - } else if (typeof t !== 'string' && `list` in t) { - newS.type = 'array' - if (t.list === 'int' || t.list === 'float') { - newS.items = { type: 'number' } - } else if (t.list === 'bytes') { - newS.items = { type: 'string', contentEncoding: 'base64' } - } else if (t.list == 'string') { - newS.items = { type: 'string' } - } else if (t.list && typeof t.list == 'object' && 'str' in t.list) { - newS.items = { type: 'string', enum: t.list.str } - } else { - newS.items = { type: 'object' } - } - } else { - newS.type = 'object' - } - - if (oldS.type != newS.type) { - for (const prop of Object.getOwnPropertyNames(newS)) { - if (prop != 'description') { - delete oldS[prop] - } - } - } else if (oldS.format == 'date-time' && newS.format != 'date-time') { - delete oldS.format - } else if (oldS.items?.type != newS.items?.type) { - delete oldS.items - } - - Object.assign(oldS, newS) - - // if (sameItems && savedItems != undefined && savedItems.enum != undefined) { - // sendUserToast(JSON.stringify(savedItems)) - // oldS.items = savedItems - // } - - if (oldS.format?.startsWith('resource-') && newS.type != 'object') { - oldS.format = undefined - } -} - export async function loadSchemaFromPath(path: string, hash?: string): Promise { if (path.startsWith('hub/')) { const { content, language, schema } = await ScriptService.getHubScriptByPath({ path }) diff --git a/frontend/src/lib/inferArgSig.ts b/frontend/src/lib/inferArgSig.ts new file mode 100644 index 0000000000..786fec84df --- /dev/null +++ b/frontend/src/lib/inferArgSig.ts @@ -0,0 +1,91 @@ +import type { SchemaProperty } from './common' + +export function argSigToJsonSchemaType( + t: + | string + | { resource: string | null } + | { list: string | { str: any } | { object: { key: string; typ: any }[] } | null } + | { str: string[] | null } + | { object: { key: string; typ: any }[] }, + oldS: SchemaProperty +): void { + const newS: SchemaProperty = { type: '' } + if (t === 'int') { + newS.type = 'integer' + } else if (t === 'float') { + newS.type = 'number' + } else if (t === 'bool') { + newS.type = 'boolean' + } else if (t === 'email') { + newS.type = 'string' + newS.format = 'email' + } else if (t === 'sql') { + newS.type = 'string' + newS.format = 'sql' + } else if (t === 'yaml') { + newS.type = 'string' + newS.format = 'yaml' + } else if (t === 'bytes') { + newS.type = 'string' + newS.contentEncoding = 'base64' + } else if (t === 'datetime') { + newS.type = 'string' + newS.format = 'date-time' + } else if (typeof t !== 'string' && `object` in t) { + newS.type = 'object' + if (t.object) { + const properties = {} + for (const prop of t.object) { + properties[prop.key] = {} + argSigToJsonSchemaType(prop.typ, properties[prop.key]) + } + newS.properties = properties + } + } else if (typeof t !== 'string' && `str` in t) { + newS.type = 'string' + if (t.str) { + newS.enum = t.str + } + } else if (typeof t !== 'string' && `resource` in t) { + newS.type = 'object' + newS.format = `resource-${t.resource}` + } else if (typeof t !== 'string' && `list` in t) { + newS.type = 'array' + if (t.list === 'int' || t.list === 'float') { + newS.items = { type: 'number' } + } else if (t.list === 'bytes') { + newS.items = { type: 'string', contentEncoding: 'base64' } + } else if (t.list == 'string') { + newS.items = { type: 'string' } + } else if (t.list && typeof t.list == 'object' && 'str' in t.list) { + newS.items = { type: 'string', enum: t.list.str } + } else { + newS.items = { type: 'object' } + } + } else { + newS.type = 'object' + } + + if (oldS.type != newS.type) { + for (const prop of Object.getOwnPropertyNames(newS)) { + if (prop != 'description') { + delete oldS[prop] + } + } + } else if (oldS.format == 'date-time' && newS.format != 'date-time') { + delete oldS.format + } else if (oldS.items?.type != newS.items?.type) { + delete oldS.items + } + + Object.assign(oldS, newS) + + // if (sameItems && savedItems != undefined && savedItems.enum != undefined) { + // sendUserToast(JSON.stringify(savedItems)) + // oldS.items = savedItems + // } + + if (oldS.format?.startsWith('resource-') && newS.type != 'object') { + oldS.format = undefined + } +}