starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 08:05:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: remove requirement on full wasm parser for row insert of db studio

Browse Source
This commit is contained in:
Ruben Fiszel
2024-04-11 01:12:41 +02:00
parent 3228147058
commit c08eb0abc8
9 changed files with 152 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+11
View File
@@ -10051,6 +10051,17 @@ dependencies = [
"windmill-common",
]
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.305.0"
dependencies = [
"serde",
"wasm-bindgen",
"wasm-bindgen-test",
"windmill-parser",
"windmill-parser-sql",
]
[[package]]
name = "windmill-worker"
version = "1.305.0"
backend/Cargo.toml
+1
View File
@@ -20,6 +20,7 @@ members = [
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-sql-datatype-parser-wasm",
]
[workspace.package]
backend/parsers/windmill-parser-sql/src/lib.rs
+1 -1
View File
@@ -5,7 +5,7 @@ use regex::Regex;
use serde_json::json;
use std::collections::HashMap;
use windmill_parser::{Arg, MainArgSignature, Typ};
pub use windmill_parser::{Arg, MainArgSignature, Typ};
pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_mysql_file(&code)?;
frontend/package-lock.json
Generated
+6
View File
@@ -52,6 +52,7 @@
"vscode-uri": "~3.0.8",
"vscode-ws-jsonrpc": "~3.1.0",
"windmill-parser-wasm": "^1.286.2",
"windmill-sql-datatype-parser-wasm": "1.305.0",
"y-monaco": "^0.1.4",
"y-websocket": "^1.5.0",
"yaml": "^2.3.4",
@@ -10095,6 +10096,11 @@
"resolved": "https://registry.npmjs.org/windmill-parser-wasm/-/windmill-parser-wasm-1.286.2.tgz",
"integrity": "sha512-xwmIRy/QJoT/p5MpZw5O1Ed4T25gOjHvR4VpkJTxeMhhzRzhXA52gBRABRGL1lZImo74kWTHdMVQ/UdKB7yO0g=="
},
"node_modules/windmill-sql-datatype-parser-wasm": {
"version": "1.305.0",
"resolved": "https://registry.npmjs.org/windmill-sql-datatype-parser-wasm/-/windmill-sql-datatype-parser-wasm-1.305.0.tgz",
"integrity": "sha512-tn0yVn61g6EQp4t7zVnH6ZeL1CbjSSPinpMB8PwbxCTcv+dj5ibWRzYmNu+bqzcLjwdMTUA6VRHsbYJ3Ne4HpQ=="
},
"node_modules/wordwrap": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz",
frontend/package.json
+1
View File
@@ -134,6 +134,7 @@
"vscode-uri": "~3.0.8",
"vscode-ws-jsonrpc": "~3.1.0",
"windmill-parser-wasm": "^1.286.2",
"windmill-sql-datatype-parser-wasm": "1.305.0",
"y-monaco": "^0.1.4",
"y-websocket": "^1.5.0",
"yaml": "^2.3.4",
frontend/src/lib/components/LightweightArgInput.svelte
+1 -1
View File
@@ -305,7 +305,7 @@
</button>
</div>
{/each}
{:else}
{:else if value != undefined}
List is not an array
{/if}
</div>
frontend/src/lib/components/apps/components/display/dbtable/InsertRow.svelte
+38 -21
View File
@@ -9,19 +9,20 @@
type ColumnDef
} from './utils'
import wasmUrl from 'windmill-parser-wasm/windmill_parser_wasm_bg.wasm?url'
import init, {
parse_sql,
parse_mysql,
parse_bigquery,
parse_snowflake,
parse_mssql
} from 'windmill-parser-wasm'
import type { MainArgSignature } from '$lib/gen'
import { makeInsertQuery } from './queries/insert'
import { argSigToJsonSchemaType } from '$lib/infer'
} from 'windmill-sql-datatype-parser-wasm'
import wasmUrl from 'windmill-sql-datatype-parser-wasm/windmill_sql_datatype_parser_wasm_bg.wasm?url'
init(wasmUrl)
import { argSigToJsonSchemaType } from '$lib/inferArgSig'
import fi from 'date-fns/locale/fi'
export let args: Record<string, any> = {}
export let dbType: DbType = 'postgresql'
@@ -63,50 +64,66 @@
}
}) as FieldMetadata[] | undefined
async function parseSQLArgs(code: string, dbType: DbType) {
await init(wasmUrl)
let rawSchema = ''
function parseSQLArgs(field: string, dbType: DbType): string {
let rawType = ''
switch (dbType) {
case 'mysql':
rawSchema = parse_mysql(code)
rawType = parse_mysql(field)
break
case 'postgresql':
rawSchema = parse_sql(code)
rawType = parse_sql(field)
break
case 'bigquery':
rawSchema = parse_bigquery(code)
rawType = parse_bigquery(field)
break
case 'snowflake':
rawSchema = parse_snowflake(code)
rawType = parse_snowflake(field)
break
case 'ms_sql_server':
rawSchema = parse_mssql(code)
rawType = parse_mssql(field)
break
default:
throw new Error('Language not supported')
}
const args: MainArgSignature = JSON.parse(rawSchema)
return args
return rawType
}
function rawTypeToSchemaType(typ: string) {
if (typ.startsWith('list-')) {
return {
list: rawTypeToSchemaType(typ.replace('list-', ''))
}
} else if (typ == 'str') {
return {
str: undefined
}
} else {
return typ
}
}
async function builtSchema(fields: FieldMetadata[], dbType: DbType) {
const properties: { [name: string]: SchemaProperty } = {}
const required: string[] = []
const insertCode = makeInsertQuery('ignoredtable', columnDefs, dbType)
const args = await parseSQLArgs(insertCode, dbType)
await init(wasmUrl)
fields.forEach((field) => {
const schemaProperty: SchemaProperty = {
type: 'string'
}
const parsedArg = args.args.find((arg) => arg.name === field.name)
const parsedArg = columnDefs.find((arg) => arg.field === field.name)
if (parsedArg) {
argSigToJsonSchemaType(parsedArg.typ, schemaProperty)
let typ: any = rawTypeToSchemaType(parseSQLArgs(parsedArg.datatype, dbType))
argSigToJsonSchemaType(typ, schemaProperty)
console.log(
'schemaProperty',
schemaProperty,
field.name,
typ,
parseSQLArgs(parsedArg.datatype, dbType)
)
}
if (field.defaultValue) {
frontend/src/lib/infer.ts
+2 -91
View File
@@ -1,6 +1,6 @@
import { ScriptService, type MainArgSignature, FlowService, Script } from '$lib/gen'
import { get, writable } from 'svelte/store'
import type { Schema, SchemaProperty, SupportedLanguage } from './common.js'
import type { Schema, SupportedLanguage } from './common.js'
import { emptySchema, sortObject } from './utils.js'
import { tick } from 'svelte'
import init, {
@@ -21,6 +21,7 @@ import init, {
} from 'windmill-parser-wasm'
import wasmUrl from 'windmill-parser-wasm/windmill_parser_wasm_bg.wasm?url'
import { workspaceStore } from './stores.js'
import { argSigToJsonSchemaType } from './inferArgSig.js'
init(wasmUrl)
@@ -151,96 +152,6 @@ export async function inferArgs(
await tick()
}
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| { list: string | { str: any } | { object: { key: string; typ: any }[] } | null }
| { str: string[] | null }
| { object: { key: string; typ: any }[] },
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: '' }
if (t === 'int') {
newS.type = 'integer'
} else if (t === 'float') {
newS.type = 'number'
} else if (t === 'bool') {
newS.type = 'boolean'
} else if (t === 'email') {
newS.type = 'string'
newS.format = 'email'
} else if (t === 'sql') {
newS.type = 'string'
newS.format = 'sql'
} else if (t === 'yaml') {
newS.type = 'string'
newS.format = 'yaml'
} else if (t === 'bytes') {
newS.type = 'string'
newS.contentEncoding = 'base64'
} else if (t === 'datetime') {
newS.type = 'string'
newS.format = 'date-time'
} else if (typeof t !== 'string' && `object` in t) {
newS.type = 'object'
if (t.object) {
const properties = {}
for (const prop of t.object) {
properties[prop.key] = {}
argSigToJsonSchemaType(prop.typ, properties[prop.key])
}
newS.properties = properties
}
} else if (typeof t !== 'string' && `str` in t) {
newS.type = 'string'
if (t.str) {
newS.enum = t.str
}
} else if (typeof t !== 'string' && `resource` in t) {
newS.type = 'object'
newS.format = `resource-${t.resource}`
} else if (typeof t !== 'string' && `list` in t) {
newS.type = 'array'
if (t.list === 'int' || t.list === 'float') {
newS.items = { type: 'number' }
} else if (t.list === 'bytes') {
newS.items = { type: 'string', contentEncoding: 'base64' }
} else if (t.list == 'string') {
newS.items = { type: 'string' }
} else if (t.list && typeof t.list == 'object' && 'str' in t.list) {
newS.items = { type: 'string', enum: t.list.str }
} else {
newS.items = { type: 'object' }
}
} else {
newS.type = 'object'
}
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != 'description') {
delete oldS[prop]
}
}
} else if (oldS.format == 'date-time' && newS.format != 'date-time') {
delete oldS.format
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items
}
Object.assign(oldS, newS)
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
if (oldS.format?.startsWith('resource-') && newS.type != 'object') {
oldS.format = undefined
}
}
export async function loadSchemaFromPath(path: string, hash?: string): Promise<Schema> {
if (path.startsWith('hub/')) {
const { content, language, schema } = await ScriptService.getHubScriptByPath({ path })
frontend/src/lib/inferArgSig.ts
+91
View File
@@ -0,0 +1,91 @@
import type { SchemaProperty } from './common'
export function argSigToJsonSchemaType(
t:
| string
| { resource: string | null }
| { list: string | { str: any } | { object: { key: string; typ: any }[] } | null }
| { str: string[] | null }
| { object: { key: string; typ: any }[] },
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: '' }
if (t === 'int') {
newS.type = 'integer'
} else if (t === 'float') {
newS.type = 'number'
} else if (t === 'bool') {
newS.type = 'boolean'
} else if (t === 'email') {
newS.type = 'string'
newS.format = 'email'
} else if (t === 'sql') {
newS.type = 'string'
newS.format = 'sql'
} else if (t === 'yaml') {
newS.type = 'string'
newS.format = 'yaml'
} else if (t === 'bytes') {
newS.type = 'string'
newS.contentEncoding = 'base64'
} else if (t === 'datetime') {
newS.type = 'string'
newS.format = 'date-time'
} else if (typeof t !== 'string' && `object` in t) {
newS.type = 'object'
if (t.object) {
const properties = {}
for (const prop of t.object) {
properties[prop.key] = {}
argSigToJsonSchemaType(prop.typ, properties[prop.key])
}
newS.properties = properties
}
} else if (typeof t !== 'string' && `str` in t) {
newS.type = 'string'
if (t.str) {
newS.enum = t.str
}
} else if (typeof t !== 'string' && `resource` in t) {
newS.type = 'object'
newS.format = `resource-${t.resource}`
} else if (typeof t !== 'string' && `list` in t) {
newS.type = 'array'
if (t.list === 'int' || t.list === 'float') {
newS.items = { type: 'number' }
} else if (t.list === 'bytes') {
newS.items = { type: 'string', contentEncoding: 'base64' }
} else if (t.list == 'string') {
newS.items = { type: 'string' }
} else if (t.list && typeof t.list == 'object' && 'str' in t.list) {
newS.items = { type: 'string', enum: t.list.str }
} else {
newS.items = { type: 'object' }
}
} else {
newS.type = 'object'
}
if (oldS.type != newS.type) {
for (const prop of Object.getOwnPropertyNames(newS)) {
if (prop != 'description') {
delete oldS[prop]
}
}
} else if (oldS.format == 'date-time' && newS.format != 'date-time') {
delete oldS.format
} else if (oldS.items?.type != newS.items?.type) {
delete oldS.items
}
Object.assign(oldS, newS)
// if (sameItems && savedItems != undefined && savedItems.enum != undefined) {
// sendUserToast(JSON.stringify(savedItems))
// oldS.items = savedItems
// }
if (oldS.format?.startsWith('resource-') && newS.type != 'object') {
oldS.format = undefined
}
}