feat(frontend): schema explorer, autocomplete and db aware AI for mysql (#1944)

* feat(frontend): schema explorer mysql
including autocomplete and AI gen

* fix: use fixed lib version for DB test and explore

frontend/src/lib/components/DBSchemaExplorer.svelte

@@ -9,30 +9,25 @@
 	import ObjectViewer from './propertyPicker/ObjectViewer.svelte'
 
 	export let resourceType: string | undefined
-	export let pg: String | undefined = undefined
+	export let resourcePath: String | undefined = undefined
 
 	let drawer: Drawer
 
-	async function getSchema() {
-		const content = `
-import { Client } from "https://deno.land/x/postgres/mod.ts";
-
-export async function main(pg: Postgresql) {
+	const content = {
+		postgresql: `import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts";
+export async function main(args: any) {
   // Create a new client with the provided connection details
   const u = new URL("postgres://")
 	u.hash = ''
-	u.search = '?sslmode=' + pg.sslmode
-	u.pathname = pg.dbname
-	u.host = pg.host
-	u.port = pg.port
-	u.password = pg.password
-	u.username = pg.user
-
+	u.search = '?sslmode=' + args.sslmode
+	u.pathname = args.dbname
+	u.host = args.host
+	u.port = args.port
+	u.password = args.password
+	u.username = args.user
   const client = new Client(u.toString())
-
   // Connect to the postgres database
   await client.connect();
-
   const result = await client.queryObject(\`SELECT 
       table_name, 
       column_name, 
@@ -44,7 +39,6 @@ export async function main(pg: Postgresql) {
       information_schema.columns
     WHERE table_schema != 'pg_catalog' AND 
       table_schema != 'information_schema'\`);
-
   const schemas = result.rows.reduce((acc, a) => {
     const table_schema = a.table_schema;
     delete a.table_schema;
@@ -52,7 +46,6 @@ export async function main(pg: Postgresql) {
     acc[table_schema].push(a);
     return acc;
   }, {});
-
   const data = {};
   for (const key in schemas) {
     data[key] = schemas[key].reduce((acc, a) => {
@@ -70,17 +63,59 @@ export async function main(pg: Postgresql) {
       return acc;
     }, {});
   }
-
+  return data;
+}`,
+		mysql: `import { Client } from "https://deno.land/x/mysql@v2.11.0/mod.ts";
+export async function main(args: any) {
+  const conn = await new Client().connect({
+    hostname: args.host,
+    port: args.port,
+    username: args.user,
+    db: args.database,
+    password: args.password,
+  });
+  const result = await conn.execute(
+    "select TABLE_SCHEMA, TABLE_NAME, DATA_TYPE, COLUMN_NAME, COLUMN_DEFAULT from information_schema.columns where table_schema != 'information_schema'",
+  );
+  const schemas = result.rows.reduce((acc, a) => {
+    const table_schema = a.TABLE_SCHEMA;
+    delete a.TABLE_SCHEMA;
+    acc[table_schema] = acc[table_schema] || [];
+    acc[table_schema].push(a);
+    return acc;
+  }, {});
+  const data = {};
+  for (const key in schemas) {
+    data[key] = schemas[key].reduce((acc, a) => {
+      const table_name = a.TABLE_NAME;
+      delete a.TABLE_NAME;
+      acc[table_name] = acc[table_name] || {};
+      const p = {
+        type: a.DATA_TYPE,
+        required: a.is_nullable === "NO",
+      };
+      if (a.column_default) {
+        p.default = a.COLUMN_DEFAULT;
+      }
+      acc[table_name][a.COLUMN_NAME] = p;
+      return acc;
+    }, {});
+  }
   return data;
 }`
+	}
+
+	async function getSchema() {
+		if (!resourceType || !resourcePath) return
+		dbSchema.set(undefined)
 		try {
 			const job = await JobService.runScriptPreview({
 				workspace: $workspaceStore!,
 				requestBody: {
 					language: 'deno' as Preview.language,
-					content,
+					content: content[resourceType],
 					args: {
-						pg: '$res:' + pg
+						args: '$res:' + resourcePath
 					}
 				}
 			})
@@ -102,15 +137,15 @@ export async function main(pg: Postgresql) {
 		}
 	}
 
-	$: pg && resourceType === 'postgresql' && getSchema()
-	$: !pg && $dbSchema && dbSchema.set(undefined)
+	$: resourcePath && resourceType && ['postgresql', 'mysql'].includes(resourceType) && getSchema()
+	$: !resourcePath && $dbSchema && dbSchema.set(undefined)
 
 	onDestroy(() => {
 		dbSchema.set(undefined)
 	})
 </script>
 
-{#if $dbSchema && pg}
+{#if $dbSchema && resourcePath}
 	<Button
 		size="xs"
 		variant="border"

frontend/src/lib/components/Editor.svelte

@@ -219,6 +219,9 @@
 	$: !$dbSchema && dbSchemaCompletor && dbSchemaCompletor.dispose()
 
 	function addDBSchemaCompletions() {
+		if (dbSchemaCompletor) {
+			dbSchemaCompletor.dispose()
+		}
 		dbSchemaCompletor = languages.registerCompletionItemProvider('sql', {
 			triggerCharacters: ['.', ' ', '('],
 			provideCompletionItems: function (model, position) {

frontend/src/lib/components/ResourcePicker.svelte

@@ -40,6 +40,7 @@
 			label: x.path
 		}))
 
+		// TODO check if this is needed
 		if (!nc.find((x) => x.value == value) && (initialValue || value)) {
 			nc.push({ value: value ?? initialValue!, label: value ?? initialValue! })
 		}
@@ -140,7 +141,7 @@
 		<Icon scale={0.8} data={faRotateRight} />
 	</Button>
 </div>
-<DbSchemaExplorer {resourceType} pg={value} />
+<DbSchemaExplorer {resourceType} resourcePath={value} />
 
 <style>
 	:global(.svelte-select-list) {

frontend/src/lib/components/TestConnection.svelte

@@ -9,13 +9,10 @@
 	export let resource_type: string | undefined
 	export let args: Record<string, any> | any = {}
 
-	let loading = false
-	async function testConnection() {
-		loading = true
-		const content = `
-import { Client } from 'https://deno.land/x/postgres/mod.ts'
+	const content = {
+		postgresql: `import { Client } from 'https://deno.land/x/postgres@v0.17.0/mod.ts'
 export async function main(args: any) {
-    const u = new URL("postgres://")
+  const u = new URL("postgres://")
 	u.hash = ''
 	u.search = '?sslmode=' + args.sslmode
 	u.pathname = args.dbname
@@ -26,13 +23,31 @@ export async function main(args: any) {
 	const client = new Client(u.toString())
 	await client.connect()
 	return 'Connection successful'
-}
-`
+}`,
+		mysql: `import { Client } from "https://deno.land/x/mysql@v2.11.0/mod.ts";
+export async function main(args: any) {
+  const conn = await new Client().connect({
+    hostname: args.host,
+    port: args.port,
+    username: args.user,
+    db: args.database,
+    password: args.password,
+  });
+  await conn.query("SELECT 1");
+  return "Connection successful";
+}`
+	}
+
+	let loading = false
+	async function testConnection() {
+		if (!resource_type) return
+		loading = true
+
 		const job = await JobService.runScriptPreview({
 			workspace: $workspaceStore!,
 			requestBody: {
 				language: 'deno' as Preview.language,
-				content,
+				content: content[resource_type],
 				args: {
 					args
 				}
@@ -53,7 +68,7 @@ export async function main(args: any) {
 	}
 </script>
 
-{#if resource_type == 'postgresql'}
+{#if resource_type == 'postgresql' || resource_type == 'mysql'}
 	<Button size="sm" on:click={testConnection}
 		>{#if loading}<Loader2 class="animate-spin mr-2" />{/if} Test connection</Button
 	>

frontend/src/lib/components/codeGen/lib.ts

@@ -71,7 +71,7 @@ async function addResourceTypes(scriptOptions: BaseOptions, workspace: string, p
 }
 
 function addDBSChema(scriptOptions: BaseOptions, prompt: string) {
-	if (scriptOptions.language === 'postgresql' && scriptOptions.dbSchema) {
+	if (['mysql', 'postgresql'].includes(scriptOptions.language) && scriptOptions.dbSchema) {
 		const { dbSchema } = scriptOptions
 		const smallerSchema = {}
 		for (const schemaKey in dbSchema) {
@@ -118,28 +118,6 @@ export async function generateScript(scriptOptions: ScriptGenerationOptions) {
 
 	prompt = addDBSChema(scriptOptions, prompt)
 
-	if (scriptOptions.language === 'postgresql' && scriptOptions.dbSchema) {
-		const { dbSchema } = scriptOptions
-		const smallerSchema = {}
-		for (const schemaKey in dbSchema) {
-			for (const tableKey in dbSchema[schemaKey]) {
-				smallerSchema[tableKey] = []
-				for (const colKey in dbSchema[schemaKey][tableKey]) {
-					const col = dbSchema[schemaKey][tableKey][colKey]
-					const p = [colKey, col.type, col.required]
-					if (col.default) {
-						p.push(col.default)
-					}
-					smallerSchema[tableKey].push(p)
-				}
-			}
-		}
-		prompt =
-			prompt +
-			"\nHere's the database schema, each column is in the format [name, type, required, default?]: " +
-			JSON.stringify(smallerSchema)
-	}
-
 	const completion = await openai.chat.completions.create({
 		model: 'gpt-4',
 		max_tokens: 2048,