feat: data table schemas (#7353)

* data tables settings ui

* install runed

* zod 4 fixes

* use new toJSONSchema

* Migrate ducklake catalogs to more generic custom instance databases

* fix compilation

* Safety conversion for old duckdb ffi

* data tables settings

* ts client basis

* inline run works

* datatables work

* Revert "datatables work"

This reverts commit 6e1588d59e.

* datatables work (without leaking pg credentials)

* println

* separate sqlUtils.ts

* nit

* Separate custom instance db Select and Wizard components

* nit

* nit wording

* add tags to custom instance dbs

* error when trying to use ducklake as datatable or opposite

* show status in dropdown

* data table instance setup works

* sqk function for ducklake

* factorize logic

* fix temp reactivity

* Data table assetexplore

* Migrate S3 permissions to modal

* Revert "Migrate S3 permissions to modal"

This reverts commit 0631d03cb0.

* nit query -> fetch

* Custom instance setup new look

* run_language_executor separate fn

* run_inline param

* nit wording

* Better typed client

* Data tables display as assets in frontend

* asset db icon

* nit

* cleaner errors

* nit

* Fix sed calls in mac

* run_inline_script_preview in python client

* basic python datatable client

* datatable and datalake parser in python

* ducklake client python

* nit fix

* Fix migration producing NULL instead of {} when no custom databases

* merge conflict fail

* python ducklake client arg fix

* parse or infer sql types in ts client

* ts asset parser, detect datatable & ducklake R/W

* fix sql repl for other read ops than select

* export type SqlTemplateFunction

* rename list_custom_instance_pg_databases

* typecheck datatable and ducklake name in Typescript

* Fix typecheck datatable and ducklake in TS

* declare module overriding instead of extending

* infer_sql_type in python client

* SqlQuery object in python

* fix merge conflicts

* update const_format

* CI fix

* factor out to var_identifiers

* sqlx prepare

* unnecessary security (admin is required)

* clearer comment

* ee repo ref

* nit snake case

* claude step 1: detect var declarations

* move detect_sql_access_type to common mod

* claude step 2: detect when saved vars are queried

* Revert "claude step 2: detect when saved vars are queried"

This reverts commit 1e1f930568.

* Revert "claude step 1: detect var declarations"

This reverts commit f866f4819d.

* remove ducklake/datatable and default

* detect data table assigns in var_identifiers

* Python parser successfully infers R/W/RW from ducklake / datatable

* still register ducklake/datatable if not used as unknown R/W

* Go to settings button in Assets Dropdown on not found

* nit

* sqlx prepare fail

* manual fix, somehow sqlx prepare won't do it

* fix frontend ci

* ee repo ref

* ducklake_user doesnt exist in unit tests

* nit fix

* ui nit

* nit

* nit missing clone

* fork ducklakes and datatables

* fix surface hover bug

* stupid mistake

* better deeply reactive mutable derived

* Ducklake picker

* Editor bar data tables

* DuckDB supports datatables

* datatable in duckdb asset parser

* duckdb asset parser var_identifiers

* Revert "duckdb asset parser var_identifiers"

This reverts commit 88068b1a77.

* sqlx prepare

* Box pin in test_workflow_as_code to fix stack overflow

* stash

* sql asset parser parses most s3 literals

* nit

* Detect attach + handle returning RW

* detect assets used with dot notation

* detect implicit access with USE dl; syntax

* Add assets as unknown if var was never used

* Support default ducklake/datatable main in parser

* ignore asset parsing errors in frontend (avoid flow layout shift)

* super weird duplication (merge conflict ?)

* nits

* fix duckdb parser detecting too much as asset when RW ctx is unknown

* fix transparent assets btn

* missing arg

* nit styling

* asset parser specific table parsing

* fix resource specific table parsing

* More concise asset display in flows + better icons

* fix assets page filtering out resources with added table

* Fix frontend to support specific table assets

* Open DB Manager to specific table

* Specific table parser in Python and TS + unit tests

* Fix UPDATE setting access to None

* fix flow edge rendering on top of output picker

* python parser fix var override bug

* add ts test

* fix compilation

* sqlx prepare

* update parsers version

* fix missing schema key onDelete

* Grant permission to create schemas in custom instance databases

* Update pg query to return empty schemas

* Create schema

* Select nits

* support schemas in sql parser

* ts parser handle schema with sql parser result

* detect .schema() syntax

* detect schema syntax in python

* support .schema() in ts and py SDK

* open db manager to specific schema

* support reassignment in ts parser

* nit better unitest

* : syntax in ts

* datatable:schema syntax in python

* fix client py

* nit select dropdown darkmode

* object | null fetchOne

* ts client nits

* parse_sql_client_name fn

* getImportWmillTsStatement refactor in EditorBar

* text to json() in python client

* update parser versions

* pkg lock

typescript-client/sqlUtils.ts

@@ -17,13 +17,13 @@ type FetchParams<ResultCollectionT extends ResultCollection> = {
 
 type SqlResult<ResultCollectionT extends ResultCollection> =
   ResultCollectionT extends "last_statement_first_row"
-    ? object
+    ? any
     : ResultCollectionT extends "all_statements_first_row"
-    ? object[]
+    ? any[]
     : ResultCollectionT extends "last_statement_all_rows"
-    ? object[]
+    ? any[]
     : ResultCollectionT extends "all_statements_all_rows"
-    ? object[][]
+    ? any[][]
     : ResultCollectionT extends "last_statement_all_rows_scalar"
     ? any[]
     : ResultCollectionT extends "all_statements_all_rows_scalar"
@@ -68,7 +68,6 @@ export type SqlStatement = {
 export interface SqlTemplateFunction {
   (strings: TemplateStringsArray, ...values: any[]): SqlStatement;
 }
-
 /**
  * Create a SQL template function for PostgreSQL/datatable queries
  * @param name - Database/datatable name (default: "main")
@@ -83,7 +82,7 @@ export interface SqlTemplateFunction {
  * `.fetch()
  */
 export function datatable(name: string = "main"): SqlTemplateFunction {
-  return sqlProviderImpl(name, "datatable");
+  return sqlProviderImpl("datatable", parseName(name));
 }
 
 /**
@@ -100,14 +99,14 @@ export function datatable(name: string = "main"): SqlTemplateFunction {
  * `.fetch()
  */
 export function ducklake(name: string = "main"): SqlTemplateFunction {
-  return sqlProviderImpl(name, "ducklake");
+  return sqlProviderImpl("ducklake", { name });
 }
 
 function sqlProviderImpl(
-  name: string,
-  provider: "datatable" | "ducklake"
+  provider: "datatable" | "ducklake",
+  { name, schema }: { name: string; schema?: string }
 ): SqlTemplateFunction {
-  let sql: SqlTemplateFunction = (
+  let sqlFn: SqlTemplateFunction = (
     strings: TemplateStringsArray,
     ...values: any[]
   ) => {
@@ -135,6 +134,10 @@ function sqlProviderImpl(
     if (provider === "ducklake")
       content += `ATTACH 'ducklake://${name}' AS dl;USE dl;\n`;
 
+    if (schema && provider === "datatable") {
+      content += `SET search_path TO "${schema}";\n`;
+    }
+
     let contentBody = "";
     for (let i = 0; i < strings.length; i++) {
       contentBody += strings[i];
@@ -189,7 +192,8 @@ function sqlProviderImpl(
         fetch({ ...params, resultCollection: "last_statement_first_row" }),
     } satisfies SqlStatement;
   };
-  return sql;
+
+  return sqlFn;
 }
 
 // DuckDB executor requires explicit argument types at declaration
@@ -240,3 +244,19 @@ function parseTypeAnnotation(
     return nextTemplateString.substring(2).trimStart().split(/\s+/)[0];
   }
 }
+
+function parseName(name: string | undefined): {
+  name: string;
+  schema?: string;
+} {
+  if (!name) return { name: "main" };
+  let [assetName, schemaName] = name.split(":");
+  if (schemaName) {
+    return {
+      name: assetName || "main",
+      schema: schemaName,
+    };
+  } else {
+    return { name };
+  }
+}