feat: add ducklake schema support to the database manager (#9633)

* feat: add ducklake schema support to the database manager

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat: support schema in wmill.ducklake("name:schema") template helper

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* fix: preserve schema when parsing ducklake asset/favorite paths

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* chore: regenerate system prompts for ducklake schema syntax doc

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Diego Imbert
2026-06-17 19:23:14 +02:00
committed by GitHub
parent 25a891041d
commit 3eeccaf968
20 changed files with 188 additions and 52 deletions
+51 -8
View File
@@ -1739,7 +1739,7 @@ struct PrimaryKeyConstraintPayload {
fn db_supports_schemas(db_type: DbType) -> bool {
matches!(
db_type,
DbType::Postgresql | DbType::Snowflake | DbType::Bigquery
DbType::Postgresql | DbType::Snowflake | DbType::Bigquery | DbType::Duckdb
)
}
@@ -2410,8 +2410,15 @@ fn make_load_table_metadata_query(
) -> Result<String, String> {
match db_type {
DbType::Duckdb => {
// For ducklake, the ducklake ATTACH is handled by the ducklake wrapper.
let mut q = String::from(
// For ducklake, the ducklake ATTACH is handled by the ducklake wrapper, so the
// ducklake catalog is the current database. information_schema spans every attached
// catalog, so we always scope to current_database() to stay within the ducklake.
let extra_col = if table.is_none() {
",\n TABLE_SCHEMA as schema_name"
} else {
""
};
let mut q = format!(
"SELECT
COLUMN_NAME as field,
DATA_TYPE as DataType,
@@ -2420,12 +2427,20 @@ fn make_load_table_metadata_query(
false as IsIdentity,
CASE WHEN IS_NULLABLE = true THEN 'YES' ELSE 'NO' END as IsNullable,
false as IsEnum,
TABLE_NAME as table_name
TABLE_NAME as table_name{}
FROM information_schema.columns c
WHERE table_schema = current_schema()",
WHERE table_catalog = current_database()",
extra_col
);
if let Some(t) = table {
q.push_str(&format!(" AND TABLE_NAME = '{}'", escape_sql_literal(t)));
let parts: Vec<&str> = t.split('.').collect();
let tname = parts[parts.len() - 1];
let schema = if parts.len() > 1 { parts[0] } else { "main" };
q.push_str(&format!(
" AND TABLE_NAME = '{}' AND TABLE_SCHEMA = '{}'",
escape_sql_literal(tname),
escape_sql_literal(schema)
));
}
Ok(q)
}
@@ -3722,9 +3737,10 @@ mod tests {
table_ref("users", Some("myschema"), DbType::Mysql),
"`users`"
);
// DuckDB (ducklake) supports schemas
assert_eq!(
table_ref("users", Some("myschema"), DbType::Duckdb),
r#""users""#
r#""myschema"."users""#
);
}
@@ -3856,6 +3872,13 @@ mod tests {
assert!(sql.contains("DROP TABLE \"users\";"));
}
#[test]
fn test_expand_drop_table_ducklake_with_schema() {
let marker = r#"-- WM_INTERNAL_DB_DROP_TABLE {"table":"events","schema":"analytics","ducklake":"my_lake"}"#;
let sql = expand_code(marker, &ScriptLang::DuckDb);
assert!(sql.contains("DROP TABLE \"analytics\".\"events\";"));
}
// -----------------------------------------------------------------------
// CREATE SCHEMA / DROP SCHEMA
// -----------------------------------------------------------------------
@@ -4355,7 +4378,27 @@ mod tests {
let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"table":"users","ducklake":"lake"}"#;
let sql = expand_code(marker, &ScriptLang::DuckDb);
assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n"));
assert!(sql.contains("TABLE_NAME = 'users'"));
assert!(sql.contains("table_catalog = current_database()"));
// Unqualified table defaults to the "main" schema.
assert!(sql.contains("TABLE_NAME = 'users' AND TABLE_SCHEMA = 'main'"));
}
#[test]
fn test_expand_load_table_metadata_ducklake_qualified_schema() {
let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"table":"analytics.events","ducklake":"lake"}"#;
let sql = expand_code(marker, &ScriptLang::DuckDb);
assert!(sql.contains("TABLE_NAME = 'events' AND TABLE_SCHEMA = 'analytics'"));
}
#[test]
fn test_expand_load_table_metadata_ducklake_all_tables() {
let marker = r#"-- WM_INTERNAL_DB_LOAD_TABLE_METADATA {"ducklake":"lake"}"#;
let sql = expand_code(marker, &ScriptLang::DuckDb);
assert!(sql.starts_with("ATTACH 'ducklake://lake' AS dl;USE dl;\n"));
// All-tables listing scopes to the ducklake catalog and exposes the schema per table.
assert!(sql.contains("table_catalog = current_database()"));
assert!(sql.contains("TABLE_SCHEMA as schema_name"));
assert!(!sql.contains("TABLE_NAME = '"));
}
// -----------------------------------------------------------------------
+12 -3
View File
@@ -946,7 +946,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -956,6 +956,9 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}
* \`.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
`,
@@ -1674,7 +1677,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -1684,6 +1687,9 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}
* \`.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
`,
@@ -2494,7 +2500,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -2504,6 +2510,9 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}
* \`.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
`,
+7 -1
View File
@@ -162,7 +162,13 @@
if (!selected.schemaKey && schemaKeys.length) {
let schemaKey =
initialSchemaKey ??
('public' in dbSchema.schema ? 'public' : 'dbo' in dbSchema.schema ? 'dbo' : schemaKeys[0])
('public' in dbSchema.schema
? 'public'
: 'dbo' in dbSchema.schema
? 'dbo'
: 'main' in dbSchema.schema
? 'main'
: schemaKeys[0])
let tableKey =
initialTableKey && dbSchema.schema?.[schemaKey]?.[initialTableKey]
? initialTableKey
@@ -150,7 +150,7 @@
{/if}
</div>
<DbManager
dbSupportsSchemas={input?.type == 'database' && dbSupportsSchemas(input.resourceType)}
dbSupportsSchemas={dbSupportsSchemas(dbType)}
databaseIsEmpty={!Object.values(dbSchema.schema).flatMap((s) => Object.values(s)).length}
{dbSchema}
colDefs={colDefs.current}
@@ -166,7 +166,7 @@
workspace: $workspaceStore
})}
initialTableKey={input.specificTable}
initialSchemaKey={input.type == 'database' ? input.specificSchema : undefined}
initialSchemaKey={input.specificSchema}
asset={_input.type == 'ducklake'
? { kind: 'ducklake', path: _input.ducklake }
: _input.resourcePath.startsWith('datatable://')
@@ -69,12 +69,17 @@
} else if (asset.kind === 's3object' && isS3Uri(assetUri)) {
s3FilePicker?.open(assetUri)
} else if (asset.kind === 'volume') {
const storage = (await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined
const storage =
(await VolumeService.getVolumeStorage({ workspace: $workspaceStore! })) ?? undefined
s3FilePicker?.open({ s3: `volumes/${$workspaceStore}/${asset.path}/`, storage })
} else if (asset.kind === 'ducklake') {
let ducklake = asset.path.split('/')[0]
let specificTable = asset.path.split('/')[1] as string | undefined
dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificTable })
let specificTableSplit = asset.path.split('/')[1]?.split('.') as string[] | undefined
let [specificSchema, specificTable] =
specificTableSplit?.length === 2
? [specificTableSplit[0], specificTableSplit[1]]
: [undefined, specificTableSplit?.[0]]
dbManagerDrawer?.openDrawer({ type: 'ducklake', ducklake, specificSchema, specificTable })
} else if (asset.kind === 'datatable') {
let datatable = asset.path.split('/')[0]
let specificTableSplit = asset.path.split('/')[1]?.split('.') as string[] | undefined
@@ -17,7 +17,7 @@ export function getDbFeatures(dbInput: DbInput): Required<DbFeatures> {
primaryKeys: true,
defaultValues: true,
enforcedForeignKeys: true,
schemas: dbInput.type !== 'ducklake' && dbSupportsSchemas(dbInput.resourceType)
schemas: dbInput.type === 'ducklake' ? true : dbSupportsSchemas(dbInput.resourceType)
}
if (dbInput.type == 'ducklake')
@@ -356,7 +356,7 @@ export async function getTablesByResource(
const paths: string[] = []
for (const key in s?.schema) {
for (const subKey in s.schema[key]) {
paths.push(`${subKey}`)
paths.push(key === 'main' ? `${subKey}` : `${key}.${subKey}`)
}
}
@@ -381,7 +381,12 @@ export function getPrimaryKeys(tableMetadata?: TableMetadata): string[] {
}
export function dbSupportsSchemas(dbType: DbType): boolean {
return dbType === 'postgresql' || dbType === 'snowflake' || dbType === 'bigquery'
return (
dbType === 'postgresql' ||
dbType === 'snowflake' ||
dbType === 'bigquery' ||
dbType === 'duckdb'
)
}
export function datatypeHasLength(datatype: string): boolean {
@@ -123,6 +123,7 @@ export function useDbManagerUriState(): DbManagerUriState {
return {
type: 'ducklake' as const,
ducklake: parsed.path,
specificSchema: parsed.schema,
specificTable: parsed.table
}
}
@@ -161,6 +162,7 @@ export function useDbManagerUriState(): DbManagerUriState {
params.dbm = buildDbm({
type: 'ducklake',
path: nInput.ducklake,
schema: nInput.specificSchema,
table: nInput.specificTable
})
}
+28 -18
View File
@@ -297,35 +297,45 @@ export async function getDucklakeSchema({
args: {}
}
})
let mainSchema = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? [])
let schemas = Array.isArray(result) && result.length && (result?.[0]?.['result'] ?? {})
// Safety for agent workers (duckdb ffi lib used to return JSON as stringified json)
if (typeof mainSchema === 'string') mainSchema = JSON.parse(mainSchema)
if (typeof schemas === 'string') schemas = JSON.parse(schemas)
if (!mainSchema) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result))
assert('mainSchema is an object', typeof mainSchema === 'object')
if (!schemas) throw new Error('Failed to get Ducklake schema: ' + JSON.stringify(result))
assert('schemas is an object', typeof schemas === 'object')
let schema: Omit<SQLSchema, 'stringified'> = {
schema: { main: mainSchema },
publicOnly: true,
schema: schemas,
publicOnly: false,
lang: 'ducklake'
}
return { ...schema, stringified: stringifySchema(schema) }
}
// Returns every schema in the ducklake (including empty ones, e.g. freshly created)
// as a nested map { schema: { table: { column: {...} } } }.
const DUCKLAKE_GET_SCHEMA_QUERY = `
SELECT json_group_object(table_name, table_data) AS result FROM (
SELECT json_group_object(schema_name, COALESCE(schema_data, json_object())) AS result FROM (
SELECT
table_name,
json_group_object(
c.column_name,
json_object(
'type', c.data_type,
'default', c.column_default,
'required', c.is_nullable == 'NO'
s.schema_name,
(
SELECT json_group_object(table_name, table_data) FROM (
SELECT
c.table_name,
json_group_object(
c.column_name,
json_object(
'type', c.data_type,
'default', c.column_default,
'required', c.is_nullable == 'NO'
)
) AS table_data
FROM information_schema.columns c
WHERE c.table_catalog = '__ducklake__' AND c.table_schema = s.schema_name
GROUP BY c.table_name
)
) AS table_data
FROM information_schema.columns c
WHERE table_catalog = '__ducklake__' AND table_schema = current_schema()
GROUP BY c.table_name
) AS schema_data
FROM information_schema.schemata s
WHERE s.catalog_name = '__ducklake__'
)`
export function getDbType(input: DbInput): DbType {
+1
View File
@@ -9,6 +9,7 @@ export type DbInput =
| {
type: 'ducklake'
ducklake: string
specificSchema?: string
specificTable?: string
}
+6 -1
View File
@@ -2113,7 +2113,12 @@ export function parseDbInputFromAssetSyntax(path: string): DbInput | null {
const [p2, _p3] = _p2.split('/')
const [p3, p4] = _p3.split('.')
return p1 === 'ducklake'
? { type: 'ducklake', ducklake: p2 || 'main', specificTable: p4 ?? p3 }
? {
type: 'ducklake',
ducklake: p2 || 'main',
specificTable: p4 ?? p3,
specificSchema: p4 ? p3 : undefined
}
: p1 === 'datatable'
? {
type: 'database',
+4 -1
View File
@@ -1472,7 +1472,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as \`name:schema\` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -1482,6 +1482,9 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = \${name} AND age = \${age}
* \`.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
`;
+4 -1
View File
@@ -1884,7 +1884,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -1894,6 +1894,9 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
@@ -542,7 +542,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -552,5 +552,8 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
@@ -713,7 +713,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -723,5 +723,8 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
@@ -713,7 +713,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -723,5 +723,8 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
@@ -713,7 +713,7 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -723,5 +723,8 @@ datatable(name: string = "main"): DatatableSqlTemplateFunction
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
ducklake(name: string = "main"): SqlTemplateFunction
+10 -4
View File
@@ -138,14 +138,16 @@ function datatableProvider(name: string, schema?: string): SqlProvider {
};
}
function ducklakeProvider(name: string): SqlProvider {
function ducklakeProvider(name: string, schema?: string): SqlProvider {
return {
providerName: "ducklake",
language: "duckdb",
extraArgs: {},
formatArgDecl: (argNum, argType) => `-- $arg${argNum} (${argType})`,
formatArgUsage: (argNum) => `$arg${argNum}`,
preamble: () => `ATTACH 'ducklake://${name}' AS dl;USE dl;\n`,
// `USE dl."schema"` sets the active schema so unqualified tables resolve there.
preamble: () =>
`ATTACH 'ducklake://${name}' AS dl;USE dl${schema ? `."${schema}"` : ""};\n`,
};
}
@@ -364,7 +366,7 @@ export function datatable(name: string = "main"): DatatableSqlTemplateFunction {
/**
* Create a SQL template function for DuckDB/ducklake queries
* @param name - DuckDB database name (default: "main")
* @param name - DuckDB database name, optionally with a schema as `name:schema` (default: "main")
* @returns SQL template function for building parameterized queries
* @example
* let sql = wmill.ducklake()
@@ -374,9 +376,13 @@ export function datatable(name: string = "main"): DatatableSqlTemplateFunction {
* SELECT * FROM friends
* WHERE name = ${name} AND age = ${age}
* `.fetch()
* @example
* // Target a specific schema within the ducklake
* let sql = wmill.ducklake("my_lake:analytics")
*/
export function ducklake(name: string = "main"): SqlTemplateFunction {
return buildSqlTemplateFunction(ducklakeProvider(name));
let { name: n, schema } = parseName(name);
return buildSqlTemplateFunction(ducklakeProvider(n, schema));
}
// ---------------------------------------------------------------------------
+29 -3
View File
@@ -51,17 +51,25 @@ function datatableProvider(name: string, schema?: string): SqlProvider {
};
}
function ducklakeProvider(name: string): SqlProvider {
function ducklakeProvider(name: string, schema?: string): SqlProvider {
return {
providerName: "ducklake",
language: "duckdb",
extraArgs: {},
formatArgDecl: (argNum, argType) => `-- $arg${argNum} (${argType})`,
formatArgUsage: (argNum) => `$arg${argNum}`,
preamble: () => `ATTACH 'ducklake://${name}' AS dl;USE dl;\n`,
preamble: () =>
`ATTACH 'ducklake://${name}' AS dl;USE dl${schema ? `."${schema}"` : ""};\n`,
};
}
function parseName(name: string | undefined): { name: string; schema?: string } {
if (!name) return { name: "main" };
let [assetName, schemaName] = name.split(":");
if (schemaName) return { name: assetName || "main", schema: schemaName };
return { name };
}
function inferSqlType(value: any): string {
if (typeof value === "bigint") return "BIGINT";
if (typeof value === "number") {
@@ -231,7 +239,10 @@ function templateTag(provider: SqlProvider) {
}
const dt = (name = "main") => templateTag(datatableProvider(name));
const dl = (name = "main") => templateTag(ducklakeProvider(name));
const dl = (name = "main") => {
const { name: n, schema } = parseName(name);
return templateTag(ducklakeProvider(n, schema));
};
const datatableQuery = (name = "main") => {
const provider = datatableProvider(name);
return (sql: string, ...params: any[]) =>
@@ -604,6 +615,21 @@ describe("ducklake() — DuckDB shape", () => {
const out = sql`SELECT 1`;
expect(out.args).not.toHaveProperty("database");
});
test("name:schema sets the active schema via USE dl.<schema>", () => {
const sql = dl("my_lake:analytics");
const out = sql`SELECT 1`;
expect(out.content).toContain(
`ATTACH 'ducklake://my_lake' AS dl;USE dl."analytics";`
);
});
test("no schema keeps the bare USE dl preamble", () => {
const sql = dl("my_lake");
const out = sql`SELECT 1`;
expect(out.content).toContain("ATTACH 'ducklake://my_lake' AS dl;USE dl;");
expect(out.content).not.toContain("USE dl.");
});
});
// =============================================================================