From cbc24fa84fa18b83e9eddd1ee620ce232a8d6c07 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Sun, 1 Nov 2026 09:15:36 -0800 Subject: [PATCH] fix: decode native SQL view metadata (#4785) View APIs now read the native `name`, `query`, and `schema` columns returned by `CREATE VIEW` and `DESCRIBE VIEW`. The schema JSON carries the defining database and namespace; the requested namespace identifies where the view lives. The unreleased JSON-envelope response is removed without a compatibility fallback. ```python view = db.create_view("adults", "SELECT name FROM people WHERE age >= 18") view = db.describe_view("adults") ``` ```sql CREATE VIEW "adults" AS SELECT name FROM people WHERE age >= 18; DESCRIBE VIEW "adults"; ``` Both Python calls execute SQL through the query-backed job adapter and wait on the resulting query. A job handle uses that query's `id` and type `query`; reading metadata consumes its completed result rather than resubmitting the DDL. Creation returns the definition from the creating execution, with no extra describe request. --- docs/web/sql/ddl/view.mdx | 17 +++-- rust/lancedb/src/remote/sql/view.rs | 113 ++++++++++++++++++++-------- rust/lancedb/src/remote/sql_test.rs | 20 +++-- 3 files changed, 105 insertions(+), 45 deletions(-) diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index b4ec8d823..33a5c6560 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -30,10 +30,14 @@ FROM "documents" AS t1 INNER JOIN "categories" AS t2 ON t1.ints = t2.ints ``` -The statement returns one UTF-8 `definition` column containing JSON with the -view name, namespace, original query, default database and namespace, and its -Arrow schema resolved at creation time. `DESCRIBE VIEW "recent_matches"` -returns the same metadata shape. Anything the +The statement returns one row with UTF-8 `name`, `query`, and `schema` columns. +`query` contains the stored query, including any declared column aliases, and +`schema` contains its Arrow schema as JSON. The schema metadata keys +`lancedb::view::default_db` and `lancedb::view::default_namespace` identify the +defining database and namespace. The root namespace is `public`; nested +namespace components are separated by `$`. + +`DESCRIBE VIEW "recent_matches"` returns the same metadata shape. Anything the [dialect](/sql/functions) supports can define one: joins, predicates, aggregates, window functions. @@ -65,9 +69,8 @@ SHOW VIEWS SHOW VIEWS IN "analytics" WHERE name > 'previous_view' LIMIT 100 ``` -`SHOW VIEWS` returns ascending `name` values and a nullable `location` column. -Locations are populated only when the deployment exposes catalog locations. -The `name` cursor is scoped to the selected namespace. +`SHOW VIEWS` returns ascending `name` values. The `name` cursor is scoped to +the selected namespace. Object commands retain their small results in shared storage even when general query-result copying is disabled, so asynchronous polling can use any query node. diff --git a/rust/lancedb/src/remote/sql/view.rs b/rust/lancedb/src/remote/sql/view.rs index 993d7dc56..0c2229684 100644 --- a/rust/lancedb/src/remote/sql/view.rs +++ b/rust/lancedb/src/remote/sql/view.rs @@ -1,11 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use super::{ - SqlClient, - materialized_view::quote_identifier, - objects::{definition, strings}, -}; +use super::{SqlClient, materialized_view::quote_identifier, objects::strings}; use crate::{ database::ExecuteQueryRequest, error::{Error, Result}, @@ -20,14 +16,13 @@ use std::sync::Arc; #[derive(Clone, serde::Serialize, serde::Deserialize)] struct ViewDefinition { name: String, - #[serde(default)] namespace: Vec, query: String, default_database: String, - #[serde(default)] default_namespace: Vec, schema: JsonArrowSchema, } + impl ViewDefinition { fn into_description(self) -> Result { let schema = @@ -46,6 +41,58 @@ impl ViewDefinition { }) } } + +fn view_description( + batches: Vec, + namespace: Vec, +) -> Result { + let one = |column: &str| -> Result { + let mut values = strings(&batches, column)?; + if values.len() != 1 { + return Err(Error::Runtime { + message: "View metadata must contain exactly one row".into(), + }); + } + Ok(values.remove(0)) + }; + let name = one("name")?; + let json: JsonArrowSchema = + serde_json::from_str(&one("schema")?).map_err(|error| Error::Runtime { + message: format!("View '{name}' has invalid schema JSON: {error}"), + })?; + let schema = lance_namespace::schema::convert_json_arrow_schema(&json).map_err(|error| { + Error::Runtime { + message: format!("View '{name}' has an undecodable schema: {error}"), + } + })?; + let default_database = schema + .metadata() + .get("lancedb::view::default_db") + .ok_or_else(|| Error::Runtime { + message: format!("View '{name}' has no defining database"), + })? + .clone(); + let default_namespace = schema + .metadata() + .get("lancedb::view::default_namespace") + .ok_or_else(|| Error::Runtime { + message: format!("View '{name}' has no defining namespace"), + })?; + let default_namespace_path = if default_namespace == "public" { + Vec::new() + } else { + default_namespace.split('$').map(str::to_owned).collect() + }; + Ok(ViewDefinition { + name, + namespace, + query: one("query")?, + default_database, + default_namespace: default_namespace_path, + schema: json, + }) +} + impl SqlClient { pub(in crate::remote) async fn create_view( &self, @@ -55,19 +102,19 @@ impl SqlClient { ) -> Result { validate_view_name(name)?; validate_namespace(namespace)?; - let definition: ViewDefinition = self - .object_job( - ExecuteQueryRequest { - query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)), - default_namespace_path: namespace.to_vec(), - parameters: None, - }, - definition, - ) - .await? - .wait() - .await?; - definition.into_description() + let namespace_path = namespace.to_vec(); + self.object_job( + ExecuteQueryRequest { + query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)), + default_namespace_path: namespace.to_vec(), + parameters: None, + }, + move |batches| view_description(batches, namespace_path), + ) + .await? + .wait() + .await? + .into_description() } pub(in crate::remote) async fn describe_view( &self, @@ -76,19 +123,19 @@ impl SqlClient { ) -> Result { validate_view_name(name)?; validate_namespace(namespace)?; - let definition: ViewDefinition = self - .object_job( - ExecuteQueryRequest { - query: format!("DESCRIBE VIEW {}", quote_identifier(name)), - default_namespace_path: namespace.to_vec(), - parameters: None, - }, - definition, - ) - .await? - .wait() - .await?; - definition.into_description() + let namespace_path = namespace.to_vec(); + self.object_job( + ExecuteQueryRequest { + query: format!("DESCRIBE VIEW {}", quote_identifier(name)), + default_namespace_path: namespace.to_vec(), + parameters: None, + }, + move |batches| view_description(batches, namespace_path), + ) + .await? + .wait() + .await? + .into_description() } pub(in crate::remote) async fn list_views( &self, diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index e943107cd..bca666356 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -2694,12 +2694,22 @@ async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() { #[tokio::test] async fn view_create_and_describe_return_metadata_through_sql() { - let definition = r#"{"name":"adults","namespace":["team"],"query":"SELECT name FROM people","default_database":"db","default_namespace":["team"],"schema":{"fields":[{"name":"name","nullable":true,"type":{"type":"utf8"}}]}}"#; + let schema = r#"{"fields":[{"name":"name","nullable":true,"type":{"type":"utf8"}}],"metadata":{"lancedb::view::default_db":"db","lancedb::view::default_namespace":"team"}}"#; for defer in [false, true] { - let result = RecordBatch::try_from_iter(vec![( - "definition", - Arc::new(StringArray::from(vec![definition])) as ArrayRef, - )]) + let result = RecordBatch::try_from_iter(vec![ + ( + "name", + Arc::new(StringArray::from(vec!["adults"])) as ArrayRef, + ), + ( + "query", + Arc::new(StringArray::from(vec!["SELECT name FROM people"])) as ArrayRef, + ), + ( + "schema", + Arc::new(StringArray::from(vec![schema])) as ArrayRef, + ), + ]) .unwrap(); let service = TestSqlService { adaptive: true,