rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-11-13 14:16:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: decode native SQL view metadata (#4785)

Browse Source 
View APIs now read the native `name`, `query`, and `schema` columns
returned by `CREATE VIEW` and `DESCRIBE VIEW`. The schema JSON carries
the defining database and namespace; the requested namespace identifies
where the view lives. The unreleased JSON-envelope response is removed
without a compatibility fallback.

```python
view = db.create_view("adults", "SELECT name FROM people WHERE age >= 18")
view = db.describe_view("adults")
```

```sql
CREATE VIEW "adults" AS SELECT name FROM people WHERE age >= 18;
DESCRIBE VIEW "adults";
```

Both Python calls execute SQL through the query-backed job adapter and
wait on the resulting query. A job handle uses that query's `id` and
type `query`; reading metadata consumes its completed result rather than
resubmitting the DDL. Creation returns the definition from the creating
execution, with no extra describe request.
This commit is contained in:
Jack Ye
2026-11-02 01:15:36 +08:00
committed by GitHub
parent 42e8e7dfca
commit cbc24fa84f
3 changed files with 105 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/web/sql/ddl/view.mdx
+10 -7
View File
@@ -30,10 +30,14 @@ FROM "documents" AS t1
INNER JOIN "categories" AS t2 ON t1.ints = t2.ints
```
The statement returns one UTF-8 `definition` column containing JSON with the
view name, namespace, original query, default database and namespace, and its
Arrow schema resolved at creation time. `DESCRIBE VIEW "recent_matches"`
returns the same metadata shape. Anything the
The statement returns one row with UTF-8 `name`, `query`, and `schema` columns.
`query` contains the stored query, including any declared column aliases, and
`schema` contains its Arrow schema as JSON. The schema metadata keys
`lancedb::view::default_db` and `lancedb::view::default_namespace` identify the
defining database and namespace. The root namespace is `public`; nested
namespace components are separated by `$`.
`DESCRIBE VIEW "recent_matches"` returns the same metadata shape. Anything the
[dialect](/sql/functions) supports can define one: joins, predicates,
aggregates, window functions.
@@ -65,9 +69,8 @@ SHOW VIEWS
SHOW VIEWS IN "analytics" WHERE name > 'previous_view' LIMIT 100
```
`SHOW VIEWS` returns ascending `name` values and a nullable `location` column.
Locations are populated only when the deployment exposes catalog locations.
The `name` cursor is scoped to the selected namespace.
`SHOW VIEWS` returns ascending `name` values. The `name` cursor is scoped to
the selected namespace.
Object commands retain their small results in shared storage even when general
query-result copying is disabled, so asynchronous polling can use any query node.
rust/lancedb/src/remote/sql/view.rs
+80 -33
View File
@@ -1,11 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use super::{
SqlClient,
materialized_view::quote_identifier,
objects::{definition, strings},
};
use super::{SqlClient, materialized_view::quote_identifier, objects::strings};
use crate::{
database::ExecuteQueryRequest,
error::{Error, Result},
@@ -20,14 +16,13 @@ use std::sync::Arc;
#[derive(Clone, serde::Serialize, serde::Deserialize)]
struct ViewDefinition {
name: String,
#[serde(default)]
namespace: Vec<String>,
query: String,
default_database: String,
#[serde(default)]
default_namespace: Vec<String>,
schema: JsonArrowSchema,
}
impl ViewDefinition {
fn into_description(self) -> Result<ViewDescription> {
let schema =
@@ -46,6 +41,58 @@ impl ViewDefinition {
})
}
}
fn view_description(
batches: Vec<arrow_array::RecordBatch>,
namespace: Vec<String>,
) -> Result<ViewDefinition> {
let one = |column: &str| -> Result<String> {
let mut values = strings(&batches, column)?;
if values.len() != 1 {
return Err(Error::Runtime {
message: "View metadata must contain exactly one row".into(),
});
}
Ok(values.remove(0))
};
let name = one("name")?;
let json: JsonArrowSchema =
serde_json::from_str(&one("schema")?).map_err(|error| Error::Runtime {
message: format!("View '{name}' has invalid schema JSON: {error}"),
})?;
let schema = lance_namespace::schema::convert_json_arrow_schema(&json).map_err(|error| {
Error::Runtime {
message: format!("View '{name}' has an undecodable schema: {error}"),
}
})?;
let default_database = schema
.metadata()
.get("lancedb::view::default_db")
.ok_or_else(|| Error::Runtime {
message: format!("View '{name}' has no defining database"),
})?
.clone();
let default_namespace = schema
.metadata()
.get("lancedb::view::default_namespace")
.ok_or_else(|| Error::Runtime {
message: format!("View '{name}' has no defining namespace"),
})?;
let default_namespace_path = if default_namespace == "public" {
Vec::new()
} else {
default_namespace.split('$').map(str::to_owned).collect()
};
Ok(ViewDefinition {
name,
namespace,
query: one("query")?,
default_database,
default_namespace: default_namespace_path,
schema: json,
})
}
impl SqlClient {
pub(in crate::remote) async fn create_view(
&self,
@@ -55,19 +102,19 @@ impl SqlClient {
) -> Result<ViewDescription> {
validate_view_name(name)?;
validate_namespace(namespace)?;
let definition: ViewDefinition = self
.object_job(
ExecuteQueryRequest {
query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
definition,
)
.await?
.wait()
.await?;
definition.into_description()
let namespace_path = namespace.to_vec();
self.object_job(
ExecuteQueryRequest {
query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
move |batches| view_description(batches, namespace_path),
)
.await?
.wait()
.await?
.into_description()
}
pub(in crate::remote) async fn describe_view(
&self,
@@ -76,19 +123,19 @@ impl SqlClient {
) -> Result<ViewDescription> {
validate_view_name(name)?;
validate_namespace(namespace)?;
let definition: ViewDefinition = self
.object_job(
ExecuteQueryRequest {
query: format!("DESCRIBE VIEW {}", quote_identifier(name)),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
definition,
)
.await?
.wait()
.await?;
definition.into_description()
let namespace_path = namespace.to_vec();
self.object_job(
ExecuteQueryRequest {
query: format!("DESCRIBE VIEW {}", quote_identifier(name)),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
move |batches| view_description(batches, namespace_path),
)
.await?
.wait()
.await?
.into_description()
}
pub(in crate::remote) async fn list_views(
&self,
rust/lancedb/src/remote/sql_test.rs
+15 -5
View File
@@ -2694,12 +2694,22 @@ async fn function_listing_uses_bounded_sql_and_scoped_version_cursors() {
#[tokio::test]
async fn view_create_and_describe_return_metadata_through_sql() {
let definition = r#"{"name":"adults","namespace":["team"],"query":"SELECT name FROM people","default_database":"db","default_namespace":["team"],"schema":{"fields":[{"name":"name","nullable":true,"type":{"type":"utf8"}}]}}"#;
let schema = r#"{"fields":[{"name":"name","nullable":true,"type":{"type":"utf8"}}],"metadata":{"lancedb::view::default_db":"db","lancedb::view::default_namespace":"team"}}"#;
for defer in [false, true] {
let result = RecordBatch::try_from_iter(vec![(
"definition",
Arc::new(StringArray::from(vec![definition])) as ArrayRef,
)])
let result = RecordBatch::try_from_iter(vec![
(
"name",
Arc::new(StringArray::from(vec!["adults"])) as ArrayRef,
),
(
"query",
Arc::new(StringArray::from(vec!["SELECT name FROM people"])) as ArrayRef,
),
(
"schema",
Arc::new(StringArray::from(vec![schema])) as ArrayRef,
),
])
.unwrap();
let service = TestSqlService {
adaptive: true,