feat(python): support exist_ok when creating views (#4984)

Add `exist_ok=False` to synchronous and asynchronous Python view
creation. Passing `True` sends `CREATE VIEW IF NOT EXISTS` and returns
the existing view's stored query and schema unchanged; the default
continues to fail on an existing name.

```python
view = db.create_view("recent_orders", query, exist_ok=True)
view = await async_db.create_view("recent_orders", query, exist_ok=True)
```

Requires a SQL server supporting `CREATE VIEW IF NOT EXISTS`. Rust gains
an awaitable builder with `.exist_ok(true)`. Existing
`.create_view(...).await` calls continue to work; callers requiring
`Future` must convert the builder, and custom `Database` implementations
must accept the new `exist_ok` argument. Includes Python round-trip
coverage and Rust coverage for direct/deferred results and metadata
preservation.
This commit is contained in:
Jack Ye
2026-11-14 10:35:52 +08:00
committed by GitHub
parent 7d50d2d28f
commit b061c3627c
12 changed files with 239 additions and 25 deletions
+10
View File
@@ -23,6 +23,16 @@ catalog removal; physical cleanup can continue afterward.
## CREATE VIEW {#create}
{/* Draft for the view exist_ok release.
`CREATE VIEW IF NOT EXISTS` creates a missing view and returns an existing
view's stored query and schema unchanged. Without `IF NOT EXISTS`, an existing
name is an error. In Python, use `db.create_view(name, query, exist_ok=True)`;
in Rust, use `db.create_view(name, query, &namespace).exist_ok(true).await?`.
Returning an existing view requires `SELECT` on that view.
*/}
<CodeGroup>
```sql SQL icon="database"
CREATE VIEW "recent_matches" AS
+6 -1
View File
@@ -253,7 +253,12 @@ class Connection(object):
self, name: str, namespace_path: Optional[List[str]] = None
) -> Tuple[str, int, int]: ...
async def create_view(
self, name: str, query: str, namespace_path: Optional[List[str]] = None
self,
name: str,
query: str,
namespace_path: Optional[List[str]] = None,
*,
exist_ok: bool = False,
) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ...
async def describe_view(
self, name: str, namespace_path: Optional[List[str]] = None
+28 -8
View File
@@ -1273,7 +1273,12 @@ class DBConnection(EnforceOverrides):
)
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
self,
name: str,
query: str,
*,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> ViewDescription:
"""Create a view: a named query the database plans on every read.
@@ -1281,9 +1286,10 @@ class DBConnection(EnforceOverrides):
is refused now rather than at the first read. A view holds no rows, and
its readers see its sources as they are at read time.
There is no replace: a name already taken is an error, and changing a
view is a drop followed by a create. LanceDB Embedded connections raise
``NotImplementedError``.
A name already taken is an error unless ``exist_ok=True``, which returns
the existing view's stored query and schema without changing them.
Changing a view is a drop followed by a create. LanceDB Embedded
connections raise ``NotImplementedError``.
>>> import lancedb
>>> db = lancedb.connect("db://my_database") # doctest: +SKIP
@@ -2592,10 +2598,17 @@ class LanceDBConnection(DBConnection):
@override
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
self,
name: str,
query: str,
*,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> ViewDescription:
return LOOP.run(
self._conn.create_view(name, query, namespace_path=namespace_path)
self._conn.create_view(
name, query, exist_ok=exist_ok, namespace_path=namespace_path
)
)
@override
@@ -4107,7 +4120,12 @@ class AsyncConnection(object):
)
async def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
self,
name: str,
query: str,
*,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> ViewDescription:
"""Create a view: a named query the database plans on every read.
@@ -4115,7 +4133,9 @@ class AsyncConnection(object):
[DBConnection.create_view][lancedb.DBConnection.create_view].
"""
return _view_description(
await self._inner.create_view(name, query, list(namespace_path or []))
await self._inner.create_view(
name, query, list(namespace_path or []), exist_ok=exist_ok
)
)
async def describe_view(
+9 -2
View File
@@ -1197,10 +1197,17 @@ class RemoteDBConnection(DBConnection):
@override
def create_view(
self, name: str, query: str, *, namespace_path: Optional[List[str]] = None
self,
name: str,
query: str,
*,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> ViewDescription:
return LOOP.run(
self._conn.create_view(name, query, namespace_path=namespace_path)
self._conn.create_view(
name, query, exist_ok=exist_ok, namespace_path=namespace_path
)
)
@override
+71
View File
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
import pyarrow as pa
import pytest
import lancedb
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
("exist_ok", "existing"),
[(None, False), (False, False), (True, False), (True, True)],
)
async def test_create_view_exist_ok_preserves_server_metadata(
asynchronous, exist_ok, existing
):
flight = pytest.importorskip("pyarrow.flight")
original_query = "SELECT 'original' AS original"
submitted_query = (
"SELECT 'replacement' AS replacement" if existing else original_query
)
schema = {
"fields": [{"name": "original", "nullable": True, "type": {"type": "utf8"}}],
"metadata": {
"lancedb::view::default_db": "analytics",
"lancedb::view::default_namespace": "team",
},
}
result = pa.table(
{"name": ["saved"], "query": [original_query], "schema": [json.dumps(schema)]}
)
class ViewServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.requests = []
def do_get(self, context, ticket):
self.requests.append(ticket.ticket)
return flight.RecordBatchStream(result)
with ViewServer() as server:
connection_options = {
"api_key": "test-key",
"host_override": "http://localhost:10024",
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
}
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
if asynchronous:
db = await lancedb.connect_async("db://analytics", **connection_options)
view = await db.create_view(
"saved", submitted_query, namespace_path=["team"], **kwargs
)
else:
db = lancedb.connect("db://analytics", **connection_options)
view = db.create_view(
"saved", submitted_query, namespace_path=["team"], **kwargs
)
(request,) = server.requests
prefix = "CREATE VIEW IF NOT EXISTS" if exist_ok else "CREATE VIEW"
assert f'{prefix} "saved" AS {submitted_query}'.encode() in request
assert view.name == "saved"
assert view.namespace_path == ["team"]
assert view.default_namespace_path == ["team"]
assert view.query == original_query
assert view.schema.names == ["original"]
+3 -1
View File
@@ -1158,18 +1158,20 @@ impl Connection {
})
}
#[pyo3(signature = (name, query, namespace_path=None))]
#[pyo3(signature = (name, query, namespace_path=None, *, exist_ok=false))]
pub fn create_view(
self_: PyRef<'_, Self>,
name: String,
query: String,
namespace_path: Option<Vec<String>>,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let namespace_path = namespace_path.unwrap_or_default();
future_into_py(self_.py(), async move {
let view = inner
.create_view(name, query, &namespace_path)
.exist_ok(exist_ok)
.await
.infer_error()?;
view_description_to_py(view)
+15 -9
View File
@@ -53,6 +53,9 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
mod create_secret;
mod create_table;
mod create_view;
pub use create_view::CreateViewBuilder;
pub use create_secret::CreateSecretBuilder;
@@ -976,9 +979,10 @@ impl Connection {
/// refused now rather than at the first read. A view holds no rows, and
/// its readers see its sources as they are at read time.
///
/// There is no replace: a name already taken is an error, and changing a
/// view is a drop followed by a create, each authorized against what it
/// actually touches. LanceDB Embedded databases return [`Error::NotSupported`].
/// A name already taken is an error unless [`CreateViewBuilder::exist_ok`] is
/// true, which returns the stored definition and schema without changing them.
/// Changing a view is a drop followed by a create, each authorized against what
/// it actually touches. LanceDB Embedded databases return [`Error::NotSupported`].
///
/// # Example
///
@@ -1011,16 +1015,18 @@ impl Connection {
/// # Ok(())
/// # }
/// ```
pub async fn create_view(
pub fn create_view(
&self,
name: impl AsRef<str>,
query: impl AsRef<str>,
namespace_path: &[String],
) -> Result<ViewDescription> {
validate_view_reference(name.as_ref(), namespace_path)?;
self.internal
.create_view(name.as_ref(), query.as_ref(), namespace_path)
.await
) -> CreateViewBuilder {
CreateViewBuilder::new(
self.internal.clone(),
name.as_ref().to_owned(),
query.as_ref().to_owned(),
namespace_path.to_vec(),
)
}
/// What this database records about one view: its defining query and the
@@ -0,0 +1,62 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use std::future::{Future, IntoFuture};
use std::pin::Pin;
use std::sync::Arc;
use crate::Result;
use crate::database::Database;
use crate::utils::validate_view_reference;
use crate::view::ViewDescription;
/// Configure creation of a view with [`super::Connection::create_view`].
#[must_use]
pub struct CreateViewBuilder {
database: Arc<dyn Database>,
name: String,
query: String,
namespace_path: Vec<String>,
exist_ok: bool,
}
impl CreateViewBuilder {
pub(super) fn new(
database: Arc<dyn Database>,
name: String,
query: String,
namespace_path: Vec<String>,
) -> Self {
Self {
database,
name,
query,
namespace_path,
exist_ok: false,
}
}
/// Return the existing view without changing its definition if it exists.
/// Defaults to false, which reports an error for an existing name.
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
self.exist_ok = exist_ok;
self
}
/// Create the view or return the existing view when `exist_ok` is true.
pub async fn execute(self) -> Result<ViewDescription> {
validate_view_reference(&self.name, &self.namespace_path)?;
self.database
.create_view(&self.name, &self.query, &self.namespace_path, self.exist_ok)
.await
}
}
impl IntoFuture for CreateViewBuilder {
type Output = Result<ViewDescription>;
type IntoFuture = Pin<Box<dyn Future<Output = Self::Output> + Send>>;
fn into_future(self) -> Self::IntoFuture {
Box::pin(self.execute())
}
}
+1
View File
@@ -693,6 +693,7 @@ pub trait Database:
_name: &str,
_query: &str,
_namespace_path: &[String],
_exist_ok: bool,
) -> Result<ViewDescription> {
view_ops_not_supported()
}
+2 -1
View File
@@ -1099,11 +1099,12 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
name: &str,
query: &str,
namespace_path: &[String],
exist_ok: bool,
) -> Result<ViewDescription> {
validate_view_name(name)?;
validate_namespace(namespace_path)?;
self.sql_client()?
.create_view(name, query, namespace_path)
.create_view(name, query, namespace_path, exist_ok)
.await
}
async fn describe_view(
+6 -1
View File
@@ -99,13 +99,18 @@ impl SqlClient {
name: &str,
query: &str,
namespace: &[String],
exist_ok: bool,
) -> Result<ViewDescription> {
validate_view_name(name)?;
validate_namespace(namespace)?;
let namespace_path = namespace.to_vec();
let if_not_exists = if exist_ok { " IF NOT EXISTS" } else { "" };
self.object_job(
ExecuteQueryRequest {
query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)),
query: format!(
"CREATE VIEW{if_not_exists} {} AS {query}",
quote_identifier(name)
),
default_namespace_path: namespace.to_vec(),
parameters: None,
},
+26 -2
View File
@@ -24,6 +24,7 @@ use super::stream::{PROTOCOL_HEADER, PROTOCOL_VERSION, QueryResponse};
use super::*;
use crate::Connection;
use crate::database::Database;
use crate::embeddings::MemoryRegistry;
use crate::materialized_view::CreateMaterializedViewRequest;
use crate::remote::RemoteCatalogOptions;
use crate::remote::client::HeaderProvider;
@@ -3710,26 +3711,49 @@ async fn view_create_and_describe_return_metadata_through_sql() {
..Default::default()
};
let statements = service.statements.clone();
let headers = service.headers.clone();
let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await;
let db = RemoteDatabase::new_mock(|_| -> http::Response<String> {
panic!("unexpected REST request");
})
.with_sql_client(client);
let created = db
let connection = Connection::new(Arc::new(db), Arc::new(MemoryRegistry::new()));
let created = connection
.create_view("adults", "SELECT name FROM people", &["team".into()])
.await
.unwrap();
let described = db.describe_view("adults", &["team".into()]).await.unwrap();
let existing = connection
.create_view(
"adults",
"SELECT replacement FROM other_people",
&["team".into()],
)
.exist_ok(true)
.await
.unwrap();
let described = connection
.describe_view("adults", &["team".into()])
.await
.unwrap();
assert_eq!(created, described);
assert_eq!(existing, described);
assert_eq!(created.schema.fields()[0].name(), "name");
assert_eq!(created.default_namespace_path, ["team"]);
assert_eq!(
statements.lock().unwrap().as_slice(),
[
"CREATE VIEW \"adults\" AS SELECT name FROM people",
"CREATE VIEW IF NOT EXISTS \"adults\" AS SELECT replacement FROM other_people",
"DESCRIBE VIEW \"adults\""
]
);
assert!(
headers
.lock()
.unwrap()
.iter()
.all(|header| header.namespace_path == "team")
);
shutdown.send(()).unwrap();
}
}