From b061c3627c46ec3d310cddb0aa45f022f8c4aad2 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Fri, 13 Nov 2026 18:35:52 -0800 Subject: [PATCH] feat(python): support exist_ok when creating views (#4984) Add `exist_ok=False` to synchronous and asynchronous Python view creation. Passing `True` sends `CREATE VIEW IF NOT EXISTS` and returns the existing view's stored query and schema unchanged; the default continues to fail on an existing name. ```python view = db.create_view("recent_orders", query, exist_ok=True) view = await async_db.create_view("recent_orders", query, exist_ok=True) ``` Requires a SQL server supporting `CREATE VIEW IF NOT EXISTS`. Rust gains an awaitable builder with `.exist_ok(true)`. Existing `.create_view(...).await` calls continue to work; callers requiring `Future` must convert the builder, and custom `Database` implementations must accept the new `exist_ok` argument. Includes Python round-trip coverage and Rust coverage for direct/deferred results and metadata preservation. --- docs/web/sql/ddl/view.mdx | 10 +++ python/python/lancedb/_lancedb.pyi | 7 ++- python/python/lancedb/db.py | 36 ++++++++--- python/python/lancedb/remote/db.py | 11 +++- python/python/tests/test_view_creation.py | 71 ++++++++++++++++++++++ python/src/connection.rs | 4 +- rust/lancedb/src/connection.rs | 24 +++++--- rust/lancedb/src/connection/create_view.rs | 62 +++++++++++++++++++ rust/lancedb/src/database.rs | 1 + rust/lancedb/src/remote/db.rs | 3 +- rust/lancedb/src/remote/sql/view.rs | 7 ++- rust/lancedb/src/remote/sql_test.rs | 28 ++++++++- 12 files changed, 239 insertions(+), 25 deletions(-) create mode 100644 python/python/tests/test_view_creation.py create mode 100644 rust/lancedb/src/connection/create_view.rs diff --git a/docs/web/sql/ddl/view.mdx b/docs/web/sql/ddl/view.mdx index e039fc43a..3d94292ab 100644 --- a/docs/web/sql/ddl/view.mdx +++ b/docs/web/sql/ddl/view.mdx @@ -23,6 +23,16 @@ catalog removal; physical cleanup can continue afterward. ## CREATE VIEW {#create} +{/* Draft for the view exist_ok release. + +`CREATE VIEW IF NOT EXISTS` creates a missing view and returns an existing +view's stored query and schema unchanged. Without `IF NOT EXISTS`, an existing +name is an error. In Python, use `db.create_view(name, query, exist_ok=True)`; +in Rust, use `db.create_view(name, query, &namespace).exist_ok(true).await?`. +Returning an existing view requires `SELECT` on that view. + +*/} + ```sql SQL icon="database" CREATE VIEW "recent_matches" AS diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 5897ddd0c..56b644abb 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -253,7 +253,12 @@ class Connection(object): self, name: str, namespace_path: Optional[List[str]] = None ) -> Tuple[str, int, int]: ... async def create_view( - self, name: str, query: str, namespace_path: Optional[List[str]] = None + self, + name: str, + query: str, + namespace_path: Optional[List[str]] = None, + *, + exist_ok: bool = False, ) -> Tuple[str, List[str], str, str, List[str], pa.Schema]: ... async def describe_view( self, name: str, namespace_path: Optional[List[str]] = None diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 7d690abe3..020c79be0 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -1273,7 +1273,12 @@ class DBConnection(EnforceOverrides): ) def create_view( - self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + self, + name: str, + query: str, + *, + exist_ok: bool = False, + namespace_path: Optional[List[str]] = None, ) -> ViewDescription: """Create a view: a named query the database plans on every read. @@ -1281,9 +1286,10 @@ class DBConnection(EnforceOverrides): is refused now rather than at the first read. A view holds no rows, and its readers see its sources as they are at read time. - There is no replace: a name already taken is an error, and changing a - view is a drop followed by a create. LanceDB Embedded connections raise - ``NotImplementedError``. + A name already taken is an error unless ``exist_ok=True``, which returns + the existing view's stored query and schema without changing them. + Changing a view is a drop followed by a create. LanceDB Embedded + connections raise ``NotImplementedError``. >>> import lancedb >>> db = lancedb.connect("db://my_database") # doctest: +SKIP @@ -2592,10 +2598,17 @@ class LanceDBConnection(DBConnection): @override def create_view( - self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + self, + name: str, + query: str, + *, + exist_ok: bool = False, + namespace_path: Optional[List[str]] = None, ) -> ViewDescription: return LOOP.run( - self._conn.create_view(name, query, namespace_path=namespace_path) + self._conn.create_view( + name, query, exist_ok=exist_ok, namespace_path=namespace_path + ) ) @override @@ -4107,7 +4120,12 @@ class AsyncConnection(object): ) async def create_view( - self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + self, + name: str, + query: str, + *, + exist_ok: bool = False, + namespace_path: Optional[List[str]] = None, ) -> ViewDescription: """Create a view: a named query the database plans on every read. @@ -4115,7 +4133,9 @@ class AsyncConnection(object): [DBConnection.create_view][lancedb.DBConnection.create_view]. """ return _view_description( - await self._inner.create_view(name, query, list(namespace_path or [])) + await self._inner.create_view( + name, query, list(namespace_path or []), exist_ok=exist_ok + ) ) async def describe_view( diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index db3ab554c..ea9468331 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -1197,10 +1197,17 @@ class RemoteDBConnection(DBConnection): @override def create_view( - self, name: str, query: str, *, namespace_path: Optional[List[str]] = None + self, + name: str, + query: str, + *, + exist_ok: bool = False, + namespace_path: Optional[List[str]] = None, ) -> ViewDescription: return LOOP.run( - self._conn.create_view(name, query, namespace_path=namespace_path) + self._conn.create_view( + name, query, exist_ok=exist_ok, namespace_path=namespace_path + ) ) @override diff --git a/python/python/tests/test_view_creation.py b/python/python/tests/test_view_creation.py new file mode 100644 index 000000000..410356da0 --- /dev/null +++ b/python/python/tests/test_view_creation.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import json + +import pyarrow as pa +import pytest + +import lancedb + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + ("exist_ok", "existing"), + [(None, False), (False, False), (True, False), (True, True)], +) +async def test_create_view_exist_ok_preserves_server_metadata( + asynchronous, exist_ok, existing +): + flight = pytest.importorskip("pyarrow.flight") + original_query = "SELECT 'original' AS original" + submitted_query = ( + "SELECT 'replacement' AS replacement" if existing else original_query + ) + schema = { + "fields": [{"name": "original", "nullable": True, "type": {"type": "utf8"}}], + "metadata": { + "lancedb::view::default_db": "analytics", + "lancedb::view::default_namespace": "team", + }, + } + result = pa.table( + {"name": ["saved"], "query": [original_query], "schema": [json.dumps(schema)]} + ) + + class ViewServer(flight.FlightServerBase): + def __init__(self): + super().__init__("grpc://127.0.0.1:0") + self.requests = [] + + def do_get(self, context, ticket): + self.requests.append(ticket.ticket) + return flight.RecordBatchStream(result) + + with ViewServer() as server: + connection_options = { + "api_key": "test-key", + "host_override": "http://localhost:10024", + "sql_host_override": f"grpc://127.0.0.1:{server.port}", + } + kwargs = {} if exist_ok is None else {"exist_ok": exist_ok} + if asynchronous: + db = await lancedb.connect_async("db://analytics", **connection_options) + view = await db.create_view( + "saved", submitted_query, namespace_path=["team"], **kwargs + ) + else: + db = lancedb.connect("db://analytics", **connection_options) + view = db.create_view( + "saved", submitted_query, namespace_path=["team"], **kwargs + ) + (request,) = server.requests + + prefix = "CREATE VIEW IF NOT EXISTS" if exist_ok else "CREATE VIEW" + assert f'{prefix} "saved" AS {submitted_query}'.encode() in request + assert view.name == "saved" + assert view.namespace_path == ["team"] + assert view.default_namespace_path == ["team"] + assert view.query == original_query + assert view.schema.names == ["original"] diff --git a/python/src/connection.rs b/python/src/connection.rs index a32c00be1..ffa59fd02 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -1158,18 +1158,20 @@ impl Connection { }) } - #[pyo3(signature = (name, query, namespace_path=None))] + #[pyo3(signature = (name, query, namespace_path=None, *, exist_ok=false))] pub fn create_view( self_: PyRef<'_, Self>, name: String, query: String, namespace_path: Option>, + exist_ok: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); let namespace_path = namespace_path.unwrap_or_default(); future_into_py(self_.py(), async move { let view = inner .create_view(name, query, &namespace_path) + .exist_ok(exist_ok) .await .infer_error()?; view_description_to_py(view) diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index cc398e32b..c7a058502 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -53,6 +53,9 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; mod create_secret; mod create_table; +mod create_view; + +pub use create_view::CreateViewBuilder; pub use create_secret::CreateSecretBuilder; @@ -976,9 +979,10 @@ impl Connection { /// refused now rather than at the first read. A view holds no rows, and /// its readers see its sources as they are at read time. /// - /// There is no replace: a name already taken is an error, and changing a - /// view is a drop followed by a create, each authorized against what it - /// actually touches. LanceDB Embedded databases return [`Error::NotSupported`]. + /// A name already taken is an error unless [`CreateViewBuilder::exist_ok`] is + /// true, which returns the stored definition and schema without changing them. + /// Changing a view is a drop followed by a create, each authorized against what + /// it actually touches. LanceDB Embedded databases return [`Error::NotSupported`]. /// /// # Example /// @@ -1011,16 +1015,18 @@ impl Connection { /// # Ok(()) /// # } /// ``` - pub async fn create_view( + pub fn create_view( &self, name: impl AsRef, query: impl AsRef, namespace_path: &[String], - ) -> Result { - validate_view_reference(name.as_ref(), namespace_path)?; - self.internal - .create_view(name.as_ref(), query.as_ref(), namespace_path) - .await + ) -> CreateViewBuilder { + CreateViewBuilder::new( + self.internal.clone(), + name.as_ref().to_owned(), + query.as_ref().to_owned(), + namespace_path.to_vec(), + ) } /// What this database records about one view: its defining query and the diff --git a/rust/lancedb/src/connection/create_view.rs b/rust/lancedb/src/connection/create_view.rs new file mode 100644 index 000000000..bd80ca6e7 --- /dev/null +++ b/rust/lancedb/src/connection/create_view.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::future::{Future, IntoFuture}; +use std::pin::Pin; +use std::sync::Arc; + +use crate::Result; +use crate::database::Database; +use crate::utils::validate_view_reference; +use crate::view::ViewDescription; + +/// Configure creation of a view with [`super::Connection::create_view`]. +#[must_use] +pub struct CreateViewBuilder { + database: Arc, + name: String, + query: String, + namespace_path: Vec, + exist_ok: bool, +} + +impl CreateViewBuilder { + pub(super) fn new( + database: Arc, + name: String, + query: String, + namespace_path: Vec, + ) -> Self { + Self { + database, + name, + query, + namespace_path, + exist_ok: false, + } + } + + /// Return the existing view without changing its definition if it exists. + /// Defaults to false, which reports an error for an existing name. + pub fn exist_ok(mut self, exist_ok: bool) -> Self { + self.exist_ok = exist_ok; + self + } + + /// Create the view or return the existing view when `exist_ok` is true. + pub async fn execute(self) -> Result { + validate_view_reference(&self.name, &self.namespace_path)?; + self.database + .create_view(&self.name, &self.query, &self.namespace_path, self.exist_ok) + .await + } +} + +impl IntoFuture for CreateViewBuilder { + type Output = Result; + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(self.execute()) + } +} diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 23ed9fc3d..39e361a39 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -693,6 +693,7 @@ pub trait Database: _name: &str, _query: &str, _namespace_path: &[String], + _exist_ok: bool, ) -> Result { view_ops_not_supported() } diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 92523ec34..a05bb07b7 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1099,11 +1099,12 @@ impl Database for RemoteDatabase { name: &str, query: &str, namespace_path: &[String], + exist_ok: bool, ) -> Result { validate_view_name(name)?; validate_namespace(namespace_path)?; self.sql_client()? - .create_view(name, query, namespace_path) + .create_view(name, query, namespace_path, exist_ok) .await } async fn describe_view( diff --git a/rust/lancedb/src/remote/sql/view.rs b/rust/lancedb/src/remote/sql/view.rs index 0c2229684..513113187 100644 --- a/rust/lancedb/src/remote/sql/view.rs +++ b/rust/lancedb/src/remote/sql/view.rs @@ -99,13 +99,18 @@ impl SqlClient { name: &str, query: &str, namespace: &[String], + exist_ok: bool, ) -> Result { validate_view_name(name)?; validate_namespace(namespace)?; let namespace_path = namespace.to_vec(); + let if_not_exists = if exist_ok { " IF NOT EXISTS" } else { "" }; self.object_job( ExecuteQueryRequest { - query: format!("CREATE VIEW {} AS {query}", quote_identifier(name)), + query: format!( + "CREATE VIEW{if_not_exists} {} AS {query}", + quote_identifier(name) + ), default_namespace_path: namespace.to_vec(), parameters: None, }, diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index d25c72830..6a3f30390 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -24,6 +24,7 @@ use super::stream::{PROTOCOL_HEADER, PROTOCOL_VERSION, QueryResponse}; use super::*; use crate::Connection; use crate::database::Database; +use crate::embeddings::MemoryRegistry; use crate::materialized_view::CreateMaterializedViewRequest; use crate::remote::RemoteCatalogOptions; use crate::remote::client::HeaderProvider; @@ -3710,26 +3711,49 @@ async fn view_create_and_describe_return_metadata_through_sql() { ..Default::default() }; let statements = service.statements.clone(); + let headers = service.headers.clone(); let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; let db = RemoteDatabase::new_mock(|_| -> http::Response { panic!("unexpected REST request"); }) .with_sql_client(client); - let created = db + let connection = Connection::new(Arc::new(db), Arc::new(MemoryRegistry::new())); + let created = connection .create_view("adults", "SELECT name FROM people", &["team".into()]) .await .unwrap(); - let described = db.describe_view("adults", &["team".into()]).await.unwrap(); + let existing = connection + .create_view( + "adults", + "SELECT replacement FROM other_people", + &["team".into()], + ) + .exist_ok(true) + .await + .unwrap(); + let described = connection + .describe_view("adults", &["team".into()]) + .await + .unwrap(); assert_eq!(created, described); + assert_eq!(existing, described); assert_eq!(created.schema.fields()[0].name(), "name"); assert_eq!(created.default_namespace_path, ["team"]); assert_eq!( statements.lock().unwrap().as_slice(), [ "CREATE VIEW \"adults\" AS SELECT name FROM people", + "CREATE VIEW IF NOT EXISTS \"adults\" AS SELECT replacement FROM other_people", "DESCRIBE VIEW \"adults\"" ] ); + assert!( + headers + .lock() + .unwrap() + .iter() + .all(|header| header.namespace_path == "team") + ); shutdown.send(()).unwrap(); } }