From 81d04b8c361b596b29f4f7988aab7df0eb9f84ca Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Fri, 13 Nov 2026 19:21:29 -0800 Subject: [PATCH] feat(python): support exist_ok for materialized views (#4991) --- docs/web/sql/ddl/materialized-view.mdx | 10 +++ python/python/lancedb/_lancedb.pyi | 2 + python/python/lancedb/db.py | 25 ++++++ python/python/lancedb/namespace.py | 8 ++ python/python/lancedb/remote/db.py | 4 + .../tests/test_materialized_view_creation.py | 84 +++++++++++++++++++ .../python/tests/test_materialized_views.py | 2 + python/src/connection.rs | 10 ++- rust/lancedb/src/materialized_view.rs | 49 ++++++++++- rust/lancedb/src/remote/db.rs | 11 ++- .../src/remote/sql/materialized_view.rs | 15 ++++ rust/lancedb/src/remote/sql_test.rs | 60 +++++++++++++ rust/lancedb/src/remote/table.rs | 7 ++ 13 files changed, 281 insertions(+), 6 deletions(-) create mode 100644 python/python/tests/test_materialized_view_creation.py diff --git a/docs/web/sql/ddl/materialized-view.mdx b/docs/web/sql/ddl/materialized-view.mdx index 2f9173ecd..a2453ba01 100644 --- a/docs/web/sql/ddl/materialized-view.mdx +++ b/docs/web/sql/ddl/materialized-view.mdx @@ -27,6 +27,16 @@ CREATE MATERIALIZED VIEW AS SELECT ... ``` +{/* Draft for the materialized-view exist_ok release. + +`IF NOT EXISTS` preserves an existing materialized view's definition, data and +refresh settings without refreshing it. The Python equivalent is +`db.create_materialized_view("saved", "source", exist_ok=True)`. A skipped +asynchronous creation returns a completed job without submitting a refresh job. +Existing names are errors by default. + +*/} + Once created, a materialized view can be queried like a table. Use the statements below to list, refresh and drop it. For SDK methods, see [Materialized views](/management/materialized-view#create). diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index d5c024959..e3f00557c 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -369,6 +369,7 @@ class Connection(object): auto_refresh_interval_seconds: Optional[float] = None, *, query: Optional[str] = None, + exist_ok: bool = False, ) -> Table: ... async def create_materialized_view_async( self, @@ -382,6 +383,7 @@ class Connection(object): auto_refresh_interval_seconds: Optional[float] = None, *, query: Optional[str] = None, + exist_ok: bool = False, ) -> Job: ... def list_materialized_views( self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 18ca9cf6f..b84e4d581 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -710,11 +710,15 @@ class DBConnection(EnforceOverrides): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + ``exist_ok=True`` preserves an existing view's definition, data and + refresh settings without refreshing it. Existing names error by default. + LanceDB Embedded creation and refresh require a source build with the ``experimental-functions`` Cargo feature. Published packages leave it disabled; LanceDB Enterprise operations remain available. @@ -780,11 +784,15 @@ class DBConnection(EnforceOverrides): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> Job[None]: """Submit materialized-view creation and return its job. + ``exist_ok=True`` preserves an existing view without refreshing it. + A skipped creation returns a completed job without a registry job ID. + The job may already be complete in LanceDB Embedded. In LanceDB Distributed, creation submits CREATE MATERIALIZED VIEW through the SQL server and the returned ``id`` is the registry job ID. That ID can be @@ -2150,11 +2158,15 @@ class LanceDBConnection(DBConnection): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. + ``exist_ok=True`` preserves an existing view's definition, data and + refresh settings without refreshing it. Existing names error by default. + LanceDB Embedded creation and refresh require a source build with the ``experimental-functions`` Cargo feature. Published packages leave it disabled; LanceDB Enterprise operations remain available. @@ -2185,6 +2197,7 @@ class LanceDBConnection(DBConnection): where=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, ) @@ -2202,6 +2215,7 @@ class LanceDBConnection(DBConnection): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> Job[None]: @@ -2214,6 +2228,7 @@ class LanceDBConnection(DBConnection): where=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, ) @@ -3453,11 +3468,15 @@ class AsyncConnection(object): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> AsyncMaterializedView: """Define a materialized view named ``name`` over the table ``source``. + ``exist_ok=True`` preserves an existing view's definition, data and + refresh settings without refreshing it. Existing names error by default. + LanceDB Embedded creation and refresh require a source build with the ``experimental-functions`` Cargo feature. Published packages leave it disabled; LanceDB Enterprise operations remain available. @@ -3480,6 +3499,7 @@ class AsyncConnection(object): filter=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, **native_options, @@ -3496,11 +3516,15 @@ class AsyncConnection(object): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> AsyncJob[None]: """Submit materialized-view creation and return its job. + ``exist_ok=True`` preserves an existing view without refreshing it. + A skipped creation returns a completed job without a registry job ID. + Wait for the returned job before opening or querying the view. In LanceDB Enterprise, creation uses SQL; ``job_type`` is ``"query"`` and ``id`` identifies that query on this connection. @@ -3521,6 +3545,7 @@ class AsyncConnection(object): filter=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, **native_options, diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 72dc78c8b..67fcc61f7 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -705,6 +705,7 @@ class LanceNamespaceDBConnection(DBConnection): with_no_data: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, + exist_ok: bool = False, ) -> "MaterializedView": """Define a materialized view over a table in the root namespace. See @@ -723,6 +724,7 @@ class LanceNamespaceDBConnection(DBConnection): with_no_data=with_no_data, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, + exist_ok=exist_ok, ) ).name ) @@ -741,6 +743,7 @@ class LanceNamespaceDBConnection(DBConnection): with_no_data: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, + exist_ok: bool = False, ) -> Job[None]: job = LOOP.run( self._inner.create_materialized_view_async( @@ -753,6 +756,7 @@ class LanceNamespaceDBConnection(DBConnection): with_no_data=with_no_data, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, + exist_ok=exist_ok, ) ) return Job(job) @@ -1352,6 +1356,7 @@ class AsyncLanceNamespaceDBConnection: with_no_data: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, + exist_ok: bool = False, ) -> "AsyncMaterializedView": """Define a materialized view over a table in the root namespace.""" view = await self._inner.create_materialized_view( @@ -1364,6 +1369,7 @@ class AsyncLanceNamespaceDBConnection: with_no_data=with_no_data, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, + exist_ok=exist_ok, ) # Reopen through the namespace so the view's table carries the # namespace client and pushdown configuration a bare inner table lacks. @@ -1381,6 +1387,7 @@ class AsyncLanceNamespaceDBConnection: with_no_data: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, + exist_ok: bool = False, ) -> AsyncJob[None]: """Submit materialized-view creation and return its job.""" return await self._inner.create_materialized_view_async( @@ -1393,6 +1400,7 @@ class AsyncLanceNamespaceDBConnection: with_no_data=with_no_data, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, + exist_ok=exist_ok, ) async def open_materialized_view(self, name: str) -> "AsyncMaterializedView": diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 69b691004..b9928d278 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -803,6 +803,7 @@ class RemoteDBConnection(DBConnection): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> MaterializedView: @@ -817,6 +818,7 @@ class RemoteDBConnection(DBConnection): where=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, ) @@ -841,6 +843,7 @@ class RemoteDBConnection(DBConnection): where: Optional[str] = None, limit: Optional[int] = None, with_no_data: bool = False, + exist_ok: bool = False, auto_refresh_enabled: Optional[bool] = None, auto_refresh_interval_seconds: Optional[float] = None, ) -> Job[None]: @@ -853,6 +856,7 @@ class RemoteDBConnection(DBConnection): where=where, limit=limit, with_no_data=with_no_data, + exist_ok=exist_ok, auto_refresh_enabled=auto_refresh_enabled, auto_refresh_interval_seconds=auto_refresh_interval_seconds, ) diff --git a/python/python/tests/test_materialized_view_creation.py b/python/python/tests/test_materialized_view_creation.py new file mode 100644 index 000000000..0d9a4c2b4 --- /dev/null +++ b/python/python/tests/test_materialized_view_creation.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import pytest + +import lancedb + + +@pytest.mark.experimental_functions +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("background", [False, True]) +@pytest.mark.parametrize("namespace", [False, True]) +@pytest.mark.parametrize("defining_query", [False, True]) +async def test_exist_ok_preserves_materialized_data( + tmp_path, asynchronous, background, namespace, defining_query +): + source_args = ( + {"query": "SELECT * FROM source"} if defining_query else {"source": "source"} + ) + replacement_args = { + **( + {"query": "SELECT * FROM missing"} + if defining_query + else {"source": "missing"} + ), + "auto_refresh_enabled": True, + "auto_refresh_interval_seconds": 1, + } + options = {"storage_options": {"new_table_enable_stable_row_ids": "true"}} + if asynchronous: + db = ( + lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}, **options) + if namespace + else await lancedb.connect_async(tmp_path, **options) + ) + source = await db.create_table("source", [{"id": 1}]) + original = await db.create_materialized_view( + "saved", **source_args, exist_ok=True + ) + definition = await original.definition() + version = await original.table.version() + await source.add([{"id": 2}]) + if background: + job = await db.create_materialized_view_async( + "saved", **replacement_args, exist_ok=True + ) + await job.wait() + kept = await db.open_materialized_view("saved") + else: + kept = await db.create_materialized_view( + "saved", **replacement_args, exist_ok=True + ) + assert (await kept.definition()).query == definition.query + assert await kept.table.version() == version + assert (await kept.table.to_arrow()).to_pylist() == [{"id": 1}] + with pytest.raises(ValueError, match="already exists"): + await db.create_materialized_view("saved", **source_args) + else: + db = ( + lancedb.connect_namespace("dir", {"root": str(tmp_path)}, **options) + if namespace + else lancedb.connect(tmp_path, **options) + ) + source = db.create_table("source", [{"id": 1}]) + original = db.create_materialized_view("saved", **source_args, exist_ok=True) + definition = original.definition + version = original.table.version + source.add([{"id": 2}]) + if background: + job = db.create_materialized_view_async( + "saved", **replacement_args, exist_ok=True + ) + job.wait() + kept = db.open_materialized_view("saved") + else: + kept = db.create_materialized_view( + "saved", **replacement_args, exist_ok=True + ) + assert kept.definition.query == definition.query + assert kept.table.version == version + assert kept.table.to_arrow().to_pylist() == [{"id": 1}] + with pytest.raises(ValueError, match="already exists"): + db.create_materialized_view("saved", **source_args) diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index 1ea0c1cb9..13950a39c 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -197,6 +197,7 @@ def test_sync_remote_create_uses_public_async_connection(query): where=None, limit=None, with_no_data=False, + exist_ok=False, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, ): @@ -729,6 +730,7 @@ def test_remote_create_forwards_refresh_options_without_interval_default(options "filter": None, "limit": None, "with_no_data": False, + "exist_ok": False, "auto_refresh_enabled": options.get("auto_refresh_enabled"), "auto_refresh_interval_seconds": options.get( "auto_refresh_interval_seconds" diff --git a/python/src/connection.rs b/python/src/connection.rs index 8763822cc..1a171032c 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -507,7 +507,7 @@ impl Connection { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))] + #[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))] pub fn create_materialized_view( self_: PyRef<'_, Self>, name: String, @@ -519,6 +519,7 @@ impl Connection { auto_refresh_enabled: Option, auto_refresh_interval_seconds: Option, query: Option, + exist_ok: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); let builder = materialized_view_builder( @@ -529,7 +530,7 @@ impl Connection { projections.is_some() || filter.is_some() || limit.is_some(), )?; future_into_py(self_.py(), async move { - let mut builder = builder; + let mut builder = builder.exist_ok(exist_ok); if let Some(enabled) = auto_refresh_enabled { builder = builder.auto_refresh_enabled(enabled); } @@ -552,7 +553,7 @@ impl Connection { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))] + #[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))] pub fn create_materialized_view_async( self_: PyRef<'_, Self>, name: String, @@ -564,6 +565,7 @@ impl Connection { auto_refresh_enabled: Option, auto_refresh_interval_seconds: Option, query: Option, + exist_ok: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); let builder = materialized_view_builder( @@ -574,7 +576,7 @@ impl Connection { projections.is_some() || filter.is_some() || limit.is_some(), )?; future_into_py(self_.py(), async move { - let mut builder = builder; + let mut builder = builder.exist_ok(exist_ok); if let Some(enabled) = auto_refresh_enabled { builder = builder.auto_refresh_enabled(enabled); } diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 9920cd893..4b2a61f2c 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -287,6 +287,8 @@ pub struct MaterializedViewInfo { #[doc(hidden)] #[derive(Debug, Clone, PartialEq)] pub struct CreateMaterializedViewRequest { + /// Preserve an existing materialized view without refreshing it. + pub exist_ok: bool, /// Name of the new view. pub name: String, /// Namespace in which to create the view. @@ -1586,6 +1588,7 @@ pub struct CreateMaterializedViewBuilder { filter: Option, limit: Option, with_no_data: bool, + exist_ok: bool, auto_refresh_enabled: Option, auto_refresh_interval_seconds: Option, } @@ -1626,6 +1629,7 @@ impl CreateMaterializedViewBuilder { filter: None, limit: None, with_no_data: false, + exist_ok: false, auto_refresh_enabled: None, auto_refresh_interval_seconds: None, } @@ -1678,6 +1682,35 @@ impl CreateMaterializedViewBuilder { self } + /// Preserve an existing view's definition, data and refresh settings. + /// Defaults to false. A skipped creation does not refresh the view. + /// + /// ```no_run + /// # #![recursion_limit = "256"] + /// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> { + /// db.create_materialized_view("saved", "source").exist_ok(true).execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn exist_ok(mut self, exist_ok: bool) -> Self { + self.exist_ok = exist_ok; + self + } + + async fn existing_view(&self) -> Result> { + match self + .connection + .open_table(&self.name) + .namespace(self.namespace.clone()) + .execute() + .await + { + Ok(table) => MaterializedView::from_table(table).await.map(Some), + Err(Error::TableNotFound { .. }) => Ok(None), + Err(error) => Err(error), + } + } + /// Enable or disable automatic refresh in LanceDB Enterprise. /// When omitted, the server determines whether automatic refresh is enabled. /// LanceDB Embedded does not support automatic refresh. @@ -1759,6 +1792,7 @@ impl CreateMaterializedViewBuilder { .connection .database() .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: self.exist_ok, name: self.name.clone(), namespace_path: self.namespace.clone(), query: self.query()?, @@ -1793,6 +1827,11 @@ impl CreateMaterializedViewBuilder { async fn execute_native(self) -> Result { crate::require_experimental_functions()?; + if self.exist_ok + && let Some(view) = self.existing_view().await? + { + return Ok(view); + } if self.auto_refresh_enabled == Some(true) || self.auto_refresh_interval_seconds.is_some() { return Err(Error::NotSupported { message: "automatic materialized-view refresh requires LanceDB Enterprise" @@ -1826,7 +1865,15 @@ impl CreateMaterializedViewBuilder { prepare_definition(&source, definition).await? } }; - let view = prepared.create_in(&self.namespace, &self.name).await?; + let view = match prepared.create_in(&self.namespace, &self.name).await { + Ok(view) => view, + Err(Error::TableAlreadyExists { .. }) if self.exist_ok => { + return self.existing_view().await?.ok_or_else(|| Error::Runtime { + message: "materialized view disappeared after concurrent creation".into(), + }); + } + Err(error) => return Err(error), + }; if !self.with_no_data { view.refresh().execute().await?; } diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 300cd6b09..7100e1b04 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -803,16 +803,24 @@ impl Database for RemoteDatabase { } else { format!(" WITH ({})", options.join(", ")) }; + let if_not_exists = if request.exist_ok { + " IF NOT EXISTS" + } else { + "" + }; let submission = client .submit_materialized_view_job( format!( - "CREATE MATERIALIZED VIEW \"{name}\"{options} AS {}", + "CREATE MATERIALIZED VIEW{if_not_exists} \"{name}\"{options} AS {}", request.query ), request.namespace_path, ) .await?; match submission { + crate::remote::sql::materialized_view::MaterializedViewJobSubmission::AlreadyExists => { + Ok(Job::new_done()) + } crate::remote::sql::materialized_view::MaterializedViewJobSubmission::Registry { job_id, } => Ok(Job::new(Box::new(RemoteJob::new( @@ -4865,6 +4873,7 @@ mod tests { for seconds in [0.0, -1.0, f64::NAN, f64::INFINITY] { let result = db .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: false, name: "adults".into(), namespace_path: vec![], query: "SELECT * FROM people".into(), diff --git a/rust/lancedb/src/remote/sql/materialized_view.rs b/rust/lancedb/src/remote/sql/materialized_view.rs index 4eaad5156..e982afac7 100644 --- a/rust/lancedb/src/remote/sql/materialized_view.rs +++ b/rust/lancedb/src/remote/sql/materialized_view.rs @@ -39,6 +39,7 @@ fn strings(batch: &RecordBatch, name: &str) -> Result> { } pub(in crate::remote) enum MaterializedViewJobSubmission { + AlreadyExists, Registry { job_id: String }, Query(Arc), } @@ -66,6 +67,7 @@ impl SqlClient { let mut reader: SendableRecordBatchStream = query.reader().await?; let mut rows = 0; let mut registry_job_id = None; + let mut already_exists = false; while let Some(batch) = reader.try_next().await? { rows += batch.num_rows(); if rows > 1 { @@ -74,6 +76,16 @@ impl SqlClient { .into(), }); } + if let Some(column) = batch.column_by_name("status") { + let values = column + .as_any() + .downcast_ref::() + .ok_or_else(|| Error::Runtime { + message: "materialized-view submission status is not UTF-8".into(), + })?; + already_exists = + values.len() == 1 && !values.is_null(0) && values.value(0) == "already_exists"; + } if let Some(column) = batch.column_by_name("job_id") { let values = column .as_any() @@ -100,6 +112,9 @@ impl SqlClient { message: "materialized-view submission must return exactly one status row".into(), }); } + if already_exists && registry_job_id.is_none() { + return Ok(MaterializedViewJobSubmission::AlreadyExists); + } let job_id = registry_job_id.ok_or_else(|| Error::Runtime { message: "server acknowledged an early materialized-view handle but returned no job_id" .into(), diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index e1780f118..1c39eda19 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -1220,6 +1220,7 @@ async fn remote_create_uses_sql_and_preserves_options_and_namespace() { ] { let job = db .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: false, name: "adults".into(), namespace_path: vec!["team".into(), "reports".into()], query: "SELECT * FROM people".into(), @@ -1302,6 +1303,7 @@ async fn remote_create_returns_the_registry_job_from_the_early_status_row() { let job = db .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: false, name: "adults".into(), namespace_path: vec!["team".into(), "reports".into()], query: "SELECT * FROM people".into(), @@ -1329,6 +1331,63 @@ async fn remote_create_returns_the_registry_job_from_the_early_status_row() { shutdown.send(()).unwrap(); } +#[tokio::test] +async fn remote_create_if_not_exists_accepts_a_completed_noop_without_a_registry_job() { + for defer in [false, true] { + let service = TestSqlService { + adaptive: true, + defer, + materialized_view_job_handle_supported: true, + ..Default::default() + }; + let statement = "CREATE MATERIALIZED VIEW IF NOT EXISTS \"adults\" AS SELECT * FROM people"; + service.result_overrides.lock().unwrap().insert( + statement.into(), + RecordBatch::try_from_iter([ + ( + "status", + Arc::new(StringArray::from(vec!["already_exists"])) as arrow_array::ArrayRef, + ), + ( + "job_id", + Arc::new(StringArray::from(vec![""])) as arrow_array::ArrayRef, + ), + ]) + .unwrap(), + ); + let statements = service.statements.clone(); + let (client, shutdown) = start_sql_service(service, ClientConfig::default()).await; + let db = RemoteDatabase::new_mock(|request| -> http::Response { + panic!( + "a skipped creation must not poll a registry job: {}", + request.url() + ); + }) + .with_sql_client(client); + let job = db + .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: true, + name: "adults".into(), + namespace_path: vec!["team".into()], + query: "SELECT * FROM people".into(), + with_no_data: false, + auto_refresh_enabled: None, + auto_refresh_interval_seconds: None, + }) + .await + .unwrap(); + if defer { + assert!(job.id().is_none()); + } else { + assert_eq!(job.job_type().as_deref(), Some("query")); + } + job.wait().await.unwrap(); + assert_eq!(job.status().await.unwrap(), "finished"); + assert_eq!(*statements.lock().unwrap(), [statement]); + shutdown.send(()).unwrap(); + } +} + #[tokio::test] async fn remote_create_registry_handle_reopens_describes_and_cancels() { let service = TestSqlService { @@ -1394,6 +1453,7 @@ async fn remote_create_registry_handle_reopens_describes_and_cancels() { let job = db .create_materialized_view_async(CreateMaterializedViewRequest { + exist_ok: false, name: "adults".into(), namespace_path: vec!["team".into(), "reports".into()], query: "SELECT * FROM people".into(), diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 9b64ab56d..f4e5c1e4a 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -2658,6 +2658,13 @@ impl BaseTable for RemoteTable { .submit_materialized_view_job(statement, self.namespace.clone()) .await?; match submission { + crate::remote::sql::materialized_view::MaterializedViewJobSubmission::AlreadyExists => { + Err(Error::Runtime { + message: + "REFRESH MATERIALIZED VIEW returned a create-only already_exists status" + .into(), + }) + } crate::remote::sql::materialized_view::MaterializedViewJobSubmission::Registry { job_id, } => Ok(Job::new_typed(Box::new(MaterializedViewRefreshJob {