feat(python): support exist_ok for materialized views (#4991)

This commit is contained in:
Jack Ye
2026-11-13 19:21:29 -08:00
committed by GitHub
parent 4db0d5c7f5
commit 81d04b8c36
13 changed files with 281 additions and 6 deletions
+2
View File
@@ -369,6 +369,7 @@ class Connection(object):
auto_refresh_interval_seconds: Optional[float] = None,
*,
query: Optional[str] = None,
exist_ok: bool = False,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -382,6 +383,7 @@ class Connection(object):
auto_refresh_interval_seconds: Optional[float] = None,
*,
query: Optional[str] = None,
exist_ok: bool = False,
) -> Job: ...
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
+25
View File
@@ -710,11 +710,15 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
``exist_ok=True`` preserves an existing view's definition, data and
refresh settings without refreshing it. Existing names error by default.
LanceDB Embedded creation and refresh require a source build with the
``experimental-functions`` Cargo feature. Published packages leave it
disabled; LanceDB Enterprise operations remain available.
@@ -780,11 +784,15 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
"""Submit materialized-view creation and return its job.
``exist_ok=True`` preserves an existing view without refreshing it.
A skipped creation returns a completed job without a registry job ID.
The job may already be complete in LanceDB Embedded. In LanceDB
Distributed, creation submits CREATE MATERIALIZED VIEW through the SQL
server and the returned ``id`` is the registry job ID. That ID can be
@@ -2150,11 +2158,15 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
``exist_ok=True`` preserves an existing view's definition, data and
refresh settings without refreshing it. Existing names error by default.
LanceDB Embedded creation and refresh require a source build with the
``experimental-functions`` Cargo feature. Published packages leave it
disabled; LanceDB Enterprise operations remain available.
@@ -2185,6 +2197,7 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
@@ -2202,6 +2215,7 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
@@ -2214,6 +2228,7 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
@@ -3453,11 +3468,15 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncMaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
``exist_ok=True`` preserves an existing view's definition, data and
refresh settings without refreshing it. Existing names error by default.
LanceDB Embedded creation and refresh require a source build with the
``experimental-functions`` Cargo feature. Published packages leave it
disabled; LanceDB Enterprise operations remain available.
@@ -3480,6 +3499,7 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
@@ -3496,11 +3516,15 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job.
``exist_ok=True`` preserves an existing view without refreshing it.
A skipped creation returns a completed job without a registry job ID.
Wait for the returned job before opening or querying the view. In LanceDB
Enterprise, creation uses SQL; ``job_type`` is ``"query"`` and ``id``
identifies that query on this connection.
@@ -3521,6 +3545,7 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
+8
View File
@@ -705,6 +705,7 @@ class LanceNamespaceDBConnection(DBConnection):
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
exist_ok: bool = False,
) -> "MaterializedView":
"""Define a materialized view over a table in the root namespace.
See
@@ -723,6 +724,7 @@ class LanceNamespaceDBConnection(DBConnection):
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
exist_ok=exist_ok,
)
).name
)
@@ -741,6 +743,7 @@ class LanceNamespaceDBConnection(DBConnection):
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
exist_ok: bool = False,
) -> Job[None]:
job = LOOP.run(
self._inner.create_materialized_view_async(
@@ -753,6 +756,7 @@ class LanceNamespaceDBConnection(DBConnection):
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
exist_ok=exist_ok,
)
)
return Job(job)
@@ -1352,6 +1356,7 @@ class AsyncLanceNamespaceDBConnection:
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
exist_ok: bool = False,
) -> "AsyncMaterializedView":
"""Define a materialized view over a table in the root namespace."""
view = await self._inner.create_materialized_view(
@@ -1364,6 +1369,7 @@ class AsyncLanceNamespaceDBConnection:
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
exist_ok=exist_ok,
)
# Reopen through the namespace so the view's table carries the
# namespace client and pushdown configuration a bare inner table lacks.
@@ -1381,6 +1387,7 @@ class AsyncLanceNamespaceDBConnection:
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
exist_ok: bool = False,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job."""
return await self._inner.create_materialized_view_async(
@@ -1393,6 +1400,7 @@ class AsyncLanceNamespaceDBConnection:
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
exist_ok=exist_ok,
)
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
+4
View File
@@ -803,6 +803,7 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
@@ -817,6 +818,7 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
@@ -841,6 +843,7 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
exist_ok: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
@@ -853,6 +856,7 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
exist_ok=exist_ok,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
@@ -0,0 +1,84 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
import lancedb
@pytest.mark.experimental_functions
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("background", [False, True])
@pytest.mark.parametrize("namespace", [False, True])
@pytest.mark.parametrize("defining_query", [False, True])
async def test_exist_ok_preserves_materialized_data(
tmp_path, asynchronous, background, namespace, defining_query
):
source_args = (
{"query": "SELECT * FROM source"} if defining_query else {"source": "source"}
)
replacement_args = {
**(
{"query": "SELECT * FROM missing"}
if defining_query
else {"source": "missing"}
),
"auto_refresh_enabled": True,
"auto_refresh_interval_seconds": 1,
}
options = {"storage_options": {"new_table_enable_stable_row_ids": "true"}}
if asynchronous:
db = (
lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}, **options)
if namespace
else await lancedb.connect_async(tmp_path, **options)
)
source = await db.create_table("source", [{"id": 1}])
original = await db.create_materialized_view(
"saved", **source_args, exist_ok=True
)
definition = await original.definition()
version = await original.table.version()
await source.add([{"id": 2}])
if background:
job = await db.create_materialized_view_async(
"saved", **replacement_args, exist_ok=True
)
await job.wait()
kept = await db.open_materialized_view("saved")
else:
kept = await db.create_materialized_view(
"saved", **replacement_args, exist_ok=True
)
assert (await kept.definition()).query == definition.query
assert await kept.table.version() == version
assert (await kept.table.to_arrow()).to_pylist() == [{"id": 1}]
with pytest.raises(ValueError, match="already exists"):
await db.create_materialized_view("saved", **source_args)
else:
db = (
lancedb.connect_namespace("dir", {"root": str(tmp_path)}, **options)
if namespace
else lancedb.connect(tmp_path, **options)
)
source = db.create_table("source", [{"id": 1}])
original = db.create_materialized_view("saved", **source_args, exist_ok=True)
definition = original.definition
version = original.table.version
source.add([{"id": 2}])
if background:
job = db.create_materialized_view_async(
"saved", **replacement_args, exist_ok=True
)
job.wait()
kept = db.open_materialized_view("saved")
else:
kept = db.create_materialized_view(
"saved", **replacement_args, exist_ok=True
)
assert kept.definition.query == definition.query
assert kept.table.version == version
assert kept.table.to_arrow().to_pylist() == [{"id": 1}]
with pytest.raises(ValueError, match="already exists"):
db.create_materialized_view("saved", **source_args)
@@ -197,6 +197,7 @@ def test_sync_remote_create_uses_public_async_connection(query):
where=None,
limit=None,
with_no_data=False,
exist_ok=False,
auto_refresh_enabled=None,
auto_refresh_interval_seconds=None,
):
@@ -729,6 +730,7 @@ def test_remote_create_forwards_refresh_options_without_interval_default(options
"filter": None,
"limit": None,
"with_no_data": False,
"exist_ok": False,
"auto_refresh_enabled": options.get("auto_refresh_enabled"),
"auto_refresh_interval_seconds": options.get(
"auto_refresh_interval_seconds"
+6 -4
View File
@@ -507,7 +507,7 @@ impl Connection {
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -519,6 +519,7 @@ impl Connection {
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
query: Option<String>,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let builder = materialized_view_builder(
@@ -529,7 +530,7 @@ impl Connection {
projections.is_some() || filter.is_some() || limit.is_some(),
)?;
future_into_py(self_.py(), async move {
let mut builder = builder;
let mut builder = builder.exist_ok(exist_ok);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}
@@ -552,7 +553,7 @@ impl Connection {
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -564,6 +565,7 @@ impl Connection {
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
query: Option<String>,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
let builder = materialized_view_builder(
@@ -574,7 +576,7 @@ impl Connection {
projections.is_some() || filter.is_some() || limit.is_some(),
)?;
future_into_py(self_.py(), async move {
let mut builder = builder;
let mut builder = builder.exist_ok(exist_ok);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}