mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok for materialized views (#4991)
This commit is contained in:
@@ -369,6 +369,7 @@ class Connection(object):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
*,
|
||||
query: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Table: ...
|
||||
async def create_materialized_view_async(
|
||||
self,
|
||||
@@ -382,6 +383,7 @@ class Connection(object):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
*,
|
||||
query: Optional[str] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Job: ...
|
||||
def list_materialized_views(
|
||||
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
|
||||
|
||||
@@ -710,11 +710,15 @@ class DBConnection(EnforceOverrides):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -780,11 +784,15 @@ class DBConnection(EnforceOverrides):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
"""Submit materialized-view creation and return its job.
|
||||
|
||||
``exist_ok=True`` preserves an existing view without refreshing it.
|
||||
A skipped creation returns a completed job without a registry job ID.
|
||||
|
||||
The job may already be complete in LanceDB Embedded. In LanceDB
|
||||
Distributed, creation submits CREATE MATERIALIZED VIEW through the SQL
|
||||
server and the returned ``id`` is the registry job ID. That ID can be
|
||||
@@ -2150,11 +2158,15 @@ class LanceDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -2185,6 +2197,7 @@ class LanceDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -2202,6 +2215,7 @@ class LanceDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
@@ -2214,6 +2228,7 @@ class LanceDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -3453,11 +3468,15 @@ class AsyncConnection(object):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> AsyncMaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
``exist_ok=True`` preserves an existing view's definition, data and
|
||||
refresh settings without refreshing it. Existing names error by default.
|
||||
|
||||
LanceDB Embedded creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; LanceDB Enterprise operations remain available.
|
||||
@@ -3480,6 +3499,7 @@ class AsyncConnection(object):
|
||||
filter=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
**native_options,
|
||||
@@ -3496,11 +3516,15 @@ class AsyncConnection(object):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> AsyncJob[None]:
|
||||
"""Submit materialized-view creation and return its job.
|
||||
|
||||
``exist_ok=True`` preserves an existing view without refreshing it.
|
||||
A skipped creation returns a completed job without a registry job ID.
|
||||
|
||||
Wait for the returned job before opening or querying the view. In LanceDB
|
||||
Enterprise, creation uses SQL; ``job_type`` is ``"query"`` and ``id``
|
||||
identifies that query on this connection.
|
||||
@@ -3521,6 +3545,7 @@ class AsyncConnection(object):
|
||||
filter=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
**native_options,
|
||||
|
||||
@@ -705,6 +705,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> "MaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace.
|
||||
See
|
||||
@@ -723,6 +724,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
).name
|
||||
)
|
||||
@@ -741,6 +743,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> Job[None]:
|
||||
job = LOOP.run(
|
||||
self._inner.create_materialized_view_async(
|
||||
@@ -753,6 +756,7 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
)
|
||||
return Job(job)
|
||||
@@ -1352,6 +1356,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> "AsyncMaterializedView":
|
||||
"""Define a materialized view over a table in the root namespace."""
|
||||
view = await self._inner.create_materialized_view(
|
||||
@@ -1364,6 +1369,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
# Reopen through the namespace so the view's table carries the
|
||||
# namespace client and pushdown configuration a bare inner table lacks.
|
||||
@@ -1381,6 +1387,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
exist_ok: bool = False,
|
||||
) -> AsyncJob[None]:
|
||||
"""Submit materialized-view creation and return its job."""
|
||||
return await self._inner.create_materialized_view_async(
|
||||
@@ -1393,6 +1400,7 @@ class AsyncLanceNamespaceDBConnection:
|
||||
with_no_data=with_no_data,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
|
||||
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
|
||||
|
||||
@@ -803,6 +803,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
@@ -817,6 +818,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
@@ -841,6 +843,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
with_no_data: bool = False,
|
||||
exist_ok: bool = False,
|
||||
auto_refresh_enabled: Optional[bool] = None,
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> Job[None]:
|
||||
@@ -853,6 +856,7 @@ class RemoteDBConnection(DBConnection):
|
||||
where=where,
|
||||
limit=limit,
|
||||
with_no_data=with_no_data,
|
||||
exist_ok=exist_ok,
|
||||
auto_refresh_enabled=auto_refresh_enabled,
|
||||
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize("namespace", [False, True])
|
||||
@pytest.mark.parametrize("defining_query", [False, True])
|
||||
async def test_exist_ok_preserves_materialized_data(
|
||||
tmp_path, asynchronous, background, namespace, defining_query
|
||||
):
|
||||
source_args = (
|
||||
{"query": "SELECT * FROM source"} if defining_query else {"source": "source"}
|
||||
)
|
||||
replacement_args = {
|
||||
**(
|
||||
{"query": "SELECT * FROM missing"}
|
||||
if defining_query
|
||||
else {"source": "missing"}
|
||||
),
|
||||
"auto_refresh_enabled": True,
|
||||
"auto_refresh_interval_seconds": 1,
|
||||
}
|
||||
options = {"storage_options": {"new_table_enable_stable_row_ids": "true"}}
|
||||
if asynchronous:
|
||||
db = (
|
||||
lancedb.connect_namespace_async("dir", {"root": str(tmp_path)}, **options)
|
||||
if namespace
|
||||
else await lancedb.connect_async(tmp_path, **options)
|
||||
)
|
||||
source = await db.create_table("source", [{"id": 1}])
|
||||
original = await db.create_materialized_view(
|
||||
"saved", **source_args, exist_ok=True
|
||||
)
|
||||
definition = await original.definition()
|
||||
version = await original.table.version()
|
||||
await source.add([{"id": 2}])
|
||||
if background:
|
||||
job = await db.create_materialized_view_async(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
await job.wait()
|
||||
kept = await db.open_materialized_view("saved")
|
||||
else:
|
||||
kept = await db.create_materialized_view(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
assert (await kept.definition()).query == definition.query
|
||||
assert await kept.table.version() == version
|
||||
assert (await kept.table.to_arrow()).to_pylist() == [{"id": 1}]
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
await db.create_materialized_view("saved", **source_args)
|
||||
else:
|
||||
db = (
|
||||
lancedb.connect_namespace("dir", {"root": str(tmp_path)}, **options)
|
||||
if namespace
|
||||
else lancedb.connect(tmp_path, **options)
|
||||
)
|
||||
source = db.create_table("source", [{"id": 1}])
|
||||
original = db.create_materialized_view("saved", **source_args, exist_ok=True)
|
||||
definition = original.definition
|
||||
version = original.table.version
|
||||
source.add([{"id": 2}])
|
||||
if background:
|
||||
job = db.create_materialized_view_async(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
job.wait()
|
||||
kept = db.open_materialized_view("saved")
|
||||
else:
|
||||
kept = db.create_materialized_view(
|
||||
"saved", **replacement_args, exist_ok=True
|
||||
)
|
||||
assert kept.definition.query == definition.query
|
||||
assert kept.table.version == version
|
||||
assert kept.table.to_arrow().to_pylist() == [{"id": 1}]
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
db.create_materialized_view("saved", **source_args)
|
||||
@@ -197,6 +197,7 @@ def test_sync_remote_create_uses_public_async_connection(query):
|
||||
where=None,
|
||||
limit=None,
|
||||
with_no_data=False,
|
||||
exist_ok=False,
|
||||
auto_refresh_enabled=None,
|
||||
auto_refresh_interval_seconds=None,
|
||||
):
|
||||
@@ -729,6 +730,7 @@ def test_remote_create_forwards_refresh_options_without_interval_default(options
|
||||
"filter": None,
|
||||
"limit": None,
|
||||
"with_no_data": False,
|
||||
"exist_ok": False,
|
||||
"auto_refresh_enabled": options.get("auto_refresh_enabled"),
|
||||
"auto_refresh_interval_seconds": options.get(
|
||||
"auto_refresh_interval_seconds"
|
||||
|
||||
@@ -507,7 +507,7 @@ impl Connection {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
|
||||
pub fn create_materialized_view(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
@@ -519,6 +519,7 @@ impl Connection {
|
||||
auto_refresh_enabled: Option<bool>,
|
||||
auto_refresh_interval_seconds: Option<f64>,
|
||||
query: Option<String>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let builder = materialized_view_builder(
|
||||
@@ -529,7 +530,7 @@ impl Connection {
|
||||
projections.is_some() || filter.is_some() || limit.is_some(),
|
||||
)?;
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut builder = builder;
|
||||
let mut builder = builder.exist_ok(exist_ok);
|
||||
if let Some(enabled) = auto_refresh_enabled {
|
||||
builder = builder.auto_refresh_enabled(enabled);
|
||||
}
|
||||
@@ -552,7 +553,7 @@ impl Connection {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None))]
|
||||
#[pyo3(signature = (name, source=None, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, *, query=None, exist_ok=false))]
|
||||
pub fn create_materialized_view_async(
|
||||
self_: PyRef<'_, Self>,
|
||||
name: String,
|
||||
@@ -564,6 +565,7 @@ impl Connection {
|
||||
auto_refresh_enabled: Option<bool>,
|
||||
auto_refresh_interval_seconds: Option<f64>,
|
||||
query: Option<String>,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
let builder = materialized_view_builder(
|
||||
@@ -574,7 +576,7 @@ impl Connection {
|
||||
projections.is_some() || filter.is_some() || limit.is_some(),
|
||||
)?;
|
||||
future_into_py(self_.py(), async move {
|
||||
let mut builder = builder;
|
||||
let mut builder = builder.exist_ok(exist_ok);
|
||||
if let Some(enabled) = auto_refresh_enabled {
|
||||
builder = builder.auto_refresh_enabled(enabled);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user