rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-23 20:22:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(python): expose materialized view auto-refresh creation options (#4621)

Browse Source 
Expose optional `auto_refresh_enabled` and
`auto_refresh_interval_seconds` when creating materialized views from
Python and Rust. Omitted intervals remain unset: the server determines
when to refresh.

```python
view = db.create_materialized_view(
    "recent_orders", "orders",
    auto_refresh_enabled=True,
    auto_refresh_interval_seconds=300,
)
```

## Architecture and workflow

- Python synchronous, asynchronous, job-returning and namespace wrappers
forward both optional values through PyO3 to the Rust creation builder.
Both default to `None`; no client interval is selected.
- Remote creation calls `POST /v1/materialized_view/{id}/create` with
the defining query and only the supplied refresh properties. A `202
Accepted` response must contain a job ID. Existing job tracking waits
for the server's ordinary SQL CREATE job before synchronous creation
opens the view.
- The server validates and authorizes the CREATE, then uses the existing
SQL worker to populate the view and persist its refresh settings. The
Lance agent subsequently uses those settings to submit REFRESH SQL jobs.
- Explicit positive finite fractional seconds are preserved. An interval
alone does not enable refresh. Embedded connections retain manual
refresh, accept explicit disabled state, and reject requests for
automatic scheduling rather than ignore them.

Includes API documentation and coverage for omission, false,
fractional/invalid intervals, namespace forwarding, server job IDs and
invalid acceptance responses.

Requires the server REST-create endpoint to be deployed before SDK
release. Existing remote `with_no_data=True` remains unsupported.
This commit is contained in:
Jack Ye
2026-10-21 23:47:09 -07:00
committed by GitHub
parent 1564ec10c8
commit f9680806c9
9 changed files with 349 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/_lancedb.pyi
+4
View File
@@ -270,6 +270,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Table: ...
async def create_materialized_view_async(
self,
@@ -279,6 +281,8 @@ class Connection(object):
filter: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job: ...
def list_materialized_views(
self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
python/python/lancedb/db.py
+27
View File
@@ -607,6 +607,8 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
@@ -631,6 +633,13 @@ class DBConnection(EnforceOverrides):
Cap the view at this many rows, in materialization order.
with_no_data: bool, default False
Skip the initial refresh and leave the backing table empty.
auto_refresh_enabled: bool, optional
Enable or disable automatic refresh on Cloud and Enterprise.
When omitted, the server determines whether automatic refresh is enabled.
Embedded databases do not support automatic refresh.
auto_refresh_interval_seconds: float, optional
Positive, finite refresh interval in seconds on Cloud and Enterprise.
When omitted, the server determines when to refresh.
Returns
-------
@@ -649,6 +658,8 @@ class DBConnection(EnforceOverrides):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
"""Submit materialized-view creation and return its job.
@@ -1706,6 +1717,8 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
@@ -1734,6 +1747,8 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return MaterializedView(self.open_table(name))
@@ -1748,6 +1763,8 @@ class LanceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._conn.create_materialized_view_async(
@@ -1757,6 +1774,8 @@ class LanceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
@@ -2767,6 +2786,8 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncMaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
@@ -2782,6 +2803,8 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
)
return AsyncMaterializedView(AsyncTable(inner))
@@ -2795,6 +2818,8 @@ class AsyncConnection(object):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job.
@@ -2810,6 +2835,8 @@ class AsyncConnection(object):
filter=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
**native_options,
)
return AsyncJob(inner)
python/python/lancedb/namespace.py
+16
View File
@@ -665,6 +665,8 @@ class LanceNamespaceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> "MaterializedView":
"""Define a materialized view over a table in the root namespace.
See
@@ -680,6 +682,8 @@ class LanceNamespaceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
).name
)
@@ -695,6 +699,8 @@ class LanceNamespaceDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._inner.create_materialized_view_async(
@@ -704,6 +710,8 @@ class LanceNamespaceDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
@@ -1288,6 +1296,8 @@ class AsyncLanceNamespaceDBConnection:
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> "AsyncMaterializedView":
"""Define a materialized view over a table in the root namespace."""
view = await self._inner.create_materialized_view(
@@ -1297,6 +1307,8 @@ class AsyncLanceNamespaceDBConnection:
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
# Reopen through the namespace so the view's table carries the
# namespace client and pushdown configuration a bare inner table lacks.
@@ -1311,6 +1323,8 @@ class AsyncLanceNamespaceDBConnection:
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> AsyncJob[None]:
"""Submit materialized-view creation and return its job."""
return await self._inner.create_materialized_view_async(
@@ -1320,6 +1334,8 @@ class AsyncLanceNamespaceDBConnection:
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
python/python/lancedb/remote/db.py
+8
View File
@@ -723,6 +723,8 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> MaterializedView:
from .table import RemoteTable
@@ -734,6 +736,8 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return MaterializedView(
@@ -755,6 +759,8 @@ class RemoteDBConnection(DBConnection):
where: Optional[str] = None,
limit: Optional[int] = None,
with_no_data: bool = False,
auto_refresh_enabled: Optional[bool] = None,
auto_refresh_interval_seconds: Optional[float] = None,
) -> Job[None]:
job = LOOP.run(
self._conn.create_materialized_view_async(
@@ -764,6 +770,8 @@ class RemoteDBConnection(DBConnection):
where=where,
limit=limit,
with_no_data=with_no_data,
auto_refresh_enabled=auto_refresh_enabled,
auto_refresh_interval_seconds=auto_refresh_interval_seconds,
)
)
return Job(job)
python/python/tests/test_materialized_views.py
+103 -2
View File
@@ -129,8 +129,21 @@ def test_sync_remote_create_uses_public_async_connection():
where=None,
limit=None,
with_no_data=False,
auto_refresh_enabled=None,
auto_refresh_interval_seconds=None,
):
calls.append((name, source, select, where, limit, with_no_data))
calls.append(
(
name,
source,
select,
where,
limit,
with_no_data,
auto_refresh_enabled,
auto_refresh_interval_seconds,
)
)
return StubAsyncMaterializedView()
async def drop_materialized_view(self, name, *, namespace_path=None):
@@ -148,9 +161,11 @@ def test_sync_remote_create_uses_public_async_connection():
where="age >= 18",
limit=10,
with_no_data=True,
auto_refresh_enabled=True,
auto_refresh_interval_seconds=0.5,
)
assert view.name == "adults"
assert calls == [("adults", "people", ["name"], "age >= 18", 10, True)]
assert calls == [("adults", "people", ["name"], "age >= 18", 10, True, True, 0.5)]
db.drop_materialized_view("adults", namespace_path=["analytics"])
assert calls[-1] == ("drop", "adults", ["analytics"])
@@ -550,3 +565,89 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
assert sorted([first, *resumed]) == ["a_view", "z_view"]
assert resumed.page_token() is None
assert list(resumed) == []
@pytest.mark.parametrize(
"options",
[
{},
{"auto_refresh_enabled": True},
{"auto_refresh_enabled": False, "auto_refresh_interval_seconds": 0.5},
],
)
def test_remote_create_forwards_refresh_options_without_interval_default(options):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect("db://dev", api_key="fake", host_override=host)
job = db.create_materialized_view_async("adults", "people", **options)
assert job.id == "mv-create-123"
assert requests == [
(
"/v1/materialized_view/adults/create",
{
"query": 'SELECT * FROM "people"',
**options,
},
)
]
@pytest.mark.asyncio
async def test_async_remote_create_keeps_interval_unspecified():
with mock_remote_materialized_view_create() as (host, requests):
db = await lancedb.connect_async("db://dev", api_key="fake", host_override=host)
job = await db.create_materialized_view_async(
"adults",
"people",
auto_refresh_enabled=True,
)
assert job.id == "mv-create-123"
assert requests[0][1] == {
"query": 'SELECT * FROM "people"',
"auto_refresh_enabled": True,
}
@pytest.mark.parametrize(
"options",
[
{"auto_refresh_enabled": True},
{"auto_refresh_interval_seconds": 300},
],
)
def test_embedded_create_refuses_automatic_refresh_before_creating_view(
tmp_path, options
):
db = make_db(tmp_path)
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
db.create_materialized_view("adults", "people", **options)
assert "adults" not in db.table_names()
@pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
def test_remote_create_rejects_invalid_interval_without_request(seconds):
with mock_remote_materialized_view_create() as (host, requests):
db = lancedb.connect("db://dev", api_key="fake", host_override=host)
with pytest.raises(ValueError, match="positive, finite"):
db.create_materialized_view_async(
"adults", "people", auto_refresh_interval_seconds=seconds
)
assert requests == []
def test_namespace_forwards_automatic_refresh_request(tmp_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
@pytest.mark.asyncio
async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
await db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False)
assert view.table.count_rows() == 3
python/src/connection.rs
+20 -2
View File
@@ -412,7 +412,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
@@ -421,10 +422,18 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}
if let Some(seconds) = auto_refresh_interval_seconds {
builder = builder.auto_refresh_interval_seconds(seconds);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}
@@ -440,7 +449,8 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
pub fn create_materialized_view_async(
self_: PyRef<'_, Self>,
name: String,
@@ -449,10 +459,18 @@ impl Connection {
filter: Option<String>,
limit: Option<u64>,
with_no_data: bool,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<f64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(enabled) = auto_refresh_enabled {
builder = builder.auto_refresh_enabled(enabled);
}
if let Some(seconds) = auto_refresh_interval_seconds {
builder = builder.auto_refresh_interval_seconds(seconds);
}
if let Some(projections) = projections {
builder = builder.select(projections);
}