mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
feat: configure automatic refresh when declaring computed columns (#4607)
Add `auto_refresh_enabled` and `auto_refresh_interval_seconds` to the
Rust builder and Python sync/async add-column APIs for computed columns
and registered Function outputs.
Python users can configure refresh when declaring a column:
```python
table.add_columns(
computed={"total": "price * quantity"},
auto_refresh_enabled=True,
auto_refresh_interval_seconds=300,
)
```
Omit the interval to let the server determine when to refresh:
```python
table.add_columns(computed={"discounted_total": "total * 0.9"})
```
The async API accepts the same options:
```python
await async_table.add_columns(
computed={"total": "price * quantity"},
auto_refresh_enabled=True,
auto_refresh_interval_seconds=300,
)
```
For an existing registered `FunctionApplication`, the options also apply
to Function columns. Disabling automatic refresh still permits explicit
refresh:
```python
table.add_columns(
{"features": function_application},
auto_refresh_enabled=False,
auto_refresh_interval_seconds=300,
)
table.refresh_column("features")
```
Options apply to every output declared by a call. Automatic refresh is
enabled by default; when the interval is omitted, the server determines
when to refresh. An explicit interval sets a minimum delay after a
successful refresh. The interval does not force recomputation of
unchanged inputs. Invalid intervals and options on one-time transforms
are rejected before declaration.
Native declarations persist `computed_column.auto_refresh_enabled` and
an explicitly supplied `computed_column.auto_refresh_interval_seconds`
in each output field’s metadata in the same commit as the declaration.
An omitted interval is neither stored nor sent in remote requests.
Remote declarations forward the settings to Cloud/Enterprise servers.
Local tables store these settings; automatic execution requires a
supporting server with scheduling enabled.
This commit is contained in:
@@ -4869,3 +4869,53 @@ async def test_refresh_column_async_job_async_table(tmp_path):
|
||||
assert result.published_version == 4
|
||||
assert await job.status() == "finished"
|
||||
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("interval", [None, 300])
|
||||
def test_computed_auto_refresh_options(tmp_path, interval):
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("refresh_options", [{"x": 1}])
|
||||
before = table.version
|
||||
table.add_columns(
|
||||
computed={"doubled": "x * 2", "tripled": "x * 3"},
|
||||
auto_refresh_enabled=False,
|
||||
auto_refresh_interval_seconds=interval,
|
||||
)
|
||||
assert table.version == before + 1
|
||||
for column in ("doubled", "tripled"):
|
||||
metadata = table.schema.field(column).metadata
|
||||
assert metadata[b"computed_column.auto_refresh_enabled"] == b"false"
|
||||
if interval is None:
|
||||
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
|
||||
else:
|
||||
assert metadata[b"computed_column.auto_refresh_interval_seconds"] == b"300"
|
||||
table.refresh_column("doubled")
|
||||
assert table.to_arrow()["doubled"].to_pylist() == [2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [0, -1, True, 1.5, "300", 2**64])
|
||||
def test_computed_auto_refresh_rejects_invalid_interval(tmp_path, value):
|
||||
table = lancedb.connect(tmp_path).create_table("invalid_options", [{"x": 1}])
|
||||
with pytest.raises(ValueError, match="auto_refresh_interval_seconds"):
|
||||
table.add_columns(
|
||||
computed={"doubled": "x * 2"}, auto_refresh_interval_seconds=value
|
||||
)
|
||||
assert table.schema.names == ["x"]
|
||||
|
||||
|
||||
def test_auto_refresh_options_require_computed_columns(tmp_path):
|
||||
table = lancedb.connect(tmp_path).create_table("invalid_kind", [{"x": 1}])
|
||||
with pytest.raises(ValueError, match="require computed"):
|
||||
table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
table = await db.create_table("server_interval", [{"x": 1}])
|
||||
before = await table.version()
|
||||
await table.add_columns(computed={"doubled": "x * 2"})
|
||||
assert await table.version() == before + 1
|
||||
metadata = (await table.schema()).field("doubled").metadata
|
||||
assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
|
||||
assert b"computed_column.auto_refresh_interval_seconds" not in metadata
|
||||
|
||||
Reference in New Issue
Block a user