mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-29 14:56:51 +00:00
fix: gate experimental local functions behind an opt-in feature (#4688)
Published SDK builds currently expose local SQL computed columns and materialized-view execution that are not ready for release. Add the opt-in `experimental-functions` Cargo feature to the Rust core and forward it through the Python and Node bindings; published packages leave it disabled. Local declarations and refreshes now return `NotSupported` before mutating tables or spawning jobs, including prepared materialized-view creation. Remote Cloud/Enterprise client operations, Python UDF packaging, ordinary eager `add_columns`, and access to existing data remain available. Stored metadata and write protections stay active, and shared server commit helpers remain usable. CI covers both feature configurations. Existing local execution tests run with the feature enabled; default-build regressions cover early rejection, unchanged table versions, existing metadata protections, and normal eager transforms. API and source-build documentation explain the opt-in. Validation: Rust library tests pass with the feature disabled (1,869 tests) and enabled (1,947 tests), plus three default-build integration regressions. Python and TypeScript bindings were rebuilt and tested in both configurations, including mocked remote requests; Rust formatting, Python package lint, TypeScript lint/build, and generated docs were checked. Repository-wide Ruff still reports 48 pre-existing errors in unchanged `docs/web-tests` files.
This commit is contained in:
@@ -48,6 +48,18 @@ Build the package
|
||||
make develop
|
||||
```
|
||||
|
||||
Local SQL computed columns and materialized-view creation/refresh are experimental
|
||||
and disabled in published packages. To opt in when building from this directory:
|
||||
|
||||
```shell
|
||||
uv run --extra tests --extra dev maturin develop --features experimental-functions --extras tests,dev
|
||||
uv run --no-sync pytest python/tests -m experimental_functions
|
||||
```
|
||||
|
||||
Rebuild without `--features experimental-functions` to test the default package.
|
||||
The tests detect the compiled feature; remote operations, metadata protections,
|
||||
and ordinary eager `add_columns` transforms remain available in both builds.
|
||||
|
||||
Format:
|
||||
|
||||
```shell
|
||||
|
||||
@@ -52,3 +52,4 @@ pyo3-build-config = { version = "0.28", features = ["abi3-py310"] }
|
||||
default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"]
|
||||
fp16kernels = ["lancedb/fp16kernels"]
|
||||
remote = ["lancedb/remote"]
|
||||
experimental-functions = ["lancedb/experimental-functions"]
|
||||
|
||||
@@ -26,6 +26,8 @@ from lance_namespace import (
|
||||
from .remote import ClientConfig
|
||||
from .types import BaseTokenizerType
|
||||
|
||||
_experimental_functions_enabled: bool
|
||||
|
||||
IvfHnswPq: type[HnswPq] = HnswPq
|
||||
IvfHnswSq: type[HnswSq] = HnswSq
|
||||
IvfHnswFlat: type[HnswFlat] = HnswFlat
|
||||
|
||||
@@ -626,6 +626,10 @@ class DBConnection(EnforceOverrides):
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
Local creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; remote Cloud/Enterprise operations remain available.
|
||||
|
||||
The view is populated before creation returns. Pass
|
||||
``with_no_data=True`` to create only its definition and empty backing
|
||||
table. The view is a normal table: it can be queried, indexed and
|
||||
@@ -1760,6 +1764,10 @@ class LanceDBConnection(DBConnection):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> MaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
Local creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; remote Cloud/Enterprise operations remain available.
|
||||
See
|
||||
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
|
||||
|
||||
@@ -1769,13 +1777,13 @@ class LanceDBConnection(DBConnection):
|
||||
>>> db = lancedb.connect("./.lancedb")
|
||||
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
|
||||
>>> table = db.create_table("people", data)
|
||||
>>> view = db.create_materialized_view(
|
||||
>>> view = db.create_materialized_view( # doctest: +SKIP
|
||||
... "adults",
|
||||
... "people",
|
||||
... select=["name", ("shout", "upper(name)")],
|
||||
... where="age >= 18",
|
||||
... )
|
||||
>>> view.table.count_rows()
|
||||
>>> view.table.count_rows() # doctest: +SKIP
|
||||
1
|
||||
"""
|
||||
LOOP.run(
|
||||
@@ -2846,6 +2854,10 @@ class AsyncConnection(object):
|
||||
auto_refresh_interval_seconds: Optional[float] = None,
|
||||
) -> AsyncMaterializedView:
|
||||
"""Define a materialized view named ``name`` over the table ``source``.
|
||||
|
||||
Local creation and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature. Published packages leave it
|
||||
disabled; remote Cloud/Enterprise operations remain available.
|
||||
See
|
||||
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
|
||||
"""
|
||||
|
||||
@@ -2,7 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
"""Materialized views: tables defined by a query over a source table and
|
||||
maintained by refresh. See ``DBConnection.create_materialized_view``."""
|
||||
maintained by refresh. See ``DBConnection.create_materialized_view``.
|
||||
|
||||
Local creation and refresh require the ``experimental-functions`` Cargo feature
|
||||
in a source build. Published packages disable it. Opening, reading, listing and
|
||||
dropping existing views and remote Cloud/Enterprise operations remain available.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -2988,6 +2988,10 @@ class Table(ABC):
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
Local declaration and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature, which published packages
|
||||
do not enable. Remote Cloud/Enterprise operations remain available.
|
||||
|
||||
A mapping from output column names to SQL expressions derives each
|
||||
output field from its expression. A direct projection of a Blob v2
|
||||
field inherits Blob v2 semantics; other expressions derive their
|
||||
@@ -3029,11 +3033,11 @@ class Table(ABC):
|
||||
>>> import lancedb
|
||||
>>> db = lancedb.connect("./.lancedb")
|
||||
>>> table = db.create_table("computed_demo", [{"x": 1}, {"x": 2}])
|
||||
>>> table.add_columns(computed={"doubled": "x * 2"})
|
||||
>>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP
|
||||
AddColumnsResult(version=2)
|
||||
>>> table.refresh_column("doubled")
|
||||
>>> table.refresh_column("doubled") # doctest: +SKIP
|
||||
RefreshColumnResult(rows_filled=2, version=4)
|
||||
>>> table.to_arrow().sort_by("x").to_pandas()
|
||||
>>> table.to_arrow().sort_by("x").to_pandas() # doctest: +SKIP
|
||||
x doubled
|
||||
0 1 2
|
||||
1 2 4
|
||||
@@ -3044,6 +3048,9 @@ class Table(ABC):
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
|
||||
Local refresh requires the ``experimental-functions`` Cargo feature
|
||||
in a source build. Published packages leave it disabled.
|
||||
|
||||
Declared with ``add_columns(computed=...)``, a column starts empty and
|
||||
gets its values here. Rows appended since the last refresh are filled
|
||||
by the next one, and rows whose inputs changed since they were computed
|
||||
@@ -3070,6 +3077,9 @@ class Table(ABC):
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
instead of blocking until it completes.
|
||||
|
||||
Local refresh requires a source build with ``experimental-functions``.
|
||||
Remote Cloud/Enterprise jobs do not require this feature.
|
||||
|
||||
The job may already be complete when returned; callers must not assume
|
||||
the column is filled until :meth:`Job.wait` returns. Invalid input --
|
||||
an unknown column, or one that is not computed -- raises here rather
|
||||
@@ -3087,13 +3097,13 @@ class Table(ABC):
|
||||
>>> import lancedb
|
||||
>>> db = lancedb.connect("./.lancedb")
|
||||
>>> table = db.create_table("computed_job_demo", [{"x": 1}, {"x": 2}])
|
||||
>>> table.add_columns(computed={"doubled": "x * 2"})
|
||||
>>> table.add_columns(computed={"doubled": "x * 2"}) # doctest: +SKIP
|
||||
AddColumnsResult(version=2)
|
||||
>>> job = table.refresh_column_async("doubled")
|
||||
>>> result = job.wait()
|
||||
>>> result.rows_assigned
|
||||
>>> job = table.refresh_column_async("doubled") # doctest: +SKIP
|
||||
>>> result = job.wait() # doctest: +SKIP
|
||||
>>> result.rows_assigned # doctest: +SKIP
|
||||
2
|
||||
>>> job.status()
|
||||
>>> job.status() # doctest: +SKIP
|
||||
'finished'
|
||||
"""
|
||||
|
||||
@@ -7195,6 +7205,10 @@ class AsyncTable:
|
||||
Function columns are supported only on LanceDB Cloud and
|
||||
Enterprise.
|
||||
computed: Dict[str, str], optional
|
||||
Local declaration and refresh require a source build with the
|
||||
``experimental-functions`` Cargo feature, which published packages
|
||||
do not enable. Remote Cloud/Enterprise operations remain available.
|
||||
|
||||
A mapping from output column names to SQL expressions derives each
|
||||
output field from its expression. A direct projection of a Blob v2
|
||||
field inherits Blob v2 semantics; other expressions derive their
|
||||
@@ -7314,6 +7328,9 @@ class AsyncTable:
|
||||
"""
|
||||
Fill the rows of a computed column that hold no value yet.
|
||||
|
||||
Local refresh requires the ``experimental-functions`` Cargo feature
|
||||
in a source build. Published packages leave it disabled.
|
||||
|
||||
Declared with ``add_columns(computed=...)``, a column starts empty and
|
||||
gets its values here. Rows appended since the last refresh are filled
|
||||
by the next one, and rows whose inputs changed since they were computed
|
||||
@@ -7384,6 +7401,9 @@ class AsyncTable:
|
||||
Like :meth:`refresh_column`, but returns a handle to the refresh job
|
||||
instead of blocking until it completes.
|
||||
|
||||
Local refresh requires a source build with ``experimental-functions``.
|
||||
Remote Cloud/Enterprise jobs do not require this feature.
|
||||
|
||||
The job may already be complete when returned; callers must not assume
|
||||
the column is filled until :meth:`AsyncJob.wait` resolves. Invalid
|
||||
input -- an unknown column, or one that is not computed -- raises here
|
||||
@@ -7409,7 +7429,7 @@ class AsyncTable:
|
||||
... result = await job.wait()
|
||||
... assert result.rows_assigned == 1
|
||||
... return await job.status()
|
||||
>>> asyncio.run(refresh_in_background())
|
||||
>>> asyncio.run(refresh_in_background()) # doctest: +SKIP
|
||||
'finished'
|
||||
"""
|
||||
return _typed_job(
|
||||
|
||||
@@ -9,6 +9,21 @@ import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line(
|
||||
"markers", "experimental_functions: requires the experimental-functions build"
|
||||
)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(items):
|
||||
if lancedb._lancedb._experimental_functions_enabled:
|
||||
return
|
||||
skip = pytest.mark.skip(reason="requires the experimental-functions Cargo feature")
|
||||
for item in items:
|
||||
if item.get_closest_marker("experimental_functions"):
|
||||
item.add_marker(skip)
|
||||
|
||||
|
||||
def pandas_string_type():
|
||||
"""Return the PyArrow string type that pandas uses for string columns.
|
||||
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import lancedb
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
lancedb._lancedb._experimental_functions_enabled,
|
||||
reason="tests the default build without experimental-functions",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("namespace", [False, True])
|
||||
def test_default_build_rejects_local_functions(tmp_path, namespace):
|
||||
db = (
|
||||
lancedb.connect_namespace("dir", {"root": str(tmp_path)})
|
||||
if namespace
|
||||
else lancedb.connect(tmp_path)
|
||||
)
|
||||
table = db.create_table("source", [{"x": 1}, {"x": 2}])
|
||||
version = table.version
|
||||
operations = [
|
||||
lambda: table.add_columns(computed={"doubled": "x * 2"}),
|
||||
lambda: table.refresh_column("x"),
|
||||
lambda: table.refresh_column_async("x"),
|
||||
lambda: db.create_materialized_view("blocked", "source"),
|
||||
lambda: db.create_materialized_view("blocked", "source", with_no_data=True),
|
||||
lambda: db.create_materialized_view_async("blocked", "source"),
|
||||
]
|
||||
for operation in operations:
|
||||
with pytest.raises(NotImplementedError, match="experimental-functions"):
|
||||
operation()
|
||||
assert table.version == version
|
||||
assert db.table_names() == ["source"]
|
||||
table.add_columns({"eager": "x * 2"})
|
||||
assert table.to_arrow()["eager"].to_pylist() == [2, 4]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_default_build_rejects_local_functions(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
table = await db.create_table("source", [{"x": 1}, {"x": 2}])
|
||||
version = await table.version()
|
||||
operations = [
|
||||
lambda: table.add_columns(computed={"doubled": "x * 2"}),
|
||||
lambda: table.refresh_column("x"),
|
||||
lambda: table.refresh_column_async("x"),
|
||||
lambda: db.create_materialized_view("blocked", "source"),
|
||||
lambda: db.create_materialized_view("blocked", "source", with_no_data=True),
|
||||
lambda: db.create_materialized_view_async("blocked", "source"),
|
||||
]
|
||||
for operation in operations:
|
||||
with pytest.raises(NotImplementedError, match="experimental-functions"):
|
||||
await operation()
|
||||
assert await table.version() == version
|
||||
assert await db.table_names() == ["source"]
|
||||
await table.add_columns({"eager": "x * 2"})
|
||||
assert (await table.query().to_arrow())["eager"].to_pylist() == [2, 4]
|
||||
@@ -173,6 +173,7 @@ def test_sync_remote_create_uses_public_async_connection():
|
||||
assert calls[-1] == ("drop", "adults", ["analytics"])
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_create_refresh_and_query(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view(
|
||||
@@ -188,6 +189,7 @@ def test_create_refresh_and_query(tmp_path):
|
||||
assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_create_and_refresh_jobs(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
create_job = db.create_materialized_view_async(
|
||||
@@ -212,6 +214,7 @@ def test_create_and_refresh_jobs(tmp_path):
|
||||
assert "adults" not in list(db.list_materialized_views())
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_definition_round_trips(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
@@ -222,6 +225,7 @@ def test_definition_round_trips(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_full_refresh_after_append(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("copy", "people", with_no_data=True)
|
||||
@@ -236,6 +240,7 @@ def test_full_refresh_after_append(tmp_path):
|
||||
assert view.refresh().mode == "rebuild"
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_full_refresh_after_update(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("copy", "people", with_no_data=True)
|
||||
@@ -249,6 +254,7 @@ def test_full_refresh_after_update(tmp_path):
|
||||
assert sorted(row["age"] for row in rows) == [8, 36, 85]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_legacy_storage_source_update_rebuilds(tmp_path):
|
||||
db = lancedb.connect(
|
||||
tmp_path,
|
||||
@@ -265,6 +271,7 @@ def test_legacy_storage_source_update_rebuilds(tmp_path):
|
||||
assert sorted(row["age"] for row in rows) == [8, 36]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_direct_writes_to_a_view_are_refused(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("copy", "people")
|
||||
@@ -290,6 +297,7 @@ def test_direct_writes_to_a_view_are_refused(tmp_path):
|
||||
assert view.refresh().rows_written == 4
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_list_and_not_a_view(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
db.create_materialized_view("adults", "people", where="age >= 18")
|
||||
@@ -304,6 +312,7 @@ def test_list_and_not_a_view(tmp_path):
|
||||
assert list(db.list_materialized_views()) == []
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_invalid_expression_fails_at_create(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
with pytest.raises(Exception, match="missing"):
|
||||
@@ -311,6 +320,7 @@ def test_invalid_expression_fails_at_create(tmp_path):
|
||||
assert "bad" not in db.list_tables().tables
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_create_refresh_and_open(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
@@ -332,6 +342,7 @@ async def test_async_create_refresh_and_open(tmp_path):
|
||||
assert [item async for item in db.list_materialized_views()] == ["shouts"]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_create_and_refresh_jobs(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
@@ -358,6 +369,7 @@ async def test_async_create_and_refresh_jobs(tmp_path):
|
||||
assert "adults" not in [item async for item in db.list_materialized_views()]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_refresh_rebuilds(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
@@ -372,6 +384,7 @@ async def test_async_refresh_rebuilds(tmp_path):
|
||||
assert result.rows_written == 2
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_source_does_not_require_stable_row_ids(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
db.create_table("plain", [{"x": 1}])
|
||||
@@ -379,6 +392,7 @@ def test_source_does_not_require_stable_row_ids(tmp_path):
|
||||
assert view.table.count_rows() == 1
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_bare_select_names_are_quoted(tmp_path):
|
||||
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
db.create_table("odd_names", [{"order item": "widget", "select": 2}])
|
||||
@@ -396,6 +410,7 @@ def test_bare_select_names_are_quoted(tmp_path):
|
||||
assert rows[0]["select"] == 2
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_scalar_select_is_one_column(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("just_name", "people", select="name")
|
||||
@@ -405,6 +420,7 @@ def test_scalar_select_is_one_column(tmp_path):
|
||||
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_scalar_select_is_one_column(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
|
||||
@@ -415,6 +431,7 @@ async def test_async_scalar_select_is_one_column(tmp_path):
|
||||
assert set(rows[0]) == {"name"}
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_limit_above_i64_max_is_refused(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
with pytest.raises(ValueError, match="exceeds the maximum"):
|
||||
@@ -434,6 +451,7 @@ def _namespace_db(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_namespace_connection_materialized_views(tmp_path):
|
||||
db = _namespace_db(tmp_path)
|
||||
db.create_table(
|
||||
@@ -464,6 +482,7 @@ def test_namespace_connection_materialized_views(tmp_path):
|
||||
assert list(db.list_materialized_views()) == []
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_namespace_connection_materialized_views(tmp_path):
|
||||
db = lancedb.connect_namespace_async(
|
||||
@@ -557,6 +576,7 @@ def test_stored_queries_and_legacy_layouts_are_read():
|
||||
read(newer)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_local_materialized_view_listing_pages_and_resume(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
db.create_materialized_view("a_view", "people", with_no_data=True)
|
||||
@@ -646,6 +666,7 @@ async def test_async_remote_create_keeps_interval_unspecified():
|
||||
assert native.calls[0][2]["auto_refresh_interval_seconds"] is None
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
@@ -673,12 +694,14 @@ def test_remote_create_rejects_invalid_interval_without_request(seconds):
|
||||
assert requests == []
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_namespace_forwards_automatic_refresh_request(tmp_path):
|
||||
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
|
||||
with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
|
||||
db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
|
||||
db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
|
||||
@@ -686,6 +709,7 @@ async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
|
||||
await db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path):
|
||||
db = make_db(tmp_path)
|
||||
view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False)
|
||||
|
||||
@@ -16,6 +16,7 @@ import pyarrow as pa
|
||||
import lancedb
|
||||
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
|
||||
from lancedb.listing import AsyncListing, Listing
|
||||
from lancedb.materialized_view import DEFINITION_FORMAT, DEFINITION_META_KEY
|
||||
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
|
||||
from lancedb.query import Query
|
||||
from lancedb.table import AsyncTable, LanceTable
|
||||
@@ -359,7 +360,17 @@ class TestNamespaceConnection:
|
||||
def test_list_materialized_views(self):
|
||||
db = lancedb.connect_namespace("dir", {"root": self.temp_dir})
|
||||
db.create_table("source", [{"id": 1}])
|
||||
db.create_materialized_view("view", "source", with_no_data=True)
|
||||
# Model a stored view from an enabled build: listing must also work
|
||||
# when local creation is disabled in the current build.
|
||||
schema = pa.schema(
|
||||
[("id", pa.int64())],
|
||||
metadata={
|
||||
DEFINITION_META_KEY: json.dumps(
|
||||
{"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"}
|
||||
).encode()
|
||||
},
|
||||
)
|
||||
db.create_table("view", schema=schema)
|
||||
|
||||
names = db.list_materialized_views(page_limit=1)
|
||||
assert isinstance(names, Listing)
|
||||
@@ -872,7 +883,16 @@ class TestAsyncNamespaceConnection:
|
||||
async def test_list_materialized_views(self):
|
||||
db = lancedb.connect_namespace_async("dir", {"root": self.temp_dir})
|
||||
await db.create_table("source", [{"id": 1}])
|
||||
await db.create_materialized_view("view", "source", with_no_data=True)
|
||||
# Model a stored view independently of the local creation feature.
|
||||
schema = pa.schema(
|
||||
[("id", pa.int64())],
|
||||
metadata={
|
||||
DEFINITION_META_KEY: json.dumps(
|
||||
{"format": DEFINITION_FORMAT, "query": "SELECT id FROM source"}
|
||||
).encode()
|
||||
},
|
||||
)
|
||||
await db.create_table("view", schema=schema)
|
||||
|
||||
names = db.list_materialized_views(page_limit=1)
|
||||
assert isinstance(names, AsyncListing)
|
||||
|
||||
@@ -5489,6 +5489,7 @@ async def test_async_search_runs_embedding_on_dedicated_executor(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_computed_column_declare_and_refresh(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("computed", [{"x": 1}, {"x": 2}])
|
||||
@@ -5512,6 +5513,7 @@ def test_computed_column_rejects_transforms_and_computed_together(tmp_path):
|
||||
table.add_columns({"a": "x + 1"}, computed={"b": "x * 2"})
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_computed_column_blob_projection_inherits_semantics(tmp_path):
|
||||
schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
|
||||
db = lancedb.connect(tmp_path)
|
||||
@@ -5535,6 +5537,7 @@ def test_computed_column_blob_projection_inherits_semantics(tmp_path):
|
||||
assert copied.to_pylist() == [b"hello", b"", None]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_computed_column_async(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
@@ -5553,6 +5556,7 @@ def test_function_errors_are_remote_only(tmp_path):
|
||||
table.function_errors()
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
def test_refresh_column_async_returns_job(tmp_path):
|
||||
db = lancedb.connect(tmp_path)
|
||||
table = db.create_table("computed_job", [{"x": 1}, {"x": 2}])
|
||||
@@ -5581,6 +5585,7 @@ def test_refresh_column_async_returns_job(tmp_path):
|
||||
table.refresh_column_async("x")
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_column_async_job_async_table(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
@@ -5597,6 +5602,7 @@ async def test_refresh_column_async_job_async_table(tmp_path):
|
||||
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.parametrize("interval", [None, 300])
|
||||
def test_computed_auto_refresh_options(tmp_path, interval):
|
||||
db = lancedb.connect(tmp_path)
|
||||
@@ -5635,6 +5641,7 @@ def test_auto_refresh_options_require_computed_columns(tmp_path):
|
||||
table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True)
|
||||
|
||||
|
||||
@pytest.mark.experimental_functions
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
|
||||
@@ -132,6 +132,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(expr_lit, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(expr_func, m)?)?;
|
||||
m.add("__version__", env!("CARGO_PKG_VERSION"))?;
|
||||
m.add(
|
||||
"_experimental_functions_enabled",
|
||||
cfg!(feature = "experimental-functions"),
|
||||
)?;
|
||||
// Give the shared runtime a coordinated, bounded shutdown at normal
|
||||
// process exit -- see `shutdown_runtime` and `runtime::shutdown` for why.
|
||||
py.import("atexit")?
|
||||
|
||||
Reference in New Issue
Block a user