rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-21 11:12:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(remote): unify unsupported operation errors and warnings

Browse Source
This commit is contained in:
Gatefixer
2026-10-03 02:24:00 +00:00
parent b0c0d8ee92
commit fe591c3b74
8 changed files with 346 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/remote/db.py
+3 -9
View File
@@ -483,12 +483,12 @@ class RemoteDBConnection(DBConnection):
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
"storage_options is ignored for remote tables"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
"index_cache_size is ignored for remote tables"
" (there is no local cache to configure)"
)
@@ -682,13 +682,6 @@ class RemoteDBConnection(DBConnection):
if namespace_path is None:
namespace_path = []
validate_table_name(name)
if embedding_functions is not None:
logging.warning(
"embedding_functions is not yet supported on LanceDB Cloud."
"Please vote https://github.com/lancedb/lancedb/issues/626 "
"for this feature."
)
from .table import RemoteTable
table = LOOP.run(
@@ -700,6 +693,7 @@ class RemoteDBConnection(DBConnection):
schema=schema,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
)
)
return RemoteTable(
python/python/lancedb/remote/table.py
+28 -47
View File
@@ -42,6 +42,7 @@ from lancedb.index import (
BTree,
Bitmap,
HnswFlat,
HnswPq,
HnswSq,
IvfFlat,
IvfPq,
@@ -237,12 +238,12 @@ class RemoteTable(Table):
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
"""to_arrow() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
"""to_arrow() is not supported for remote tables."""
raise NotImplementedError("to_arrow() is not supported for remote tables.")
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
"""to_pandas() is not supported for remote tables."""
raise NotImplementedError("to_pandas() is not supported for remote tables.")
def checkout(self, version: Union[int, str]):
result = LOOP.run(self._table.checkout(version))
@@ -499,14 +500,8 @@ class RemoteTable(Table):
if accelerator is not None:
logging.warning(
"GPU accelerator is not yet supported on LanceDB cloud."
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
"GPU accelerator is not supported for remote tables "
"and will be ignored."
)
idx_type = index_type.upper()
@@ -526,9 +521,11 @@ class RemoteTable(Table):
elif idx_type == "IVF_SQ":
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
elif idx_type == "IVF_HNSW_PQ":
raise ValueError(
"IVF_HNSW_PQ is not supported on LanceDB cloud."
"Please use IVF_HNSW_SQ instead."
config = HnswPq(
distance_type=metric,
num_partitions=num_partitions,
num_sub_vectors=num_sub_vectors,
num_bits=num_bits,
)
elif idx_type == "IVF_HNSW_SQ":
config = HnswSq(distance_type=metric, num_partitions=num_partitions)
@@ -657,7 +654,7 @@ class RemoteTable(Table):
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Not supported for remote tables. Setting this raises.
Returns
-------
@@ -933,26 +930,20 @@ class RemoteTable(Table):
def cleanup_old_versions(self, *_):
"""
cleanup_old_versions() is a no-op on LanceDB Cloud.
Tables are automatically cleaned up and optimized.
cleanup_old_versions() is a no-op for remote tables.
"""
warnings.warn(
"cleanup_old_versions() is a no-op on LanceDB Cloud. "
"Tables are automatically cleaned up and optimized.",
"cleanup_old_versions() is a no-op for remote tables.",
stacklevel=2,
)
pass
def compact_files(self, *_):
"""
compact_files() is a no-op on LanceDB Cloud.
Tables are automatically compacted and optimized.
compact_files() is a no-op for remote tables.
"""
warnings.warn(
"compact_files() is a no-op on LanceDB Cloud. "
"Tables are automatically compacted and optimized.",
"compact_files() is a no-op for remote tables.",
stacklevel=2,
)
pass
@@ -963,17 +954,13 @@ class RemoteTable(Table):
cleanup_older_than: Optional[timedelta] = None,
delete_unverified: bool = False,
):
"""
optimize() is a no-op on LanceDB Cloud.
Indices are optimized automatically.
"""
warnings.warn(
"optimize() is a no-op on LanceDB Cloud. "
"Indices are optimized automatically.",
stacklevel=2,
"""optimize() is not supported for remote tables."""
return LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
)
)
pass
def count_rows(self, filter: Optional[str] = None) -> int:
return LOOP.run(self._table.count_rows(filter))
@@ -1021,7 +1008,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Not supported on LanceDB Cloud."""
"""Not supported for remote tables."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
@@ -1067,7 +1054,7 @@ class RemoteTable(Table):
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
def drop_index(self, index_name: str):
@@ -1135,14 +1122,10 @@ class RemoteTable(Table):
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.uses_v2_manifest_paths())
def migrate_v2_manifest_paths(self):
raise NotImplementedError(
"migrate_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.migrate_manifest_paths_v2())
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
@@ -1153,9 +1136,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
python/python/tests/test_remote_db.py
+161 -4
View File
@@ -17,6 +17,8 @@ from packaging.version import Version
import lancedb
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.embeddings import EmbeddingFunctionConfig
from lancedb.index import HnswPq
from lancedb.query import ColumnOrdering
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
@@ -1099,7 +1101,7 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
def test_remote_create_index_new_api():
def test_remote_create_index_new_api(caplog):
received_requests = []
def handler(request):
@@ -1160,7 +1162,8 @@ def test_remote_create_index_new_api():
from lancedb.index import BTree, FTS, IvfPq, IvfRq
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
table = db.create_table("test", [{"id": 1}], embedding_functions=[])
assert "embedding_functions is not yet supported" not in caplog.text
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -1182,6 +1185,8 @@ def test_remote_create_index_new_api():
vector_column_name="vector",
index_type="IVF_RQ",
num_partitions=8,
replace=False,
accelerator="cuda",
)
assert len(received_requests) == 6
@@ -1195,6 +1200,152 @@ def test_remote_create_index_new_api():
]
assert received_requests[2]["block_size"] == 256
assert received_requests[4]["replace"] is False
assert received_requests[5]["replace"] is False
assert "replace is not supported" not in caplog.text
assert caplog.messages == [
"GPU accelerator is not supported for remote tables and will be ignored."
]
def remote_unsupported_handler(request):
assert request.path == "/v1/table/test/describe/"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
version=1,
schema=dict(
fields=[
dict(name="id", type={"type": "int64"}, nullable=False),
dict(
name="vector",
type={
"type": "fixed_size_list",
"fields": [
dict(
name="item",
type={"type": "float"},
nullable=True,
)
],
"length": 2,
},
nullable=False,
),
]
),
)
).encode()
)
REMOTE_UNSUPPORTED_OPERATIONS = [
("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"),
("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"),
("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"),
("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"),
("optimize", (), {}, "optimize"),
("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"),
]
@pytest.mark.parametrize("method,args,kwargs,operation", REMOTE_UNSUPPORTED_OPERATIONS)
def test_remote_unsupported_operations_sync(method, args, kwargs, operation):
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.raises(NotImplementedError) as error:
getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("method,args,kwargs,operation", REMOTE_UNSUPPORTED_OPERATIONS)
async def test_remote_unsupported_operations_async(method, args, kwargs, operation):
async with mock_lancedb_connection_async(remote_unsupported_handler) as db:
table = await db.open_table("test")
if method == "migrate_v2_manifest_paths":
method = "migrate_manifest_paths_v2"
with pytest.raises(NotImplementedError) as error:
await getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
def test_remote_unsupported_legacy_hnsw_pq():
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.warns(DeprecationWarning, match="create_index"):
with pytest.raises(
NotImplementedError,
match="IVF_HNSW_PQ is not supported for remote tables. "
"Please use IVF_HNSW_SQ instead.",
):
table.create_index(
vector_column_name="vector", index_type="IVF_HNSW_PQ"
)
def test_remote_python_only_messages(caplog):
with mock_lancedb_connection(remote_unsupported_handler) as db:
with caplog.at_level("INFO"):
table = db.open_table("test", storage_options={}, index_cache_size=1)
assert "storage_options is ignored for remote tables" in caplog.text
assert "index_cache_size is ignored for remote tables" in caplog.text
for method in ["to_arrow", "to_pandas"]:
with pytest.raises(NotImplementedError) as error:
getattr(table, method)()
assert str(error.value) == f"{method}() is not supported for remote tables."
for method in ["compact_files", "cleanup_old_versions"]:
with pytest.warns(UserWarning) as warning:
getattr(table, method)()
assert (
str(warning[0].message) == f"{method}() is a no-op for remote tables."
)
def test_remote_embedding_config_does_not_warn_or_drop_metadata(caplog):
received_tables = []
def handler(request):
assert request.path == "/v1/table/test/create/?mode=create"
if request.headers.get("Transfer-Encoding") == "chunked":
body = bytearray()
while size := int(request.rfile.readline().split(b";")[0], 16):
body.extend(request.rfile.read(size))
assert request.rfile.read(2) == b"\r\n"
assert request.rfile.read(2) == b"\r\n"
else:
body = request.rfile.read(int(request.headers["Content-Length"]))
received_tables.append(pa.ipc.open_stream(body).read_all())
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
config = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=MockTextEmbeddingFunction.create(),
)
schema = pa.schema(
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
)
with mock_lancedb_connection(handler) as db:
db.create_table("test", schema=schema, embedding_functions=[config])
assert caplog.messages == []
assert len(received_tables) == 1
metadata = json.loads(received_tables[0].schema.metadata[b"embedding_functions"])
assert metadata[0]["source_column"] == "text"
assert metadata[0]["vector_column"] == "vector"
def test_table_wait_for_index_timeout():
@@ -2567,9 +2718,15 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
def test_remote_blob_byte_apis_not_supported_on_old_server():
with blob_remote_table(server_version=Version("0.1.0")) as table:
assert table.blob_columns() == ["image"]
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blobs is not supported by this LanceDB server",
):
table.fetch_blobs("image", [1])
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blob_files requires LanceDB server 0.5.0 or newer",
):
table.fetch_blob_files("image", [1])
python/src/table.rs
+8 -2
View File
@@ -1709,7 +1709,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
@@ -1721,7 +1724,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()