rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-20 18:52:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(remote): unify unsupported operation errors and warnings

Browse Source
This commit is contained in:
Gatefixer
2026-10-03 02:24:00 +00:00
parent b0c0d8ee92
commit fe591c3b74
8 changed files with 346 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/__test__/remote.test.ts
+53
View File
@@ -20,6 +20,7 @@ import {
StaticHeaderProvider,
} from "../lancedb/header";
import { Index } from "../lancedb/indices";
import { LocalTable } from "../lancedb/table";
// Test-only header providers
class CustomProvider extends HeaderProvider {
@@ -85,6 +86,58 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("reports unsupported operations with backend-neutral errors", async () => {
await withMockDatabase(
(req, res) => {
expect(req.url).toBe("/v1/table/test/describe/");
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: false },
{
name: "vector",
type: {
type: "fixed_size_list",
fields: [
{ name: "item", type: { type: "float" }, nullable: true },
],
length: 2,
},
nullable: false,
},
],
},
}),
);
},
async (db) => {
const table = (await db.openTable("test")) as LocalTable;
const cases: [() => Promise<unknown>, string][] = [
[() => table.usesV2ManifestPaths(), "uses_v2_manifest_paths"],
[() => table.migrateManifestPathsV2(), "migrate_manifest_paths_v2"],
[
() => table.setUnenforcedPrimaryKey("id"),
"set_unenforced_primary_key",
],
[() => table.optimize(), "optimize"],
[
() => table.createIndex("vector", { config: Index.hnswPq() }),
"IVF_HNSW_PQ",
],
];
for (const [run, operation] of cases) {
let message = `LanceDBError: not supported: ${operation} is not supported for remote tables.`;
if (operation === "IVF_HNSW_PQ") {
message += " Please use IVF_HNSW_SQ instead.";
}
await expect(run()).rejects.toThrow(new Error(message));
}
},
);
});
it("rejects the external blob opt-in without blocking regular adds", async () => {
let insertRequests = 0;
const describeRequests: string[] = [];
nodejs/src/table.rs
+8 -2
View File
@@ -839,7 +839,10 @@ impl Table {
pub async fn uses_v2_manifest_paths(&self) -> napi::Result<bool> {
self.inner_ref()?
.as_native()
.ok_or_else(|| napi::Error::from_reason("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.default_error()?
.uses_v2_manifest_paths()
.await
.default_error()
@@ -849,7 +852,10 @@ impl Table {
pub async fn migrate_manifest_paths_v2(&self) -> napi::Result<()> {
self.inner_ref()?
.as_native()
.ok_or_else(|| napi::Error::from_reason("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.default_error()?
.migrate_manifest_paths_v2()
.await
.default_error()
python/python/lancedb/remote/db.py
+3 -9
View File
@@ -483,12 +483,12 @@ class RemoteDBConnection(DBConnection):
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
"storage_options is ignored for remote tables"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
"index_cache_size is ignored for remote tables"
" (there is no local cache to configure)"
)
@@ -682,13 +682,6 @@ class RemoteDBConnection(DBConnection):
if namespace_path is None:
namespace_path = []
validate_table_name(name)
if embedding_functions is not None:
logging.warning(
"embedding_functions is not yet supported on LanceDB Cloud."
"Please vote https://github.com/lancedb/lancedb/issues/626 "
"for this feature."
)
from .table import RemoteTable
table = LOOP.run(
@@ -700,6 +693,7 @@ class RemoteDBConnection(DBConnection):
schema=schema,
on_bad_vectors=on_bad_vectors,
fill_value=fill_value,
embedding_functions=embedding_functions,
)
)
return RemoteTable(
python/python/lancedb/remote/table.py
+28 -47
View File
@@ -42,6 +42,7 @@ from lancedb.index import (
BTree,
Bitmap,
HnswFlat,
HnswPq,
HnswSq,
IvfFlat,
IvfPq,
@@ -237,12 +238,12 @@ class RemoteTable(Table):
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
"""to_arrow() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
"""to_arrow() is not supported for remote tables."""
raise NotImplementedError("to_arrow() is not supported for remote tables.")
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
"""to_pandas() is not supported for remote tables."""
raise NotImplementedError("to_pandas() is not supported for remote tables.")
def checkout(self, version: Union[int, str]):
result = LOOP.run(self._table.checkout(version))
@@ -499,14 +500,8 @@ class RemoteTable(Table):
if accelerator is not None:
logging.warning(
"GPU accelerator is not yet supported on LanceDB cloud."
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
"GPU accelerator is not supported for remote tables "
"and will be ignored."
)
idx_type = index_type.upper()
@@ -526,9 +521,11 @@ class RemoteTable(Table):
elif idx_type == "IVF_SQ":
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
elif idx_type == "IVF_HNSW_PQ":
raise ValueError(
"IVF_HNSW_PQ is not supported on LanceDB cloud."
"Please use IVF_HNSW_SQ instead."
config = HnswPq(
distance_type=metric,
num_partitions=num_partitions,
num_sub_vectors=num_sub_vectors,
num_bits=num_bits,
)
elif idx_type == "IVF_HNSW_SQ":
config = HnswSq(distance_type=metric, num_partitions=num_partitions)
@@ -657,7 +654,7 @@ class RemoteTable(Table):
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Not supported for remote tables. Setting this raises.
Returns
-------
@@ -933,26 +930,20 @@ class RemoteTable(Table):
def cleanup_old_versions(self, *_):
"""
cleanup_old_versions() is a no-op on LanceDB Cloud.
Tables are automatically cleaned up and optimized.
cleanup_old_versions() is a no-op for remote tables.
"""
warnings.warn(
"cleanup_old_versions() is a no-op on LanceDB Cloud. "
"Tables are automatically cleaned up and optimized.",
"cleanup_old_versions() is a no-op for remote tables.",
stacklevel=2,
)
pass
def compact_files(self, *_):
"""
compact_files() is a no-op on LanceDB Cloud.
Tables are automatically compacted and optimized.
compact_files() is a no-op for remote tables.
"""
warnings.warn(
"compact_files() is a no-op on LanceDB Cloud. "
"Tables are automatically compacted and optimized.",
"compact_files() is a no-op for remote tables.",
stacklevel=2,
)
pass
@@ -963,17 +954,13 @@ class RemoteTable(Table):
cleanup_older_than: Optional[timedelta] = None,
delete_unverified: bool = False,
):
"""
optimize() is a no-op on LanceDB Cloud.
Indices are optimized automatically.
"""
warnings.warn(
"optimize() is a no-op on LanceDB Cloud. "
"Indices are optimized automatically.",
stacklevel=2,
"""optimize() is not supported for remote tables."""
return LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
)
)
pass
def count_rows(self, filter: Optional[str] = None) -> int:
return LOOP.run(self._table.count_rows(filter))
@@ -1021,7 +1008,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Not supported on LanceDB Cloud."""
"""Not supported for remote tables."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
@@ -1067,7 +1054,7 @@ class RemoteTable(Table):
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
def drop_index(self, index_name: str):
@@ -1135,14 +1122,10 @@ class RemoteTable(Table):
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.uses_v2_manifest_paths())
def migrate_v2_manifest_paths(self):
raise NotImplementedError(
"migrate_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.migrate_manifest_paths_v2())
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
@@ -1153,9 +1136,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
python/python/tests/test_remote_db.py
+161 -4
View File
@@ -17,6 +17,8 @@ from packaging.version import Version
import lancedb
from lancedb.conftest import MockTextEmbeddingFunction
from lancedb.embeddings import EmbeddingFunctionConfig
from lancedb.index import HnswPq
from lancedb.query import ColumnOrdering
from lancedb.remote import ClientConfig
from lancedb.remote.errors import HttpError, RetryError
@@ -1099,7 +1101,7 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
def test_remote_create_index_new_api():
def test_remote_create_index_new_api(caplog):
received_requests = []
def handler(request):
@@ -1160,7 +1162,8 @@ def test_remote_create_index_new_api():
from lancedb.index import BTree, FTS, IvfPq, IvfRq
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
table = db.create_table("test", [{"id": 1}], embedding_functions=[])
assert "embedding_functions is not yet supported" not in caplog.text
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -1182,6 +1185,8 @@ def test_remote_create_index_new_api():
vector_column_name="vector",
index_type="IVF_RQ",
num_partitions=8,
replace=False,
accelerator="cuda",
)
assert len(received_requests) == 6
@@ -1195,6 +1200,152 @@ def test_remote_create_index_new_api():
]
assert received_requests[2]["block_size"] == 256
assert received_requests[4]["replace"] is False
assert received_requests[5]["replace"] is False
assert "replace is not supported" not in caplog.text
assert caplog.messages == [
"GPU accelerator is not supported for remote tables and will be ignored."
]
def remote_unsupported_handler(request):
assert request.path == "/v1/table/test/describe/"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
version=1,
schema=dict(
fields=[
dict(name="id", type={"type": "int64"}, nullable=False),
dict(
name="vector",
type={
"type": "fixed_size_list",
"fields": [
dict(
name="item",
type={"type": "float"},
nullable=True,
)
],
"length": 2,
},
nullable=False,
),
]
),
)
).encode()
)
REMOTE_UNSUPPORTED_OPERATIONS = [
("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"),
("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"),
("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"),
("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"),
("optimize", (), {}, "optimize"),
("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"),
]
@pytest.mark.parametrize("method,args,kwargs,operation", REMOTE_UNSUPPORTED_OPERATIONS)
def test_remote_unsupported_operations_sync(method, args, kwargs, operation):
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.raises(NotImplementedError) as error:
getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
@pytest.mark.asyncio
@pytest.mark.parametrize("method,args,kwargs,operation", REMOTE_UNSUPPORTED_OPERATIONS)
async def test_remote_unsupported_operations_async(method, args, kwargs, operation):
async with mock_lancedb_connection_async(remote_unsupported_handler) as db:
table = await db.open_table("test")
if method == "migrate_v2_manifest_paths":
method = "migrate_manifest_paths_v2"
with pytest.raises(NotImplementedError) as error:
await getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
def test_remote_unsupported_legacy_hnsw_pq():
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.warns(DeprecationWarning, match="create_index"):
with pytest.raises(
NotImplementedError,
match="IVF_HNSW_PQ is not supported for remote tables. "
"Please use IVF_HNSW_SQ instead.",
):
table.create_index(
vector_column_name="vector", index_type="IVF_HNSW_PQ"
)
def test_remote_python_only_messages(caplog):
with mock_lancedb_connection(remote_unsupported_handler) as db:
with caplog.at_level("INFO"):
table = db.open_table("test", storage_options={}, index_cache_size=1)
assert "storage_options is ignored for remote tables" in caplog.text
assert "index_cache_size is ignored for remote tables" in caplog.text
for method in ["to_arrow", "to_pandas"]:
with pytest.raises(NotImplementedError) as error:
getattr(table, method)()
assert str(error.value) == f"{method}() is not supported for remote tables."
for method in ["compact_files", "cleanup_old_versions"]:
with pytest.warns(UserWarning) as warning:
getattr(table, method)()
assert (
str(warning[0].message) == f"{method}() is a no-op for remote tables."
)
def test_remote_embedding_config_does_not_warn_or_drop_metadata(caplog):
received_tables = []
def handler(request):
assert request.path == "/v1/table/test/create/?mode=create"
if request.headers.get("Transfer-Encoding") == "chunked":
body = bytearray()
while size := int(request.rfile.readline().split(b";")[0], 16):
body.extend(request.rfile.read(size))
assert request.rfile.read(2) == b"\r\n"
assert request.rfile.read(2) == b"\r\n"
else:
body = request.rfile.read(int(request.headers["Content-Length"]))
received_tables.append(pa.ipc.open_stream(body).read_all())
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
config = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=MockTextEmbeddingFunction.create(),
)
schema = pa.schema(
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
)
with mock_lancedb_connection(handler) as db:
db.create_table("test", schema=schema, embedding_functions=[config])
assert caplog.messages == []
assert len(received_tables) == 1
metadata = json.loads(received_tables[0].schema.metadata[b"embedding_functions"])
assert metadata[0]["source_column"] == "text"
assert metadata[0]["vector_column"] == "vector"
def test_table_wait_for_index_timeout():
@@ -2567,9 +2718,15 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
def test_remote_blob_byte_apis_not_supported_on_old_server():
with blob_remote_table(server_version=Version("0.1.0")) as table:
assert table.blob_columns() == ["image"]
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blobs is not supported by this LanceDB server",
):
table.fetch_blobs("image", [1])
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blob_files requires LanceDB server 0.5.0 or newer",
):
table.fetch_blob_files("image", [1])
python/src/table.rs
+8 -2
View File
@@ -1709,7 +1709,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
@@ -1721,7 +1724,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()
rust/lancedb/src/remote/table.rs
+83 -7
View File
@@ -620,9 +620,9 @@ impl<S: HttpSend> RemoteTable<S> {
});
}
}
_ => {
Index::IvfHnswPq(_) => {
return Err(Error::NotSupported {
message: "Index type not supported".into(),
message: "IVF_HNSW_PQ is not supported for remote tables. Please use IVF_HNSW_SQ instead.".into(),
});
}
};
@@ -1082,7 +1082,7 @@ impl<S: HttpSend> RemoteTable<S> {
if let Some(full_text_search) = &params.full_text_search {
if full_text_search.wand_factor.is_some() {
return Err(Error::NotSupported {
message: "Wand factor is not yet supported in LanceDB Cloud".into(),
message: "Wand factor is not supported for remote tables.".into(),
});
}
@@ -2848,6 +2848,16 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.fetch_blobs_impl(column, row_ids).await
}
async fn fetch_blob_ranges(
&self,
_column: &str,
_requests: &[crate::blob::BlobRangeRequest],
) -> Result<LargeBinaryArray> {
Err(Error::NotSupported {
message: "fetch_blob_ranges is not supported for remote tables.".into(),
})
}
async fn fetch_blob_files(
&self,
column: &str,
@@ -3156,7 +3166,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn set_unenforced_primary_key(&self, _columns: &[&str]) -> Result<()> {
Err(Error::NotSupported {
message: "set_unenforced_primary_key is not supported on LanceDB cloud.".into(),
message: "set_unenforced_primary_key is not supported for remote tables.".into(),
})
}
@@ -3347,7 +3357,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn optimize(&self, _action: OptimizeAction) -> Result<OptimizeStats> {
self.check_mutable().await?;
Err(Error::NotSupported {
message: "optimize is not supported on LanceDB cloud.".into(),
message: "optimize is not supported for remote tables.".into(),
})
}
async fn add_columns(
@@ -5577,6 +5587,72 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_unsupported_operations() {
let table = Table::new_with_handler("my_table", |_| -> http::Response<String> {
panic!("unsupported operations must not send requests to the server");
});
let mut fts = FullTextSearchQuery::new("test".into());
fts.wand_factor = Some(1.0);
let results = [
(
"set_unenforced_primary_key",
table.set_unenforced_primary_key(["id"]).await,
),
(
"optimize",
table.optimize(OptimizeAction::All).await.map(|_| ()),
),
(
"fetch_blob_ranges",
table.fetch_blob_ranges("image", []).await.map(|_| ()),
),
(
"Wand factor",
table
.query()
.full_text_search(fts)
.execute()
.await
.map(|_| ()),
),
];
for (operation, result) in results {
match result.unwrap_err() {
Error::NotSupported { message } => assert_eq!(
message,
format!("{operation} is not supported for remote tables.")
),
error => panic!("expected not-supported error, got {error}"),
}
}
}
#[tokio::test]
async fn test_remote_unsupported_hnsw_pq() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
let schema = Schema::new(vec![Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
false,
)]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
});
let error = table
.create_index(&["vector"], Index::IvfHnswPq(Default::default()))
.execute()
.await
.unwrap_err();
assert_not_supported_error(
error,
"IVF_HNSW_PQ is not supported for remote tables. Please use IVF_HNSW_SQ instead.",
);
}
#[tokio::test]
async fn test_fetch_blobs_rejects_missing_column() {
let batch = RecordBatch::try_new(
@@ -5690,12 +5766,12 @@ mod tests {
assert_not_supported_error(
table.fetch_blobs("image", &[1]).await.unwrap_err(),
"fetch_blobs",
"fetch_blobs is not supported by this LanceDB server.",
);
assert_not_supported_error(
table.fetch_blob_files("image", &[1]).await.unwrap_err(),
"requires LanceDB Cloud server 0.5.0 or newer",
"requires LanceDB server 0.5.0 or newer",
);
}
rust/lancedb/src/remote/table/blobs.rs
+2 -2
View File
@@ -393,7 +393,7 @@ impl<S: HttpSend> RemoteTable<S> {
}
if !self.server_version.support_blobs() {
return Err(Error::NotSupported {
message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
message: "fetch_blobs is not supported by this LanceDB server.".into(),
});
}
let mut read_snapshot = self.snapshot_read_state().await;
@@ -716,7 +716,7 @@ impl<S: HttpSend> RemoteTable<S> {
Ok(())
} else {
Err(Error::NotSupported {
message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(),
message: "fetch_blob_files requires LanceDB server 0.5.0 or newer.".into(),
})
}
}