fix(remote): unify unsupported operation errors and warnings (#4414)

Remote SDKs report Cloud-specific errors against self-hosted servers,
and some Python wrappers raise different exception types for the same
unsupported operation. The sync index wrapper also warns that `replace`
is ignored even though Rust forwards it to the server.

Use backend-neutral table/server diagnostics and the existing
`NotSupported` error conversions for unsupported manifest operations,
blob ranges, primary keys, optimization, and HNSW-PQ indexes. Sync
Python delegates to the async paths, so manifest operations and
`optimize()` consistently raise `NotImplementedError`. Legacy and
config-based HNSW-PQ creation share the same error and alternative-index
advice.

Remove the false replacement warning and simplify the ignored
GPU-accelerator warning. Keep the deprecated cleanup/compaction no-ops,
with backend-neutral warnings that do not assume automatic server
maintenance.

Embedding configuration forwarding and stale warning removal are now
supplied by #4408 in `main`. Retain the regression that verifies full
embedding metadata and warning-free remote creation alongside the
diagnostics tests.

Validation against the merged `main` revision `bbc4aebe` and Lance
14.0.0-beta.1:

- Rebuilt the Python extension with `uv run --extra tests --extra dev
maturin develop --extras tests,dev`; all 122 remote Python tests passed.
Root Ruff checks and formatting passed using the uv-managed environment.
- `pnpm build`, `pnpm lint`, formatting, and `pnpm run docs` passed; all
47 remote Node tests and the focused local vector-index regression
passed.
- All 583 Rust remote tests and 54 query unit tests passed, along with
`cargo check --quiet --features remote --tests --examples`, `cargo
clippy --quiet --features remote --tests --examples`, and `cargo fmt
--all`.

Fixes #4393

<!-- lance-gatekeeper-fix:v1 agent=10bafd337999f4910855c3a9b71e7dcf
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
lancedb-gatefixer[bot]
2026-10-08 18:42:42 -07:00
committed by GitHub
parent 74f473012a
commit 1e4662ac23
8 changed files with 351 additions and 66 deletions
+53
View File
@@ -28,6 +28,7 @@ import {
StaticHeaderProvider,
} from "../lancedb/header";
import { Index } from "../lancedb/indices";
import { LocalTable } from "../lancedb/table";
// Test-only header providers
class CustomProvider extends HeaderProvider {
@@ -93,6 +94,58 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("reports unsupported operations with backend-neutral errors", async () => {
await withMockDatabase(
(req, res) => {
expect(req.url).toBe("/v1/table/test/describe/");
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
version: 1,
schema: {
fields: [
{ name: "id", type: { type: "int64" }, nullable: false },
{
name: "vector",
type: {
type: "fixed_size_list",
fields: [
{ name: "item", type: { type: "float" }, nullable: true },
],
length: 2,
},
nullable: false,
},
],
},
}),
);
},
async (db) => {
const table = (await db.openTable("test")) as LocalTable;
const cases: [() => Promise<unknown>, string][] = [
[() => table.usesV2ManifestPaths(), "uses_v2_manifest_paths"],
[() => table.migrateManifestPathsV2(), "migrate_manifest_paths_v2"],
[
() => table.setUnenforcedPrimaryKey("id"),
"set_unenforced_primary_key",
],
[() => table.optimize(), "optimize"],
[
() => table.createIndex("vector", { config: Index.hnswPq() }),
"IVF_HNSW_PQ",
],
];
for (const [run, operation] of cases) {
let message = `LanceDBError: not supported: ${operation} is not supported for remote tables.`;
if (operation === "IVF_HNSW_PQ") {
message += " Please use IVF_HNSW_SQ instead.";
}
await expect(run()).rejects.toThrow(new Error(message));
}
},
);
});
it("rejects nprobes(0) before sending a query", async () => {
const requests: string[] = [];
await withMockDatabase(
+8 -2
View File
@@ -836,7 +836,10 @@ impl Table {
pub async fn uses_v2_manifest_paths(&self) -> napi::Result<bool> {
self.inner_ref()?
.as_native()
.ok_or_else(|| napi::Error::from_reason("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.default_error()?
.uses_v2_manifest_paths()
.await
.default_error()
@@ -846,7 +849,10 @@ impl Table {
pub async fn migrate_manifest_paths_v2(&self) -> napi::Result<()> {
self.inner_ref()?
.as_native()
.ok_or_else(|| napi::Error::from_reason("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.default_error()?
.migrate_manifest_paths_v2()
.await
.default_error()
+2 -2
View File
@@ -486,12 +486,12 @@ class RemoteDBConnection(DBConnection):
namespace_path = []
if storage_options is not None:
logging.info(
"storage_options is ignored in LanceDb Cloud"
"storage_options is ignored for remote tables"
" (storage is managed; set storage_options on connect() instead)"
)
if index_cache_size is not None:
logging.info(
"index_cache_size is ignored in LanceDb Cloud"
"index_cache_size is ignored for remote tables"
" (there is no local cache to configure)"
)
+28 -47
View File
@@ -42,6 +42,7 @@ from lancedb.index import (
BTree,
Bitmap,
HnswFlat,
HnswPq,
HnswSq,
IvfFlat,
IvfPq,
@@ -239,12 +240,12 @@ class RemoteTable(Table):
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
"""to_arrow() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_arrow() is not yet supported on LanceDB cloud.")
"""to_arrow() is not supported for remote tables."""
raise NotImplementedError("to_arrow() is not supported for remote tables.")
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not yet supported on LanceDB cloud."""
raise NotImplementedError("to_pandas() is not yet supported on LanceDB cloud.")
"""to_pandas() is not supported for remote tables."""
raise NotImplementedError("to_pandas() is not supported for remote tables.")
def checkout(self, version: Union[int, str]):
result = LOOP.run(self._table.checkout(version))
@@ -501,14 +502,8 @@ class RemoteTable(Table):
if accelerator is not None:
logging.warning(
"GPU accelerator is not yet supported on LanceDB cloud."
"If you have 100M+ vectors to index,"
"please contact us at contact@lancedb.com"
)
if replace is not None:
logging.warning(
"replace is not supported on LanceDB cloud."
"Existing indexes will always be replaced."
"GPU accelerator is not supported for remote tables "
"and will be ignored."
)
idx_type = index_type.upper()
@@ -528,9 +523,11 @@ class RemoteTable(Table):
elif idx_type == "IVF_SQ":
config = IvfSq(distance_type=metric, num_partitions=num_partitions)
elif idx_type == "IVF_HNSW_PQ":
raise ValueError(
"IVF_HNSW_PQ is not supported on LanceDB cloud."
"Please use IVF_HNSW_SQ instead."
config = HnswPq(
distance_type=metric,
num_partitions=num_partitions,
num_sub_vectors=num_sub_vectors,
num_bits=num_bits,
)
elif idx_type == "IVF_HNSW_SQ":
config = HnswSq(distance_type=metric, num_partitions=num_partitions)
@@ -659,7 +656,7 @@ class RemoteTable(Table):
capped at the number of CPU cores. Lower this if bulk ingestion is
using too much memory.
allow_external_blob_outside_bases: bool, default False
Not supported on LanceDB Cloud. Setting this raises.
Not supported for remote tables. Setting this raises.
Returns
-------
@@ -956,26 +953,20 @@ class RemoteTable(Table):
def cleanup_old_versions(self, *_):
"""
cleanup_old_versions() is a no-op on LanceDB Cloud.
Tables are automatically cleaned up and optimized.
cleanup_old_versions() is a no-op for remote tables.
"""
warnings.warn(
"cleanup_old_versions() is a no-op on LanceDB Cloud. "
"Tables are automatically cleaned up and optimized.",
"cleanup_old_versions() is a no-op for remote tables.",
stacklevel=2,
)
pass
def compact_files(self, *_):
"""
compact_files() is a no-op on LanceDB Cloud.
Tables are automatically compacted and optimized.
compact_files() is a no-op for remote tables.
"""
warnings.warn(
"compact_files() is a no-op on LanceDB Cloud. "
"Tables are automatically compacted and optimized.",
"compact_files() is a no-op for remote tables.",
stacklevel=2,
)
pass
@@ -986,17 +977,13 @@ class RemoteTable(Table):
cleanup_older_than: Optional[timedelta] = None,
delete_unverified: bool = False,
):
"""
optimize() is a no-op on LanceDB Cloud.
Indices are optimized automatically.
"""
warnings.warn(
"optimize() is a no-op on LanceDB Cloud. "
"Indices are optimized automatically.",
stacklevel=2,
"""optimize() is not supported for remote tables."""
return LOOP.run(
self._table.optimize(
cleanup_older_than=cleanup_older_than,
delete_unverified=delete_unverified,
)
)
pass
def count_rows(self, filter: Optional[str] = None) -> int:
return LOOP.run(self._table.count_rows(filter))
@@ -1044,7 +1031,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Not supported on LanceDB Cloud."""
"""Not supported for remote tables."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
@@ -1090,7 +1077,7 @@ class RemoteTable(Table):
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
"""No-op for remote tables (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
def drop_index(self, index_name: str):
@@ -1158,14 +1145,10 @@ class RemoteTable(Table):
return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
def uses_v2_manifest_paths(self) -> bool:
raise NotImplementedError(
"uses_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.uses_v2_manifest_paths())
def migrate_v2_manifest_paths(self):
raise NotImplementedError(
"migrate_v2_manifest_paths() is not supported on the LanceDB Cloud"
)
return LOOP.run(self._table.migrate_manifest_paths_v2())
def blob_columns(self) -> list[str]:
return LOOP.run(self._table.blob_columns())
@@ -1176,9 +1159,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(self, column: str, requests) -> pa.LargeBinaryArray:
raise NotImplementedError(
"fetch_blob_ranges() is not supported on LanceDB Cloud"
)
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
+167 -4
View File
@@ -1272,7 +1272,7 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
def test_remote_create_index_new_api():
def test_remote_create_index_new_api(caplog):
received_requests = []
def handler(request):
@@ -1333,7 +1333,8 @@ def test_remote_create_index_new_api():
from lancedb.index import BTree, FTS, IvfPq, IvfRq
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
table = db.create_table("test", [{"id": 1}], embedding_functions=[])
assert "embedding_functions is not yet supported" not in caplog.text
# New API: column-first, config= kwarg. Should NOT emit DeprecationWarning.
import warnings as _warnings
@@ -1355,6 +1356,8 @@ def test_remote_create_index_new_api():
vector_column_name="vector",
index_type="IVF_RQ",
num_partitions=8,
replace=False,
accelerator="cuda",
)
assert len(received_requests) == 6
@@ -1368,6 +1371,160 @@ def test_remote_create_index_new_api():
]
assert received_requests[2]["block_size"] == 256
assert received_requests[4]["replace"] is False
assert received_requests[5]["replace"] is False
assert "replace is not supported" not in caplog.text
assert caplog.messages == [
"GPU accelerator is not supported for remote tables and will be ignored."
]
def remote_unsupported_handler(request):
assert request.path == "/v1/table/test/describe/"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
dict(
version=1,
schema=dict(
fields=[
dict(name="id", type={"type": "int64"}, nullable=False),
dict(
name="vector",
type={
"type": "fixed_size_list",
"fields": [
dict(
name="item",
type={"type": "float"},
nullable=True,
)
],
"length": 2,
},
nullable=False,
),
]
),
)
).encode()
)
def remote_unsupported_operations():
from lancedb.index import HnswPq
return [
("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"),
("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"),
("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"),
("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"),
("optimize", (), {}, "optimize"),
("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"),
]
@pytest.mark.parametrize(
"method,args,kwargs,operation", remote_unsupported_operations()
)
def test_remote_unsupported_operations_sync(method, args, kwargs, operation):
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.raises(NotImplementedError) as error:
getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method,args,kwargs,operation", remote_unsupported_operations()
)
async def test_remote_unsupported_operations_async(method, args, kwargs, operation):
async with mock_lancedb_connection_async(remote_unsupported_handler) as db:
table = await db.open_table("test")
if method == "migrate_v2_manifest_paths":
method = "migrate_manifest_paths_v2"
with pytest.raises(NotImplementedError) as error:
await getattr(table, method)(*args, **kwargs)
expected = f"LanceDBError: not supported: {operation} is not supported "
expected += "for remote tables."
if operation == "IVF_HNSW_PQ":
expected += " Please use IVF_HNSW_SQ instead."
assert str(error.value) == expected
def test_remote_unsupported_legacy_hnsw_pq():
with mock_lancedb_connection(remote_unsupported_handler) as db:
table = db.open_table("test")
with pytest.warns(DeprecationWarning, match="create_index"):
with pytest.raises(
NotImplementedError,
match="IVF_HNSW_PQ is not supported for remote tables. "
"Please use IVF_HNSW_SQ instead.",
):
table.create_index(
vector_column_name="vector", index_type="IVF_HNSW_PQ"
)
def test_remote_python_only_messages(caplog):
with mock_lancedb_connection(remote_unsupported_handler) as db:
with caplog.at_level("INFO"):
table = db.open_table("test", storage_options={}, index_cache_size=1)
assert "storage_options is ignored for remote tables" in caplog.text
assert "index_cache_size is ignored for remote tables" in caplog.text
for method in ["to_arrow", "to_pandas"]:
with pytest.raises(NotImplementedError) as error:
getattr(table, method)()
assert str(error.value) == f"{method}() is not supported for remote tables."
for method in ["compact_files", "cleanup_old_versions"]:
with pytest.warns(UserWarning) as warning:
getattr(table, method)()
assert (
str(warning[0].message) == f"{method}() is a no-op for remote tables."
)
def test_remote_embedding_config_does_not_warn_or_drop_metadata(caplog):
received_tables = []
def handler(request):
assert request.path == "/v1/table/test/create/?mode=create"
if request.headers.get("Transfer-Encoding") == "chunked":
body = bytearray()
while size := int(request.rfile.readline().split(b";")[0], 16):
body.extend(request.rfile.read(size))
assert request.rfile.read(2) == b"\r\n"
assert request.rfile.read(2) == b"\r\n"
else:
body = request.rfile.read(int(request.headers["Content-Length"]))
received_tables.append(pa.ipc.open_stream(body).read_all())
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
config = EmbeddingFunctionConfig(
source_column="text",
vector_column="vector",
function=MockNonNormTextEmbeddingFunction.create(),
)
schema = pa.schema(
[pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 10))]
)
with mock_lancedb_connection(handler) as db:
db.create_table("test", schema=schema, embedding_functions=[config])
assert caplog.messages == []
assert len(received_tables) == 1
metadata = EmbeddingFunctionRegistry.get_instance().get_table_metadata([config])
assert (
received_tables[0].schema.metadata == pa.schema([], metadata=metadata).metadata
)
def test_table_wait_for_index_timeout():
@@ -2995,9 +3152,15 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
def test_remote_blob_byte_apis_not_supported_on_old_server():
with blob_remote_table(server_version=Version("0.1.0")) as table:
assert table.blob_columns() == ["image"]
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blobs is not supported by this LanceDB server",
):
table.fetch_blobs("image", [1])
with pytest.raises(NotImplementedError, match="not supported"):
with pytest.raises(
NotImplementedError,
match="fetch_blob_files requires LanceDB server 0.5.0 or newer",
):
table.fetch_blob_files("image", [1])
+8 -2
View File
@@ -1708,7 +1708,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported for remote tables.".into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
@@ -1720,7 +1723,10 @@ impl Table {
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| PyValueError::new_err("This cannot be run on a remote table"))?
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported for remote tables.".into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()
+83 -7
View File
@@ -620,9 +620,9 @@ impl<S: HttpSend> RemoteTable<S> {
});
}
}
_ => {
Index::IvfHnswPq(_) => {
return Err(Error::NotSupported {
message: "Index type not supported".into(),
message: "IVF_HNSW_PQ is not supported for remote tables. Please use IVF_HNSW_SQ instead.".into(),
});
}
};
@@ -1085,7 +1085,7 @@ impl<S: HttpSend> RemoteTable<S> {
if let Some(full_text_search) = &params.full_text_search {
if full_text_search.wand_factor.is_some() {
return Err(Error::NotSupported {
message: "Wand factor is not yet supported in LanceDB Cloud".into(),
message: "Wand factor is not supported for remote tables.".into(),
});
}
@@ -2825,6 +2825,16 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
self.fetch_blobs_impl(column, row_ids).await
}
async fn fetch_blob_ranges(
&self,
_column: &str,
_requests: &[crate::blob::BlobRangeRequest],
) -> Result<LargeBinaryArray> {
Err(Error::NotSupported {
message: "fetch_blob_ranges is not supported for remote tables.".into(),
})
}
async fn fetch_blob_files(
&self,
column: &str,
@@ -3139,7 +3149,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn set_unenforced_primary_key(&self, _columns: &[&str]) -> Result<()> {
Err(Error::NotSupported {
message: "set_unenforced_primary_key is not supported on LanceDB cloud.".into(),
message: "set_unenforced_primary_key is not supported for remote tables.".into(),
})
}
@@ -3331,7 +3341,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn optimize(&self, _action: OptimizeAction) -> Result<OptimizeStats> {
self.check_mutable().await?;
Err(Error::NotSupported {
message: "optimize is not supported on LanceDB cloud.".into(),
message: "optimize is not supported for remote tables.".into(),
})
}
async fn add_columns(
@@ -5748,6 +5758,72 @@ mod tests {
}
}
#[tokio::test]
async fn test_remote_unsupported_operations() {
let table = Table::new_with_handler("my_table", |_| -> http::Response<String> {
panic!("unsupported operations must not send requests to the server");
});
let mut fts = FullTextSearchQuery::new("test".into());
fts.wand_factor = Some(1.0);
let results = [
(
"set_unenforced_primary_key",
table.set_unenforced_primary_key(["id"]).await,
),
(
"optimize",
table.optimize(OptimizeAction::All).await.map(|_| ()),
),
(
"fetch_blob_ranges",
table.fetch_blob_ranges("image", []).await.map(|_| ()),
),
(
"Wand factor",
table
.query()
.full_text_search(fts)
.execute()
.await
.map(|_| ()),
),
];
for (operation, result) in results {
match result.unwrap_err() {
Error::NotSupported { message } => assert_eq!(
message,
format!("{operation} is not supported for remote tables.")
),
error => panic!("expected not-supported error, got {error}"),
}
}
}
#[tokio::test]
async fn test_remote_unsupported_hnsw_pq() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
let schema = Schema::new(vec![Field::new(
"vector",
DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 2),
false,
)]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
});
let error = table
.create_index(&["vector"], Index::IvfHnswPq(Default::default()))
.execute()
.await
.unwrap_err();
assert_not_supported_error(
error,
"IVF_HNSW_PQ is not supported for remote tables. Please use IVF_HNSW_SQ instead.",
);
}
#[tokio::test]
async fn test_fetch_blobs_rejects_missing_column() {
let batch = RecordBatch::try_new(
@@ -5861,12 +5937,12 @@ mod tests {
assert_not_supported_error(
table.fetch_blobs("image", &[1]).await.unwrap_err(),
"fetch_blobs",
"fetch_blobs is not supported by this LanceDB server.",
);
assert_not_supported_error(
table.fetch_blob_files("image", &[1]).await.unwrap_err(),
"requires LanceDB Cloud server 0.5.0 or newer",
"requires LanceDB server 0.5.0 or newer",
);
}
+2 -2
View File
@@ -393,7 +393,7 @@ impl<S: HttpSend> RemoteTable<S> {
}
if !self.server_version.support_blobs() {
return Err(Error::NotSupported {
message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
message: "fetch_blobs is not supported by this LanceDB server.".into(),
});
}
let mut read_snapshot = self.snapshot_read_state().await;
@@ -716,7 +716,7 @@ impl<S: HttpSend> RemoteTable<S> {
Ok(())
} else {
Err(Error::NotSupported {
message: "fetch_blob_files requires LanceDB Cloud server 0.5.0 or newer".into(),
message: "fetch_blob_files requires LanceDB server 0.5.0 or newer.".into(),
})
}
}