rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-15 16:22:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into gatekeeper/fix-4392-1

Browse Source 
Preserve upstream blob-mode validation alongside explicit remote conversion diagnostics. Refresh regression coverage for server-backed blob queries and the upstream closed-file error contract.
This commit is contained in:
Gatefixer
2026-10-14 18:42:55 +00:00
parent 1ca717948a eabec507a8
commit cea4c2a759
125 changed files with 11636 additions and 1399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.41.0-beta.1"
version = "0.41.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/pyproject.toml
+3 -2
View File
@@ -10,7 +10,7 @@ dependencies = [
"pyarrow>=16",
"pydantic>=2.7.4,<3",
"tqdm>=4.27.0",
"lance-namespace>=0.3.2"
"lance-namespace>=0.13.0"
]
description = "lancedb"
authors = [{ name = "LanceDB Devs", email = "dev@lancedb.com" }]
@@ -63,7 +63,7 @@ tests = [
"polars>=0.19, <=1.32.3",
"pyarrow<25",
"pyarrow-stubs>=16.0",
"pylance==14.0.0b6",
"pylance==14.0.0b12",
"requests>=2.31.0",
"datafusion>=54,<55",
"opentelemetry-sdk>=1.30.0",
@@ -73,6 +73,7 @@ dev = [
"pre-commit>=3.5.0",
"pyright>=1.1.350",
'typing-extensions>=4.0.0; python_version < "3.11"',
"maturin>=1.10",
]
docs = ["mkdocs", "mkdocs-jupyter", "mkdocs-material", "mkdocstrings-python"]
clip = ["torch", "pillow>=12.1.1", "open-clip-torch"]
python/python/lancedb/_blob.py
+250 -53
View File
@@ -6,13 +6,14 @@
from __future__ import annotations
import io
import operator
from collections.abc import Awaitable, Callable, Iterable
from typing import TYPE_CHECKING, Optional, Union
import pyarrow as pa
from .expr import Expr
from .schema import row_addressable_blob_v2_paths
from .schema import blob_column_paths, row_addressable_blob_v2_paths
from .types import BlobMode, QueryProjection, QueryProjectionSpec
if TYPE_CHECKING:
@@ -26,21 +27,24 @@ BLOB_MODE_TO_HANDLING = {
ROW_ID_FIELD_NAME = "_lance_row_id"
FetchBlobsSync = Callable[[str, pa.Table], pa.Array | pa.ChunkedArray]
FetchBlobsAsync = Callable[[str, pa.Table], Awaitable[pa.Array | pa.ChunkedArray]]
FetchBlobsSync = Callable[[str, list[int]], pa.Array | pa.ChunkedArray]
FetchBlobsAsync = Callable[[str, list[int]], Awaitable[pa.Array | pa.ChunkedArray]]
class BlobFile(io.RawIOBase):
"""Seekable lazy handle from :meth:`~lancedb.table.Table.fetch_blob_files`.
Bytes load on ``read`` or ``read_range``, not when the handle is opened.
Use :meth:`aread` from async code.
Bytes load on ``read``, ``read1``, ``read_range``, or ``read_ranges``, not
when the handle is opened. Range reads leave the cursor unchanged. Use
:meth:`aread` from async code.
"""
def __init__(self, inner) -> None:
self._inner = inner
async def aread(self) -> bytes:
if self.closed:
raise ValueError("read of closed file")
return await self._inner.read()
def close(self) -> None:
@@ -58,13 +62,16 @@ class BlobFile(io.RawIOBase):
def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
if whence == io.SEEK_SET:
self._inner.seek(offset)
position = offset
elif whence == io.SEEK_CUR:
self._inner.seek(self._inner.tell() + offset)
position = self._inner.tell() + offset
elif whence == io.SEEK_END:
self._inner.seek(self._inner.size() + offset)
position = self._inner.size() + offset
else:
raise ValueError(f"invalid whence: {whence}")
if position < 0:
raise ValueError(f"negative seek value {position}")
self._inner.seek(position)
return self._inner.tell()
def tell(self) -> int:
@@ -74,17 +81,39 @@ class BlobFile(io.RawIOBase):
return self._inner.size()
def readall(self) -> bytes:
if self.closed:
raise ValueError("read of closed file")
return self._inner.read_bytes()
def read(self, size: int = -1) -> bytes:
if size == -1:
def read(self, size: Optional[int] = -1) -> bytes:
if self.closed:
raise ValueError("read of closed file")
if size is None or size < 0:
return self._inner.read_bytes()
return super().read(size)
def read1(self, size: Optional[int] = -1) -> bytes:
"""Read up to ``size`` bytes with one underlying read."""
if self.closed:
raise ValueError("read of closed file")
if size is None or size < 0:
size = io.DEFAULT_BUFFER_SIZE
return self._inner.read_up_to(size)
def read_range(self, offset: int, length: int) -> bytes:
if self.closed:
raise ValueError("read of closed file")
return self._inner.read_range(offset, length)
def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]:
"""Read ``(offset, length)`` pairs without moving the cursor."""
if self.closed:
raise ValueError("read of closed file")
return self._inner.read_ranges(ranges)
def readinto(self, b: WriteableBuffer) -> int:
if self.closed:
raise ValueError("readinto of closed file")
view = memoryview(b).cast("B")
chunk = self._inner.read_up_to(len(view))
view[: len(chunk)] = chunk
@@ -107,10 +136,12 @@ def projection_includes_blob_column(
columns = set(blob_columns)
if not columns:
return False
if projection is None:
if not projection:
return True
for output, source in _iter_projection_pairs(projection):
if output in columns or source in columns:
if output in columns or any(
column == source or column.startswith(f"{source}.") for column in columns
):
return True
return False
@@ -119,17 +150,69 @@ def blob_v2_projection_sources(
schema: pa.Schema,
projection: QueryProjection,
) -> dict[str, str]:
blob_columns = row_addressable_blob_v2_paths(schema)
if not blob_columns:
return {}
columns = set(blob_columns)
if projection is None:
return _project_blob_paths(row_addressable_blob_v2_paths(schema), projection)
def reject_unfetchable_blob_columns(
schema: pa.Schema, projection: QueryProjection, blob_sources: dict[str, str]
) -> None:
# Rerankers can drop field metadata, so validate against the source schema.
unfetchable = [
output
for output in _project_blob_paths(blob_column_paths(schema), projection)
if output not in blob_sources
]
if unfetchable:
raise ValueError(
f"blob_mode='bytes' cannot fetch blob columns {unfetchable}. Bytes are "
"fetched by row id, which covers blob v2 columns with one blob per "
"row but not legacy blob columns or blobs inside lists. Use "
"blob_mode='descriptions' or remove these columns from the projection."
)
def reject_unmatched_blob_outputs(
output_schema: pa.Schema, blob_sources: dict[str, str]
) -> None:
# Another spelling of a blob column, such as a quoted or recased name, is a
# blob in the query's output schema but maps to no source by name.
unmatched = [
path for path in blob_column_paths(output_schema) if path not in blob_sources
]
if unmatched:
raise ValueError(
f"blob_mode='bytes' could not match result columns {unmatched} to a blob "
"column of the table. Select blob columns by their plain names, or "
"use blob_mode='descriptions'."
)
def _project_blob_paths(
blob_columns: list[str], projection: QueryProjection
) -> dict[str, str]:
"""Map each projected output path to the blob column path it reads."""
# An empty projection selects all columns.
if not projection:
return {column: column for column in blob_columns}
return {
output: source
for output, source in _iter_projection_pairs(projection)
if source in columns
}
sources: dict[str, str] = {}
entries: dict[str, str] = {}
for output, source in _iter_projection_pairs(projection):
for column in blob_columns:
if column == source:
path = output
elif column.startswith(f"{source}."):
path = f"{output}{column[len(source) :]}"
else:
continue
if path in sources:
raise ValueError(
f"projection entries {entries[path]!r} and {output!r} both "
f"produce blob column '{path}'. Rename one of them or drop "
"the redundant selection."
)
sources[path] = column
entries[path] = output
return sources
def v2_projection_needs_row_id(
@@ -179,11 +262,14 @@ async def replace_v2_blob_columns_with_bytes(
blob_sources: dict[str, str],
fetch_blobs: FetchBlobsAsync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
continue
blobs = await fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
present = _present_blob_sources(tbl, blob_sources)
if not present:
return tbl
row_ids = _hit_row_ids(tbl, present)
for output_name, source_name in present.items():
tbl = _set_blob_column(
tbl, output_name, await fetch_blobs(source_name, row_ids)
)
return tbl
@@ -192,19 +278,48 @@ def replace_v2_blob_columns_with_bytes_sync(
blob_sources: dict[str, str],
fetch_blobs: FetchBlobsSync,
) -> pa.Table:
for output_name, source_name in blob_sources.items():
if output_name not in tbl.column_names:
continue
blobs = fetch_blobs(source_name, tbl)
tbl = _set_blob_column(tbl, output_name, blobs)
present = _present_blob_sources(tbl, blob_sources)
if not present:
return tbl
row_ids = _hit_row_ids(tbl, present)
for output_name, source_name in present.items():
tbl = _set_blob_column(tbl, output_name, fetch_blobs(source_name, row_ids))
return tbl
def _present_blob_sources(
tbl: pa.Table, blob_sources: dict[str, str]
) -> dict[str, str]:
return {
output: source
for output, source in blob_sources.items()
if _has_column_path(tbl.schema, output)
}
def _hit_row_ids(tbl: pa.Table, blob_sources: dict[str, str]) -> list[int]:
"""Read row ids before replacing columns, since a swapped alias can overwrite
the lookup source."""
if "_rowid" in tbl.column_names:
return tbl["_rowid"].to_pylist()
for output in blob_sources:
descriptor = _leaf_struct_column(tbl, output)
if pa.types.is_struct(descriptor.type) and (
ROW_ID_FIELD_NAME in descriptor.type.names
):
return descriptor.field(ROW_ID_FIELD_NAME).to_pylist()
raise ValueError(
f"query result has no '_rowid' column and no '{ROW_ID_FIELD_NAME}' "
f"field on blob columns {list(blob_sources)}. Call .with_row_id(True) "
"to fetch blob bytes for this query."
)
def stash_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table:
if "_rowid" not in tbl.column_names:
raise ValueError("query result has no '_rowid' column to hide")
present_paths = [p for p in blob_paths if p.split(".")[0] in tbl.column_names]
present_paths = [p for p in blob_paths if _has_column_path(tbl.schema, p)]
if not present_paths:
raise ValueError("query result has no blob v2 column to carry a row id")
@@ -224,7 +339,7 @@ def read_row_ids_from_hits(hits: pa.Table, blob_column: str) -> list[int]:
try:
leaf = _leaf_struct_column(hits, blob_column)
if ROW_ID_FIELD_NAME in leaf.type.names:
if pa.types.is_struct(leaf.type) and ROW_ID_FIELD_NAME in leaf.type.names:
return leaf.field(ROW_ID_FIELD_NAME).to_pylist()
except KeyError:
pass
@@ -287,9 +402,37 @@ def _iter_projection_pairs(
yield name, source if source is not None else expr.to_sql()
def _set_blob_column(tbl: pa.Table, output_name: str, blobs: pa.Array) -> pa.Table:
index = tbl.schema.get_field_index(output_name)
return tbl.set_column(index, pa.field(output_name, blobs.type), [blobs])
def _has_column_path(schema: pa.Schema, path: str) -> bool:
top_name, rest = _split_column_path(schema, path)
index = schema.get_field_index(top_name)
return (
index >= 0 and _walk_struct_fields(schema.field(index).type, rest) is not None
)
def _set_blob_column(
tbl: pa.Table, output_name: str, blobs: pa.Array | pa.ChunkedArray
) -> pa.Table:
top_name, rest = _split_column_path(tbl.schema, output_name)
if not rest:
index = tbl.schema.get_field_index(top_name)
return tbl.set_column(index, pa.field(top_name, blobs.type), [blobs])
leaf_name = rest[-1]
if isinstance(blobs, pa.ChunkedArray):
blobs = blobs.combine_chunks()
def replace_blob(children: list, child_fields: list) -> None:
for i, field in enumerate(child_fields):
if field.name == leaf_name:
children[i] = blobs
child_fields[i] = pa.field(
leaf_name, blobs.type, nullable=field.nullable
)
return
raise KeyError(f"Blob field '{leaf_name}' not found in '{output_name}'")
return _transform_struct_column(tbl, top_name, rest[:-1], replace_blob)
def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.Table:
@@ -297,7 +440,9 @@ def _embed_row_id_in_column(tbl: pa.Table, path: str, row_ids: pa.Array) -> pa.T
children.append(row_ids)
child_fields.append(pa.field(ROW_ID_FIELD_NAME, pa.uint64(), nullable=False))
return _transform_struct_column(tbl, path, add_row_id)
return _transform_struct_column(
tbl, *_split_column_path(tbl.schema, path), add_row_id
)
def strip_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table:
@@ -314,16 +459,20 @@ def strip_auto_row_ids(tbl: pa.Table, blob_paths: Iterable[str]) -> pa.Table:
return
for path in blob_paths:
if path.split(".")[0] not in tbl.column_names:
if not _has_column_path(tbl.schema, path):
continue
tbl = _transform_struct_column(tbl, path, drop_row_id)
tbl = _transform_struct_column(
tbl, *_split_column_path(tbl.schema, path), drop_row_id
)
return tbl
def _transform_struct_column(
tbl: pa.Table, path: str, leaf_transform: Callable[[list, list], None]
tbl: pa.Table,
top_name: str,
rest: list[str],
leaf_transform: Callable[[list, list], None],
) -> pa.Table:
top_name, *rest = path.split(".")
top_index = tbl.schema.get_field_index(top_name)
top_field = tbl.schema.field(top_index)
top_array = tbl.column(top_name)
@@ -381,26 +530,74 @@ def _rebuild_struct(
def _leaf_struct_column(tbl: pa.Table, path: str) -> pa.StructArray:
parts = path.split(".")
column = tbl.column(parts[0])
top_name, rest = _split_column_path(tbl.schema, path)
column = tbl.column(top_name)
if isinstance(column, pa.ChunkedArray):
column = column.combine_chunks()
for part in parts[1:]:
for part in rest:
if not pa.types.is_struct(column.type):
raise KeyError(f"blob path '{path}' crosses non-struct column")
column = column.field(part)
return column
def _split_column_path(schema: pa.Schema, path: str) -> tuple[str, list[str]]:
"""Prefer a path ending in a struct, since a flat alias can shadow a nested
blob path."""
parts = path.split(".")
fallback = None
for cut in range(len(parts), 0, -1):
top_name, rest = ".".join(parts[:cut]), parts[cut:]
index = schema.get_field_index(top_name)
if index < 0:
continue
leaf_type = _walk_struct_fields(schema.field(index).type, rest)
if leaf_type is None:
continue
if pa.types.is_struct(leaf_type):
return top_name, rest
fallback = fallback or (top_name, rest)
return fallback or (parts[0], parts[1:])
def _walk_struct_fields(
data_type: pa.DataType, names: list[str]
) -> Optional[pa.DataType]:
for name in names:
if not pa.types.is_struct(data_type):
return None
index = data_type.get_field_index(name)
if index < 0:
return None
data_type = data_type.field(index).type
return data_type
def _normalize_blob_row_ids(
row_ids: Union[list[int], pa.Table], blob_column: str
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table], blob_column: str
) -> list[int]:
if isinstance(row_ids, pa.Table):
return read_row_ids_from_hits(row_ids, blob_column)
if isinstance(row_ids, (pa.Array, pa.ChunkedArray)):
raise ValueError(
"pass a query table with _rowid, not a column array "
"(use fetch_blobs('image', hits), not fetch_blobs('image', hits['image']))"
)
return list(row_ids)
values = read_row_ids_from_hits(row_ids, blob_column)
elif isinstance(row_ids, (pa.Array, pa.ChunkedArray)):
if not pa.types.is_integer(row_ids.type):
raise ValueError(
"row id arrays must have an integer type; pass a query table with "
"_rowid when using a blob descriptor column"
)
values = row_ids.to_pylist()
else:
values = list(row_ids)
normalized = []
for value in values:
try:
row_id = operator.index(value)
except TypeError as exc:
raise ValueError("row ids must be integers without nulls") from exc
if row_id < 0:
raise ValueError("row ids must be non-negative")
normalized.append(row_id)
return normalized
def _wrap_blob_files(handles: Iterable[object]) -> list[Optional[BlobFile]]:
python/python/lancedb/_lancedb.pyi
+66 -2
View File
@@ -1,6 +1,6 @@
from datetime import date, datetime, timedelta
from decimal import Decimal
from typing import Dict, List, Optional, Tuple, Any, TypedDict, Union, Literal
from typing import Dict, List, Optional, Sequence, Tuple, Any, TypedDict, Union, Literal
from uuid import UUID
import pyarrow as pa
@@ -311,6 +311,7 @@ class BlobFile:
def tell(self) -> int: ...
def size(self) -> int: ...
def read_range(self, offset: int, length: int) -> bytes: ...
def read_ranges(self, ranges: list[tuple[int, int]]) -> list[bytes]: ...
def read_up_to(self, length: int) -> bytes: ...
class Job:
@@ -563,7 +564,7 @@ class Table:
async def fetch_blob_ranges(
self,
column: str,
requests: List[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray: ...
async def fetch_blob_files(
self, column: str, row_ids: list[int]
@@ -727,6 +728,7 @@ class VectorQuery:
def use_lsm(self, enable: bool): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -752,6 +754,7 @@ class HybridQuery:
def distance_type(self, distance_type: str): ...
def refine_factor(self, refine_factor: int): ...
def nprobes(self, nprobes: int): ...
def search_effort(self, search_effort: Optional[float]): ...
def minimum_nprobes(self, minimum_nprobes: int): ...
def maximum_nprobes(self, maximum_nprobes: int): ...
def bypass_vector_index(self): ...
@@ -778,8 +781,10 @@ class PyQueryRequest:
use_lsm: Optional[bool]
column: Optional[str]
query_vector: Optional[List[pa.Array]]
nprobes: Optional[int]
minimum_nprobes: Optional[int]
maximum_nprobes: Optional[int]
search_effort: Optional[float]
lower_bound: Optional[float]
upper_bound: Optional[float]
ef: Optional[int]
@@ -951,6 +956,8 @@ class PermutationReader:
def __init__(self, base_table: Table, permutation_table: Table): ...
class Catalog:
@property
def authz(self) -> AuthorizationClient: ...
@property
def uri(self) -> str: ...
async def create_database(
@@ -997,3 +1004,60 @@ class JobListing:
def page_token(self) -> Optional[str]: ...
def __aiter__(self) -> JobListing: ...
async def __anext__(self) -> JobInfo: ...
class AuthorizationClient:
async def list_acls(self, request: str) -> str: ...
async def add_acl(self, request: str) -> None: ...
async def delete_acl(self, request: str) -> None: ...
async def list_role_bindings(self, request: str) -> str: ...
async def add_role_binding(self, request: str) -> None: ...
async def delete_role_binding(self, request: str) -> None: ...
async def get_principal(self, request: str) -> str: ...
async def get_group(self, request: str) -> str: ...
class AuthzSubject:
def __init__(self, kind: str, value: str): ...
@staticmethod
def from_canonical(value: str) -> AuthzSubject: ...
@property
def kind(self) -> str: ...
@property
def value(self) -> str: ...
@property
def display_value(self) -> str: ...
def to_wire(self, context: Optional[str] = None) -> str: ...
def __str__(self) -> str: ...
def __repr__(self) -> str: ...
class AuthzPrivilege:
def __init__(self, value: str): ...
@staticmethod
def known() -> list[str]: ...
def __str__(self) -> str: ...
class AuthzObject:
def __init__(self, value: str): ...
@staticmethod
def system() -> AuthzObject: ...
@staticmethod
def database(name: str) -> AuthzObject: ...
@staticmethod
def namespace(database: str, namespace_path: Sequence[str]) -> AuthzObject: ...
@staticmethod
def table(
database: str, namespace_path: Sequence[str], name: str
) -> AuthzObject: ...
@staticmethod
def view(
database: str, namespace_path: Sequence[str], name: str
) -> AuthzObject: ...
@staticmethod
def secret(
database: str, namespace_path: Sequence[str], name: str
) -> AuthzObject: ...
@staticmethod
def function(
database: str, namespace_path: Sequence[str], name: str
) -> AuthzObject: ...
def __str__(self) -> str: ...
def __repr__(self) -> str: ...
python/python/lancedb/authz.py
+588
View File
@@ -0,0 +1,588 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Deployment-wide authorization through ``catalog.authz``.
ACLs grant one privilege on an object to a principal, group, or role. Role
bindings grant a role to a principal or group. Identity-provider group membership
is read-only. Name resolution and permission checks are performed by the server.
"""
from collections.abc import AsyncIterator, Iterator, Sequence
from dataclasses import dataclass, field
from enum import Enum
import json
from typing import Literal, Optional, TypeVar, Union, cast
from . import _lancedb
from .background_loop import LOOP
__all__ = [
"Authorization",
"AsyncAuthorization",
"Subject",
"Object",
"Privilege",
"AccessControlEntry",
"RoleBinding",
"IdentityInfo",
"PrincipalInfo",
"GroupInfo",
]
_PAGE_SIZE = 100
_T = TypeVar("_T")
_SubjectKind = Literal[
"principal_id",
"principal_name",
"group_id",
"group_name",
"role",
"principal_api_key",
"unknown",
]
@dataclass(frozen=True)
class Subject:
"""An immutable identity selector. Prefer the named factory methods.
IDs are stable; names must resolve unambiguously on the server. API-key
selectors identify the subject of an operation, not its authenticated caller.
Secret values are omitted from representations. Unknown canonical subject
kinds returned by newer servers use ``kind="unknown"`` and preserve the full
wire string in ``value``; they can be passed back to ACL methods.
"""
kind: _SubjectKind
_value: str = field(repr=False)
_inner: _lancedb.AuthzSubject = field(init=False, repr=False, compare=False)
def __post_init__(self):
object.__setattr__(
self, "_inner", _lancedb.AuthzSubject(self.kind, self._value)
)
@classmethod
def principal_id(cls, id: str) -> "Subject":
"""Select a principal by canonical identity-provider ID."""
return cls("principal_id", id)
@classmethod
def principal_name(cls, name: str) -> "Subject":
"""Select a principal by an unambiguous provider name."""
return cls("principal_name", name)
@classmethod
def group_id(cls, id: str) -> "Subject":
"""Select a group by canonical identity-provider ID."""
return cls("group_id", id)
@classmethod
def group_name(cls, name: str) -> "Subject":
"""Select a group by an unambiguous provider name."""
return cls("group_name", name)
@classmethod
def role(cls, name: str) -> "Subject":
"""Select a LanceDB-managed role, without an ``r:`` prefix."""
return cls("role", name)
@classmethod
def principal_api_key(cls, api_key: str) -> "Subject":
"""Select the principal holding this secret API key."""
return cls("principal_api_key", api_key)
def __repr__(self) -> str:
value = self._inner.display_value
return f"Subject.{self.kind}({value!r})"
@property
def value(self) -> str:
"""The selected value, or full wire string for unknown subject kinds.
API-key values are secrets.
"""
return self._value
def __reduce__(self):
return type(self), (self.kind, self._value)
def _wire(self) -> str:
return self._inner.to_wire()
@classmethod
def _canonical(cls, value: str) -> "Subject":
inner = _lancedb.AuthzSubject.from_canonical(value)
return cls(cast(_SubjectKind, inner.kind), inner.value)
@dataclass(frozen=True)
class Object:
"""An immutable resource name in the authorization syntax.
Use the builders to avoid interpreting separators as part of a name.
Methods also accept raw object strings, validated by the same Rust parser.
Unknown resource types are preserved for compatibility with newer servers.
Omitted or empty namespace paths select the database's public namespace.
"""
_value: str
_inner: _lancedb.AuthzObject = field(init=False, repr=False, compare=False)
def __post_init__(self):
object.__setattr__(self, "_inner", _lancedb.AuthzObject(self._value))
@classmethod
def system(cls) -> "Object":
"""The deployment-wide system object."""
return cls(str(_lancedb.AuthzObject.system()))
@classmethod
def database(cls, name: str) -> "Object":
"""A database; slash-containing names remain one logical database."""
return cls(str(_lancedb.AuthzObject.database(name)))
@classmethod
def namespace(
cls, *, database: str, namespace_path: Sequence[str] = ("public",)
) -> "Object":
"""A namespace path, defaulting to the database's public namespace."""
return cls(str(_lancedb.AuthzObject.namespace(database, namespace_path)))
@classmethod
def table(
cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",)
) -> "Object":
"""A table, defaulting to the database's public namespace."""
return cls(str(_lancedb.AuthzObject.table(database, namespace_path, name)))
@classmethod
def view(
cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",)
) -> "Object":
"""A view, defaulting to the database's public namespace."""
return cls(str(_lancedb.AuthzObject.view(database, namespace_path, name)))
@classmethod
def secret(
cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",)
) -> "Object":
"""A secret, defaulting to the database's public namespace."""
return cls(str(_lancedb.AuthzObject.secret(database, namespace_path, name)))
@classmethod
def function(
cls, *, database: str, name: str, namespace_path: Sequence[str] = ("public",)
) -> "Object":
"""A function, defaulting to the database's public namespace."""
return cls(str(_lancedb.AuthzObject.function(database, namespace_path, name)))
def __str__(self) -> str:
return str(self._inner)
def __reduce__(self):
return type(self), (self._value,)
class Privilege(str, Enum):
"""Known privileges, validated and serialized by Rust.
ACL methods also accept strings such as ``"SELECT"`` and future privilege
names such as ``"PRIVILEGE20"``. Future names are preserved verbatim;
their meaning and validity are determined by the server.
Grant ``CREATE_SECRET`` or ``CREATE_FUNCTION`` on a namespace to delegate
creation of those resources without transferring namespace ownership.
"""
def __new__(cls, value: str):
canonical = str(_lancedb.AuthzPrivilege(value))
member = str.__new__(cls, canonical)
member._value_ = canonical
return member
OWNERSHIP = "OWNERSHIP"
USAGE = "USAGE"
SELECT = "SELECT"
INSERT = "INSERT"
UPDATE = "UPDATE"
DELETE = "DELETE"
CREATE_DATABASE = "CREATE_DATABASE"
CREATE_NAMESPACE = "CREATE_NAMESPACE"
CREATE_TABLE = "CREATE_TABLE"
CREATE_MATERIALIZED_VIEW = "CREATE_MATERIALIZED_VIEW"
CREATE_VIEW = "CREATE_VIEW"
CREATE_SECRET = "CREATE_SECRET"
CREATE_FUNCTION = "CREATE_FUNCTION"
OPERATE = "OPERATE"
@dataclass(frozen=True)
class AccessControlEntry:
"""One object/subject/privilege grant; subject contains a canonical ID or role."""
object: str
subject: Subject
subject_name: Optional[str]
privilege: str
@dataclass(frozen=True)
class RoleBinding:
"""A direct principal/group role binding, not an expanded effective role."""
subject: Subject
subject_name: Optional[str]
role: str
@dataclass(frozen=True)
class IdentityInfo:
"""A canonical identity-provider ID and its optional display name."""
id: str
name: Optional[str] = None
@dataclass(frozen=True)
class PrincipalInfo:
"""A principal's groups and effective roles, including group-inherited roles."""
principal_id: str
principal_name: Optional[str]
groups: list[IdentityInfo]
roles: list[str]
@dataclass(frozen=True)
class GroupInfo:
"""A group's provider-managed members and LanceDB-managed direct roles."""
group_id: str
group_name: Optional[str]
principals: list[IdentityInfo]
roles: list[str]
def _subject(subject: Subject, context: Optional[str] = None) -> str:
if not isinstance(subject, Subject):
raise ValueError("subject must be a Subject selector")
return subject._inner.to_wire(context)
def _object(value: Union[str, Object]) -> str:
if not isinstance(value, (str, Object)) or not str(value):
raise ValueError("object must be a nonempty string or Object")
return str(value) if isinstance(value, Object) else str(_lancedb.AuthzObject(value))
def _limit(limit: Optional[int]) -> None:
if limit is not None and (
isinstance(limit, bool) or not isinstance(limit, int) or limit < 0
):
raise ValueError("limit must be a nonnegative integer or None")
def _privilege(value: Union[str, Privilege]) -> str:
if not isinstance(value, str):
raise ValueError("privilege must be a string or Privilege")
return str(_lancedb.AuthzPrivilege(value))
def _entry(value: dict) -> AccessControlEntry:
return AccessControlEntry(
value["object"],
Subject._canonical(value["subject_id"]),
value.get("subject_name"),
value["privilege"],
)
def _binding(value: dict) -> RoleBinding:
return RoleBinding(
Subject._canonical(value["subject_id"]),
value.get("subject_name"),
value["role"],
)
class AsyncAuthorization:
"""Async authorization returned by ``AsyncCatalog.authz``.
HTTP failures raise [HttpError][lancedb.remote.errors.HttpError], preserving
status and request ID. Invalid local arguments raise ValueError. Lists are
lazy async iterators: failures can occur during iteration after some results
have been yielded. Iteration does not provide a consistent snapshot.
"""
def __init__(self, inner: _lancedb.AuthorizationClient):
self._inner = inner
async def _pages(self, method, request: dict, key: str, limit: Optional[int]):
yielded = 0
while limit is None or yielded < limit:
request["max_results"] = (
_PAGE_SIZE if limit is None else min(_PAGE_SIZE, limit - yielded)
)
response = json.loads(await method(json.dumps(request)))
for value in response[key]:
yield value
yielded += 1
if limit is not None and yielded >= limit:
return
token = response.get("continuation_token")
if not token:
return
if token == request.get("continuation_token"):
raise RuntimeError(
"Authorization listing returned a repeated continuation token"
)
request["continuation_token"] = token
def list_acls(
self,
*,
object: Optional[Union[str, Object]] = None,
subject: Optional[Subject] = None,
limit: Optional[int] = None,
) -> AsyncIterator[AccessControlEntry]:
"""Iterate grants, optionally filtered by exact object and/or subject.
Requires USAGE on the supplied object, or on system if object is omitted.
``limit`` bounds yielded entries across all pages; None lists all and zero
makes no requests. Missing matches produce an empty iterator.
"""
_limit(limit)
request = {}
if object is not None:
request["object"] = _object(object)
if subject is not None:
request["subject"] = _subject(subject)
async def entries():
async for value in self._pages(
self._inner.list_acls, request, "entries", limit
):
yield _entry(value)
return entries()
async def add_acl(
self,
*,
object: Union[str, Object],
subject: Subject,
privilege: Union[str, Privilege],
) -> None:
"""Grant one privilege; requires OWNERSHIP on the object.
A duplicate grant raises HTTP 409. Granting OWNERSHIP transfers ownership.
Writes are not automatically retried after ambiguous failures.
"""
await self._inner.add_acl(
json.dumps(
{
"object": _object(object),
"subject": _subject(subject),
"privilege": _privilege(privilege),
}
)
)
async def delete_acl(
self,
*,
object: Union[str, Object],
subject: Subject,
privilege: Union[str, Privilege],
) -> None:
"""Revoke one privilege; requires OWNERSHIP on the object.
A missing grant raises HTTP 404. OWNERSHIP itself cannot be deleted.
"""
await self._inner.delete_acl(
json.dumps(
{
"object": _object(object),
"subject": _subject(subject),
"privilege": _privilege(privilege),
}
)
)
def list_role_bindings(
self,
*,
subject: Optional[Subject] = None,
role: Optional[str] = None,
limit: Optional[int] = None,
) -> AsyncIterator[RoleBinding]:
"""Iterate direct bindings, optionally filtered by principal/group and role.
Requires USAGE on system. Limit semantics match ``list_acls``. Principals'
roles inherited through groups are available through ``get_principal``.
"""
_limit(limit)
request = {}
if subject is not None:
request["subject"] = _subject(subject, "role_member")
if role is not None:
request["role"] = role
async def bindings():
async for value in self._pages(
self._inner.list_role_bindings, request, "bindings", limit
):
yield _binding(value)
return bindings()
async def add_role_binding(self, *, subject: Subject, role: str) -> None:
"""Grant a role idempotently; requires OPERATE on system."""
await self._inner.add_role_binding(
json.dumps({"subject": _subject(subject, "role_member"), "role": role})
)
async def delete_role_binding(self, *, subject: Subject, role: str) -> None:
"""Idempotently revoke a role; requires OPERATE on system."""
await self._inner.delete_role_binding(
json.dumps({"subject": _subject(subject, "role_member"), "role": role})
)
async def get_principal(self, subject: Subject) -> PrincipalInfo:
"""Look up groups and effective roles; requires USAGE on system.
Select by ID, name, or API key. Missing/ambiguous identities raise HTTP
404/409; unavailable or unconfigured resolution raises HTTP 503/501.
"""
value = json.loads(
await self._inner.get_principal(
json.dumps({"principal": _subject(subject, "principal")})
)
)
return PrincipalInfo(
value["principal_id"],
value.get("principal_name"),
[IdentityInfo(**group) for group in value["groups"]],
value["roles"],
)
async def get_group(self, subject: Subject) -> GroupInfo:
"""Look up a group's members and roles by ID/name; requires USAGE on system.
Membership is read-only and comes from the identity provider. Lookup
errors have the same HTTP semantics as ``get_principal``.
"""
value = json.loads(
await self._inner.get_group(
json.dumps({"group": _subject(subject, "group")})
)
)
return GroupInfo(
value["group_id"],
value.get("group_name"),
[IdentityInfo(**principal) for principal in value["principals"]],
value["roles"],
)
async def _next(iterator: AsyncIterator[_T]) -> _T:
return await iterator.__anext__()
def _sync_iter(iterator: AsyncIterator[_T]) -> Iterator[_T]:
try:
while True:
yield LOOP.run(_next(iterator))
except StopAsyncIteration:
return
finally:
close = getattr(iterator, "aclose", None)
if close is not None:
LOOP.run(close())
class Authorization:
"""Synchronous authorization returned by ``Catalog.authz``.
Shares the async API's permissions, errors, and lazy pagination behavior.
Group membership comes from the identity provider and cannot be edited here.
"""
def __init__(self, inner: AsyncAuthorization):
self._inner = inner
def list_acls(
self,
*,
object: Optional[Union[str, Object]] = None,
subject: Optional[Subject] = None,
limit: Optional[int] = None,
) -> Iterator[AccessControlEntry]:
"""Iterate grants.
See [list_acls][lancedb.authz.AsyncAuthorization.list_acls] for permissions.
"""
return _sync_iter(
self._inner.list_acls(object=object, subject=subject, limit=limit)
)
def add_acl(
self,
*,
object: Union[str, Object],
subject: Subject,
privilege: Union[str, Privilege],
) -> None:
"""Grant a privilege; duplicates raise HTTP 409.
OWNERSHIP transfers ownership.
"""
LOOP.run(
self._inner.add_acl(object=object, subject=subject, privilege=privilege)
)
def delete_acl(
self,
*,
object: Union[str, Object],
subject: Subject,
privilege: Union[str, Privilege],
) -> None:
"""Revoke a privilege. Missing grants raise HTTP 404.
OWNERSHIP cannot be deleted.
"""
LOOP.run(
self._inner.delete_acl(object=object, subject=subject, privilege=privilege)
)
def list_role_bindings(
self,
*,
subject: Optional[Subject] = None,
role: Optional[str] = None,
limit: Optional[int] = None,
) -> Iterator[RoleBinding]:
"""Iterate direct role bindings, using the same limit semantics as list_acls."""
return _sync_iter(
self._inner.list_role_bindings(subject=subject, role=role, limit=limit)
)
def add_role_binding(self, *, subject: Subject, role: str) -> None:
"""Grant a role idempotently; requires OPERATE on system."""
LOOP.run(self._inner.add_role_binding(subject=subject, role=role))
def delete_role_binding(self, *, subject: Subject, role: str) -> None:
"""Idempotently revoke a role; requires OPERATE on system."""
LOOP.run(self._inner.delete_role_binding(subject=subject, role=role))
def get_principal(self, subject: Subject) -> PrincipalInfo:
"""Look up groups and effective roles; requires USAGE on system."""
return LOOP.run(self._inner.get_principal(subject))
def get_group(self, subject: Subject) -> GroupInfo:
"""Look up a group's members and roles; requires USAGE on system."""
return LOOP.run(self._inner.get_group(subject))
python/python/lancedb/catalog.py
+12
View File
@@ -3,11 +3,13 @@
"""Remote catalogs manage databases through a server's root namespace."""
from functools import cached_property
from datetime import timedelta
from typing import Any, AsyncIterator, Iterator, Optional, Union
from . import _lancedb
from .background_loop import LOOP
from .authz import AsyncAuthorization, Authorization
from .db import AsyncConnection, DBConnection
from .remote import ClientConfig, OAuthConfig
from .remote.db import RemoteDBConnection
@@ -86,6 +88,11 @@ class AsyncCatalog:
def __init__(self, inner: _lancedb.Catalog):
self._inner = inner
@cached_property
def authz(self) -> AsyncAuthorization:
"""Authorization sharing this catalog's scope and credentials."""
return AsyncAuthorization(self._inner.authz)
@property
def uri(self) -> str:
"""The catalog's root namespace endpoint."""
@@ -157,6 +164,11 @@ class Catalog:
self._sql_host_override = sql_host_override
self._oauth_config = oauth_config
@cached_property
def authz(self) -> Authorization:
"""Authorization sharing this catalog's scope and credentials."""
return Authorization(self._inner.authz)
@property
def uri(self) -> str:
"""The catalog's root namespace endpoint."""
python/python/lancedb/db.py
+20 -14
View File
@@ -317,7 +317,7 @@ class DBConnection(EnforceOverrides):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -328,7 +328,8 @@ class DBConnection(EnforceOverrides):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -343,7 +344,7 @@ class DBConnection(EnforceOverrides):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -357,8 +358,9 @@ class DBConnection(EnforceOverrides):
page_token: str, optional
The token to use for pagination. If not present, start from the beginning.
Typically, this token is last table name from the previous page.
limit: int, default 10
The size of the page to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -1507,7 +1509,7 @@ class LanceDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -1518,7 +1520,8 @@ class LanceDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -1537,7 +1540,7 @@ class LanceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -1552,8 +1555,9 @@ class LanceDBConnection(DBConnection):
The namespace to list tables in.
page_token: str, optional
The token to use for pagination.
limit: int, default 10
The maximum number of tables to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
@@ -2383,7 +2387,7 @@ class AsyncConnection(object):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -2394,7 +2398,8 @@ class AsyncConnection(object):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -2431,8 +2436,9 @@ class AsyncConnection(object):
This can be combined with limit to implement pagination by setting this to
the last table name from the previous page.
limit: int, default 10
The number of results to return.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
python/python/lancedb/functions.py
+1 -1
View File
@@ -372,7 +372,7 @@ class FunctionVersion(_RemoteValue):
image: FunctionImage
signature: FunctionSignature
secret_bindings: tuple[SecretBinding, ...] = ()
created_at: str
created_at_millis: int
metadata: Mapping[str, str]
disabled: bool
python/python/lancedb/index.py
+35 -2
View File
@@ -112,9 +112,42 @@ class Fm:
class FTS:
"""Describe a FTS index configuration.
`FTS` is a full-text search index that can be used on `String` columns
`FTS` supports string columns, lists of strings, and JSON columns created
from PyArrow ``pa.json_()``. JSON columns use Lance's JSON-aware tokenizer
and are stored as ``LargeBinary`` with the ``lance.json`` logical type.
Arbitrary binary columns are not supported.
For example, it works with `title`, `description`, `content`, etc.
JSON queries use ``path,str,value`` triples. Paths are relative to the JSON
column and use dots for nested object keys. For example,
``meta.tag,str,nature`` matches the string value at ``$.meta.tag``;
it does not search other paths or JSON key names. The value is processed
with the configured text tokenizer. Use ``str``, not ``string``.
Create and query a JSON index through the modern API::
import pyarrow as pa
from lancedb.index import FTS
table = db.create_table(
"documents",
pa.table({"doc": pa.array(
['{"meta":{"tag":"nature"}}'], type=pa.json_()
)}),
)
table.create_index("doc", config=FTS())
table.search(
"meta.tag,str,nature", query_type="fts", fts_columns="doc"
).to_list()
Create the index on the JSON column itself, not a JSON subpath such as
``doc.meta.tag``. JSON-path scalar indexes do not provide FTS path scoping.
Plain text such as ``nature`` is not a valid JSON FTS query. The current
Lance tokenizer can panic on malformed triples; see
https://github.com/lance-format/lance/issues/8812.
Remote tables use the same ``FTS()`` configuration and require a server
that supports JSON FTS. Wait for ``table.wait_for_index(["doc_idx"])``
before searching a newly created remote index with its default name.
Examples
--------
python/python/lancedb/namespace.py
+23 -3
View File
@@ -105,6 +105,11 @@ def _query_to_namespace_request(
if not isinstance(query, Query):
raise TypeError(f"Expected Query, got {type(query)}")
if query.search_effort is not None:
raise ValueError(
"namespace QueryTable does not support search_effort; "
"execute the query locally"
)
vector_request = None
if query.vector is not None:
@@ -190,8 +195,12 @@ def _query_to_namespace_request(
kwargs["with_row_id"] = query.with_row_id
if query.ef is not None:
kwargs["ef"] = query.ef
if query.nprobes is not None:
kwargs["nprobes"] = query.nprobes
if query.minimum_nprobes is not None:
kwargs["nprobes"] = query.minimum_nprobes
kwargs["minimum_nprobes"] = query.minimum_nprobes
if query.maximum_nprobes is not None:
kwargs["maximum_nprobes"] = query.maximum_nprobes
if query.refine_factor is not None:
kwargs["refine_factor"] = query.refine_factor
if query.lower_bound is not None:
@@ -205,6 +214,13 @@ def _query_to_namespace_request(
if query.distance_type is not None:
kwargs["distance_type"] = query.distance_type
if kwargs.get("maximum_nprobes") == 0:
# The generated 0.13 model rejects LanceDB's unbounded sentinel;
# model_copy preserves direct wire pass-through to the Lance receiver.
maximum_nprobes = kwargs.pop("maximum_nprobes")
return QueryTableRequest(**kwargs).model_copy(
update={"maximum_nprobes": maximum_nprobes}
)
return QueryTableRequest(**kwargs)
@@ -522,13 +538,15 @@ class LanceNamespaceDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the database.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
@@ -1157,13 +1175,15 @@ class AsyncLanceNamespaceDBConnection:
async def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
"""
List table names in the namespace.
If limit is None, return all table names; zero returns an empty list.
.. deprecated::
Use :meth:`list_tables` instead, which provides proper pagination support.
"""
python/python/lancedb/query.py
+463 -114
View File
@@ -12,6 +12,7 @@ from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Literal,
Optional,
@@ -40,7 +41,7 @@ from .rerankers.base import Reranker
from .rerankers.rrf import RRFReranker
from .rerankers.util import check_reranker_result
from .schema import blob_column_paths, is_blob_like_field, schema_has_blob_field
from .util import _validate_query_vector, flatten_columns
from .util import _validate_query_vector, flatten_columns, get_uri_scheme
from . import _wal_hybrid # WAL-PK-FUSION: delete.
from ._blob import (
BLOB_MODE_TO_HANDLING,
@@ -49,6 +50,8 @@ from ._blob import (
blob_auto_row_id_for_scan,
blob_v2_projection_sources,
finalize_blob_query_table,
reject_unfetchable_blob_columns,
reject_unmatched_blob_outputs,
replace_v2_blob_columns_with_bytes,
replace_v2_blob_columns_with_bytes_sync,
strip_auto_row_ids,
@@ -99,15 +102,45 @@ class _LanceScanner(Protocol):
def to_reader(self): ...
def _blob_mode_requires_native_pandas(blob_mode: BlobMode, schema: pa.Schema) -> bool:
return blob_mode in BLOB_MODE_TO_HANDLING and schema_has_blob_field(schema)
def _unsupported_blob_pandas_error(reason: str) -> RuntimeError:
return RuntimeError(
"blob columns require Lance native scanner conversion for query "
f"to_pandas(), but {reason}. Use a plain scan query or remove blob "
"columns from the projection."
f"to_pandas(), but {reason}. Use a plain scan query, pass "
"blob_mode='bytes' or blob_mode='descriptions', or remove blob columns "
"from the projection."
)
def _supports_local_blob_fetch(
async_table: "AsyncTable", pushdown_operations: Iterable[str]
) -> bool:
# Remote tables and QueryTable pushdown answer queries on a server, so a
# fetch through this handle could read another version.
return async_table._inner._is_native() and "QueryTable" not in pushdown_operations
def _blob_bytes_without_snapshot_error() -> NotImplementedError:
return NotImplementedError(
"blob_mode='bytes' cannot safely materialize blob columns without a "
"stable table snapshot. Remote tables and namespace QueryTable pushdown "
"run the query on a server, so the blob fetch could read another table "
"version. Use blob_mode='descriptions'."
)
def _query_needs_lance_scanner(builder: Any) -> bool:
"""Whether the query sets options that to_arrow ignores."""
to_query_object = getattr(builder, "to_query_object", None)
if to_query_object is None:
return False
try:
query = to_query_object()
except NotImplementedError:
return False
return (
query.fragments is not None
or query.fragment_ids is not None
or bool(query.with_row_address)
)
@@ -151,7 +184,8 @@ def _combine_where(
def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
if columns is None:
# An empty projection selects all columns.
if not columns:
return {}
if isinstance(columns, list):
if all(isinstance(column, str) for column in columns):
@@ -175,7 +209,8 @@ def _projection_to_scanner_kwargs(columns: QueryProjection) -> Dict[str, Any]:
def _query_request_projection(req: "PyQueryRequest") -> QueryProjection:
if req.select_source_columns is not None:
# Treat empty map as a calculated select
if req.select_source_columns:
return req.select_source_columns
return req.select
@@ -475,8 +510,8 @@ class MatchQuery(FullTextQuery):
----------
query : str
The query string to match against.
column : str
The name of the column to match against.
column : str, optional
The column to match against. None leaves column selection to query planning.
boost : float, default 1.0
The boost factor for the query.
The score of each matching document is multiplied by this value.
@@ -505,7 +540,7 @@ class MatchQuery(FullTextQuery):
"""
query: str
column: str
column: Optional[str] = None
boost: float = pydantic.Field(1.0, kw_only=True)
fuzziness: Optional[int] = pydantic.Field(0, kw_only=True)
max_expansions: int = pydantic.Field(50, kw_only=True)
@@ -693,8 +728,9 @@ class Query(pydantic.BaseModel):
if True then apply the filter after vector / FTS search. This is ignored for
plain SQL filtering.
nprobes : Optional[int]
The number of IVF partitions to search. If this is None then a default
number of partitions will be used.
The legacy number of IVF partitions to search. Lance sets both probe
bounds to this value. If this is None then Lance's default probe settings
will be used.
- A higher number makes search more accurate but also slower.
@@ -773,22 +809,28 @@ class Query(pydantic.BaseModel):
# distance type to use for vector search
distance_type: Optional[str] = None
# legacy number of IVF partitions to search
#
# Lance sets both probe bounds to this value. Explicit bounds can override it.
nprobes: Optional[int] = None
# which columns to return in the results (dict values may be str or Expr)
columns: QueryProjection = None
# minimum number of IVF partitions to search
#
# If None then a default value (20) will be used.
# If None then Lance's default will be used.
minimum_nprobes: Optional[int] = None
# maximum number of IVF partitions to search
#
# If None then a default value (20) will be used.
#
# If 0 then no limit will be applied and all partitions could be searched
# if needed to satisfy the limit.
# If None then Lance's default will be used. If 0 then no limit will be applied
# and all partitions could be searched if needed to satisfy the limit.
maximum_nprobes: Optional[int] = None
# Optional initial IVF effort; None preserves Auto and explicit nprobes.
search_effort: Optional[float] = None
# lower bound for distance search
lower_bound: Optional[float] = None
@@ -847,8 +889,10 @@ class Query(pydantic.BaseModel):
query.vector_column = req.column
query.vector = req.query_vector
query.distance_type = req.distance_type
query.nprobes = req.nprobes
query.minimum_nprobes = req.minimum_nprobes
query.maximum_nprobes = req.maximum_nprobes
query.search_effort = req.search_effort
query.lower_bound = req.lower_bound
query.upper_bound = req.upper_bound
query.ef = req.ef
@@ -1063,44 +1107,42 @@ class LanceQueryBuilder(ABC):
The maximum time to wait for the query to complete.
If None, wait indefinitely.
blob_mode: str, default "lazy"
Controls how blob columns are returned for plain scan queries.
Vector, FTS, hybrid, and other non-native query shapes keep the
existing Arrow conversion path and only support blob descriptions.
Controls how blob columns are returned. ``"lazy"`` returns
:class:`~lancedb._blob.BlobFile` handles and needs a plain scan.
``"descriptions"`` and ``"bytes"`` work for every query shape. A
plain scan without ``timeout`` reads the local dataset through
Lance's scanner, which also returns legacy blob columns as bytes.
Other shapes use :meth:`to_arrow` with the same ``blob_mode``.
**kwargs
Forwarded to pyarrow.Table.to_pandas after query execution and
optional flattening.
"""
validate_blob_mode(blob_mode)
output_schema = getattr(self, "output_schema", None)
if output_schema is not None:
schema = output_schema()
if _blob_mode_requires_native_pandas(blob_mode, schema):
native_error = None
if (flatten is None or blob_mode == "descriptions") and timeout is None:
try:
df = self._plain_scan_to_pandas(
blob_mode, flatten=flatten, **kwargs
)
if df is not None:
return df
except Exception as err:
native_error = err
reason = (
if output_schema is not None and schema_has_blob_field(output_schema()):
if (flatten is None or blob_mode != "lazy") and timeout is None:
df = self._plain_scan_to_pandas(blob_mode, flatten=flatten, **kwargs)
if df is not None:
return df
if blob_mode == "lazy" or _query_needs_lance_scanner(self):
raise _unsupported_blob_pandas_error(
"this query shape cannot use Lance native pandas conversion"
if native_error is None
else str(native_error)
)
raise _unsupported_blob_pandas_error(reason) from native_error
tbl = flatten_columns(self.to_arrow(timeout=timeout), flatten)
if _blob_mode_requires_native_pandas(blob_mode, tbl.schema):
tbl = flatten_columns(self._to_arrow_in_mode(timeout, blob_mode), flatten)
if blob_mode == "lazy" and schema_has_blob_field(tbl.schema):
raise _unsupported_blob_pandas_error(
"this query shape cannot use Lance native pandas conversion"
)
return tbl.to_pandas(**kwargs)
@abstractmethod
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
"""
Execute the query and return the results as an
[Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
@@ -1114,6 +1156,11 @@ class LanceQueryBuilder(ABC):
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If None, wait indefinitely.
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
"""
raise NotImplementedError
@@ -1144,7 +1191,12 @@ class LanceQueryBuilder(ABC):
"""
raise NotImplementedError
def to_list(self, *, timeout: Optional[timedelta] = None) -> List[dict]:
def to_list(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> List[dict]:
"""
Execute the query and return the results as a list of dictionaries.
@@ -1157,8 +1209,19 @@ class LanceQueryBuilder(ABC):
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If None, wait indefinitely.
blob_mode: str, default "lazy"
Passed through to :meth:`to_arrow`.
"""
return self.to_arrow(timeout=timeout).to_pylist()
validate_blob_mode(blob_mode)
return self._to_arrow_in_mode(timeout, blob_mode).to_pylist()
def _to_arrow_in_mode(
self, timeout: Optional[timedelta], blob_mode: BlobMode
) -> pa.Table:
# Older subclasses may not accept blob_mode. Omit it when it has no effect.
if blob_mode == "lazy" or not schema_has_blob_field(self._table.schema):
return self.to_arrow(timeout=timeout)
return self.to_arrow(timeout=timeout, blob_mode=blob_mode)
def to_pydantic(
self, model: type[T], *, timeout: Optional[timedelta] = None
@@ -1331,19 +1394,62 @@ class LanceQueryBuilder(ABC):
query.with_row_id = True
return query
def _finalize_blob_query_table(self, tbl: pa.Table) -> pa.Table:
blob_auto_row_id = self._blob_auto_row_id_enabled()
blob_paths = (
blob_v2_projection_sources(self._table.schema, self._columns).keys()
if blob_auto_row_id
else ()
def _validated_blob_sources(self, blob_mode: BlobMode) -> dict[str, str]:
"""Map projected blob v2 outputs to their source columns.
Bytes mode is checked here, before the query runs.
"""
validate_blob_mode(blob_mode)
schema = self._table.schema
blob_sources = blob_v2_projection_sources(schema, self._columns)
if blob_mode != "bytes":
return blob_sources
reject_unfetchable_blob_columns(schema, self._columns, blob_sources)
if blob_sources and not _supports_local_blob_fetch(
self._table._table, getattr(self._table, "_pushdown_operations", ())
):
raise _blob_bytes_without_snapshot_error()
reject_unmatched_blob_outputs(self._blob_output_schema(), blob_sources)
return blob_sources
def _blob_output_schema(self) -> pa.Schema:
return self.output_schema()
def _apply_arrow_blob_mode(
self, tbl: pa.Table, blob_mode: BlobMode, blob_sources: dict[str, str]
) -> pa.Table:
if blob_mode == "lazy":
return tbl
if blob_mode == "descriptions":
return strip_auto_row_ids(tbl, blob_sources)
return replace_v2_blob_columns_with_bytes_sync(
tbl, blob_sources, self._table.fetch_blobs
)
return finalize_blob_query_table(
def _finalize_blob_query_table(
self,
tbl: pa.Table,
*,
blob_mode: BlobMode,
blob_sources: dict[str, str],
) -> pa.Table:
blob_auto_row_id = self._blob_auto_row_id_enabled()
finalized = finalize_blob_query_table(
tbl,
user_requested_row_id=self._user_requested_row_id(),
blob_auto_row_id=blob_auto_row_id,
blob_paths=blob_paths,
blob_paths=blob_sources.keys() if blob_auto_row_id else (),
)
finalized = self._apply_arrow_blob_mode(finalized, blob_mode, blob_sources)
# with_row_id(False) does not reach the scan, which adds _rowid anyway.
# Lazy keeps it because fetch_blobs(hits) reads it there.
if (
blob_mode != "lazy"
and self._with_row_id is False
and "_rowid" in finalized.column_names
):
finalized = finalized.drop_columns(["_rowid"])
return finalized
def with_row_address(self, with_row_address: bool = True) -> Self:
"""Set whether to return row addresses.
@@ -1556,6 +1662,11 @@ class LanceQueryBuilder(ABC):
if not _query_is_plain_scan(query):
return None
# Remote and in-memory tables have no Lance dataset to scan.
if not self._table._table._inner._is_native() or (
get_uri_scheme(self._table._dataset_path) == "memory"
):
return None
dataset = self._table.to_lance()
blob_auto_row_id = self._blob_auto_row_id_enabled()
blob_sources = (
@@ -1563,6 +1674,10 @@ class LanceQueryBuilder(ABC):
if blob_mode == "bytes"
else {}
)
if blob_sources:
reject_unfetchable_blob_columns(
self._table.schema, query.columns, blob_sources
)
scanner = dataset.scanner(
**_scanner_kwargs_for_query(
query,
@@ -1576,7 +1691,7 @@ class LanceQueryBuilder(ABC):
blob_mode=blob_mode,
blob_sources=blob_sources,
fetch_blobs=self._table.fetch_blobs,
strip_auto_row_id=blob_auto_row_id,
strip_auto_row_id=not self._user_requested_row_id(),
flatten=flatten,
**kwargs,
)
@@ -1627,8 +1742,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
super().__init__(table)
self._query = query
self._distance_type = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._lower_bound = None
self._upper_bound = None
self._refine_factor = None
@@ -1691,9 +1808,9 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
See discussion in [Querying an ANN Index](https://lancedb.com/docs/indexing/)
for tuning advice.
This method sets both the minimum and maximum number of probes to the same
value. See `minimum_nprobes` and `maximum_nprobes` for more fine-grained
control.
The value is retained as `nprobes` through client and server request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
Parameters
----------
@@ -1707,8 +1824,28 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceVectorQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table("effort_vectors", [{"vector": [0.1, 0.2]}])
>>> query = table.search([0.1, 0.2]).search_effort(0.75)
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceVectorQueryBuilder:
@@ -1815,7 +1952,12 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
"""
return self._table._output_schema(self.to_query_object())
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
"""
Execute the query and return the results as an
[Apache Arrow Table](https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table).
@@ -1829,9 +1971,17 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
timeout: Optional[timedelta]
The maximum time to wait for the query to complete.
If None, wait indefinitely.
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
"""
blob_sources = self._validated_blob_sources(blob_mode)
return self._finalize_blob_query_table(
self.to_batches(timeout=timeout).read_all()
self.to_batches(timeout=timeout).read_all(),
blob_mode=blob_mode,
blob_sources=blob_sources,
)
def to_query_object(self) -> Query:
@@ -1850,8 +2000,10 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
limit=self._limit,
distance_type=self._distance_type,
columns=self._columns,
nprobes=self._nprobes,
minimum_nprobes=self._minimum_nprobes,
maximum_nprobes=self._maximum_nprobes,
search_effort=self._search_effort,
lower_bound=self._lower_bound,
upper_bound=self._upper_bound,
refine_factor=self._refine_factor,
@@ -2102,16 +2254,23 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
"""
return self._table._output_schema(self.to_query_object())
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
blob_sources = self._validated_blob_sources(blob_mode)
self._table._ensure_no_legacy_fts_index()
query = self._query_for_scan()
results = self._table._execute_query(query, timeout=timeout)
results = results.read_all()
results = self._table._execute_query(query, timeout=timeout).read_all()
if self._reranker is not None:
results = self._reranker.rerank_fts(self._query, results)
check_reranker_result(results)
return self._finalize_blob_query_table(results)
return self._finalize_blob_query_table(
results, blob_mode=blob_mode, blob_sources=blob_sources
)
def to_batches(
self, /, batch_size: Optional[int] = None, timeout: Optional[timedelta] = None
@@ -2138,9 +2297,17 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
class LanceEmptyQueryBuilder(LanceQueryBuilder):
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
blob_sources = self._validated_blob_sources(blob_mode)
return self._finalize_blob_query_table(
self.to_batches(timeout=timeout).read_all()
self.to_batches(timeout=timeout).read_all(),
blob_mode=blob_mode,
blob_sources=blob_sources,
)
def to_query_object(self) -> Query:
@@ -2208,8 +2375,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_columns = fts_columns
self._norm = None
self._reranker = None
self._nprobes = None
self._minimum_nprobes = None
self._maximum_nprobes = None
self._search_effort = None
self._refine_factor = None
self._distance_type = None
self._phrase_query = None
@@ -2271,16 +2440,30 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
def to_query_object(self) -> Query:
raise NotImplementedError("to_query_object not yet supported on a hybrid query")
def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
blob_sources = self._validated_blob_sources(blob_mode)
# WAL-PK-FUSION: without the fallback, this body is `_run_hybrid`'s.
return _wal_hybrid.with_pk_fallback(
lambda: self._run_hybrid(timeout=timeout),
lambda: self._run_hybrid(
timeout=timeout, blob_mode=blob_mode, blob_sources=blob_sources
),
self._table,
fused=lambda: self._pk_fusion is not None,
caller_requested_row_id=self._caller_requested_row_id,
)
def _run_hybrid(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
def _run_hybrid(
self,
*,
timeout: Optional[timedelta] = None,
blob_mode: BlobMode,
blob_sources: dict[str, str],
) -> pa.Table:
self._create_query_builders()
fts_query, vector_query = self._fts_query, self._vector_query
# WAL-PK-FUSION: without the fallback, both legs always ask for row ids.
@@ -2307,18 +2490,33 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
with_row_ids=True,
offset=self._offset,
)
return self._finish_hybrid_results(results)
return self._finish_hybrid_results(
results, blob_mode=blob_mode, blob_sources=blob_sources
)
def _finish_hybrid_results(self, results: pa.Table) -> pa.Table:
def _finish_hybrid_results(
self,
results: pa.Table,
*,
blob_mode: BlobMode,
blob_sources: dict[str, str],
) -> pa.Table:
if self._pk_fusion is not None: # WAL-PK-FUSION: delete.
return self._pk_fusion.strip(results)
return self._apply_arrow_blob_mode(
self._pk_fusion.strip(results), blob_mode, blob_sources
)
if self._user_requested_row_id():
return results
return self._apply_arrow_blob_mode(results, blob_mode, blob_sources)
if self._blob_auto_row_id_enabled():
return self._finalize_blob_query_table(results)
return self._finalize_blob_query_table(
results, blob_mode=blob_mode, blob_sources=blob_sources
)
if "_rowid" in results.column_names:
return results.drop(["_rowid"])
return results
# Fetch bytes before dropping _rowid.
return self._apply_arrow_blob_mode(results, blob_mode, blob_sources).drop(
["_rowid"]
)
return self._apply_arrow_blob_mode(results, blob_mode, blob_sources)
@staticmethod
def _combine_hybrid_results(
@@ -2499,6 +2697,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
Higher values will yield better recall (more likely to find vectors if
they exist) at the expense of latency.
The value is retained as `nprobes` until Lance sets both probe bounds to it.
Explicit minimum or maximum settings can override their respective bounds.
Parameters
----------
nprobes: int
@@ -2511,8 +2712,33 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
"""
if nprobes <= 0:
raise ValueError("Invalid input, nprobes must be greater than 0")
self._minimum_nprobes = nprobes
self._maximum_nprobes = nprobes
self._nprobes = nprobes
return self
def search_effort(self, search_effort: Optional[float]) -> LanceHybridQueryBuilder:
"""Set the initial IVF search effort to a finite value in [0, 1].
None (the default) uses Auto, internally 0.5. Zero starts at the caller
minimum; one searches all partitions within the caller maximum. Later
probing may continue when too few candidates are found. Any explicit
effort, including 0.5, overrides nprobes in either call order. Explicit
minimum and maximum bounds still apply. This is not a recall guarantee
and does not change refinement or HNSW ef. MemWAL LSM queries do not
yet support this option.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("memory://")
>>> table = db.create_table(
... "effort_hybrid", [{"vector": [0.1, 0.2], "text": "dog"}]
... )
>>> query = (
... table.search(query_type="hybrid").vector([0.1, 0.2])
... .text("dog").search_effort(0.75)
... )
"""
self._search_effort = search_effort
return self
def minimum_nprobes(self, minimum_nprobes: int) -> LanceHybridQueryBuilder:
@@ -2741,6 +2967,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
)
return "\n".join(results)
def _blob_output_schema(self) -> pa.Schema:
self._create_query_builders()
return self._vector_query.output_schema()
def _create_query_builders(self):
"""Set up and configure the vector and FTS query builders."""
vector_query, fts_query = self._validate_query(
@@ -2791,6 +3021,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
self._fts_query.phrase_query(True)
if self._distance_type:
self._vector_query.metric(self._distance_type)
if self._nprobes is not None:
self._vector_query.nprobes(self._nprobes)
if self._search_effort is not None:
self._vector_query.search_effort(self._search_effort)
if self._minimum_nprobes is not None:
self._vector_query.minimum_nprobes(self._minimum_nprobes)
if self._maximum_nprobes is not None:
@@ -2997,7 +3231,48 @@ class AsyncQueryBase(object):
"""
return await self._inner.output_schema()
async def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
async def _validated_blob_sources(self, blob_mode: BlobMode) -> dict[str, str]:
"""Map projected blob v2 outputs to their source columns.
Bytes mode is checked here, before the query runs.
"""
validate_blob_mode(blob_mode)
# Lazy needs no mapping, and skipping it saves remote tables a request.
if blob_mode == "lazy" or self._table is None:
return {}
schema = await self._table.schema()
projection = _query_request_projection(self._inner.to_query_request())
blob_sources = blob_v2_projection_sources(schema, projection)
if blob_mode != "bytes":
return blob_sources
reject_unfetchable_blob_columns(schema, projection, blob_sources)
if blob_sources and not _supports_local_blob_fetch(
self._table, self._table._pushdown_operations
):
raise _blob_bytes_without_snapshot_error()
reject_unmatched_blob_outputs(await self._blob_output_schema(), blob_sources)
return blob_sources
async def _blob_output_schema(self) -> pa.Schema:
return await self.output_schema()
async def _apply_arrow_blob_mode(
self, tbl: pa.Table, blob_mode: BlobMode, blob_sources: dict[str, str]
) -> pa.Table:
if blob_mode == "lazy":
return tbl
if blob_mode == "descriptions":
return strip_auto_row_ids(tbl, blob_sources)
return await replace_v2_blob_columns_with_bytes(
tbl, blob_sources, self._table.fetch_blobs
)
async def to_arrow(
self,
timeout: Optional[timedelta] = None,
*,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
"""
Execute the query and collect the results into an Apache Arrow Table.
@@ -3011,13 +3286,26 @@ class AsyncQueryBase(object):
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
Remote tables and namespace query pushdown refuse ``"bytes"``.
"""
blob_sources = await self._validated_blob_sources(blob_mode)
batch_iter = await self.to_batches(timeout=timeout)
return self._finalize_blob_query_table(
tbl = self._finalize_blob_query_table(
pa.Table.from_batches(await batch_iter.read_all(), schema=batch_iter.schema)
)
return await self._apply_arrow_blob_mode(tbl, blob_mode, blob_sources)
async def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]:
async def to_list(
self,
timeout: Optional[timedelta] = None,
*,
blob_mode: BlobMode = "lazy",
) -> List[dict]:
"""
Execute the query and return the results as a list of dictionaries.
@@ -3031,8 +3319,10 @@ class AsyncQueryBase(object):
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Passed through to :meth:`to_arrow`.
"""
return (await self.to_arrow(timeout=timeout)).to_pylist()
return (await self.to_arrow(timeout=timeout, blob_mode=blob_mode)).to_pylist()
async def to_pandas(
self,
@@ -3074,40 +3364,42 @@ class AsyncQueryBase(object):
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Controls how blob columns are returned. Remote queries support
"descriptions"; "bytes" and "lazy" are not yet supported.
Controls how blob columns are returned. ``"lazy"`` returns
:class:`~lancedb._blob.BlobFile` handles and needs a plain scan.
``"descriptions"`` and ``"bytes"`` work for every query shape. A
plain scan without ``timeout`` reads the local dataset through
Lance's scanner, which also returns legacy blob columns as bytes.
Other shapes use :meth:`to_arrow` with the same ``blob_mode``.
Remote queries support ``"descriptions"`` only.
**kwargs
Forwarded to pyarrow.Table.to_pandas after query execution and
optional flattening.
"""
validate_blob_mode(blob_mode)
if self._table is not None and not self._table._inner._is_native():
await self._validated_blob_sources(blob_mode)
return await self._remote_to_pandas(
flatten=flatten, timeout=timeout, blob_mode=blob_mode, **kwargs
)
if hasattr(self._inner, "output_schema"):
schema = await self.output_schema()
if _blob_mode_requires_native_pandas(blob_mode, schema):
native_error = None
if (flatten is None or blob_mode == "descriptions") and timeout is None:
try:
df = await self._plain_scan_to_pandas(
blob_mode, flatten=flatten, **kwargs
)
if df is not None:
return df
except Exception as err:
native_error = err
reason = (
if hasattr(self._inner, "output_schema") and schema_has_blob_field(
await self.output_schema()
):
if (flatten is None or blob_mode != "lazy") and timeout is None:
df = await self._plain_scan_to_pandas(
blob_mode, flatten=flatten, **kwargs
)
if df is not None:
return df
if blob_mode == "lazy" or _query_needs_lance_scanner(self):
raise _unsupported_blob_pandas_error(
"this query shape cannot use Lance native pandas conversion"
if native_error is None
else str(native_error)
)
raise _unsupported_blob_pandas_error(reason) from native_error
tbl = flatten_columns(await self.to_arrow(timeout=timeout), flatten)
if _blob_mode_requires_native_pandas(blob_mode, tbl.schema):
tbl = flatten_columns(
await self.to_arrow(timeout=timeout, blob_mode=blob_mode), flatten
)
if blob_mode == "lazy" and schema_has_blob_field(tbl.schema):
raise _unsupported_blob_pandas_error(
"this query shape cannot use Lance native pandas conversion"
)
@@ -3162,6 +3454,13 @@ class AsyncQueryBase(object):
if blob_mode == "bytes"
else {}
)
if blob_sources:
reject_unfetchable_blob_columns(schema, query.columns, blob_sources)
# Remote and in-memory tables have no Lance dataset to scan.
if not self._table._inner._is_native() or (
get_uri_scheme(await self._table.uri()) == "memory"
):
return None
dataset = await self._table.to_lance()
scanner = dataset.scanner(
**_scanner_kwargs_for_query(
@@ -3176,7 +3475,7 @@ class AsyncQueryBase(object):
blob_mode=blob_mode,
blob_sources=blob_sources,
fetch_blobs=self._table.fetch_blobs,
strip_auto_row_id=blob_auto_row_id,
strip_auto_row_id=self._with_row_id is not True,
flatten=flatten,
**kwargs,
)
@@ -3690,7 +3989,7 @@ class AsyncVectorQueryBase:
def nprobes(self, nprobes: int) -> Self:
"""
Set the number of partitions to search (probe)
Set the legacy IVF probe parameter
The number of probes must be greater than 0.
@@ -3702,20 +4001,49 @@ class AsyncVectorQueryBase:
The partition whose centroids are closest to the query vector will be
exhaustiely searched to find matches. This parameter controls how many
partitions should be searched.
partitions will be searched.
Increasing this value will increase the recall of your query but will
also increase the latency of your query. The default value is 20. This
default is good for many cases but the best value to use will depend on
your data and the recall that you need to achieve.
also increase the latency of your query. If this method is not called,
Lance's adaptive probe defaults are used.
For best results we recommend tuning this parameter with a benchmark against
your actual data to find the smallest possible value that will still give
you the desired recall.
LanceDB retains this as `nprobes` through local and remote request
construction. Lance sets both probe bounds to this value. Explicit minimum
or maximum settings can override their respective bounds.
"""
self._inner.nprobes(nprobes)
return self
def search_effort(self, search_effort: Optional[float]) -> Self:
"""Set the initial IVF search effort to a finite value in [0, 1].
None preserves Auto (internally 0.5) and fixed nprobes. Any explicit
effort, including 0.5, overrides nprobes in either call order; explicit
minimum and maximum bounds remain effective. Zero starts at the caller
minimum and one searches all partitions within the maximum. Later
probing may continue to obtain enough candidates. This is not a recall
guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
do not yet support this option.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def build_query():
... db = await lancedb.connect_async("memory://")
... table = await db.create_table(
... "effort_async", [{"vector": [0.1, 0.2]}]
... )
... return table.vector_search([0.1, 0.2]).search_effort(0.75)
>>> query = asyncio.run(build_query())
"""
self._inner.search_effort(search_effort)
return self
def minimum_nprobes(self, minimum_nprobes: int) -> Self:
"""Set the minimum number of probes to use.
@@ -3987,6 +4315,10 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
return self
async def _blob_output_schema(self) -> pa.Schema:
vector_query = AsyncVectorQuery(self._inner.to_vector_query(), self._table)
return await vector_query.output_schema()
async def _create_child_queries(
self,
) -> Tuple["AsyncFTSQuery", "AsyncVectorQuery", int, int]:
@@ -4348,7 +4680,12 @@ class BaseQueryBuilder(object):
return pa.RecordBatchReader.from_batches(async_reader.schema, iter_sync())
def to_arrow(self, timeout: Optional[timedelta] = None) -> pa.Table:
def to_arrow(
self,
timeout: Optional[timedelta] = None,
*,
blob_mode: BlobMode = "lazy",
) -> pa.Table:
"""
Execute the query and collect the results into an Apache Arrow Table.
@@ -4362,10 +4699,20 @@ class BaseQueryBuilder(object):
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
"""
return LOOP.run(self._inner.to_arrow(timeout))
return LOOP.run(self._inner.to_arrow(timeout, blob_mode=blob_mode))
def to_list(self, timeout: Optional[timedelta] = None) -> List[dict]:
def to_list(
self,
timeout: Optional[timedelta] = None,
*,
blob_mode: BlobMode = "lazy",
) -> List[dict]:
"""
Execute the query and return the results as a list of dictionaries.
@@ -4379,8 +4726,10 @@ class BaseQueryBuilder(object):
The maximum time to wait for the query to complete.
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Passed through to :meth:`to_arrow`.
"""
return LOOP.run(self._inner.to_list(timeout))
return LOOP.run(self._inner.to_list(timeout, blob_mode=blob_mode))
def to_pandas(
self,
@@ -4422,7 +4771,7 @@ class BaseQueryBuilder(object):
If not specified, no timeout is applied. If the query does not
complete within the specified time, an error will be raised.
blob_mode: str, default "lazy"
Controls how blob columns are returned for plain scan queries.
Passed through to :meth:`AsyncQueryBase.to_pandas`.
**kwargs
Forwarded to pyarrow.Table.to_pandas after query execution and
optional flattening.
python/python/lancedb/remote/__init__.py
+5
View File
@@ -170,6 +170,10 @@ class ClientConfig:
Configuration for retrying failed requests.
timeout_config: Optional[TimeoutConfig]
Configuration for request timeouts.
blob_request_concurrency: Optional[int]
Maximum concurrent blob HTTP requests across all handles on this
connection. Defaults to 8. Set a positive integer here or via
`LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`.
extra_headers: Optional[dict]
Additional headers to include in requests.
id_delimiter: Optional[str]
@@ -193,6 +197,7 @@ class ClientConfig:
user_agent: str = f"LanceDB-Python-Client/{__version__}"
retry_config: RetryConfig = field(default_factory=RetryConfig)
timeout_config: Optional[TimeoutConfig] = field(default_factory=TimeoutConfig)
blob_request_concurrency: Optional[int] = None
extra_headers: Optional[dict] = None
id_delimiter: Optional[str] = None
tls_config: Optional[TlsConfig] = None
python/python/lancedb/remote/db.py
+8 -5
View File
@@ -112,6 +112,7 @@ def _client_config_to_dict(config: ClientConfig) -> dict[str, Any]:
"user_agent": config.user_agent,
"retry_config": _retry_config_to_dict(config.retry_config),
"timeout_config": _timeout_config_to_dict(config.timeout_config),
"blob_request_concurrency": config.blob_request_concurrency,
"extra_headers": config.extra_headers,
"id_delimiter": config.id_delimiter,
"tls_config": _tls_config_to_dict(config.tls_config),
@@ -379,7 +380,7 @@ class RemoteDBConnection(DBConnection):
page_token: Optional[str] = None,
limit: Optional[int] = None,
) -> ListTablesResponse:
"""List all tables in this database with pagination support.
"""List a page of tables in this database.
Parameters
----------
@@ -390,7 +391,8 @@ class RemoteDBConnection(DBConnection):
Token for pagination. Use the token from a previous response
to get the next page of results.
limit: int, optional
The maximum number of results to return.
The maximum number of results to return, default 100.
Zero returns an empty page without a continuation token.
Returns
-------
@@ -409,7 +411,7 @@ class RemoteDBConnection(DBConnection):
def table_names(
self,
page_token: Optional[str] = None,
limit: int = 10,
limit: Optional[int] = None,
*,
namespace_path: Optional[List[str]] = None,
) -> Iterable[str]:
@@ -425,8 +427,9 @@ class RemoteDBConnection(DBConnection):
Empty list represents root namespace.
page_token: str
The last token to start the new page.
limit: int, default 10
The maximum number of tables to return for each page.
limit: int, optional
The maximum number of tables to return. None returns all tables;
zero returns an empty list.
Returns
-------
python/python/lancedb/remote/table.py
+10 -3
View File
@@ -58,6 +58,7 @@ from lancedb.table import IndexConfigType, KNOWN_METRICS
import pyarrow as pa
from lancedb.common import DATA, VEC, VECTOR_COLUMN_NAME
from lancedb._blob import validate_blob_mode
from lancedb.merge import LanceMergeInsertBuilder
from lancedb.embeddings import EmbeddingFunctionRegistry
from lancedb.table import _normalize_progress
@@ -244,8 +245,9 @@ class RemoteTable(Table):
"""List all versions of the table"""
return LOOP.run(self._table.list_versions())
def to_arrow(self) -> pa.Table:
def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table:
"""to_arrow() is not supported for remote tables."""
validate_blob_mode(blob_mode)
raise NotImplementedError(
"to_arrow() is not supported for remote tables. "
"Use table.search().to_arrow() instead."
@@ -253,6 +255,7 @@ class RemoteTable(Table):
def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
"""to_pandas() is not supported for remote tables."""
validate_blob_mode(blob_mode)
raise NotImplementedError(
"to_pandas() is not supported for remote tables. "
"Use table.search().to_pandas() instead."
@@ -1179,7 +1182,9 @@ class RemoteTable(Table):
return LOOP.run(self._table.blob_columns())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
@@ -1187,7 +1192,9 @@ class RemoteTable(Table):
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
return LOOP.run(self._table.fetch_blob_files(column, row_ids))
python/python/lancedb/schema.py
+51 -4
View File
@@ -5,7 +5,7 @@
"""Schema helpers for Lance blob columns."""
import importlib
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional
import pyarrow as pa
import pyarrow.ipc
@@ -181,13 +181,60 @@ def _resolve_blob_type():
return _resolved_blob_type
def blob(name: str, nullable: bool = True) -> pa.Field:
def blob(
name: str,
nullable: bool = True,
*,
inline_size_threshold: Optional[int] = None,
dedicated_size_threshold: Optional[int] = None,
pack_file_size_threshold: Optional[int] = None,
) -> pa.Field:
"""Create a Lance blob v2 column field.
When pylance is installed this is ``lance.blob.BlobType``.
Parameters
----------
name:
Field name.
nullable:
Whether the column accepts null values.
inline_size_threshold:
Maximum payload size in bytes to keep inline in the data file
before using packed blob storage. Zero is allowed.
dedicated_size_threshold:
Maximum payload size in bytes to store in packed blob storage
before using dedicated blob storage. Must be positive.
pack_file_size_threshold:
Maximum size in bytes of a single packed blob sidecar file before
a new one is started. Must be positive.
The thresholds are forwarded to ``lance.blob_field``, which validates
them and stores them as field metadata, so pylance is required whenever
any threshold is set. When all thresholds are omitted the behavior is
unchanged.
"""
blob_type = _resolve_blob_type()
return pa.field(name, blob_type(), nullable=nullable)
if (
inline_size_threshold is None
and dedicated_size_threshold is None
and pack_file_size_threshold is None
):
blob_type = _resolve_blob_type()
return pa.field(name, blob_type(), nullable=nullable)
try:
lance = importlib.import_module("lance")
except ModuleNotFoundError as err:
raise ValueError(
"Setting blob tier thresholds requires pylance to be installed"
) from err
_resolve_blob_type()
return lance.blob_field(
name,
nullable=nullable,
inline_size_threshold=inline_size_threshold,
dedicated_size_threshold=dedicated_size_threshold,
pack_file_size_threshold=pack_file_size_threshold,
)
def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:
python/python/lancedb/table.py
+191 -39
View File
@@ -576,6 +576,55 @@ def _cast_to_target_schema(
return pa.RecordBatchReader.from_batches(reordered_schema, gen())
def _preserve_blob_v2_fields(
inferred_schema: pa.Schema, existing_schema: pa.Schema
) -> pa.Schema:
"""Keep existing blob fields when an overwrite infers a replacement schema."""
existing_fields = {field.name: field for field in existing_schema}
fields = [
_preserve_blob_v2_field(field, existing_fields[field.name])
if field.name in existing_fields
else field
for field in inferred_schema
]
return pa.schema(fields, metadata=inferred_schema.metadata)
def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field:
if is_blob_v2_field(existing):
return existing
inferred_type, existing_type = inferred.type, existing.type
if pa.types.is_struct(inferred_type) and pa.types.is_struct(existing_type):
existing_children = {field.name: field for field in existing_type}
children = [
_preserve_blob_v2_field(field, existing_children[field.name])
if field.name in existing_children
else field
for field in inferred_type
]
return inferred.with_type(pa.struct(children))
if _is_list_like(inferred_type) and _is_list_like(existing_type):
value_field = _preserve_blob_v2_field(
inferred_type.value_field, existing_type.value_field
)
if value_field == inferred_type.value_field:
return inferred
value_field = value_field.with_name(existing_type.value_field.name)
if pa.types.is_fixed_size_list(inferred_type):
value_type = pa.list_(value_field, inferred_type.list_size)
elif pa.types.is_large_list(inferred_type):
value_type = pa.large_list(value_field)
else:
value_type = pa.list_(value_field)
return inferred.with_type(value_type)
if blob_v2_column_paths(pa.schema([existing])):
return existing
return inferred
def _coerce_blob_write_columns(
batch: pa.RecordBatch, target_schema: pa.Schema
) -> pa.RecordBatch:
@@ -691,12 +740,7 @@ def _coerce_blob_list_values(
def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
if pa.types.is_null(values.type):
data = pa.nulls(len(values), type=pa.large_binary())
elif pa.types.is_large_binary(values.type):
data = values
else:
data = values.cast(pa.large_binary())
value_field = "uri" if _is_string_like(values.type) else "data"
length = len(values)
storage_type = target_field.type
if isinstance(storage_type, pa.ExtensionType):
@@ -704,8 +748,11 @@ def _coerce_value_to_blob(values: pa.Array, target_field: pa.Field) -> pa.Array:
storage_fields = list(storage_type)
children = []
for storage_field in storage_fields:
if storage_field.name == "data":
children.append(data)
if storage_field.name == value_field and not pa.types.is_null(values.type):
if values.type == storage_field.type:
children.append(values)
else:
children.append(values.cast(storage_field.type))
else:
children.append(pa.nulls(length, type=storage_field.type))
storage = pa.StructArray.from_arrays(
@@ -725,7 +772,19 @@ def _physical_array_and_type(array: pa.Array) -> tuple[pa.Array, pa.DataType]:
def _can_coerce_to_blob(data_type: pa.DataType) -> bool:
return _is_binary_like(data_type) or pa.types.is_null(data_type)
return (
_is_binary_like(data_type)
or _is_string_like(data_type)
or pa.types.is_null(data_type)
)
def _is_string_like(data_type: pa.DataType) -> bool:
return (
pa.types.is_string(data_type)
or pa.types.is_large_string(data_type)
or pa.types.is_string_view(data_type)
)
def _is_binary_like(data_type: pa.DataType) -> bool:
@@ -887,6 +946,24 @@ def _arrow_json_storage_type(input_type: pa.DataType) -> Optional[pa.DataType]:
def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
if is_blob_v2_field(target_field):
source_type = field.type
if pa.types.is_struct(source_type):
if not any(child.name in ("data", "uri") for child in source_type):
raise ValueError(
f"blob struct input for column '{field.name}' must contain "
"a 'data' or 'uri' child"
)
elif not _can_coerce_to_blob(source_type) and not isinstance(
source_type, pa.ExtensionType
):
raise ValueError(
f"cannot coerce column '{field.name}' with type {source_type} "
"into a blob v2 struct. expected binary bytes "
"(Binary, LargeBinary, BinaryView), strings "
"(Utf8, LargeUtf8, Utf8View), or a Struct with a "
"'data' or 'uri' child"
)
# LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but
# casting the input to that storage type here merely relabels the raw JSON bytes as
# JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
@@ -1322,9 +1399,17 @@ class Table(ABC):
return self.to_arrow().to_pandas(**kwargs)
@abstractmethod
def to_arrow(self) -> pa.Table:
def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table:
"""Return the table as a pyarrow Table.
Parameters
----------
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
Returns
-------
pa.Table
@@ -1750,7 +1835,8 @@ class Table(ABC):
- pyarrow.Table or pyarrow.RecordBatch
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite".
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill".
@@ -2066,19 +2152,30 @@ class Table(ABC):
@abstractmethod
def blob_columns(self) -> list[str]:
"""Names of the blob v2 columns declared on this table."""
"""Names of blob v2 columns that can be fetched by row id.
Struct-nested blobs use dotted paths. Blobs inside lists are omitted
because a row id does not identify an individual list element.
"""
@abstractmethod
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
"""Materialize full blob bytes for ``column`` at the given rows.
The result has the same length and order as ``row_ids``. Null blobs
produce null slots; valid empty blobs produce ``b""``.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
``row_ids`` may be a list, an integer Arrow array or chunked array,
or a query table with row ids. ``_rowid`` values stay valid after
compaction when the table has stable row ids.
One missing or deleted row ID rejects the entire batch; no partial
results are returned.
Blobs inside lists cannot be fetched by row id yet.
Remote servers limit each request to 1024 row IDs and 64 MiB of blob
bytes. The client splits requests automatically and reads an individual
@@ -2090,17 +2187,20 @@ class Table(ABC):
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
"""Materialize row-specific byte ranges from a blob v2 column.
Each request is a ``(row_id, offset, length)`` tuple. Requests may be
repeated or reordered, including multiple ranges for the same blob.
Each request is a three-item ``(row_id, offset, length)`` sequence.
Requests may be repeated or reordered, including multiple ranges for
the same blob.
The result has the same length and order as ``requests``; null blobs
produce null slots and empty ranges on non-null blobs produce ``b""``.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
One missing or deleted row ID rejects the entire batch; no partial
results are returned.
Row IDs can be obtained from a query with ``with_row_id(True)``. This
API is currently supported only by local tables.
@@ -2108,18 +2208,22 @@ class Table(ABC):
@abstractmethod
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
"""Open lazy, seekable :class:`~lancedb._blob.BlobFile` handles.
Prefer this over :meth:`fetch_blobs` for large payloads. ``row_ids`` is
a ``list[int]`` or a query ``pyarrow.Table`` carrying row identity via
``_rowid`` or a ``_lance_row_id`` field on the blob descriptor. Null
rows are ``None``. Remote tables require LanceDB Cloud server 0.5.0 or
newer.
a ``list[int]``, an integer Arrow array or chunked array, or a query
``pyarrow.Table`` carrying row identity via ``_rowid`` or a
``_lance_row_id`` field on the blob descriptor. Null blob values return
``None``. Remote tables require LanceDB Cloud server 0.5.0 or newer.
``_rowid`` values stay valid after compaction when the table has stable
row ids.
One missing or deleted row ID rejects the entire batch; no partial
handles are returned.
"""
@abstractmethod
@@ -2877,6 +2981,8 @@ class LanceTable(Table):
managed_versioning=managed_versioning,
)
)
# Take queries read the pushdown set from the inner table.
self._table._pushdown_operations = self._pushdown_operations
@property
def name(self) -> str:
@@ -3025,19 +3131,23 @@ class LanceTable(Table):
return LOOP.run(self._table.blob_columns())
def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blobs(column, row_ids))
def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
return LOOP.run(self._table.fetch_blob_ranges(column, list(requests)))
def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
return LOOP.run(self._table.fetch_blob_files(column, row_ids))
@@ -3251,12 +3361,23 @@ class LanceTable(Table):
return self.to_lance().to_pandas(blob_mode=blob_mode, **kwargs)
def to_arrow(self) -> pa.Table:
def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table:
"""Return the table as a pyarrow Table.
Parameters
----------
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
Returns
-------
pa.Table"""
validate_blob_mode(blob_mode)
if blob_mode != "lazy" and schema_has_blob_field(self.schema):
return self.search().to_arrow(blob_mode=blob_mode)
if (
_should_push_down_query_table(
self._namespace_client, self._pushdown_operations
@@ -3992,7 +4113,8 @@ class LanceTable(Table):
The data to insert into the table.
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite".
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill", "null".
@@ -4452,6 +4574,7 @@ class LanceTable(Table):
and not self._route_pushdown_to_rust
and self.current_branch() is None
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -5698,22 +5821,37 @@ class AsyncTable:
return await self.query().to_pandas(blob_mode=blob_mode, **kwargs)
return (await self.to_lance()).to_pandas(blob_mode=blob_mode, **kwargs)
async def to_arrow(self) -> pa.Table:
async def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table:
"""Return the table as a pyarrow Table.
Parameters
----------
blob_mode: str, default "lazy"
``"lazy"`` returns blob v2 descriptors that carry the row id
``fetch_blobs`` reads. ``"descriptions"`` returns them without it.
``"bytes"`` returns payload bytes and raises for legacy blob columns
and blobs inside lists.
Returns
-------
pa.Table
"""
validate_blob_mode(blob_mode)
schema = await self.schema()
fetches_bytes = blob_mode == "bytes" and schema_has_blob_field(schema)
if (
_should_push_down_query_table(
not fetches_bytes
and _should_push_down_query_table(
self._namespace_client, self._pushdown_operations
)
and not self._route_pushdown_to_rust
):
return (await self._execute_query(Query())).read_all()
return await self.query().to_arrow()
tbl = (await self._execute_query(Query())).read_all()
if blob_mode == "descriptions":
tbl = strip_auto_row_ids(tbl, row_addressable_blob_v2_paths(schema))
return tbl
# The query builder refuses bytes under pushdown before running the query.
return await self.query().to_arrow(blob_mode=blob_mode)
async def create_index(
self,
@@ -6009,7 +6147,8 @@ class AsyncTable:
- pyarrow.Table or pyarrow.RecordBatch
mode: str
The mode to use when writing the data. Valid values are
"append" and "overwrite".
"append" and "overwrite". Overwrite infers a replacement schema
but preserves existing blob v2 fields present in the new data.
on_bad_vectors: str, default "error"
What to do if any of the vectors are not the same size or contains NaNs.
One of "error", "drop", "fill", "null".
@@ -6047,6 +6186,11 @@ class AsyncTable:
data, _ = sanitize_create_table(
data, None, on_bad_vectors=on_bad_vectors, fill_value=fill_value
)
if blob_v2_column_paths(schema):
# Plain bytes cannot be inferred as blob v2. Keep matching blob
# fields while allowing other columns to use the inferred schema.
target_schema = _preserve_blob_v2_fields(data.schema, schema)
data = _cast_to_target_schema(data, target_schema)
elif on_bad_vectors != "error" or (
schema.metadata is not None and b"embedding_functions" in schema.metadata
):
@@ -6451,9 +6595,12 @@ class AsyncTable:
)
if query.distance_type is not None:
async_query = async_query.distance_type(query.distance_type)
if query.nprobes is not None:
async_query = async_query.nprobes(query.nprobes)
if query.search_effort is not None:
async_query = async_query.search_effort(query.search_effort)
if query.minimum_nprobes is not None and query.maximum_nprobes is not None:
# Set both to the minimum first to avoid min > max error.
async_query = async_query.nprobes(
async_query = async_query.minimum_nprobes(
query.minimum_nprobes
).maximum_nprobes(query.maximum_nprobes)
elif query.minimum_nprobes is not None:
@@ -6492,6 +6639,7 @@ class AsyncTable:
)
and not self._route_pushdown_to_rust
and query.take_offsets is None
and query.search_effort is None
):
from lancedb.namespace import _execute_server_side_query
@@ -7086,7 +7234,9 @@ class AsyncTable:
return await self._inner.blob_columns()
async def fetch_blobs(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blobs(
column, _normalize_blob_row_ids(row_ids, column)
@@ -7095,12 +7245,14 @@ class AsyncTable:
async def fetch_blob_ranges(
self,
column: str,
requests: Sequence[Tuple[int, int, int]],
requests: Sequence[Sequence[int]],
) -> pa.LargeBinaryArray:
return await self._inner.fetch_blob_ranges(column, list(requests))
async def fetch_blob_files(
self, column: str, row_ids: Union[list[int], pa.Table]
self,
column: str,
row_ids: Union[list[int], pa.Array, pa.ChunkedArray, pa.Table],
) -> "list[Optional[BlobFile]]":
handles = await self._inner.fetch_blob_files(
column, _normalize_blob_row_ids(row_ids, column)
python/python/lancedb/util.py
+9
View File
@@ -10,6 +10,7 @@ import os
import pathlib
import warnings
from datetime import date, datetime
from decimal import Decimal
from functools import singledispatch
from typing import Tuple, Union, Optional, Any, List
from urllib.parse import urlparse
@@ -373,6 +374,14 @@ def _(value: float):
return str(value)
@value_to_sql.register(Decimal)
def _(value: Decimal):
if not value.is_finite():
raise ValueError("Non-finite Decimal values cannot be converted to SQL")
# Quote exact decimal text so the target-column cast avoids float rounding.
return value_to_sql(format(value, "f"))
@value_to_sql.register(bool)
def _(value: bool):
return str(value).upper()
python/python/tests/test_authz.py
+821
View File
@@ -0,0 +1,821 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import asyncio
from dataclasses import FrozenInstanceError
import json
from unittest.mock import AsyncMock
import pytest
import lancedb
from lancedb.authz import (
AccessControlEntry,
AsyncAuthorization,
Authorization,
GroupInfo,
IdentityInfo,
Object,
PrincipalInfo,
Privilege,
RoleBinding,
Subject,
)
from lancedb.remote import HeaderProvider
from lancedb.remote.errors import HttpError
from test_catalog import catalog_server as _catalog_server
catalog_server = _catalog_server
class DynamicHeaders(HeaderProvider):
def get_headers(self):
return {
"Authorization": "Bearer rotating-token",
"X-LanceDB-Database": "wrong-dynamic-db",
"X-LanceDB-Database-Prefix": "wrong-prefix",
}
@pytest.mark.parametrize("asynchronous", [False, True])
def test_authz_operations_and_scope(catalog_server, asynchronous):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {}),
(200, {}),
(200, {}),
(200, {}),
(
200,
{
"entries": [
{
"subject_id": "g:team-id",
"object": "table:tenant/db:a$b$events",
"privilege": "PRIVILEGE20",
}
]
},
),
(
200,
{
"bindings": [
{
"subject_id": "p:alice-id",
"subject_name": "Alice",
"role": "reader",
}
]
},
),
(
200,
{
"principal_id": "alice-id",
"groups": [{"id": "team-id"}],
"roles": ["reader"],
},
),
(
200,
{
"group_id": "team-id",
"group_name": "Engineering",
"principals": [{"id": "alice-id", "name": "Alice"}],
"roles": ["reader"],
},
),
]
)
config = {
"header_provider": DynamicHeaders(),
"extra_headers": {"x-lancedb-database": "wrong-static-db"},
}
obj = Object.table(database="tenant/db", namespace_path=["a", "b"], name="events")
async def exercise():
if asynchronous:
catalog = await lancedb.connect_catalog_async(
endpoint, client_config=config
)
else:
catalog = lancedb.connect_catalog(endpoint, client_config=config)
authz = catalog.authz
assert authz is catalog.authz
assert requests == [] # Constructing handles does not probe a database.
async def call(method, *args, **kwargs):
value = method(*args, **kwargs)
return await value if asynchronous else value
await call(
authz.add_acl,
object=obj,
subject=Subject.group_name("Engineering"),
privilege=Privilege.SELECT,
)
await call(
authz.delete_acl,
object=obj,
subject=Subject.group_id("team-id"),
privilege="SELECT",
)
await call(
authz.add_role_binding,
subject=Subject.principal_name("Alice"),
role="reader",
)
await call(
authz.delete_role_binding,
subject=Subject.principal_api_key("subject-secret"),
role="reader",
)
iterator = authz.list_acls(object=obj, subject=Subject.role("reader"))
entries = (
[entry async for entry in iterator] if asynchronous else list(iterator)
)
assert entries == [
AccessControlEntry(
str(obj), Subject.group_id("team-id"), None, "PRIVILEGE20"
)
]
iterator = authz.list_role_bindings(
subject=Subject.group_id("team-id"), role="reader"
)
bindings = (
[binding async for binding in iterator] if asynchronous else list(iterator)
)
assert bindings == [
RoleBinding(Subject.principal_id("alice-id"), "Alice", "reader")
]
assert await call(
authz.get_principal, Subject.principal_id("alice-id")
) == PrincipalInfo("alice-id", None, [IdentityInfo("team-id")], ["reader"])
assert await call(
authz.get_group, Subject.group_name("Engineering")
) == GroupInfo(
"team-id", "Engineering", [IdentityInfo("alice-id", "Alice")], ["reader"]
)
asyncio.run(exercise())
assert [path for path, _, _ in requests] == [
"/admin/authz/acl/add",
"/admin/authz/acl/delete",
"/admin/authz/role_binding/add",
"/admin/authz/role_binding/delete",
"/admin/authz/acl/list",
"/admin/authz/role_binding/list",
"/admin/authz/principal",
"/admin/authz/group",
]
assert [body for _, _, body in requests] == [
{"object": str(obj), "subject": "G:Engineering", "privilege": "SELECT"},
{"object": str(obj), "subject": "g:team-id", "privilege": "SELECT"},
{"subject": "P:Alice", "role": "reader"},
{"subject": "a:subject-secret", "role": "reader"},
{"object": str(obj), "subject": "r:reader", "max_results": 100},
{"subject": "g:team-id", "role": "reader", "max_results": 100},
{"principal": "p:alice-id"},
{"group": "G:Engineering"},
]
for _, headers, _ in requests:
headers = {key.lower(): value for key, value in headers.items()}
assert "x-lancedb-database" not in headers
assert "x-lancedb-database-prefix" not in headers
assert headers["authorization"] == "Bearer rotating-token"
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("limit", [None, 0, 1, 3])
def test_lazy_expanded_acl_pagination(catalog_server, asynchronous, limit):
endpoint, requests, responses = catalog_server
def entry(privilege):
return {
"object": "database:db",
"subject_id": "r:reader",
"privilege": privilege,
}
responses.extend(
[
(
200,
{
"entries": [entry("SELECT"), entry("INSERT")],
"continuation_token": "opaque/first",
},
),
(200, {"entries": [], "continuation_token": "opaque/second"}),
(200, {"entries": [entry("UPDATE"), entry("DELETE")]}),
]
)
async def exercise():
catalog = (
await lancedb.connect_catalog_async(endpoint, api_key="caller-secret")
if asynchronous
else lancedb.connect_catalog(endpoint, api_key="caller-secret")
)
iterator = catalog.authz.list_acls(subject=Subject.role("reader"), limit=limit)
assert requests == []
return [entry async for entry in iterator] if asynchronous else list(iterator)
values = asyncio.run(exercise())
expected = ["SELECT", "INSERT", "UPDATE", "DELETE"]
assert [entry.privilege for entry in values] == (
expected if limit is None else expected[:limit]
)
assert len(requests) == (0 if limit == 0 else 1 if limit == 1 else 3)
for _, headers, body in requests:
assert headers["x-api-key"] == "caller-secret"
assert body["subject"] == "r:reader"
if len(requests) == 3:
assert requests[1][2]["continuation_token"] == "opaque/first"
assert requests[2][2]["continuation_token"] == "opaque/second"
@pytest.mark.parametrize("status", [400, 401, 403, 404, 409, 501, 503])
def test_http_errors_preserve_status_without_replaying_writes(catalog_server, status):
endpoint, requests, responses = catalog_server
responses.append((status, {"error": "authorization failure"}))
authz = lancedb.connect_catalog(endpoint).authz
with pytest.raises(HttpError) as error:
authz.add_acl(
object="system", subject=Subject.role("operator"), privilege="OPERATE"
)
assert error.value.status_code == status
assert error.value.request_id
assert len(requests) == 1
@pytest.mark.parametrize("asynchronous", [False, True])
def test_role_pagination_and_late_failure(catalog_server, asynchronous):
endpoint, requests, responses = catalog_server
responses.extend(
[
(
200,
{
"bindings": [{"subject_id": "g:team", "role": "reader"}],
"continuation_token": "next",
},
),
(403, {"error": "denied"}),
]
)
async def exercise():
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
iterator = catalog.authz.list_role_bindings(role="reader")
first = await anext(iterator) if asynchronous else next(iterator)
assert first.subject == Subject.group_id("team")
assert len(requests) == 1
with pytest.raises(HttpError) as error:
if asynchronous:
await anext(iterator)
else:
next(iterator)
assert error.value.status_code == 403
asyncio.run(exercise())
assert requests[1][2] == {
"role": "reader",
"max_results": 100,
"continuation_token": "next",
}
def test_early_close_does_not_fetch_next_page(catalog_server):
endpoint, requests, responses = catalog_server
responses.append(
(
200,
{
"bindings": [{"subject_id": "g:team", "role": "reader"}],
"continuation_token": "next",
},
)
)
iterator = lancedb.connect_catalog(endpoint).authz.list_role_bindings()
next(iterator)
iterator.close()
assert len(requests) == 1
def test_selectors_objects_and_redaction():
secret = Subject.principal_api_key("do-not-print-this")
assert "do-not-print-this" not in repr(secret)
assert "do-not-print-this" not in str(secret)
with pytest.raises(FrozenInstanceError):
secret.kind = "role"
assert str(Object.system()) == "system"
assert str(Object.database("tenant/db")) == "database:tenant/db"
assert (
str(Object.namespace(database="db", namespace_path=["a", "b"]))
== "namespace:db:a$b"
)
assert str(Object.table(database="db", name="events")) == "table:db:public$events"
assert str(Object.view(database="db", name="recent")) == "view:db:public$recent"
assert Subject.principal_id("p:id")._wire() == "p:p:id"
assert Subject.group_id("team-id").value == "team-id"
for namespace in (["a$b"], [""], "public", [".."]):
with pytest.raises(ValueError):
Object.table(database="db", namespace_path=namespace, name="events")
for database in ("", "a//b", "a/../b", "a:b", "a%2Fb", "a$b"):
with pytest.raises(ValueError):
Object.database(database)
@pytest.mark.asyncio
async def test_validation_before_io_and_empty_lists():
inner = AsyncMock()
inner.list_acls.return_value = json.dumps({"entries": []})
inner.list_role_bindings.return_value = json.dumps({"bindings": []})
authz = AsyncAuthorization(inner)
for limit in (-1, 1.5, True):
with pytest.raises(ValueError):
authz.list_acls(limit=limit)
with pytest.raises(ValueError):
Authorization(authz).list_role_bindings(limit=limit)
with pytest.raises(ValueError):
await authz.get_group(Subject.principal_id("alice"))
with pytest.raises(ValueError):
await authz.get_principal(Subject.group_id("team"))
for method in (authz.add_role_binding, authz.delete_role_binding):
with pytest.raises(ValueError):
await method(subject=Subject.role("reader"), role="writer")
with pytest.raises(ValueError):
authz.list_role_bindings(subject=Subject.role("reader"))
assert [entry async for entry in authz.list_acls(limit=0)] == []
assert inner.mock_calls == []
assert [entry async for entry in authz.list_acls()] == []
assert [entry async for entry in authz.list_role_bindings()] == []
def test_api_key_subject_is_not_logged(catalog_server):
import os
import subprocess
import sys
endpoint, requests, responses = catalog_server
responses.append((200, {"principal_id": "public-id", "groups": [], "roles": []}))
result = subprocess.run(
[
sys.executable,
"-c",
"import lancedb, sys; from lancedb.authz import Subject; "
"lancedb.connect_catalog(sys.argv[1]).authz.get_principal("
"Subject.principal_api_key('test-subject-secret-never-log'))",
endpoint,
],
env={**os.environ, "LANCEDB_LOG": "debug"},
capture_output=True,
text=True,
check=True,
timeout=30,
)
assert requests[0][2] == {"principal": "a:test-subject-secret-never-log"}
assert "test-subject-secret-never-log" not in result.stdout + result.stderr
@pytest.mark.parametrize(
"builder, kwargs, wire",
[
("system", {}, "system"),
("secret", {"database": "db", "name": "api-key"}, "secret:db:public$api-key"),
(
"function",
{"database": "db", "name": "caption"},
"function:db:public$caption",
),
(
"secret",
{"database": "tenant/db", "namespace_path": ["a", "b"], "name": "api-key"},
"secret:tenant/db:a$b$api-key",
),
(
"function",
{"database": "tenant/db", "namespace_path": ["a", "b"], "name": "caption"},
"function:tenant/db:a$b$caption",
),
("database", {"name": "tenant/db"}, "database:tenant/db"),
(
"namespace",
{"database": "tenant/db", "namespace_path": ["a", "b"]},
"namespace:tenant/db:a$b",
),
(
"table",
{"database": "tenant/db", "namespace_path": ["a", "b"], "name": "events"},
"table:tenant/db:a$b$events",
),
(
"view",
{"database": "db", "namespace_path": ["public"], "name": "recent"},
"view:db:public$recent",
),
],
)
def test_rust_backed_object_builders(builder, kwargs, wire):
import pickle
from lancedb import _lancedb
value = getattr(Object, builder)(**kwargs)
assert str(value) == wire
assert str(value._inner) == wire
assert str(_lancedb.AuthzObject(wire)) == wire
assert value == Object(wire)
assert hash(value) == hash(Object(wire))
assert pickle.loads(pickle.dumps(value)) == value
@pytest.mark.parametrize(
"kind, prefix",
[
("principal_id", "p:"),
("principal_name", "P:"),
("group_id", "g:"),
("group_name", "G:"),
("role", "r:"),
("principal_api_key", "a:"),
],
)
def test_rust_backed_subject_selectors(kind, prefix):
import pickle
from lancedb import _lancedb
value = "p:literal@example.com / 名前"
subject = getattr(Subject, kind)(value)
assert subject.kind == kind
assert subject.value == value
assert subject._wire() == prefix + value
assert _lancedb.AuthzSubject(kind, value).to_wire() == prefix + value
assert pickle.loads(pickle.dumps(subject)) == subject
assert hash(subject) == hash(getattr(Subject, kind)(value))
with pytest.raises(ValueError):
getattr(Subject, kind)("")
if kind == "principal_api_key":
assert value not in repr(subject._inner)
assert value not in str(subject._inner)
if kind in ("principal_id", "group_id", "role"):
assert Subject._canonical(prefix + value) == subject
else:
with pytest.raises(ValueError) as error:
Subject._canonical(prefix + value)
assert value not in str(error.value)
@pytest.mark.parametrize(
"wire",
[
"",
"database:",
"database:a//b",
"database:a/../b",
"database:a%2Fb",
"namespace:db:a$$b",
"table:db:events",
"table:db:public$",
"table:db:public$..",
"table:db:a/b$events",
"view:db:public$café",
"secret:db:public$",
"function:db:public$..",
"secret:db:a$$key",
"function:db:caption",
"secret:db:public$bad/name",
"function:db:public$bad:name",
"system:db",
],
)
def test_invalid_raw_objects_fail_before_io(catalog_server, wire):
endpoint, requests, _ = catalog_server
authz = lancedb.connect_catalog(endpoint).authz
with pytest.raises(ValueError):
Object(wire)
with pytest.raises(ValueError):
authz.list_acls(object=wire)
with pytest.raises(ValueError):
authz.add_acl(object=wire, subject=Subject.role("reader"), privilege="SELECT")
assert requests == []
@pytest.mark.parametrize(
"wire",
[
"future:db:resource",
"table_extension:db:resource",
"secret:tenant/db:a$b$key",
"function:tenant/db:a$b$caption",
],
)
def test_objects_round_trip_through_acl_requests(catalog_server, wire):
endpoint, requests, responses = catalog_server
responses.extend(
[
(200, {}),
(
200,
{
"entries": [
{
"object": wire,
"subject_id": "r:reader",
"privilege": "SELECT",
}
]
},
),
(200, {}),
]
)
authz = lancedb.connect_catalog(endpoint).authz
obj = Object(wire)
assert str(obj) == wire
authz.add_acl(object=obj, subject=Subject.role("reader"), privilege="SELECT")
entries = list(authz.list_acls(object=wire))
assert entries[0].object == wire
authz.delete_acl(
object=entries[0].object, subject=Subject.role("reader"), privilege="SELECT"
)
assert len(requests) == 3
for _, _, body in requests:
assert body["object"] == wire
@pytest.mark.asyncio
async def test_native_requests_validate_selector_context_before_io(catalog_server):
endpoint, requests, _ = catalog_server
catalog = await lancedb.connect_catalog_async(endpoint)
native = catalog.authz._inner
for method, request in [
(native.list_role_bindings, {"subject": "r:reader"}),
(native.add_role_binding, {"subject": "r:reader", "role": "writer"}),
(native.delete_role_binding, {"subject": "r:reader", "role": "writer"}),
(native.get_principal, {"principal": "g:group"}),
(native.get_group, {"group": "a:never-log-this-key"}),
]:
with pytest.raises(ValueError) as error:
await method(json.dumps(request))
assert "never-log-this-key" not in str(error.value)
assert requests == []
def test_privilege_values_use_rust_serialization():
import pickle
from lancedb import _lancedb
names = [value.value for value in Privilege]
assert names == _lancedb.AuthzPrivilege.known()
for privilege in Privilege:
assert json.loads(json.dumps(privilege)) == privilege.value
assert str(_lancedb.AuthzPrivilege(privilege)) == privilege.value
assert pickle.loads(pickle.dumps(privilege)) is privilege
for index in range(64):
alias = f"PRIVILEGE{index}"
expected = names[index] if index < len(names) else alias
assert str(_lancedb.AuthzPrivilege(alias)) == expected
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"creation_privilege", [None, "CREATE_SECRET", "CREATE_FUNCTION"]
)
def test_privilege_names_and_future_values_round_trip(
catalog_server, asynchronous, creation_privilege
):
endpoint, requests, responses = catalog_server
inputs = [
Privilege.SELECT,
"SELECT",
"PRIVILEGE2",
"PRIVILEGE20",
"PRIVILEGE63",
"PRIVILEGE64",
"PRIVILEGE256",
"PRIVILEGE18446744073709551616",
]
expected = ["SELECT", "SELECT", "SELECT", *inputs[3:]]
object_id = "table:db:public$t"
if creation_privilege is not None:
inputs = [Privilege[creation_privilege], creation_privilege]
expected = [creation_privilege, creation_privilege]
object_id = "namespace:db:public"
for privilege in inputs:
responses.extend(
[
(200, {}),
(
200,
{
"entries": [
{
"object": object_id,
"subject_id": "r:reader",
"privilege": privilege,
}
]
},
),
(200, {}),
]
)
async def exercise():
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
authz = catalog.authz
for privilege, canonical in zip(inputs, expected):
added = authz.add_acl(
object=object_id,
subject=Subject.role("reader"),
privilege=privilege,
)
if asynchronous:
await added
iterator = authz.list_acls()
entries = (
[entry async for entry in iterator] if asynchronous else list(iterator)
)
assert entries[0].privilege == canonical
deleted = authz.delete_acl(
object=entries[0].object,
subject=entries[0].subject,
privilege=entries[0].privilege,
)
if asynchronous:
await deleted
asyncio.run(exercise())
assert len(requests) == 3 * len(inputs)
for index, canonical in enumerate(expected):
assert requests[index * 3][2]["object"] == object_id
assert requests[index * 3][2]["privilege"] == canonical
assert requests[index * 3 + 2][2]["object"] == object_id
assert requests[index * 3 + 2][2]["privilege"] == canonical
@pytest.mark.parametrize("asynchronous", [False, True])
def test_invalid_privileges_fail_before_io(catalog_server, asynchronous):
endpoint, requests, _ = catalog_server
invalid = [
"",
"select",
"SELECT ",
"UNKNOWN",
"PRIVILEGE",
"PRIVILEGE-1",
"PRIVILEGE+1",
"PRIVILEGE1x",
None,
2,
True,
["SELECT"],
]
async def exercise():
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
async_authz = catalog.authz if asynchronous else catalog.authz._inner
native = async_authz._inner
for privilege in invalid:
for method in [catalog.authz.add_acl, catalog.authz.delete_acl]:
with pytest.raises(ValueError, match="privilege"):
result = method(
object="system",
subject=Subject.role("reader"),
privilege=privilege,
)
if asynchronous:
await result
for method in [
native.add_acl,
native.delete_acl,
]:
with pytest.raises(ValueError):
await method(
json.dumps(
{
"object": "system",
"subject": "r:reader",
"privilege": privilege,
}
)
)
asyncio.run(exercise())
assert requests == []
@pytest.mark.parametrize("asynchronous", [False, True])
def test_acl_page_preserves_unknown_subjects(catalog_server, asynchronous):
import pickle
endpoint, requests, responses = catalog_server
responses.extend(
[
(
200,
{
"entries": [
{
"subject_id": "p:known",
"object": "system",
"privilege": "USAGE",
},
{
"subject_id": "x:future-subject",
"object": "system",
"privilege": "USAGE",
},
]
},
),
(200, {}),
]
)
async def exercise():
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
iterator = catalog.authz.list_acls()
entries = (
[entry async for entry in iterator] if asynchronous else list(iterator)
)
assert len(entries) == 2
assert entries[0].subject == Subject.principal_id("known")
subject = entries[1].subject
assert subject.kind == "unknown"
assert subject.value == "x:future-subject"
assert subject._wire() == "x:future-subject"
assert pickle.loads(pickle.dumps(subject)) == subject
result = catalog.authz.delete_acl(
object=entries[1].object, subject=subject, privilege=entries[1].privilege
)
if asynchronous:
await result
asyncio.run(exercise())
assert len(requests) == 2
assert requests[1][2]["subject"] == "x:future-subject"
@pytest.mark.parametrize("builder", ["secret", "function"])
def test_secret_and_function_builders_validate_components(builder):
from lancedb import _lancedb
for database, namespace, name in [
("a//b", ["public"], "valid"),
("db", ["a$b"], "valid"),
("db", "public", "valid"),
("db", ["public"], "bad$name"),
("db", ["public"], ".."),
]:
with pytest.raises(ValueError):
getattr(Object, builder)(
database=database, namespace_path=namespace, name=name
)
with pytest.raises(ValueError):
getattr(_lancedb.AuthzObject, builder)(database, namespace, name)
@pytest.mark.parametrize(
"builder", ["namespace", "table", "view", "secret", "function"]
)
def test_empty_and_omitted_namespace_paths_select_public(builder):
from lancedb import _lancedb
kwargs = {"database": "db"}
native_args = ()
wire = "namespace:db:public"
if builder != "namespace":
kwargs["name"] = "item"
native_args = ("item",)
wire = f"{builder}:db:public$item"
build = getattr(Object, builder)
expected = build(**kwargs, namespace_path=["public"])
assert str(expected) == wire
assert build(**kwargs) == expected
for namespace in [[], ()]:
assert build(**kwargs, namespace_path=namespace) == expected
assert (
str(getattr(_lancedb.AuthzObject, builder)("db", namespace, *native_args))
== wire
)
python/python/tests/test_blob.py
+875 -5
View File
File diff suppressed because it is too large Load Diff
python/python/tests/test_catalog.py
+42
View File
@@ -263,3 +263,45 @@ async def test_database_names_invalid_limit(catalog_server, asynchronous, page_l
next(names)
assert names.num_page_results() == 0
assert requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_catalog_rejects_invalid_database_names_before_io(
catalog_server, asynchronous
):
endpoint, requests, _ = catalog_server
catalog = (
await lancedb.connect_catalog_async(endpoint)
if asynchronous
else lancedb.connect_catalog(endpoint)
)
for name in [
"",
".",
"..",
"/db",
"db/",
"a//b",
"a/./b",
"a/../b",
"a b",
"a:b",
"a$b",
"a%b",
"a?b",
"a#b",
"a\\b",
"café",
"a\nb",
]:
for method in [
catalog.create_database,
catalog.connect_database,
catalog.drop_database,
]:
with pytest.raises(ValueError, match="Invalid database name"):
result = method(name)
if asynchronous:
await result
assert requests == []
python/python/tests/test_db.py
+34
View File
@@ -245,6 +245,40 @@ def test_db_contains_and_len_include_all_table_name_pages(tmp_db: lancedb.DBConn
assert "does_not_exist" not in tmp_db
@pytest.mark.asyncio
async def test_table_listing_defaults_and_zero_limit(tmp_path):
names = [f"t{i:03}" for i in range(130)]
for name in names:
(tmp_path / f"{name}.lance").mkdir()
db = lancedb.connect(tmp_path)
assert db.table_names() == names
assert db.table_names(limit=None) == names
assert db.table_names(limit=0) == []
first = db.list_tables()
assert first.tables == names[:100]
assert first.page_token
second = db.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
assert db.list_tables(limit=200).tables == names
empty = db.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
adb = await lancedb.connect_async(tmp_path)
assert await adb.table_names() == names
assert await adb.table_names(limit=0) == []
first = await adb.list_tables()
assert first.tables == names[:100]
second = await adb.list_tables(page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
empty = await adb.list_tables(limit=0)
assert empty.tables == []
assert empty.page_token is None
def test_db_contains_stops_after_matching_table_page(
tmp_db: lancedb.DBConnection, monkeypatch
):
python/python/tests/test_e2e_remote_db.py
+65
View File
@@ -2,13 +2,78 @@
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import os
from datetime import timedelta
from uuid import uuid4
import lancedb
import numpy as np
import pyarrow as pa
import pytest
from lancedb import LanceDBConnection
from lancedb.index import FTS
# TODO: setup integ test mark and script
@pytest.mark.skipif(
not os.environ.get("LANCEDB_URI"), reason="Set LANCEDB_URI to a JSON FTS server"
)
def test_json_fts_against_server():
"""Requires a writable database and capacity to run the distributed indexer.
Configure LANCEDB_URI, LANCEDB_API_KEY, and optionally LANCEDB_REGION and
LANCEDB_HOST_OVERRIDE before running this test.
"""
uri = os.environ["LANCEDB_URI"]
assert uri.startswith("db://"), "LANCEDB_URI must identify a remote database"
conn = lancedb.connect(
uri,
api_key=os.environ.get("LANCEDB_API_KEY"),
region=os.environ.get("LANCEDB_REGION", "us-east-1"),
host_override=os.environ.get("LANCEDB_HOST_OVERRIDE"),
)
name = f"json_fts_{uuid4().hex}"
table = conn.create_table(
name,
pa.table(
{
"id": [0, 1, 2],
"doc": pa.array(
[
'{"meta":{"tag":"nature"},"author":"alice"}',
'{"meta":{"tag":"city"},"author":"nature"}',
'{"meta":{"tag":"nature"},"author":"bob"}',
],
type=pa.json_(),
),
"raw": pa.array([b"nature"] * 3, type=pa.large_binary()),
}
),
)
try:
table.create_index("doc", config=FTS(), name="doc_fts")
table.wait_for_index(["doc_fts"], timeout=timedelta(minutes=5))
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 3
assert stats.num_unindexed_rows == 0
assert stats.index_type == "FTS"
for query, expected in [
("meta.tag,str,nature", [0, 2]),
("meta.tag,str,alice", []),
("meta.tag,str,tag", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
finally:
conn.drop_table(name)
@pytest.mark.skip(reason="Need to set up a local server")
def test_against_local_server():
conn = LanceDBConnection("lancedb+http://localhost:10024")
python/python/tests/test_first_class_function_slice2.py
+1 -1
View File
@@ -1555,7 +1555,7 @@ def _mock_remote_function_catalog():
)["image"],
"signature": body["signature"],
"secret_bindings": body.get("secret_bindings", []),
"created_at": "2026-08-21T00:00:00Z",
"created_at_millis": 1787270400000, # 2026-08-21T00:00:00Z
}
response = {"job_id": "job-register"}
status = 202
python/python/tests/test_fts.py
+69
View File
@@ -45,6 +45,75 @@ from utils import exception_output
TEST_LANGUAGE_MODEL_HOME = Path(__file__).parent / "models"
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.parametrize("block_size", [128, 256])
def test_json_fts(tmp_path, block_size):
db = ldb.connect(tmp_path)
data = pa.table(
{
"id": [0, 1, 2, 3],
"doc": pa.array(
[
'{"title":"brown fox","meta":{"tag":"nature"},"author":"alice"}',
'{"title":"blue car","meta":{"tag":"city"}}',
'{"title":"brown bear","meta":{"tag":"nature"}}',
'{"title":"red fox","author":"alice"}',
],
type=pa.json_(),
),
}
)
table = db.create_table("json_fts", data)
table.create_index("doc", config=FTS(block_size=block_size), name="doc_fts")
table = db.open_table("json_fts")
index = table.list_indices()[0]
assert index.name == "doc_fts"
assert index.index_type == "FTS"
assert index.columns == ["doc"]
stats = table.index_stats("doc_fts")
assert stats.num_indexed_rows == 4
assert stats.num_unindexed_rows == 0
for query, expected in [
("title,str,brown", [0, 2]),
("meta.tag,str,nature", [0, 2]),
("author,str,alice", [0, 3]),
("title,str,alice", []),
("title,str,title", []),
]:
search = table.search(query, query_type="fts", fts_columns="doc")
assert sorted(row["id"] for row in search.to_list()) == expected
plan = search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type")
@pytest.mark.asyncio
async def test_json_fts_async(tmp_path):
db = await ldb.connect_async(tmp_path)
table = await db.create_table(
"json_fts",
pa.table({"doc": pa.array(['{"meta":{"tag":"nature"}}'], type=pa.json_())}),
)
await table.create_index("doc", config=FTS())
search = table.query().nearest_to_text(MatchQuery("meta.tag,str,nature", "doc"))
assert len(await search.to_list()) == 1
plan = await search.explain_plan(True)
assert "MatchQuery" in plan, plan
assert "FlatMatchQuery" not in plan, plan
@pytest.mark.parametrize("dtype", [pa.binary(), pa.large_binary()])
def test_fts_rejects_raw_binary(tmp_path, dtype):
table = ldb.connect(tmp_path).create_table(
"binary_fts", pa.table({"doc": pa.array([b'{"title":"brown"}'], type=dtype)})
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("doc", config=FTS())
assert table.list_indices() == []
@pytest.fixture
def table(tmp_path) -> ldb.table.LanceTable:
# Use local random state to avoid affecting other tests
python/python/tests/test_hybrid_query.py
+14
View File
@@ -355,6 +355,20 @@ def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
)
def test_hybrid_query_nprobes_remains_independent(sync_table: Table):
query = sync_table.search(query_type="hybrid").nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes is None
assert query._maximum_nprobes is None
query = query.minimum_nprobes(5).nprobes(20)
assert query._nprobes == 20
assert query._minimum_nprobes == 5
assert query._maximum_nprobes is None
def test_hybrid_query_distance_range(sync_table: Table):
reranker = RRFReranker(return_score="all")
result = (
python/python/tests/test_namespace.py
+167 -1
View File
@@ -6,12 +6,17 @@
import tempfile
import shutil
import importlib
import http.server
import json
import threading
from urllib.parse import parse_qs, unquote, urlparse
import pytest
import pyarrow as pa
import lancedb
from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from lancedb.listing import AsyncListing, Listing
from lancedb.namespace import _MAX_QUERY_K
from lancedb.namespace import _MAX_QUERY_K, _query_to_namespace_request
from lancedb.query import Query
from lancedb.table import AsyncTable, LanceTable
@@ -73,6 +78,132 @@ def _namespace_lance_table(namespace_client: _NamespaceClient) -> LanceTable:
return table
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["test_ns"]])
async def test_namespace_table_names_defaults_and_limits(tmp_path, namespace_path):
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
if namespace_path:
db.create_namespace(namespace_path)
names = [f"t{idx:03}" for idx in range(12)]
schema = pa.schema([pa.field("id", pa.int64())])
for name in names:
db.create_table(name, schema=schema, namespace_path=namespace_path)
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(db.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
assert (
list(
db.table_names(page_token=names[1], limit=2, namespace_path=namespace_path)
)
== names[2:4]
)
adb = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert (
list(await adb.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
)
assert (
list(
await adb.table_names(
page_token=names[1], limit=2, namespace_path=namespace_path
)
)
== names[2:4]
)
@pytest.fixture
def paginated_rest_namespace():
names = [f"t{idx:03}" for idx in reversed(range(12))]
requests = []
class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
url = urlparse(self.path)
query = parse_qs(url.query)
requests.append((unquote(url.path), query))
token = query.get("page_token", [None])[0]
if token not in {None, "opaque-next"}:
self.send_error(400, "Unknown page token")
return
start = 0 if token is None else 10
body = {"tables": names[start : start + 10]}
if start == 0:
body["page_token"] = "opaque-next"
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.end_headers()
self.wfile.write(json.dumps(body).encode())
def log_message(self, *args):
pass
with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever)
thread.start()
try:
yield {"uri": f"http://127.0.0.1:{server.server_port}"}, requests
finally:
server.shutdown()
thread.join()
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace_path", [[], ["n"]])
async def test_rest_namespace_table_names_collects_pages(
paginated_rest_namespace, namespace_path
):
properties, requests = paginated_rest_namespace
names = [f"t{idx:03}" for idx in range(12)]
db = lancedb.connect_namespace("rest", properties)
first = db.list_tables(namespace_path=namespace_path)
second = db.list_tables(namespace_path=namespace_path, page_token=first.page_token)
assert len(first.tables) == 10
assert first.page_token == "opaque-next"
assert sorted(first.tables + second.tables) == names
assert second.page_token is None
assert list(db.table_names(namespace_path=namespace_path)) == names
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
assert list(db.table_names(limit=11, namespace_path=namespace_path)) == names[:11]
assert (
list(db.table_names(page_token="t005", limit=2, namespace_path=namespace_path))
== names[6:8]
)
adb = lancedb.connect_namespace_async("rest", properties)
assert list(await adb.table_names(namespace_path=namespace_path)) == names
assert (
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
)
assert (
list(await adb.table_names(limit=11, namespace_path=namespace_path))
== names[:11]
)
assert (
list(
await adb.table_names(
page_token="t005", limit=2, namespace_path=namespace_path
)
)
== names[6:8]
)
request_count = len(requests)
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
assert len(requests) == request_count
identifier = "$" if not namespace_path else "n"
assert all(path == f"/v1/namespace/{identifier}/table/list" for path, _ in requests)
class TestNamespaceConnection:
"""Test namespace-based LanceDB connection using DirectoryNamespace."""
@@ -981,6 +1112,29 @@ class TestPushdownOperations:
]
assert all(r.k <= 2**31 - 1 for r in namespace_client.requests)
def test_probe_fields_pass_through_independently(self):
request = _query_to_namespace_request(
["geneva", "hist"],
Query(
vector=[1.0, 2.0],
nprobes=20,
minimum_nprobes=3,
maximum_nprobes=10,
),
)
assert request.nprobes == 20
assert request.minimum_nprobes == 3
assert request.maximum_nprobes == 10
request = _query_to_namespace_request(
["geneva", "hist"],
Query(vector=[1.0, 2.0], nprobes=5, maximum_nprobes=0),
)
assert request.nprobes == 5
assert request.maximum_nprobes == 0
assert request.to_dict()["maximum_nprobes"] == 0
@pytest.mark.asyncio
class TestAsyncPushdownOperations:
@@ -1055,3 +1209,15 @@ def test_local_table_to_arrow_and_to_pandas_are_unchanged(tmp_path):
assert table.to_arrow().column("id").to_pylist() == [1, 2]
assert table.to_pandas()["id"].tolist() == [1, 2]
@pytest.mark.parametrize("effort", [0.0, 0.5, 1.0])
def test_namespace_request_rejects_search_effort(effort):
from lancedb.query import Query
with pytest.raises(
ValueError, match="namespace QueryTable does not support search_effort"
):
_query_to_namespace_request(
["test"], Query(vector=[1.0, 0.0], search_effort=effort)
)
python/python/tests/test_query.py
+229 -33
View File
@@ -509,18 +509,30 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
)
def test_vector_query_to_pandas_blob_descriptions_requires_plain_scan(tmp_db):
def test_vector_query_to_pandas_legacy_blob_descriptions_and_bytes(tmp_db):
pytest.importorskip("lance")
table = tmp_db.create_table(
"test_vector_query_blob_descriptions",
_blob_query_data(),
storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
)
query = table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1)
with pytest.raises(RuntimeError, match="plain scan query"):
table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas(
blob_mode="descriptions"
)
df = query.to_pandas(blob_mode="descriptions")
assert df["blob"].tolist() == [{"position": 0, "size": 3}]
with pytest.raises(ValueError, match="legacy blob columns"):
query.to_pandas(blob_mode="bytes")
with pytest.raises(ValueError, match="legacy blob columns"):
query.to_arrow(blob_mode="bytes")
with pytest.raises(ValueError, match="legacy blob columns"):
table.to_arrow(blob_mode="bytes")
assert table.to_pandas(blob_mode="bytes")["blob"].tolist() == [
b"one",
b"two",
b"three",
b"four",
]
def test_order_by_plain_query(mem_db):
@@ -995,12 +1007,22 @@ def test_invalid_nprobes_sync(table):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").maximum_nprobes(5).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
LanceVectorQueryBuilder(table, [0, 0], "vector").minimum_nprobes(100).to_list()
(
LanceVectorQueryBuilder(table, [0, 0], "vector")
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_nprobes_works_sync(table):
@@ -1039,12 +1061,22 @@ async def test_invalid_nprobes_async(table_async: AsyncTable):
ValueError,
match="maximum_nprobes must be greater than or equal to minimum_nprobes",
):
await table_async.vector_search([0, 0]).maximum_nprobes(5).to_list()
await (
table_async.vector_search([0, 0])
.minimum_nprobes(10)
.maximum_nprobes(5)
.to_list()
)
with pytest.raises(
ValueError,
match="minimum_nprobes must be less than or equal to maximum_nprobes",
):
await table_async.vector_search([0, 0]).minimum_nprobes(100).to_list()
await (
table_async.vector_search([0, 0])
.maximum_nprobes(5)
.minimum_nprobes(100)
.to_list()
)
def test_query_builder_with_prefilter(table):
@@ -1635,8 +1667,9 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
)
@@ -1654,8 +1687,19 @@ def test_query_serialization_sync(table: lancedb.table.Table):
q,
vector_column="vector",
vector=[5.0, 6.0],
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).minimum_nprobes(5).nprobes(50).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=50,
minimum_nprobes=5,
maximum_nprobes=None,
)
q = table.search([5.0, 6.0]).maximum_nprobes(10).to_query_object()
@@ -1667,6 +1711,15 @@ def test_query_serialization_sync(table: lancedb.table.Table):
minimum_nprobes=None,
)
q = table.search([5.0, 6.0]).nprobes(5).maximum_nprobes(0).to_query_object()
check_set_props(
q,
vector_column="vector",
vector=[5.0, 6.0],
nprobes=5,
maximum_nprobes=0,
)
q = table.search([5.0, 6.0]).distance_range(0.0, 1.0).to_query_object()
check_set_props(
q, vector_column="vector", vector=[5.0, 6.0], lower_bound=0.0, upper_bound=1.0
@@ -1716,8 +1769,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
limit=10,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1727,8 +1780,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1739,8 +1792,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
postfilter=False,
minimum_nprobes=50,
maximum_nprobes=50,
nprobes=50,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1759,8 +1813,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
filter="id = 1",
postfilter=True,
vector=sample_vector,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
)
@@ -1774,8 +1828,9 @@ async def test_query_serialization_async(table_async: AsyncTable):
check_set_props(
q,
vector=sample_vector,
minimum_nprobes=10,
maximum_nprobes=10,
nprobes=10,
minimum_nprobes=None,
maximum_nprobes=None,
refine_factor=5,
postfilter=False,
with_row_id=False,
@@ -1788,7 +1843,24 @@ async def test_query_serialization_async(table_async: AsyncTable):
q,
vector=sample_vector,
minimum_nprobes=5,
maximum_nprobes=20,
maximum_nprobes=None,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
limit=10,
)
q = (
(await table_async.search([5.0, 6.0]))
.nprobes(5)
.maximum_nprobes(0)
.to_query_object()
)
check_set_props(
q,
vector=sample_vector,
nprobes=5,
maximum_nprobes=0,
postfilter=False,
with_row_id=False,
bypass_vector_index=False,
@@ -1806,8 +1878,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
lower_bound=0.0,
upper_bound=1.0,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1819,8 +1891,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
distance_type="cosine",
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1832,8 +1904,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
ef=7,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
bypass_vector_index=False,
limit=10,
@@ -1845,8 +1917,8 @@ async def test_query_serialization_async(table_async: AsyncTable):
bypass_vector_index=True,
vector=sample_vector,
postfilter=False,
minimum_nprobes=20,
maximum_nprobes=20,
minimum_nprobes=None,
maximum_nprobes=None,
with_row_id=False,
limit=10,
)
@@ -2205,3 +2277,127 @@ def test_blob_v2_to_batches_row_id(tmp_db):
assert "_rowid" in hits.column_names
blobs = table.fetch_blobs("blob", hits)
assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"]
@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"])
@pytest.mark.parametrize(
"effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)]
)
def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes):
import re
from lancedb.index import IvfFlat
monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0")
monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4")
monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4")
rng = np.random.default_rng(42)
vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0)
vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32)
data = pa.table(
{
"id": np.arange(512),
"text": ["dog"] * 512,
"vector": pa.FixedSizeListArray.from_arrays(pa.array(vectors.ravel()), 16),
}
)
table = lancedb.connect(tmp_path).create_table("effort", data.slice(0, 256))
table.add(data.slice(256))
table.create_index(
"vector", config=IvfFlat(distance_type=metric, num_partitions=16)
)
vector = np.eye(16, dtype=np.float32)[0]
def search(value=effort):
return table.search(vector).distance_type(metric).search_effort(value).limit(8)
def partitions(query):
vector_plan = query.analyze_plan().split("FTS Search Plan:", 1)[0]
values = re.findall(r"partitions_searched=(\d+)", vector_plan)
assert values
return sum(map(int, values))
assert partitions(search()) == probes
exact = set(
table.search(vector)
.distance_type(metric)
.bypass_vector_index()
.limit(8)
.to_arrow()["id"]
.to_pylist()
)
actual = set(search().to_arrow()["id"].to_pylist())
assert len(actual & exact) / 8 >= 0.5
expected = 1 if effort is None else probes
assert partitions(search().nprobes(1)) == expected
assert (
partitions(
table.search(vector)
.distance_type(metric)
.nprobes(1)
.search_effort(effort)
.limit(8)
)
== expected
)
assert partitions(search().nprobes(1).search_effort(None)) == 1
assert partitions(search().maximum_nprobes(2)) <= 2
assert partitions(search().minimum_nprobes(3).maximum_nprobes(3)) == 3
assert partitions(search(0).nprobes(16).maximum_nprobes(0)) == 1
if effort is None:
assert search().to_arrow() == search(0.5).to_arrow()
filtered = search(0).maximum_nprobes(2).where("id % 8 = 0", prefilter=True)
assert partitions(filtered) <= 2
assert all(row["id"] % 8 == 0 for row in filtered.to_list())
if metric == "l2" and effort == 0.5:
table.create_index("text", config=FTS())
hybrid = (
table.search(query_type="hybrid")
.vector(vector)
.text("dog")
.nprobes(1)
.search_effort(0.5)
.limit(8)
)
assert partitions(hybrid) == 4
assert partitions(hybrid.search_effort(None)) == 1
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
def test_search_effort_invalid_sync(table, effort):
with pytest.raises(ValueError, match="search_effort must be finite and in"):
table.search([0, 0]).search_effort(effort).to_list()
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1])
async def test_search_effort_async_roundtrip(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
query = query.nprobes(5).search_effort(effort).minimum_nprobes(2).maximum_nprobes(3)
request = query.to_query_object()
assert request.search_effort == effort
assert request.nprobes == 5
assert request.minimum_nprobes == 2
assert request.maximum_nprobes == 3
assert query.search_effort(None).to_query_object().search_effort is None
query.search_effort(effort)
if not hybrid:
assert len(await query.limit(1).to_list()) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("hybrid", [False, True])
@pytest.mark.parametrize(
"effort", [-0.1, 1.1, float("nan"), float("inf"), -float("inf")]
)
async def test_search_effort_invalid_async(table_async, hybrid, effort):
query = table_async.vector_search([0, 0])
if hybrid:
query = query.nearest_to_text("dog")
with pytest.raises(ValueError, match="search_effort must be finite and in"):
query.search_effort(effort)
python/python/tests/test_remote_db.py
+246 -20
View File
@@ -4,6 +4,7 @@ from concurrent.futures import ThreadPoolExecutor
import contextlib
from datetime import timedelta
import http.server
import io
import json
import multiprocessing as mp
import pickle
@@ -13,6 +14,7 @@ import threading
import time
from unittest.mock import MagicMock, patch
import uuid
from urllib.parse import parse_qs, urlparse
from packaging.version import Version
import lancedb
@@ -105,6 +107,67 @@ async def mock_lancedb_connection_async(handler, **client_config):
handle.join()
def paginated_table_listing_handler(request):
url = urlparse(request.path)
assert url.path in {
"/v1/table/",
"/v1/namespace/$/table/list",
"/v1/namespace/n/table/list",
}
query = parse_qs(url.query)
limit = int(query.get("limit", [10])[0])
assert limit > 0
token = query.get("page_token", ["page-0"])[0]
start = int(token.removeprefix("page-"))
end = min(start + limit, 130)
body = {"tables": [f"t{i:03}" for i in range(start, end)]}
if end < 130:
body["page_token"] = f"page-{end}"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(body).encode())
@pytest.mark.parametrize("namespace", [[], ["n"]])
def test_remote_table_listing_defaults_and_zero_limit(namespace):
names = [f"t{i:03}" for i in range(130)]
with mock_lancedb_connection(paginated_table_listing_handler) as db:
assert db.table_names(namespace_path=namespace) == names
assert db.table_names(limit=None, namespace_path=namespace) == names
assert db.table_names(limit=0, namespace_path=namespace) == []
first = db.list_tables(namespace_path=namespace)
assert first.tables == names[:100]
assert first.page_token == "page-100"
second = db.list_tables(namespace_path=namespace, page_token=first.page_token)
assert second.tables == names[100:]
assert second.page_token is None
assert db.list_tables(namespace_path=namespace, limit=200).tables == names
empty = db.list_tables(namespace_path=namespace, limit=0)
assert empty.tables == []
assert empty.page_token is None
@pytest.mark.asyncio
@pytest.mark.parametrize("namespace", [[], ["n"]])
async def test_async_remote_table_listing_defaults_and_zero_limit(namespace):
names = [f"t{i:03}" for i in range(130)]
async with mock_lancedb_connection_async(paginated_table_listing_handler) as db:
assert await db.table_names(namespace_path=namespace) == names
assert await db.table_names(limit=0, namespace_path=namespace) == []
first = await db.list_tables(namespace_path=namespace)
assert first.tables == names[:100]
assert first.page_token == "page-100"
second = await db.list_tables(
namespace_path=namespace, page_token=first.page_token
)
assert second.tables == names[100:]
assert second.page_token is None
empty = await db.list_tables(namespace_path=namespace, limit=0)
assert empty.tables == []
assert empty.page_token is None
@pytest.mark.asyncio
async def test_async_remote_db():
def handler(request):
@@ -805,6 +868,19 @@ def test_table_unimplemented_functions():
assert f"search().{method}()" in message
@pytest.mark.parametrize("method", ["to_arrow", "to_pandas"])
@pytest.mark.parametrize("blob_mode", ["lazy", "descriptions", "bytes", "invalid"])
def test_remote_conversion_blob_mode_errors(method, blob_mode):
with blob_remote_table() as table:
if blob_mode == "invalid":
with pytest.raises(ValueError, match="blob_mode must be one of"):
getattr(table, method)(blob_mode=blob_mode)
else:
with pytest.raises(NotImplementedError) as error:
getattr(table, method)(blob_mode=blob_mode)
assert f"search().{method}()" in str(error.value)
@pytest.mark.asyncio
@pytest.mark.parametrize("version", [None, 1])
@pytest.mark.parametrize("expose_location", [False, True])
@@ -1441,6 +1517,72 @@ def test_remote_job_wait_raises_on_failure():
job.wait()
@pytest.mark.parametrize("server_version", ["0.2.0", "0.6.0"])
def test_remote_json_fts_config(server_version):
from lancedb.index import FTS
received_requests = []
def handler(request):
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.send_header("phalanx-version", server_version)
request.end_headers()
if request.path == "/v1/table/test/describe/":
fields = [
{
"name": "doc",
"type": {"type": "large_binary"},
"nullable": True,
"metadata": {"ARROW:extension:name": "lance.json"},
},
{
"name": "raw",
"type": {"type": "large_binary"},
"nullable": True,
},
]
request.wfile.write(
json.dumps({"version": 1, "schema": {"fields": fields}}).encode()
)
elif request.path == "/v1/table/test/create_index/":
received_requests.append(read_json_body(request))
request.wfile.write(b"{}")
else:
raise AssertionError(f"Unexpected request: {request.path}")
with mock_lancedb_connection(handler) as db:
table = db.open_table("test")
table.create_index(
"doc",
config=FTS(
with_position=True,
base_tokenizer="whitespace",
stem=False,
remove_stop_words=False,
lower_case=False,
block_size=256,
),
name="doc_fts",
replace=False,
)
with pytest.raises(ValueError, match="FTS.*lance.json.*raw binary"):
table.create_index("raw", config=FTS())
assert len(received_requests) == 1
request = received_requests[0]
assert request["column"] == "doc"
assert request["index_type"] == "FTS"
assert request["name"] == "doc_fts"
assert request["replace"] is False
assert request["with_position"] is True
assert request["base_tokenizer"] == "whitespace"
assert request["stem"] is False
assert request["remove_stop_words"] is False
assert request["lower_case"] is False
assert request["block_size"] == 256
def test_remote_create_index_new_api(caplog):
received_requests = []
@@ -2132,9 +2274,6 @@ def test_query_sync_minimal():
"ef": None,
"vector": [1.0, 2.0, 3.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"version": None,
}
@@ -2306,8 +2445,6 @@ def test_query_sync_maximal():
"refine_factor": 10,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 5,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -2403,7 +2540,7 @@ async def test_query_async_nprobes_zero(hybrid):
assert requests == ["/v1/table/test/describe/"]
def test_query_sync_nprobes():
def test_query_sync_probe_bounds():
def handler(body):
assert body == {
"k": 10,
@@ -2415,7 +2552,6 @@ def test_query_sync_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 15,
"version": None,
@@ -2432,7 +2568,28 @@ def test_query_sync_nprobes():
)
def test_query_sync_no_max_nprobes():
def test_query_sync_nprobes_passthrough():
def handler(body):
assert body["nprobes"] == 20
assert "minimum_nprobes" not in body
assert "maximum_nprobes" not in body
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).to_list()
def test_query_sync_nprobes_with_unbounded_maximum():
def handler(body):
assert body["nprobes"] == 20
assert body["maximum_nprobes"] == 0
return pa.table({"id": [1]})
with query_test_table(handler) as table:
table.search([1, 2, 3]).nprobes(20).maximum_nprobes(0).to_list()
def test_query_sync_minimum_nprobes_only():
def handler(body):
assert body == {
"k": 10,
@@ -2444,9 +2601,7 @@ def test_query_sync_no_max_nprobes():
"upper_bound": None,
"ef": None,
"vector": [1.0, 2.0, 3.0],
"nprobes": 5,
"minimum_nprobes": 5,
"maximum_nprobes": 0,
"version": None,
}
@@ -2456,7 +2611,6 @@ def test_query_sync_no_max_nprobes():
(
table.search([1, 2, 3], vector_column_name="vector2", fast_search=True)
.minimum_nprobes(5)
.maximum_nprobes(0)
.to_list()
)
@@ -2611,9 +2765,6 @@ def test_query_sync_hybrid():
"refine_factor": None,
"vector": [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
"vector_column": "vector",
"nprobes": 20,
"minimum_nprobes": 20,
"maximum_nprobes": 20,
"lower_bound": None,
"upper_bound": None,
"ef": None,
@@ -2687,6 +2838,11 @@ def test_create_client():
assert isinstance(db.client_config, ClientConfig)
assert db.client_config.retry_config.retries == 42
db = lancedb.connect(
**mandatory_args, client_config={"blob_request_concurrency": 3}
)
assert db.client_config.blob_request_concurrency == 3
with pytest.warns(DeprecationWarning):
db = lancedb.connect(**mandatory_args, connection_timeout=42)
assert db.client_config.timeout_config.connect_timeout == timedelta(seconds=42)
@@ -3359,7 +3515,7 @@ def test_remote_blob_columns_and_fetch():
@pytest.mark.parametrize("blob_mode", ["descriptions", "bytes", "lazy"])
def test_sync_remote_blob_pandas_rejects_storage_access(
def test_sync_remote_blob_pandas_avoids_storage_access(
tmp_path, monkeypatch, blob_mode
):
import lance
@@ -3372,10 +3528,19 @@ def test_sync_remote_blob_pandas_rejects_storage_access(
monkeypatch.setattr(lance, "dataset", fail_dataset)
with blob_remote_table(location=location) as table:
with pytest.raises(RuntimeError, match="to_lance.*remote tables") as error:
table.search().select(["id", "image"]).to_pandas(blob_mode=blob_mode)
assert isinstance(error.value.__cause__, NotImplementedError)
monkeypatch.setattr(table, "to_lance", fail_dataset)
query = table.search().select(["id", "image"])
if blob_mode == "descriptions":
df = query.to_pandas(blob_mode=blob_mode)
assert df["id"].tolist() == [1, 2, 3]
assert "_rowid" not in df.columns
assert df["image"][0]["size"] == 5
elif blob_mode == "bytes":
with pytest.raises(NotImplementedError, match="stable table snapshot"):
query.to_pandas(blob_mode=blob_mode)
else:
with pytest.raises(RuntimeError, match="cannot use Lance native"):
query.to_pandas(blob_mode=blob_mode)
@pytest.mark.asyncio
@@ -3446,6 +3611,45 @@ async def test_async_remote_blob_pandas_rejects_unpinned_fetch(monkeypatch, blob
await table.to_pandas(**pandas_kwargs)
@pytest.mark.parametrize("method", ["to_arrow", "to_list"])
def test_remote_search_bytes_refused_before_query(monkeypatch, method):
with blob_remote_table() as table:
query = table.search().select(["id", "image"])
descriptions = getattr(query, method)(blob_mode="descriptions")
images = (
descriptions["image"].to_pylist()
if method == "to_arrow"
else [row["image"] for row in descriptions]
)
assert "_lance_row_id" not in images[0]
def fail_query(*args, **kwargs):
raise AssertionError("bytes must be refused before the query runs")
monkeypatch.setattr(table, "_execute_query", fail_query)
with pytest.raises(NotImplementedError, match="stable table snapshot"):
getattr(query, method)(blob_mode="bytes")
with pytest.raises(ValueError, match="blob_mode must be one of"):
table.to_pandas(blob_mode="handles")
@pytest.mark.asyncio
async def test_async_remote_to_arrow_bytes_refused_before_query(monkeypatch):
with blob_remote_table() as remote_table:
table = remote_table._table
async def fail_fetch(*args, **kwargs):
raise AssertionError("blob fetch must not use an unpinned remote table")
monkeypatch.setattr(table, "fetch_blobs", fail_fetch)
query = table.query().select(["id", "image"])
monkeypatch.setattr(query, "to_batches", fail_fetch)
with pytest.raises(NotImplementedError, match="stable table snapshot"):
await query.to_arrow(blob_mode="bytes")
with pytest.raises(NotImplementedError, match="stable table snapshot"):
await table.to_arrow(blob_mode="bytes")
@pytest.mark.asyncio
@pytest.mark.parametrize("blob_mode", ["bytes", "lazy"])
async def test_async_remote_nonblob_projection_to_pandas(blob_mode):
@@ -3477,10 +3681,32 @@ def test_remote_blob_files_are_lazy_seekable_handles():
assert empty.read_range(0, 0) == b""
alpha.close()
assert alpha.closed
with pytest.raises(RuntimeError, match="already closed"):
with pytest.raises(ValueError, match="read of closed file"):
alpha.read_range(0, 1)
def test_remote_blob_file_matches_file_error_and_range_contract():
with blob_remote_table() as table:
handle = table.fetch_blob_files("image", [10])[0]
handle.seek(2)
with pytest.raises(ValueError, match="negative seek value -1"):
handle.seek(-3, io.SEEK_CUR)
assert handle.tell() == 2
assert handle.read(None) == b"pha"
handle.seek(2)
assert handle.read1(2) == b"ph"
assert handle.read_ranges([(0, 2), (3, 2)]) == [b"al", b"ha"]
assert handle.tell() == 4
with pytest.raises(ValueError):
handle.read_range(4, 2)
handle.close()
with pytest.raises(ValueError, match="read of closed file"):
handle.read(1)
def test_remote_blob_fetch_accepts_query_table():
hits = pa.table({"_rowid": pa.array([10, 20, 30], type=pa.uint64())})
python/python/tests/test_table.py
+46 -1
View File
@@ -12,7 +12,7 @@ import warnings
import weakref
from concurrent.futures import ThreadPoolExecutor
from datetime import date, datetime, timedelta
from decimal import Decimal
from decimal import Decimal, localcontext
from time import sleep
from typing import List
from unittest.mock import patch
@@ -2612,6 +2612,51 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
@pytest.mark.parametrize("use_arrow_scalar", [False, True])
@pytest.mark.parametrize(
"decimal_type, value",
[
(pa.decimal128(10, 2), "9.99"),
(pa.decimal128(10, 2), "-9.99"),
(pa.decimal128(10, 2), "0.00"),
(pa.decimal128(10, 2), "1.23E+5"),
(pa.decimal128(38, 18), "1E-18"),
(pa.decimal128(38, 18), "12345678901234567.123456789012345678"),
(pa.decimal128(38, 18), "12345678901234567890.123456789012345678"),
(pa.decimal128(38, 18), "-12345678901234567890.123456789012345678"),
(
pa.decimal256(76, 38),
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
(
pa.decimal256(76, 38),
"-12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
),
],
)
def test_update_decimal(mem_db: DBConnection, decimal_type, value, use_arrow_scalar):
expected = Decimal(value)
table = mem_db.create_table(
"my_table",
pa.table(
{
"id": [1, 2],
"price": pa.array([Decimal("1"), expected], type=decimal_type),
}
),
)
update = table.to_arrow()["price"][1] if use_arrow_scalar else expected
with localcontext() as context:
context.prec = 6
result = table.update(where="id = 1", values={"price": update})
assert result.rows_updated == 1
actual = table.to_arrow()
assert actual.schema.field("price").type == decimal_type
assert actual["price"].to_pylist() == [expected, expected]
def test_update_types(mem_db: DBConnection):
table = mem_db.create_table(
"my_table",
python/python/tests/test_util.py
+69
View File
@@ -4,6 +4,7 @@
import os
import pathlib
from decimal import Decimal, localcontext
from typing import Optional
import lance
@@ -157,6 +158,41 @@ def test_value_to_sql_string(tmp_path):
assert table.to_pandas().query("search == @value")["replace"].item() == value
@pytest.mark.parametrize(
"value, expected",
[
("9.99", "'9.99'"),
("-9.99", "'-9.99'"),
("0", "'0'"),
("-0.00", "'-0.00'"),
("1.23E+5", "'123000'"),
("1E-18", "'0.000000000000000001'"),
(
"12345678901234567890.123456789012345678",
"'12345678901234567890.123456789012345678'",
),
(
"12345678901234567890123456789012345678."
"12345678901234567890123456789012345678",
"'12345678901234567890123456789012345678."
"12345678901234567890123456789012345678'",
),
],
)
def test_value_to_sql_decimal(value, expected):
with localcontext() as context:
context.prec = 6
assert value_to_sql(Decimal(value)) == expected
@pytest.mark.parametrize("value", ["NaN", "sNaN", "NaN123456", "Infinity", "-Infinity"])
def test_value_to_sql_decimal_non_finite(value):
with pytest.raises(
ValueError, match="^Non-finite Decimal values cannot be converted to SQL$"
):
value_to_sql(Decimal(value))
def test_value_to_sql_dict():
# Simple flat struct
assert value_to_sql({"a": 1, "b": "hello"}) == "named_struct('a', 1, 'b', 'hello')"
@@ -951,6 +987,39 @@ def test_cast_to_target_schema_coerces_binary_to_blob_v2():
]
@pytest.mark.parametrize(
"string_type", [pa.string(), pa.large_string(), pa.string_view()]
)
def test_cast_to_target_schema_coerces_string_to_blob_v2_uri(string_type):
data = pa.table({"image": pa.array(["file:///payload", None], type=string_type)})
target = pa.schema([lancedb.blob("image")])
output = _cast_to_target_schema(data.to_reader(), target).read_all()
image = output["image"].chunk(0)
assert type(image.type) is lancedb.BlobType
assert image.storage.to_pylist() == [
{"data": None, "uri": "file:///payload", "position": None, "size": None},
None,
]
@pytest.mark.parametrize(
"source_type, value",
[
(pa.binary(4), b"abcd"),
(pa.list_(pa.uint8()), [1, 2]),
(pa.int64(), 42),
],
)
def test_cast_to_target_schema_rejects_unsupported_blob_input(source_type, value):
data = pa.table({"image": pa.array([value], type=source_type)})
target = pa.schema([lancedb.blob("image")])
with pytest.raises(ValueError, match="cannot coerce column 'image' with type"):
_cast_to_target_schema(data.to_reader(), target)
def test_cast_to_target_schema_coerces_binary_to_metadata_blob_struct():
storage = lancedb.blob("image").type.storage_type
target = pa.schema(
python/python/tests/test_wal_hybrid.py
+3 -1
View File
@@ -119,7 +119,9 @@ def test_wal_hybrid_fuses_on_the_key_and_drops_it_again():
limit=10,
with_row_ids=True,
)
results = builder._finish_hybrid_results(combined)
results = builder._finish_hybrid_results(
combined, blob_mode="lazy", blob_sources={}
)
assert "_rowid" not in results.column_names, "the surrogate is internal"
assert "id" not in results.column_names, "the caller selected only `text`"
python/src/authz.rs
+280
View File
@@ -0,0 +1,280 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use crate::error::PythonErrorExt;
use crate::runtime::future_into_py;
use lancedb::authz::*;
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::types::{PyAnyMethods, PyBytes, PyString};
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
use std::sync::Arc;
#[pyclass]
pub struct AuthorizationClient {
pub(crate) inner: Arc<dyn Authorization>,
}
#[pymethods]
impl AuthorizationClient {
fn list_acls<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: ListAccessControlEntriesRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.list_acls(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn add_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: AccessControlEntryRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.add_acl(request).await.infer_error()
})
}
fn delete_acl<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: AccessControlEntryRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.delete_acl(request).await.infer_error()
})
}
fn list_role_bindings<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: ListRoleBindingsRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.list_role_bindings(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn add_role_binding<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: RoleBindingRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.add_role_binding(request).await.infer_error()
})
}
fn delete_role_binding<'py>(
self_: PyRef<'py, Self>,
request: &str,
) -> PyResult<Bound<'py, PyAny>> {
let request: RoleBindingRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.delete_role_binding(request).await.infer_error()
})
}
fn get_principal<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: GetPrincipalRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.get_principal(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
fn get_group<'py>(self_: PyRef<'py, Self>, request: &str) -> PyResult<Bound<'py, PyAny>> {
let request: GetGroupRequest = serde_json::from_str(request)
.map_err(|_| PyValueError::new_err("Invalid authorization request"))?;
request.validate().infer_error()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner.get_group(request).await.infer_error()?;
serde_json::to_string(&result).map_err(|err| PyRuntimeError::new_err(err.to_string()))
})
}
}
/// The shared privilege parser and canonical wire names.
#[pyclass(frozen)]
pub struct AuthzPrivilege {
inner: Privilege,
}
#[pymethods]
impl AuthzPrivilege {
#[new]
fn new(value: &str) -> PyResult<Self> {
Ok(Self {
inner: value.parse().map_err(PyValueError::new_err)?,
})
}
#[staticmethod]
fn known() -> Vec<String> {
Privilege::KNOWN.iter().map(ToString::to_string).collect()
}
fn __str__(&self) -> String {
self.inner.to_string()
}
}
#[pyclass(frozen)]
pub struct AuthzSubject {
inner: Subject,
}
#[pymethods]
impl AuthzSubject {
#[new]
fn new(kind: &str, value: &Bound<'_, PyAny>) -> PyResult<Self> {
let value: String = value
.extract()
.map_err(|_| PyValueError::new_err("Subject value must be a nonempty string"))?;
Ok(Self {
inner: Subject::new(kind.parse().infer_error()?, value).infer_error()?,
})
}
#[staticmethod]
fn from_canonical(value: &str) -> PyResult<Self> {
Ok(Self {
inner: Subject::from_canonical(value).infer_error()?,
})
}
#[getter]
fn kind(&self) -> &str {
self.inner.kind().as_str()
}
#[getter]
fn value(&self) -> &str {
self.inner.value()
}
#[getter]
fn display_value(&self) -> &str {
self.inner.display_value()
}
#[pyo3(signature = (context=None))]
fn to_wire(&self, context: Option<&str>) -> PyResult<String> {
match context {
None => {}
Some("principal") => self.inner.validate_principal().infer_error()?,
Some("group") => self.inner.validate_group().infer_error()?,
Some("role_member") => self.inner.validate_role_member().infer_error()?,
Some(_) => return Err(PyValueError::new_err("Invalid subject context")),
}
Ok(self.inner.to_wire())
}
fn __repr__(&self) -> String {
format!("{:?}", self.inner)
}
fn __str__(&self) -> String {
self.inner.to_string()
}
}
#[pyclass(frozen)]
pub struct AuthzObject {
inner: Object,
}
impl AuthzObject {
fn validated(inner: Object) -> PyResult<Self> {
inner.validate().map_err(PyValueError::new_err)?;
Ok(Self { inner })
}
}
fn namespace_path(value: &Bound<'_, PyAny>) -> PyResult<NamespacePath> {
if value.is_instance_of::<PyString>() || value.is_instance_of::<PyBytes>() {
return Err(PyValueError::new_err(
"namespace must be a sequence of components",
));
}
let components = value
.extract()
.map_err(|_| PyValueError::new_err("namespace must be a sequence of string components"))?;
NamespacePath::new(components).map_err(PyValueError::new_err)
}
#[pymethods]
impl AuthzObject {
#[new]
fn new(value: &str) -> PyResult<Self> {
Ok(Self {
inner: value.parse().infer_error()?,
})
}
#[staticmethod]
fn system() -> Self {
Self {
inner: Object::System,
}
}
#[staticmethod]
fn database(name: &str) -> PyResult<Self> {
Self::validated(Object::Database(DatabaseObject::new(name.into())))
}
#[staticmethod]
fn namespace(database: &str, namespace: &Bound<'_, PyAny>) -> PyResult<Self> {
Self::validated(Object::Namespace(NamespaceObject {
database: database.into(),
namespace: namespace_path(namespace)?,
}))
}
#[staticmethod]
fn table(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Table(TableObject {
database: database.into(),
namespace: namespace_path(namespace)?,
table: name.into(),
}))
}
#[staticmethod]
fn view(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::View(ViewObject {
database: database.into(),
namespace: namespace_path(namespace)?,
view: name.into(),
}))
}
#[staticmethod]
fn secret(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Secret(SecretObject {
database: database.into(),
namespace: namespace_path(namespace)?,
secret: name.into(),
}))
}
#[staticmethod]
fn function(database: &str, namespace: &Bound<'_, PyAny>, name: &str) -> PyResult<Self> {
Self::validated(Object::Function(FunctionObject {
database: database.into(),
namespace: namespace_path(namespace)?,
function: name.into(),
}))
}
fn __str__(&self) -> String {
self.inner.to_string()
}
fn __repr__(&self) -> String {
format!("{:?}", self.inner)
}
}
python/src/catalog.rs
+7
View File
@@ -21,6 +21,13 @@ pub struct Catalog {
#[pymethods]
impl Catalog {
#[getter]
fn authz(&self) -> PyResult<crate::authz::AuthorizationClient> {
Ok(crate::authz::AuthorizationClient {
inner: self.inner.authz().infer_error()?,
})
}
#[getter]
fn uri(&self) -> &str {
self.inner.uri()
python/src/connection.rs
+2
View File
@@ -1220,6 +1220,7 @@ pub struct PyClientConfig {
user_agent: String,
retry_config: Option<PyClientRetryConfig>,
timeout_config: Option<PyClientTimeoutConfig>,
blob_request_concurrency: Option<usize>,
extra_headers: Option<HashMap<String, String>>,
id_delimiter: Option<String>,
tls_config: Option<PyClientTlsConfig>,
@@ -1305,6 +1306,7 @@ impl From<PyClientConfig> for lancedb::remote::ClientConfig {
user_agent: value.user_agent,
retry_config: value.retry_config.map(Into::into).unwrap_or_default(),
timeout_config: value.timeout_config.map(Into::into).unwrap_or_default(),
blob_request_concurrency: value.blob_request_concurrency,
extra_headers: value.extra_headers.unwrap_or_default(),
id_delimiter: value.id_delimiter,
tls_config: value.tls_config.map(Into::into),
python/src/lib.rs
+5
View File
@@ -22,6 +22,7 @@ use table::{
};
pub mod arrow;
pub mod authz;
pub mod catalog;
pub mod connection;
pub mod error;
@@ -70,6 +71,10 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<listing::FunctionListing>()?;
m.add_class::<listing::JobListing>()?;
m.add_class::<authz::AuthorizationClient>()?;
m.add_class::<authz::AuthzSubject>()?;
m.add_class::<authz::AuthzObject>()?;
m.add_class::<authz::AuthzPrivilege>()?;
m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?;
m.add_class::<Session>()?;
m.add_class::<Table>()?;
python/src/query.rs
+36 -10
View File
@@ -71,7 +71,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
match class_name(&ob)?.as_str() {
"MatchQuery" => {
let query = ob.getattr("query")?.extract()?;
let column = ob.getattr("column")?.extract()?;
let column = ob.getattr("column")?.extract::<Option<String>>()?;
let boost = ob.getattr("boost")?.extract()?;
let fuzziness = ob.getattr("fuzziness")?.extract()?;
let max_expansions = ob.getattr("max_expansions")?.extract()?;
@@ -88,7 +88,7 @@ impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
let mut query =
MatchQuery::new(query)
.with_column(Some(column))
.with_column(column)
.with_boost(boost)
.with_fuzziness(fuzziness)
.with_max_expansions(max_expansions)
@@ -199,7 +199,7 @@ impl<'py> IntoPyObject<'py> for PyLanceDB<FtsQuery> {
}
namespace
.getattr(intern!(py, "MatchQuery"))?
.call((query.terms, query.column.unwrap()), Some(&kwargs))
.call((query.terms, query.column), Some(&kwargs))
}
FtsQuery::Phrase(query) => {
let kwargs = PyDict::new(py);
@@ -336,10 +336,10 @@ pub struct PyQueryRequest {
pub use_lsm: Option<bool>,
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub nprobes: Option<usize>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub search_effort: Option<f64>,
pub lower_bound: Option<f32>,
pub upper_bound: Option<f32>,
pub ef: Option<usize>,
@@ -369,8 +369,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: query_request.use_lsm,
column: None,
query_vector: None,
nprobes: None,
minimum_nprobes: None,
maximum_nprobes: None,
search_effort: None,
lower_bound: None,
upper_bound: None,
ef: None,
@@ -396,11 +398,10 @@ impl From<AnyQuery> for PyQueryRequest {
use_lsm: vector_query.base.use_lsm,
column: vector_query.column,
query_vector: Some(PyQueryVectors(vector_query.query_vector)),
minimum_nprobes: Some(vector_query.minimum_nprobes),
maximum_nprobes: match vector_query.maximum_nprobes {
None => Some(0),
Some(value) => Some(value),
},
nprobes: vector_query.nprobes,
minimum_nprobes: vector_query.minimum_nprobes,
maximum_nprobes: vector_query.maximum_nprobes,
search_effort: vector_query.search_effort,
lower_bound: vector_query.lower_bound,
upper_bound: vector_query.upper_bound,
ef: vector_query.ef,
@@ -1000,6 +1001,15 @@ impl VectorQuery {
Ok(())
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.infer_error()?;
Ok(())
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner = self
.inner
@@ -1193,6 +1203,18 @@ impl HybridQuery {
self.inner_vec.nprobes(nprobe)
}
pub fn search_effort(&mut self, search_effort: Option<f64>) -> PyResult<()> {
self.inner_vec.search_effort(search_effort)
}
pub fn minimum_nprobes(&mut self, minimum_nprobes: u32) -> PyResult<()> {
self.inner_vec.minimum_nprobes(minimum_nprobes)
}
pub fn maximum_nprobes(&mut self, maximum_nprobes: u32) -> PyResult<()> {
self.inner_vec.maximum_nprobes(maximum_nprobes)
}
pub fn ef(&mut self, ef: u32) {
self.inner_vec.ef(ef);
}
@@ -1237,6 +1259,10 @@ impl HybridQuery {
req.query_vector = vec_req.query_vector;
req.column = vec_req.column;
req.distance_type = vec_req.distance_type;
req.nprobes = vec_req.nprobes;
req.minimum_nprobes = vec_req.minimum_nprobes;
req.maximum_nprobes = vec_req.maximum_nprobes;
req.search_effort = vec_req.search_effort;
req.ef = vec_req.ef;
req.refine_factor = vec_req.refine_factor;
req.lower_bound = vec_req.lower_bound;
python/src/table.rs
+58 -15
View File
@@ -697,6 +697,14 @@ impl PyBlobFile {
}
}
fn blob_file_error(operation: &str, error: lance_core::Error) -> pyo3::PyErr {
let message = format!("blob {operation} failed: {error}");
match error {
lance_core::Error::InvalidInput { .. } => PyValueError::new_err(message),
_ => PyRuntimeError::new_err(message),
}
}
#[pymethods]
impl PyBlobFile {
fn read_bytes(self_: PyRef<'_, Self>) -> PyResult<Py<PyBytes>> {
@@ -705,7 +713,7 @@ impl PyBlobFile {
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read().await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
.map_err(|e| blob_file_error("read", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
@@ -713,10 +721,7 @@ impl PyBlobFile {
self_.ensure_open()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let bytes = inner
.read()
.await
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
let bytes = inner.read().await.map_err(|e| blob_file_error("read", e))?;
Python::attach(|py| Ok(PyBytes::new(py, bytes.as_ref()).unbind()))
})
}
@@ -739,7 +744,7 @@ impl PyBlobFile {
self_
.py()
.detach(move || block_on(async move { inner.close().await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob close failed: {e}")))
.map_err(|e| blob_file_error("close", e))
}
fn is_closed(self_: PyRef<'_, Self>) -> bool {
@@ -752,7 +757,7 @@ impl PyBlobFile {
self_
.py()
.detach(move || block_on(async move { inner.seek(position).await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob seek failed: {e}")))
.map_err(|e| blob_file_error("seek", e))
}
fn tell(self_: PyRef<'_, Self>) -> PyResult<u64> {
@@ -761,7 +766,7 @@ impl PyBlobFile {
self_
.py()
.detach(move || block_on(async move { inner.tell().await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob tell failed: {e}")))
.map_err(|e| blob_file_error("tell", e))
}
fn size(self_: PyRef<'_, Self>) -> u64 {
@@ -778,17 +783,39 @@ impl PyBlobFile {
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_range(offset..end).await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob read_range failed: {e}")))?;
.map_err(|e| blob_file_error("read_range", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
/// Read blob-local byte ranges without moving the cursor.
fn read_ranges(self_: PyRef<'_, Self>, ranges: Vec<(u64, u64)>) -> PyResult<Vec<Py<PyBytes>>> {
let ranges = ranges
.into_iter()
.map(|(offset, length)| {
offset
.checked_add(length)
.map(|end| offset..end)
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))
})
.collect::<PyResult<Vec<_>>>()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_ranges(&ranges).await }))
.map_err(|e| blob_file_error("read_ranges", e))?;
Ok(bytes
.iter()
.map(|bytes| PyBytes::new(py, bytes.as_ref()).unbind())
.collect())
}
fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_up_to(length).await }))
.map_err(|e| PyRuntimeError::new_err(format!("blob read_up_to failed: {e}")))?;
.map_err(|e| blob_file_error("read_up_to", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
}
@@ -1462,14 +1489,30 @@ impl Table {
pub fn fetch_blob_ranges(
self_: PyRef<'_, Self>,
column: String,
requests: Vec<(u64, u64, u64)>,
requests: Vec<Vec<i128>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let requests = requests
.into_iter()
.map(|values| {
let [row_id, offset, length] = <[i128; 3]>::try_from(values).map_err(|_| {
PyValueError::new_err("each blob range request must have three values")
})?;
let to_u64 = |value, name| {
u64::try_from(value).map_err(|_| {
PyValueError::new_err(format!(
"blob range {name} must be a non-negative 64-bit integer"
))
})
};
Ok(BlobRangeRequest::new(
to_u64(row_id, "row id")?,
to_u64(offset, "offset")?,
to_u64(length, "length")?,
))
})
.collect::<PyResult<Vec<_>>>()?;
future_into_py(self_.py(), async move {
let requests = requests
.into_iter()
.map(|(row_id, offset, length)| BlobRangeRequest::new(row_id, offset, length))
.collect::<Vec<_>>();
let blobs: LargeBinaryArray = inner
.fetch_blob_ranges(column, requests)
.await
python/uv.lock
Generated
+41 -15
View File
@@ -1849,19 +1849,19 @@ wheels = [
[[package]]
name = "lance-namespace"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "lance-namespace-urllib3-client" },
]
sdist = { url = "https://files.pythonhosted.org/packages/bf/93/da5f7fcac690db9b282a3439ed9e34960c147619a0d6e1f4eb8cd240e7a5/lance_namespace-0.11.1.tar.gz", hash = "sha256:f67cfbbe0647b7cb42f23b673e7edf8a75b7d8a047265a916492f8d247ee1bc2", size = 11631, upload-time = "2026-08-18T17:40:06.294Z" }
sdist = { url = "https://files.pythonhosted.org/packages/c0/e7/d5d46594678ee479c0eda830c47b2f5c46133bd100e84e9aa6e01306eca7/lance_namespace-0.13.0.tar.gz", hash = "sha256:24554a0997bdb39595c6e4cb3ac6722069f6cd3bd1a74e7acbba7bfaf774a40d", size = 11636, upload-time = "2026-09-14T02:47:03.871523Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fa/bc/601f2b3cc4cfa0070d858a33223bc823fffdd7981a25c45984a5216ca952/lance_namespace-0.11.1-py3-none-any.whl", hash = "sha256:07643fce9a42ad4d58cc8bf91e3f592bc7f4cbd8d0ad5233223506debf67551c", size = 13507, upload-time = "2026-08-18T17:40:03.561Z" },
{ url = "https://files.pythonhosted.org/packages/e7/47/cfa33cca1ba7c749fd2918cfc5c8ded788f378cc6424d23e4fead5a14125/lance_namespace-0.13.0-py3-none-any.whl", hash = "sha256:438c7b17aef421c21c138196e715f2510d62f07e865372047f87c1e75e618c7a", size = 13507, upload-time = "2026-09-14T02:47:01.765655Z" },
]
[[package]]
name = "lance-namespace-urllib3-client"
version = "0.11.1"
version = "0.13.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pydantic" },
@@ -1869,9 +1869,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "urllib3" },
]
sdist = { url = "https://files.pythonhosted.org/packages/e4/c5/2bdd0ff98b469894c8a73be809d26ffdad5402517b0e5f9e758026cba29e/lance_namespace_urllib3_client-0.11.1.tar.gz", hash = "sha256:145a9e9424d7597487249b5b95ee274423bf2910e1a9160b6a07b676b61ea46a", size = 237345, upload-time = "2026-08-18T17:40:07.308Z" }
sdist = { url = "https://files.pythonhosted.org/packages/69/25/9aaa4a5e8999693fb0f227c2c0c4b97bd8f0539066408cdff0b89d41b2d5/lance_namespace_urllib3_client-0.13.0.tar.gz", hash = "sha256:1e8a79c6e4e6277033597fd76aa0e2f33d909ca1436c935936e9e569221f43ef", size = 238348, upload-time = "2026-09-14T02:47:04.521878Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/2a/eaaefd55d1190291207049fedc6b3eb22b506e57d6de91bae46bbaaa9c60/lance_namespace_urllib3_client-0.11.1-py3-none-any.whl", hash = "sha256:36537f529294da6d884ba0fe783704483f0a75463497c7705fd083a4d0257990", size = 406311, upload-time = "2026-08-18T17:40:04.842Z" },
{ url = "https://files.pythonhosted.org/packages/60/4c/7b8f0712a7fe1b8655b711342bc4989696635a79e7027ee56ba3cae23e99/lance_namespace_urllib3_client-0.13.0-py3-none-any.whl", hash = "sha256:fb361eb4f6c7f2d1f9e92809609657b73e38241393e9c4516e12d6b6ee643a0a", size = 406848, upload-time = "2026-09-14T02:47:02.715954Z" },
]
[[package]]
@@ -1900,6 +1900,7 @@ clip = [
{ name = "torch" },
]
dev = [
{ name = "maturin" },
{ name = "pre-commit" },
{ name = "pyright" },
{ name = "ruff" },
@@ -1980,7 +1981,8 @@ requires-dist = [
{ name = "huggingface-hub", marker = "extra == 'embeddings'", specifier = ">=0.19.0" },
{ name = "ibm-watsonx-ai", marker = "python_full_version >= '3.10' and extra == 'embeddings'", specifier = ">=1.1.2" },
{ name = "instructorembedding", marker = "extra == 'embeddings'", specifier = ">=1.0.1" },
{ name = "lance-namespace", specifier = ">=0.3.2" },
{ name = "lance-namespace", specifier = ">=0.13.0" },
{ name = "maturin", marker = "extra == 'dev'", specifier = ">=1.10" },
{ name = "mkdocs", marker = "extra == 'docs'" },
{ name = "mkdocs-jupyter", marker = "extra == 'docs'" },
{ name = "mkdocs-material", marker = "extra == 'docs'" },
@@ -2005,7 +2007,7 @@ requires-dist = [
{ name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
{ name = "pydantic", specifier = ">=2.7.4,<3" },
{ name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b6" },
{ name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
{ name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
{ name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
{ name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -2156,6 +2158,30 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
[[package]]
name = "maturin"
version = "1.15.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "tomli", marker = "python_full_version < '3.11'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b9/c8/22e5e21b2679c9bce6415ca578034ca2cc9316be0642ae21e051a2d5198c/maturin-1.15.0.tar.gz", hash = "sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a", size = 385504, upload-time = "2026-08-24T12:11:22.665Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/69/5c01b461044eb1f45ddcce006706eb88110c793cdb11c7ae0b5e08492e94/maturin-1.15.0-py3-none-linux_armv6l.whl", hash = "sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82", size = 10206220, upload-time = "2026-08-24T12:10:53.327Z" },
{ url = "https://files.pythonhosted.org/packages/eb/1f/2b431554e11687cdb1077e0cdadcc118c53f611086b3af00c8545a67c6a5/maturin-1.15.0-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7", size = 19416513, upload-time = "2026-08-24T12:10:56.126Z" },
{ url = "https://files.pythonhosted.org/packages/51/36/e23a21cb34a648b711036b9b2fe1d4f3f4ee24f8db54215d73f1a9a3a3ec/maturin-1.15.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd", size = 10014962, upload-time = "2026-08-24T12:10:58.486Z" },
{ url = "https://files.pythonhosted.org/packages/8c/80/33b15cb2d8f30f12c807955e8f2fd775027692904e30ec0784744ce8cd83/maturin-1.15.0-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55", size = 10196223, upload-time = "2026-08-24T12:11:00.696Z" },
{ url = "https://files.pythonhosted.org/packages/fe/91/b495e19e2f5c503b540452b2039115e7b2363867e8c5ad4179eb752fa92c/maturin-1.15.0-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e", size = 10541186, upload-time = "2026-08-24T12:11:02.857Z" },
{ url = "https://files.pythonhosted.org/packages/5d/2b/2abff58037188d852b124871b1f0d720e1c2bfb3d4f1b03d87c52cd66488/maturin-1.15.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136", size = 10083468, upload-time = "2026-08-24T12:11:05.012Z" },
{ url = "https://files.pythonhosted.org/packages/3f/07/b7e9f8be99a6627849e81ac7b6694876bce8f50a92995fe17e3cf2610f0a/maturin-1.15.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1", size = 10047786, upload-time = "2026-08-24T12:11:07.042Z" },
{ url = "https://files.pythonhosted.org/packages/ca/ab/167e3cb7accee11b507dbe53e0e87aeccb376d44ae66284c96ee4df3a9fd/maturin-1.15.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d", size = 13315332, upload-time = "2026-08-24T12:11:09.433Z" },
{ url = "https://files.pythonhosted.org/packages/14/4d/801379f646cbc6b00998e5289b0630a886be3a4ee4c75b6bc9b87478a7f1/maturin-1.15.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15", size = 10807183, upload-time = "2026-08-24T12:11:11.73Z" },
{ url = "https://files.pythonhosted.org/packages/89/27/2e612e1cbd1580e9e94d4722c227b5180dca27b32b955a34b79918aa1292/maturin-1.15.0-py3-none-manylinux_2_31_riscv64.musllinux_1_1_riscv64.whl", hash = "sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176", size = 10413274, upload-time = "2026-08-24T12:11:14.04Z" },
{ url = "https://files.pythonhosted.org/packages/70/d8/202a7b4d75a51f20f84ec9ce3b7345b12b822207072164e2b1c6ef665125/maturin-1.15.0-py3-none-win32.whl", hash = "sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca", size = 8928744, upload-time = "2026-08-24T12:11:16.269Z" },
{ url = "https://files.pythonhosted.org/packages/40/dc/4e90da594986ba78dd3bc8a5921ecdcdb11085b22b02a412caab3b225601/maturin-1.15.0-py3-none-win_amd64.whl", hash = "sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1", size = 10335085, upload-time = "2026-08-24T12:11:18.326Z" },
{ url = "https://files.pythonhosted.org/packages/8b/10/15d4314edf130955edf2dc237aa393a8a7c10f2b9b57b89fa2f61f915659/maturin-1.15.0-py3-none-win_arm64.whl", hash = "sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48", size = 9713795, upload-time = "2026-08-24T12:11:20.83Z" },
]
[[package]]
name = "mdit-py-plugins"
version = "0.6.1"
@@ -3912,7 +3938,7 @@ crypto = [
[[package]]
name = "pylance"
version = "14.0.0b6"
version = "14.0.0b12"
source = { registry = "https://pypi.fury.io/lance-format/" }
dependencies = [
{ name = "lance-namespace" },
@@ -3922,12 +3948,12 @@ dependencies = [
{ name = "pyarrow" },
]
wheels = [
{ url = "https://pypi.fury.io/lance-format/-/ver_DuQEn/pylance-14.0.0b6-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:3eade86fa30d9ff3248ee7ffb0b27ce29495394d12327da4d22b5c54bc2b59fe" },
{ url = "https://pypi.fury.io/lance-format/-/ver_K5sqj/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10e291974ee738a1d5d7e9dad17a634394a91cb60080b5ceac209077fbaec43c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_EnbkT/pylance-14.0.0b6-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bac10f5081c1210edd017f81c8c784d800b89446bd0ed05d4a24e7001965bc" },
{ url = "https://pypi.fury.io/lance-format/-/ver_19ffYg/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c627ad4a9870611becf84409d8e5ecddc6be64e1dd53bee4bd5fb10ab561e5c" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2eLLY5/pylance-14.0.0b6-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:bc603a7dd690233f1fe000600171e1c092f153bc9060cdf198825c4bbe1418aa" },
{ url = "https://pypi.fury.io/lance-format/-/ver_t8ggc/pylance-14.0.0b6-cp310-abi3-win_amd64.whl", hash = "sha256:1d32df32a3d157611450def07eae8ace3a7401286eac68cad9a24e0d5deed2db" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
{ url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
{ url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
{ url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
{ url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
{ url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
]
[[package]]