Merge main into gatekeeper/fix-2325-1

python/README.md

@@ -40,6 +40,33 @@ Stable releases are created about every 2 weeks. For the latest features and bug
 pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb
 ```
 
+If you also need the optional `lance` Python API, install the `pylance` extra
+and include the Lance preview index:
+
+```bash
+pip install --pre \
+  --extra-index-url https://pypi.fury.io/lancedb/ \
+  --extra-index-url https://pypi.fury.io/lance-format/ \
+  "lancedb[pylance]"
+```
+
+With uv, also select the best matching version across the indexes:
+
+```bash
+uv pip install --prerelease allow \
+  --index https://pypi.fury.io/lancedb/ \
+  --index https://pypi.fury.io/lance-format/ \
+  --index-strategy unsafe-best-match \
+  "lancedb[pylance]"
+```
+
+The LanceDB preview index contains old `pylance` releases; current Lance previews
+are published to the `lance-format` index. By default, uv only considers versions
+from the first index containing a package, which can select an old `pylance` or
+backtrack to an older LanceDB release. `unsafe-best-match` searches all configured
+indexes, like pip; use it only with indexes you trust. Install the extra rather
+than an unconstrained `pylance` so LanceDB's minimum supported version is enforced.
+
 ### Threading in CPU-limited containers
 
 LanceDB uses separate pools for compute work and storage I/O. On a container with
@@ -74,3 +101,52 @@ print(results)
 ### Development
 
 See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute to LanceDB.
+
+### Opening SQL blob descriptors
+
+A SQL `blob_descriptor(video)` result keeps the Blob V2 storage metadata and a
+`source` struct containing its table URI, version, commit timestamp, branch,
+column and row identity. This example uses a remote SQL connection whose SQL
+service produces those source-bearing descriptors. Open the reference on the
+source table to read incrementally:
+
+```python
+rows = db.execute_query("""
+    SELECT f.timestamp, blob_descriptor(v.video) AS video
+    FROM videos v JOIN frames f ON v.id = f.video_id
+""").read_all()
+
+videos = db.open_table("videos")
+for row in rows.to_pylist():
+    if row["video"] is not None:
+        with videos.open_blob_file(row["video"]) as video:
+            frame = extract_frame(video, row["timestamp"])
+```
+
+`open_blob_file` accepts the dictionary or an Arrow struct scalar directly. It
+keeps the query's version and branch without changing the table handle's checkout.
+Payload bytes load on reads. An external blob with an unspecified size may need a
+metadata-only HEAD request to open.
+
+`videos.open_blob_files(rows["video"])` returns an iterator over the same handles,
+preserving order, duplicates and nulls (`None`). It accepts Arrow arrays/chunked
+arrays or another iterable and opens each handle as iteration advances. Close each
+non-null handle after use. Async tables offer `await videos.open_blob_file(...)`
+and `async for video in videos.open_blob_files(...)`.
+
+```python
+for video in videos.open_blob_files(rows["video"]):
+    if video is not None:
+        with video:
+            header = video.read(4096)
+```
+
+To extract a frame while retaining each row's timestamp, iterate over the
+handles and rows together:
+
+```python
+for row, video in zip(rows.to_pylist(), videos.open_blob_files(rows["video"])):
+    if video is not None:
+        with video:
+            frame = extract_frame(video, row["timestamp"])
+```

python/pyproject.toml

@@ -45,7 +45,8 @@ repository = "https://github.com/lancedb/lancedb"
 
 [project.optional-dependencies]
 pylance = [
-    "pylance>=5.0.0b5",
+    # Keep the minimum in the bundled Lance major and compatible with tests below.
+    "pylance>=15.0.0b1",
 ]
 # A library only needs the OpenTelemetry API; the application supplies and
 # configures the SDK (the actual exporter/reader). See
@@ -63,7 +64,7 @@ tests = [
     "polars>=0.19, <=1.32.3",
     "pyarrow<25",
     "pyarrow-stubs>=16.0",
-    "pylance==14.0.0b12",
+    "pylance==15.0.0b1",
     "requests>=2.31.0",
     "datafusion>=54,<55",
     "opentelemetry-sdk>=1.30.0",

python/python/lancedb/__init__.py

@@ -24,6 +24,7 @@ from .remote import ClientConfig
 from .remote.db import RemoteDBConnection
 from .expr import Expr, col, lit, func
 from .schema import blob, vector
+from ._blob_reader import BlobReader as BlobReader
 from .job import AsyncJob, Job
 from .sql import AsyncQuery as AsyncSqlQuery
 from .sql import Query as SqlQuery

python/python/lancedb/_blob.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 
 import io
 import operator
-from collections.abc import Awaitable, Callable, Iterable
+from collections.abc import Awaitable, Callable, Iterable, Mapping
 from typing import TYPE_CHECKING, Optional, Union
 
 import pyarrow as pa
@@ -31,6 +31,18 @@ FetchBlobsSync = Callable[[str, list[int]], pa.Array | pa.ChunkedArray]
 FetchBlobsAsync = Callable[[str, list[int]], Awaitable[pa.Array | pa.ChunkedArray]]
 
 
+def _normalize_blob_descriptor(value):
+    if isinstance(value, pa.Scalar):
+        value = value.as_py()
+    if value is None:
+        return None
+    if not isinstance(value, Mapping):
+        raise TypeError(
+            "blob descriptor must be a mapping, an Arrow struct scalar, or None"
+        )
+    return dict(value)
+
+
 class BlobFile(io.RawIOBase):
     """Seekable lazy handle from :meth:`~lancedb.table.Table.fetch_blob_files`.
 

python/python/lancedb/_blob_reader.py

@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""Seekable binary input for Functions that consume Blob V2 values lazily."""
+
+import io
+import operator
+import urllib.request
+from collections.abc import Callable
+from http import HTTPStatus
+
+_READ_CHUNK_BYTES = 1024 * 1024
+_READ_TIMEOUT_SECONDS = 300
+
+
+class BlobReader(io.RawIOBase):
+    """Read a blob incrementally inside a Function.
+
+    Annotating a Function input with ``BlobReader`` requests a lazy, seekable
+    reader. Annotating it with ``bytes`` requests materialized bytes instead.
+    ``read_range`` does not change the sequential cursor. Readers are scoped
+    to the invocation and must not be retained for a later call.
+
+    For local calls, construct a reader from a range-reading callable and the
+    blob's size. The callable accepts an offset and length and returns bytes.
+    """
+
+    def __init__(self, read_range: Callable[[int, int], bytes], size: int):
+        super().__init__()
+        self._read_range = read_range
+        self._size = operator.index(size)
+        if self._size < 0:
+            raise ValueError("blob size must be nonnegative")
+        self._position = 0
+
+    @classmethod
+    def _from_uri(cls, uri: str, size: int):
+        opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
+
+        def read_range(offset, length):
+            request = urllib.request.Request(
+                uri, headers={"Range": f"bytes={offset}-{offset + length - 1}"}
+            )
+            with opener.open(request, timeout=_READ_TIMEOUT_SECONDS) as response:
+                if response.status != HTTPStatus.PARTIAL_CONTENT:
+                    raise OSError(
+                        "blob range service did not return a partial response"
+                    )
+                data = response.read(length + 1)
+                if len(data) != length:
+                    raise OSError(
+                        "blob range service returned an invalid payload length"
+                    )
+                return data
+
+        return cls(read_range, size)
+
+    def readable(self) -> bool:
+        return not self.closed
+
+    def seekable(self) -> bool:
+        return not self.closed
+
+    def size(self) -> int:
+        """Return the blob size without reading its payload."""
+        self._checkClosed()
+        return self._size
+
+    def tell(self) -> int:
+        self._checkClosed()
+        return self._position
+
+    def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
+        self._checkClosed()
+        offset = operator.index(offset)
+        if whence == io.SEEK_SET:
+            position = offset
+        elif whence == io.SEEK_CUR:
+            position = self._position + offset
+        elif whence == io.SEEK_END:
+            position = self._size + offset
+        else:
+            raise ValueError(f"invalid whence: {whence}")
+        if position < 0:
+            raise ValueError("negative seek position")
+        self._position = position
+        return position
+
+    def read_range(self, offset: int, length: int) -> bytes:
+        """Read a byte range without changing the cursor; truncate at EOF."""
+        self._checkClosed()
+        offset, length = operator.index(offset), operator.index(length)
+        if offset < 0 or length < 0:
+            raise ValueError("offset and length must be nonnegative")
+        remaining = min(length, max(0, self._size - offset))
+        chunks = []
+        while remaining:
+            count = min(remaining, _READ_CHUNK_BYTES)
+            chunk = self._read_range(offset, count)
+            if len(chunk) != count:
+                raise OSError("blob range reader returned an invalid payload length")
+            chunks.append(chunk)
+            offset += count
+            remaining -= count
+        return b"".join(chunks)
+
+    def read(self, size: int = -1) -> bytes:
+        self._checkClosed()
+        size = -1 if size is None else operator.index(size)
+        if size < 0:
+            size = max(0, self._size - self._position)
+        data = self.read_range(self._position, size)
+        self._position += len(data)
+        return data
+
+    def readinto(self, buffer) -> int:
+        view = memoryview(buffer).cast("B")
+        data = self.read(len(view))
+        view[: len(data)] = data
+        return len(data)

python/python/lancedb/_lancedb.pyi

@@ -19,11 +19,9 @@ from .index import (
     FTS,
 )
 from lance_namespace import (
-    ListNamespacesResponse,
     CreateNamespaceResponse,
     DropNamespaceResponse,
     DescribeNamespaceResponse,
-    ListTablesResponse,
 )
 from .remote import ClientConfig
 from .types import BaseTokenizerType
@@ -114,12 +112,12 @@ class Connection(object):
     uri: str
     async def is_open(self): ...
     async def close(self): ...
-    async def list_namespaces(
+    def list_namespaces(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse: ...
+    ) -> NameListing: ...
     async def create_namespace(
         self,
         namespace_path: List[str],
@@ -136,12 +134,12 @@ class Connection(object):
         self,
         namespace_path: List[str],
     ) -> DescribeNamespaceResponse: ...
-    async def list_tables(
+    def list_tables(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse: ...
+    ) -> NameListing: ...
     async def table_names(
         self,
         namespace_path: Optional[List[str]],
@@ -322,6 +320,8 @@ class Job:
     @property
     def id(self) -> Optional[str]: ...
     @property
+    def job_type(self) -> Optional[str]: ...
+    @property
     def _state(self) -> Optional[str]: ...
     @property
     def _description(self) -> Optional[JobDescription]: ...
@@ -581,6 +581,7 @@ class Table:
     async def fetch_blob_files(
         self, column: str, row_ids: list[int]
     ) -> list[Optional[BlobFile]]: ...
+    async def open_blob_file(self, descriptor: str) -> BlobFile: ...
     def vector_search(self) -> VectorQuery: ...
 
 class Tags:
@@ -1011,6 +1012,7 @@ async def connect_catalog(
 class NameListing:
     def num_page_results(self) -> int: ...
     def page_token(self) -> Optional[str]: ...
+    async def next_page(self) -> Optional[dict[str, Any]]: ...
     def __aiter__(self) -> NameListing: ...
     async def __anext__(self) -> str: ...
 

python/python/lancedb/db.py

@@ -32,18 +32,24 @@ if sys.version_info >= (3, 12):
 else:
     from overrides import EnforceOverrides, override  # type: ignore
 
-from lancedb.listing import AsyncListing, Listing
+from lancedb.listing import (
+    AsyncListing,
+    AsyncNamespaceListing,
+    AsyncTableListing,
+    Listing,
+    NamespaceListing,
+    TableListing,
+    validate_page_limit,
+)
 from lancedb.embeddings.registry import EmbeddingFunctionRegistry
 
 from lancedb.common import data_to_reader, sanitize_uri, validate_schema
 from lancedb.background_loop import LOOP
 from lance_namespace import (
     LanceNamespace,
-    ListNamespacesResponse,
     CreateNamespaceResponse,
     DropNamespaceResponse,
     DescribeNamespaceResponse,
-    ListTablesResponse,
     connect as namespace_connect,
 )
 from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
@@ -171,7 +177,7 @@ class DBConnection(EnforceOverrides):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> NamespaceListing:
         """List immediate child namespace names in the given namespace.
 
         Parameters
@@ -187,12 +193,12 @@ class DBConnection(EnforceOverrides):
 
         Returns
         -------
-        ListNamespacesResponse
+        NamespaceListing
             Response containing namespace names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return ListNamespacesResponse(namespaces=[], page_token=None)
+        return NamespaceListing(AsyncNamespaceListing(None, empty=True))
 
     def create_namespace(
         self,
@@ -317,8 +323,8 @@ class DBConnection(EnforceOverrides):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
-        """List a page of tables in this database.
+    ) -> TableListing:
+        """List tables in this database, one page at a time.
 
         Parameters
         ----------
@@ -334,7 +340,7 @@ class DBConnection(EnforceOverrides):
 
         Returns
         -------
-        ListTablesResponse
+        TableListing
             Response containing table names and optional page_token for pagination.
         """
         raise NotImplementedError(
@@ -542,7 +548,9 @@ class DBConnection(EnforceOverrides):
         LanceTable(name='table4', ...)
 
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "create_table is not supported for this connection type"
+        )
 
     def __getitem__(self, name: str) -> LanceTable:
         return self.open_table(name)
@@ -597,7 +605,9 @@ class DBConnection(EnforceOverrides):
         -------
         A LanceTable object representing the table.
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "open_table is not supported for this connection type"
+        )
 
     def create_materialized_view(
         self,
@@ -664,10 +674,11 @@ class DBConnection(EnforceOverrides):
     ) -> Job[None]:
         """Submit materialized-view creation and return its job.
 
-        The job may already be complete for a local database. On LanceDB
-        Cloud and Enterprise, its ``id`` is the server job identifier from
-        the ``202 Accepted`` create response. Wait for the job before opening
-        or querying the view.
+        The job may already be complete for a local database. Remote creation
+        submits CREATE MATERIALIZED VIEW through the SQL server. Its job has
+        ``job_type == "query"`` and ``id`` identifies the connection-scoped
+        query. Registry event history is unavailable.
+        Wait for the job before opening or querying the view.
         """
         raise NotImplementedError(
             "materialized views are not supported on this connection type"
@@ -737,13 +748,17 @@ class DBConnection(EnforceOverrides):
         """
         if namespace_path is None:
             namespace_path = []
-        raise NotImplementedError
+        raise NotImplementedError(
+            "drop_table is not supported for this connection type"
+        )
 
     def drop_table_async(
         self, name: str, namespace_path: Optional[List[str]] = None
     ) -> Job:
         """Start dropping a table and return its cleanup job."""
-        raise NotImplementedError
+        raise NotImplementedError(
+            "drop_table_async is not supported for this connection type"
+        )
 
     def rename_table(
         self,
@@ -771,19 +786,26 @@ class DBConnection(EnforceOverrides):
             cur_namespace_path = []
         if new_namespace_path is None:
             new_namespace_path = []
-        raise NotImplementedError
+        raise NotImplementedError(
+            "rename_table is not supported for this connection type"
+        )
 
     def drop_database(self):
         """
         Drop database
         This is the same thing as dropping all the tables
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "drop_database is not supported for this connection type"
+        )
 
     def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
         """
         Drop all tables from the database
 
+        Remote connections do not currently support this operation. Use
+        ``list_tables()`` and ``drop_table()`` to drop tables individually.
+
         Parameters
         ----------
         namespace_path: List[str], optional
@@ -792,7 +814,9 @@ class DBConnection(EnforceOverrides):
         """
         if namespace_path is None:
             namespace_path = []
-        raise NotImplementedError
+        raise NotImplementedError(
+            "drop_all_tables is not supported for this connection type"
+        )
 
     @property
     def uri(self) -> str:
@@ -1230,18 +1254,19 @@ class DBConnection(EnforceOverrides):
                 parameters=[np.array([0.1, 0.2, 0.3], dtype=np.float32), 10],
             )
 
-        Parameters are supported in queries, not in DDL or DML. A
-        parameterized query runs on the call that returns its rows:
-        cancelling it, or dropping its handle before reading, stops it on the
-        server. Its status follows that stream as well: ``describe()``
-        reports ``running`` until the reader has received every row, so open
-        ``reader()`` directly rather than waiting for ``finished``.
+        Parameters are supported in queries, not in DDL or DML. With or
+        without parameters, the server chooses whether to stream rows on the
+        initial call or return a polling continuation. The reader follows
+        either path transparently. Dropping an unread handle or its reader
+        stops a directly streamed query; deferred execution requires explicit
+        cancellation. Open ``reader()`` directly: a streamed query needs its
+        reader to make progress and reports ``running`` until the stream ends.
 
         Local connections do not support SQL.
         """
         raise NotImplementedError("SQL is not supported for this connection type")
 
-    def describe_query(self, query_id: UUID) -> QueryDescription:
+    def describe_query(self, query_id: Union[UUID, str]) -> QueryDescription:
         """Describe a submitted SQL query by its connection-scoped id."""
         raise NotImplementedError("SQL is not supported for this connection type")
 
@@ -1300,6 +1325,10 @@ class LanceDBConnection(DBConnection):
         self.storage_options = storage_options
         self._manifest_enabled = manifest_enabled
         self._namespace_client_properties = namespace_client_properties
+        # The native connection does not expose the Python Session wrapper.
+        # Keep the object supplied by the caller so this property stays
+        # synchronous for debugger inspection.
+        self._session = session
         if _inner is not None:
             self._conn = _inner
             # Native-derived wrappers resolve this in their async reconstruction
@@ -1366,7 +1395,12 @@ class LanceDBConnection(DBConnection):
 
     @property
     def session(self) -> Optional[Session]:
-        return self._conn.session
+        """Return the session supplied when this connection was opened.
+
+        None when the caller did not pass a session, including connections
+        reconstructed with :meth:`from_inner`.
+        """
+        return self._session
 
     @property
     def uri(self) -> str:
@@ -1427,7 +1461,7 @@ class LanceDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> NamespaceListing:
         """List immediate child namespace names in the given namespace.
 
         Parameters
@@ -1443,16 +1477,14 @@ class LanceDBConnection(DBConnection):
 
         Returns
         -------
-        ListNamespacesResponse
+        NamespaceListing
             Response containing namespace names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return NamespaceListing(
             self._conn.list_namespaces(
-                namespace_path=namespace_path,
-                page_token=page_token,
-                limit=limit,
+                namespace_path=namespace_path, page_token=page_token, limit=limit
             )
         )
 
@@ -1503,8 +1535,8 @@ class LanceDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
-        """List a page of tables in this database.
+    ) -> TableListing:
+        """List tables in this database, one page at a time.
 
         Parameters
         ----------
@@ -1520,12 +1552,12 @@ class LanceDBConnection(DBConnection):
 
         Returns
         -------
-        ListTablesResponse
+        TableListing
             Response containing table names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return TableListing(
             self._conn.list_tables(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
             )
@@ -2277,12 +2309,12 @@ class AsyncConnection(object):
         else:
             return None
 
-    async def list_namespaces(
+    def list_namespaces(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> AsyncNamespaceListing:
         """List immediate child namespace names in the given namespace.
 
         Parameters
@@ -2297,15 +2329,18 @@ class AsyncConnection(object):
 
         Returns
         -------
-        ListNamespacesResponse
+        AsyncNamespaceListing
             Response containing namespace names and optional pagination token
         """
         if namespace_path is None:
             namespace_path = []
-        result = await self._inner.list_namespaces(
+        validate_page_limit(limit)
+        inner = self._inner.list_namespaces(
             namespace_path=namespace_path, page_token=page_token, limit=limit
         )
-        return ListNamespacesResponse(**result)
+        if limit == 0:
+            return AsyncNamespaceListing(None, empty=True)
+        return AsyncNamespaceListing(inner, page_token)
 
     async def create_namespace(
         self,
@@ -2384,13 +2419,13 @@ class AsyncConnection(object):
         result = await self._inner.describe_namespace(namespace_path)
         return DescribeNamespaceResponse(**result)
 
-    async def list_tables(
+    def list_tables(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
-        """List a page of tables in this database.
+    ) -> AsyncTableListing:
+        """List tables in this database, one page at a time.
 
         Parameters
         ----------
@@ -2406,15 +2441,18 @@ class AsyncConnection(object):
 
         Returns
         -------
-        ListTablesResponse
+        AsyncTableListing
             Response containing table names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        result = await self._inner.list_tables(
+        validate_page_limit(limit)
+        inner = self._inner.list_tables(
             namespace_path=namespace_path, page_token=page_token, limit=limit
         )
-        return ListTablesResponse(**result)
+        if limit == 0:
+            return AsyncTableListing(None, empty=True)
+        return AsyncTableListing(inner, page_token)
 
     async def table_names(
         self,
@@ -2832,7 +2870,9 @@ class AsyncConnection(object):
     ) -> AsyncJob[None]:
         """Submit materialized-view creation and return its job.
 
-        Wait for the returned job before opening or querying the view.
+        Wait for the returned job before opening or querying the view. Remote
+        creation uses SQL; ``job_type`` is ``"query"`` and ``id``
+        identifies that query on this connection.
         """
         source, native_options = await self._materialized_view_source(
             source, select, where, limit
@@ -3372,8 +3412,14 @@ class AsyncConnection(object):
             )
         )
 
-    async def describe_query(self, query_id: UUID) -> QueryDescription:
-        """Describe a submitted SQL query by its connection-scoped id."""
+    async def describe_query(self, query_id: Union[UUID, str]) -> QueryDescription:
+        """Describe a submitted SQL query by its UUID or string ID.
+
+        For a handle with ``job_type == "query"``, pass its ``id`` on the
+        connection that submitted it. Registry job IDs are not query IDs.
+        """
+        if isinstance(query_id, str):
+            query_id = UUID(query_id)
         return await self._inner.describe_query(query_id)
 
     async def namespace_client(self) -> LanceNamespace:

python/python/lancedb/embeddings/base.py

@@ -38,9 +38,11 @@ class EmbeddingFunction(BaseModel, ABC):
     3. ndims() which returns the number of dimensions of the vector column
     """
 
-    max_retries: int = (
-        7  # Setting 0 disables retires. Maybe this should not be enabled by default,
-    )
+    # Maximum number of retries for transient errors (rate limits, 5xx responses,
+    # connection errors, ...), with exponential backoff capped at 60 seconds between
+    # attempts. Permanent errors (HTTP 4xx other than 408/409/429) are never retried.
+    # Setting 0 disables retries.
+    max_retries: int = 7
     _ndims: int = PrivateAttr()
     _original_args: dict = PrivateAttr()
 
@@ -239,7 +241,9 @@ class EmbeddingFunction(BaseModel, ABC):
         Creates a pydantic Field that can automatically annotate
         the source column for this embedding function
         """
-        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
+        json_schema_extra = dict(kwargs.pop("json_schema_extra", None) or {})
+        json_schema_extra["source_column_for"] = self
+        return Field(json_schema_extra=json_schema_extra, **kwargs)
 
     def VectorField(self, **kwargs):
         """
@@ -250,7 +254,9 @@ class EmbeddingFunction(BaseModel, ABC):
         static type checkers. LanceDB will infer the fixed vector dimension from
         this embedding function.
         """
-        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
+        json_schema_extra = dict(kwargs.pop("json_schema_extra", None) or {})
+        json_schema_extra["vector_column_for"] = self
+        return Field(json_schema_extra=json_schema_extra, **kwargs)
 
     def __eq__(self, __value: object) -> bool:
         if not hasattr(__value, "__dict__"):

python/python/lancedb/embeddings/jinaai.py

@@ -5,9 +5,10 @@
 import os
 import io
 import base64
+from functools import cached_property
 from urllib.parse import urlparse
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, List, Union, Optional, Any, Dict
+from typing import TYPE_CHECKING, List, Union, Optional, Any, Dict
 
 import numpy as np
 import pyarrow as pa
@@ -51,7 +52,6 @@ class JinaEmbeddings(EmbeddingFunction):
 
     name: str = "jina-clip-v1"
     api_key: Optional[str] = None
-    _session: ClassVar = None
 
     def ndims(self):
         # TODO: fix hardcoding
@@ -206,8 +206,7 @@ class JinaEmbeddings(EmbeddingFunction):
         texts: list[str] or np.ndarray (of str)
             The texts to embed
         """
-        self._init_client()
-        resp = JinaEmbeddings._session.post(  # type: ignore
+        resp = self._session.post(
             API_URL, json={"input": input, "model": self.name}
         ).json()
         if "data" not in resp:
@@ -220,14 +219,15 @@ class JinaEmbeddings(EmbeddingFunction):
 
         return [result["embedding"] for result in sorted_embeddings]
 
-    def _init_client(self):
+    @cached_property
+    def _session(self):
         import requests
 
-        if JinaEmbeddings._session is None:
-            if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
-                api_key_not_found_help("jina")
-            api_key = self.api_key or os.environ.get("JINA_API_KEY")
-            JinaEmbeddings._session = requests.Session()
-            JinaEmbeddings._session.headers.update(
-                {"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}
-            )
+        if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
+            api_key_not_found_help("jina")
+        api_key = self.api_key or os.environ.get("JINA_API_KEY")
+        session = requests.Session()
+        session.headers.update(
+            {"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}
+        )
+        return session

python/python/lancedb/embeddings/ollama.py

@@ -70,6 +70,8 @@ class OllamaEmbeddings(TextEmbeddingFunction):
             The texts to embed
         """
         # TODO retry, rate limit, token limit
+        if isinstance(texts, np.ndarray):
+            texts = texts.tolist()
         embeddings = self._compute_embedding(texts)
         return list(embeddings)
 

python/python/lancedb/embeddings/registry.py

@@ -69,11 +69,15 @@ class EmbeddingFunctionRegistry:
         def decorator(cls):
             if not issubclass(cls, EmbeddingFunction):
                 raise TypeError("Must be a subclass of EmbeddingFunction")
-            if cls.__name__ in self._functions:
-                raise KeyError(f"{cls.__name__} was already registered")
             key = alias or cls.__name__
+            if key in self._functions and (
+                not alias or self._functions[key] is not cls
+            ):
+                raise KeyError(f"{key} was already registered")
             self._functions[key] = cls
-            cls.__embedding_function_registry_alias__ = alias
+            # Store the name the class is registered under, so the table
+            # metadata written for it can be looked up again when reading.
+            cls.__embedding_function_registry_alias__ = key
             return cls
 
         return decorator

python/python/lancedb/embeddings/utils.py

@@ -241,15 +241,30 @@ def retry_with_exponential_backoff(
     exponential_base: float = 2,
     jitter: bool = True,
     max_retries: int = 7,
+    max_delay: float = 60,
 ):
     """Retry a function with exponential backoff.
 
+    Only errors that may succeed when repeated are retried. Errors that carry an
+    HTTP status code are retried when the status is 408, 409, 429 or >= 500; any
+    other 4xx status (for example 400, 401, 403, 404 or 422) is a permanent failure
+    that is raised immediately. Errors without a recognisable status code (for
+    example connection errors, timeouts, or exceptions raised by a custom embedding
+    function) are retried.
+
+    The delay before the n-th retry is ``initial_delay * exponential_base ** n``,
+    never more than ``max_delay``. With ``jitter`` enabled, a random amount of up to
+    half of that delay is removed ("equal jitter"), so the actual wait is always in
+    the range ``[delay / 2, delay]``.
+
     Args:
         func (function): The function to be retried.
         initial_delay (float): Initial delay in seconds (default is 1).
         exponential_base (float): The base for exponential backoff (default is 2).
         jitter (bool): Whether to add jitter to the delay (default is True).
         max_retries (int): Maximum number of retries (default is 7).
+        max_delay (float): Maximum delay between two attempts, in seconds
+            (default is 60).
 
     Returns:
         function: The decorated function.
@@ -265,13 +280,12 @@ def retry_with_exponential_backoff(
             try:
                 return func(*args, **kwargs)
 
-            # Currently retrying on all exceptions as there is no way to know the
-            # format of the error msgs used by different APIs. We'll log the error
-            # and say that it is assumed that if this portion errors out, it's due
-            # to rate limit but the user should check the error message to be sure.
+            # Errors are retried unless they are known to be permanent (see
+            # `_is_non_retryable_error`). Custom embedding functions can raise
+            # arbitrary exceptions, so unknown errors are assumed to be transient.
             except Exception as e:  # noqa: PERF203
-                # Don't retry on authentication errors (e.g., OpenAI 401)
-                # These are permanent failures that won't be fixed by retrying
+                # Don't retry permanent failures (e.g., OpenAI 401 or 404). They
+                # won't be fixed by retrying and would only delay the real error.
                 if _is_non_retryable_error(e):
                     raise
 
@@ -280,24 +294,72 @@ def retry_with_exponential_backoff(
                 if num_retries > max_retries:
                     raise Exception(
                         f"Maximum number of retries ({max_retries}) exceeded.", e
-                    )
+                    ) from e
 
-                delay *= exponential_base * (1 + jitter * random.random())
+                delay = min(delay * exponential_base, max_delay)
+                sleep_for = delay
+                if jitter:
+                    sleep_for = delay / 2 + random.random() * delay / 2
                 logging.warning(
                     "Error occurred: %s \n Retrying in %s seconds (retry %s of %s) \n",
                     e,
-                    delay,
+                    sleep_for,
                     num_retries,
                     max_retries,
                 )
-                time.sleep(delay)
+                time.sleep(sleep_for)
 
     return wrapper
 
 
+# HTTP statuses in the 4xx range that are worth retrying: request timeout,
+# conflict (the OpenAI SDK treats it as transient) and rate limiting.
+_RETRYABLE_CLIENT_STATUSES = (408, 409, 429)
+
+# AWS error codes that are reported with a 4xx status but are transient.
+_RETRYABLE_AWS_ERROR_CODES = (
+    "Throttl",
+    "TooManyRequests",
+    "RequestLimitExceeded",
+    "ProvisionedThroughputExceeded",
+    "ModelNotReady",
+    "ModelTimeout",
+)
+
+
+def _get_http_status(error: Exception):
+    """Best-effort extraction of an HTTP status code from an exception.
+
+    Provider SDKs expose it in different ways: ``status_code`` (openai, cohere),
+    ``http_status`` (voyageai), ``response.status_code`` (requests, httpx),
+    ``code`` (google-api-core, urllib) or ``response["ResponseMetadata"]``
+    (botocore). Returns None if no status can be found.
+    """
+    candidates = [
+        getattr(error, "status_code", None),
+        getattr(error, "http_status", None),
+    ]
+    response = getattr(error, "response", None)
+    if isinstance(response, dict):
+        candidates.append(response.get("ResponseMetadata", {}).get("HTTPStatusCode"))
+    else:
+        candidates.append(getattr(response, "status_code", None))
+    candidates.append(getattr(error, "code", None))
+
+    for status in candidates:
+        if isinstance(status, int) and not isinstance(status, bool):
+            if 100 <= status <= 599:
+                return status
+    return None
+
+
 def _is_non_retryable_error(error: Exception) -> bool:
     """Check if an error should not be retried.
 
+    An error is non-retryable if it is an authentication error or if it carries an
+    HTTP client-error status (4xx) other than 408, 409 and 429. Errors without a
+    recognisable status are treated as retryable.
+
     Args:
         error: The exception to check
 
@@ -314,13 +376,25 @@ def _is_non_retryable_error(error: Exception) -> bool:
     if error_type == "AuthenticationError":
         return True
 
-    # Check for other common non-retryable HTTP status codes
-    # 401 Unauthorized, 403 Forbidden
-    if hasattr(error, "status_code"):
-        if error.status_code in (401, 403):
-            return True
+    # A missing API key or a missing optional dependency fails the same way on
+    # every attempt
+    if isinstance(error, (MissingApiKeyError, ImportError)):
+        return True
 
-    return False
+    status = _get_http_status(error)
+    if status is None or not 400 <= status < 500:
+        return False
+    if status in _RETRYABLE_CLIENT_STATUSES:
+        return False
+
+    # AWS reports throttling with a 400 status; the error code tells them apart.
+    response = getattr(error, "response", None)
+    if isinstance(response, dict):
+        aws_code = str(response.get("Error", {}).get("Code", ""))
+        if aws_code.startswith(_RETRYABLE_AWS_ERROR_CODES):
+            return False
+
+    return True
 
 
 def url_retrieve(url: str):
@@ -337,9 +411,15 @@ def url_retrieve(url: str):
         raise ConnectionError("could not download {} due to {}".format(url, err))
 
 
+class MissingApiKeyError(ValueError):
+    """Raised when an embedding provider's API key is not set."""
+
+
 def api_key_not_found_help(provider):
     logging.error("Could not find API key for %s", provider)
-    raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")
+    raise MissingApiKeyError(
+        f"Please set the {provider.upper()}_API_KEY environment variable."
+    )
 
 
 def is_flash_attn_2_available():

python/python/lancedb/embeddings/voyageai.py

@@ -75,7 +75,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
             content = {"type": "text", "text": input_data}
     elif isinstance(input_data, PIL_Image.Image):
         buffered = BytesIO()
-        input_data.save(buffered, format="JPEG")
+        input_data.convert("RGB").save(buffered, format="JPEG")
         img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
         content = {
             "type": "image_base64",
@@ -84,7 +84,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
     elif isinstance(input_data, bytes):
         img = PIL_Image.open(BytesIO(input_data))
         buffered = BytesIO()
-        img.save(buffered, format="JPEG")
+        img.convert("RGB").save(buffered, format="JPEG")
         img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
         content = {
             "type": "image_base64",
@@ -103,7 +103,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
         else:
             img = PIL_Image.open(input_data)
             buffered = BytesIO()
-            img.save(buffered, format="JPEG")
+            img.convert("RGB").save(buffered, format="JPEG")
             img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
             content = {
                 "type": "image_base64",
@@ -145,6 +145,8 @@ def sanitize_text_input(inputs: TEXT) -> List[str]:
     """
     if isinstance(inputs, str):
         inputs = [inputs]
+    elif isinstance(inputs, (list, np.ndarray)):
+        inputs = list(inputs)
     elif isinstance(inputs, pa.Array):
         inputs = inputs.to_pylist()
     elif isinstance(inputs, pa.ChunkedArray):

python/python/lancedb/functions.py

@@ -33,6 +33,7 @@ from typing import (
     Annotated,
     Any,
     Callable,
+    Literal,
     Optional,
     Union,
     get_args,
@@ -52,7 +53,9 @@ from pydantic import (
     model_validator,
 )
 
+from .schema import blob as _blob_field
 from .schema import is_blob_v2_field as _is_blob_v2_field
+from ._blob_reader import BlobReader
 from .secrets import EnvVarSecret
 
 _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1)
@@ -213,6 +216,7 @@ class FunctionParameter(_RemoteValue):
     name: str
     arrow_type: str
     nullable: bool
+    blob_access: Optional[Literal["bytes", "reader"]] = None
 
 
 class FunctionResultField(_OpenRemoteValue):
@@ -1096,6 +1100,43 @@ def _infer_signature(
     if (input_schema is None) != (output_schema is None):
         raise ValueError("input_schema and output_schema must be provided together")
 
+    try:
+        annotations = get_type_hints(function, include_extras=True)
+    except Exception as error:
+        raise TypeError(f"failed to resolve Function annotations: {error}") from error
+
+    def input_parameter(field):
+        annotation = annotations.get(field.name)
+        if get_origin(annotation) in (Union, types.UnionType):
+            choices = [
+                value for value in get_args(annotation) if value is not type(None)
+            ]
+            if len(choices) == 1:
+                annotation = choices[0]
+        arrow_type = _canonical_arrow_field(field)
+        if annotation is BlobReader:
+            if arrow_type != _FUNCTION_BLOB_V2_TYPE:
+                raise TypeError(
+                    f"BlobReader input '{field.name}' requires a Blob V2 field"
+                )
+            access = "reader"
+        elif annotation is bytes:
+            if arrow_type != _FUNCTION_BLOB_V2_TYPE and not (
+                pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type)
+            ):
+                raise TypeError(
+                    f"bytes input '{field.name}' requires binary or Blob V2"
+                )
+            access = "bytes"
+        else:
+            access = "bytes" if arrow_type == _FUNCTION_BLOB_V2_TYPE else None
+        return FunctionParameter(
+            name=field.name,
+            arrow_type=arrow_type,
+            nullable=field.nullable,
+            blob_access=access,
+        )
+
     if input_schema is not None:
         if not isinstance(input_schema, pa.Schema):
             raise TypeError("input_schema must be a PyArrow Schema")
@@ -1110,20 +1151,9 @@ def _infer_signature(
                 "input_schema fields must exactly match callable parameters in order: "
                 f"expected {expected!r}, got {actual!r}"
             )
-        inputs = tuple(
-            FunctionParameter(
-                name=field.name,
-                arrow_type=_canonical_arrow_field(field),
-                nullable=field.nullable,
-            )
-            for field in input_schema
-        )
+        inputs = tuple(input_parameter(field) for field in input_schema)
         return FunctionSignature(inputs=inputs, output=_function_output(output_schema))
 
-    try:
-        annotations = get_type_hints(function, include_extras=True)
-    except Exception as error:
-        raise TypeError(f"failed to resolve Function annotations: {error}") from error
     missing = [
         parameter.name for parameter in parameters if parameter.name not in annotations
     ]
@@ -1132,15 +1162,21 @@ def _infer_signature(
         raise TypeError(f"missing Function annotations: {names!r}")
     inputs = []
     for parameter in parameters:
-        data_type, nullable = _annotation_type(annotations[parameter.name])
-        inputs.append(
-            FunctionParameter(
-                name=parameter.name,
-                arrow_type=_canonical_arrow_field(
-                    pa.field(parameter.name, data_type, nullable=nullable)
-                ),
-                nullable=nullable,
+        annotation = annotations[parameter.name]
+        base = annotation
+        nullable = False
+        if get_origin(base) in (Union, types.UnionType):
+            choices = [value for value in get_args(base) if value is not type(None)]
+            if len(choices) == 1:
+                base, nullable = choices[0], True
+        if base is BlobReader:
+            inputs.append(
+                input_parameter(_blob_field(parameter.name, nullable=nullable))
             )
+            continue
+        data_type, nullable = _annotation_type(annotation)
+        inputs.append(
+            input_parameter(pa.field(parameter.name, data_type, nullable=nullable))
         )
     output_type, output_nullable = _annotation_type(annotations["return"])
     if output_nullable:
@@ -1239,7 +1275,7 @@ _PYTHON_CALLABLE_ARTIFACT = "python_callable"
 _PYTHON_BUNDLE_ARTIFACT = "python_bundle"
 # Modules the Function image places beside user code.
 _RESERVED_SOURCE_MODULES = frozenset(
-    {"user_function", "lance_source", "lance_callable"}
+    {"user_function", "lance_source", "lance_callable", "lance_blob"}
 )
 # The service accepts bundle paths made of these components only.
 _SOURCE_PATH_COMPONENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
@@ -1499,6 +1535,8 @@ class _SourcePackager:
         """One module-level line that rebinds `name` to `value` in the artifact:
         an import for modules and importable classes/functions, a literal
         otherwise."""
+        if value is BlobReader:
+            return f"from lance_blob import BlobReader as {name}"
         if isinstance(value, types.ModuleType):
             if value.__name__.split(".")[0] in _NAMESPACE_MODULES:
                 raise ValueError(

python/python/lancedb/integrations/pyarrow.py

@@ -52,6 +52,9 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
             raise NotImplementedError("memory_pool not supported")
 
     def count_rows(self):
+        if isinstance(self.filter, pa.compute.Expression):
+            # stream the batches so memory does not grow with the matching rows
+            return sum(batch.num_rows for batch in self.to_reader())
         return self.table.count_rows(self.filter)
 
     def from_batches(self, **kwargs):
@@ -83,6 +86,8 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
         return self.to_reader().read_all()
 
     def to_reader(self, *, limit: Optional[int] = None):
+        if isinstance(self.filter, pa.compute.Expression):
+            return self._to_reader_with_expression(limit=limit)
         query = self.table.search()
         # Disable the builtin limit
         if limit is None:
@@ -98,13 +103,30 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
             query = query.where(self.filter, prefilter=True)
         return query.to_batches(batch_size=self.batch_size)
 
+    def _to_reader_with_expression(self, *, limit: Optional[int] = None):
+        # LanceDB filters are SQL strings, but engines such as DuckDB push down
+        # pyarrow compute expressions.  Scan the table and apply the filter and
+        # projection with pyarrow, since the filter may reference columns that
+        # are not part of the projection.
+        if limit is not None and limit <= 0:
+            raise ValueError("limit must be positive")
+        query = self.table.search().limit(self.table.count_rows())
+        scanner = pa.dataset.Scanner.from_batches(
+            query.to_batches(batch_size=self.batch_size),
+            columns=self.columns,
+            filter=self.filter,
+        )
+        if limit is None:
+            return scanner.to_reader()
+        return scanner.head(limit).to_reader()
+
 
 class PyarrowDatasetAdapter(pa.dataset.Dataset):
     def __init__(self, table: Table):
         self.table = table
 
     def count_rows(self, filter: Optional[Filter] = None):
-        return self.table.count_rows(filter)
+        return self.scanner(filter=filter).count_rows()
 
     def get_fragments(self, filter: Optional[Filter] = None):
         raise NotImplementedError

python/python/lancedb/job.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-"""Handles to operations a server may run asynchronously."""
+"""Handles to asynchronous registry jobs, SQL queries, and local operations."""
 
 import asyncio
 import json
@@ -44,12 +44,12 @@ class AsyncJob(Generic[T]):
 
     @property
     def id(self) -> Optional[str]:
-        """Identifies the operation on the server that is running it.
+        """The operation's opaque ID, interpreted according to :attr:`job_type`.
 
-        Returned for correlating with server logs or the jobs API. Operations
-        that run in this process have no server id and return `None`. The value
-        is opaque: parsing it or storing it to resume the job later is not
-        supported.
+        Registry jobs expose a registry job ID; SQL-backed jobs expose a
+        connection-scoped query ID as a string. Local operations return ``None``.
+        Query IDs cannot be used with registry job APIs or to resume after a
+        client restart.
         """
         return self._inner.id if self._inner is not None else None
 
@@ -94,9 +94,11 @@ class AsyncJob(Generic[T]):
         """Ask the backend for this job's current state, and for a server-side
         job its full record, then cache it for the properties below.
 
-        The properties are all `None` until this runs, because submitting an
-        operation returns only a job id. `status` fetches the whole record too;
-        `wait` records only the terminal state it establishes.
+        Registry details are `None` until this runs, because submitting an
+        operation does not fetch its full registry record. Query-backed jobs
+        expose only lifecycle state through this method. For registry jobs,
+        `status` fetches the whole record too; `wait` records only the terminal
+        state it establishes.
         """
         if self._inner is None:
             return
@@ -114,11 +116,13 @@ class AsyncJob(Generic[T]):
 
     @property
     def job_type(self) -> Optional[str]:
-        """The job's type, as the server names it.
+        """The job's kind of work, without contacting the backend.
 
-        `None` for an in-process job, which has no server-side record.
+        SQL-backed handles report ``"query"`` immediately. Registry handles
+        report the server's job type after :meth:`refresh` or :meth:`status`.
+        In-process handles return ``None``.
         """
-        return self._field("job_type")
+        return self._inner.job_type if self._inner is not None else None
 
     @property
     def creation_ms(self) -> Optional[int]:
@@ -294,7 +298,7 @@ class Job(Generic[T]):
     @property
     def job_type(self) -> Optional[str]:
         """The job's type. See :attr:`AsyncJob.job_type`."""
-        return self._field("job_type")
+        return self._inner.job_type if self._inner is not None else None
 
     @property
     def creation_ms(self) -> Optional[int]:

python/python/lancedb/listing.py

@@ -5,6 +5,7 @@
 
 from __future__ import annotations
 
+import asyncio
 from typing import Any, AsyncIterator, Callable, Generic, Iterator, Optional, TypeVar
 
 from .background_loop import LOOP
@@ -83,3 +84,173 @@ class Listing(Iterator[T], Generic[T]):
         for initial, cached, final-page, and error behavior.
         """
         return self._inner.page_token()
+
+
+def validate_page_limit(limit: Optional[int]) -> None:
+    """Validate the legacy limit argument before passing it to the bindings."""
+    if limit is not None and (
+        not isinstance(limit, int) or limit < 0 or limit > 2**31 - 1
+    ):
+        raise ValueError("limit must be between 0 and 2147483647")
+
+
+class _AsyncPagedNameListing(AsyncIterator[str]):
+    """An async listing with an inspectable copy of its current backend page."""
+
+    def __init__(
+        self, inner: Any, page_token: Optional[str] = None, *, empty: bool = False
+    ):
+        self._inner = inner
+        self._token = page_token
+        self._page: Optional[list[str]] = [] if empty else None
+        self._position = 0
+        self._done = empty
+        self._failure: Optional[BaseException] = None
+        self._lock = asyncio.Lock()
+
+    @property
+    def page_token(self) -> Optional[str]:
+        """The continuation token for the next backend request."""
+        return self._token
+
+    def num_page_results(self) -> int:
+        """Count the remaining names in the current page."""
+        return 0 if self._page is None else len(self._page) - self._position
+
+    def _page_copy(self) -> list[str]:
+        if self._page is None:
+            raise RuntimeError("Await or iterate the listing before reading its page")
+        return self._page.copy()
+
+    async def _fetch_page(self) -> None:
+        try:
+            page = await self._inner.next_page()
+        except asyncio.CancelledError:
+            raise
+        except BaseException as exc:
+            self._failure = exc
+            self._done = True
+            raise
+        if page is None:
+            self._page = []
+            self._token = None
+            self._done = True
+        else:
+            self._page = page["items"]
+            self._token = page["page_token"]
+            self._done = self._token is None
+        self._position = 0
+
+    async def _load_initial(self):
+        async with self._lock:
+            if self._failure is not None:
+                raise self._failure
+            if self._page is None:
+                await self._fetch_page()
+        return self
+
+    def __await__(self):
+        return self._load_initial().__await__()
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> str:
+        async with self._lock:
+            if self._failure is not None:
+                raise StopAsyncIteration
+            if self._page is None:
+                await self._fetch_page()
+            assert self._page is not None
+            while self._position >= len(self._page):
+                if self._done:
+                    raise StopAsyncIteration
+                await self._fetch_page()
+                assert self._page is not None
+            item = self._page[self._position]
+            self._position += 1
+            return item
+
+
+class AsyncTableListing(_AsyncPagedNameListing):
+    """Awaitable table iterator; awaiting loads one page.
+
+    For example::
+
+        page = await db.list_tables(limit=20)
+        print(page.tables, page.page_token)
+        async for name in db.list_tables(limit=20):
+            print(name)
+    """
+
+    @property
+    def tables(self) -> list[str]:
+        """A copy of every table name in the most recently fetched page."""
+        return self._page_copy()
+
+
+class AsyncNamespaceListing(_AsyncPagedNameListing):
+    """Awaitable namespace iterator; awaiting loads one page.
+
+    For example::
+
+        page = await db.list_namespaces(limit=20)
+        print(page.namespaces, page.page_token)
+    """
+
+    @property
+    def namespaces(self) -> list[str]:
+        """A copy of every name in the most recently fetched page."""
+        return self._page_copy()
+
+
+class _PagedNameListing(Iterator[str]):
+    def __init__(self, inner: _AsyncPagedNameListing):
+        self._inner = inner
+        LOOP.run(inner._load_initial())
+
+    def __iter__(self):
+        return self
+
+    def __next__(self) -> str:
+        try:
+            return LOOP.run(self._inner.__anext__())
+        except StopAsyncIteration:
+            raise StopIteration from None
+
+    @property
+    def page_token(self) -> Optional[str]:
+        return self._inner.page_token
+
+    def num_page_results(self) -> int:
+        return self._inner.num_page_results()
+
+
+class TableListing(_PagedNameListing):
+    """Table iterator retaining the current page as a list.
+
+    For example::
+
+        page = db.list_tables(limit=20)
+        print(page.tables, page.page_token)
+        for name in db.list_tables(limit=20):
+            print(name)
+    """
+
+    @property
+    def tables(self) -> list[str]:
+        return self._inner.tables
+
+
+class NamespaceListing(_PagedNameListing):
+    """Namespace iterator retaining the current page as a list.
+
+    For example::
+
+        page = db.list_namespaces(limit=20)
+        print(page.namespaces, page.page_token)
+    """
+
+    @property
+    def namespaces(self) -> list[str]:
+        return self._inner.namespaces

python/python/lancedb/namespace.py

@@ -11,6 +11,7 @@ through a namespace abstraction.
 from __future__ import annotations
 
 import sys
+import warnings
 from typing import Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 
@@ -51,7 +52,14 @@ from lancedb.background_loop import LOOP
 from lancedb.arrow import AsyncRecordBatchReader
 from lancedb.db import AsyncConnection, DBConnection
 from lancedb.job import AsyncJob, Job
-from lancedb.listing import AsyncListing, Listing
+from lancedb.listing import (
+    AsyncListing,
+    AsyncNamespaceListing,
+    AsyncTableListing,
+    Listing,
+    NamespaceListing,
+    TableListing,
+)
 from lancedb.sql import AsyncQuery as AsyncSqlQuery
 from lancedb.sql import QueryDescription, QueryParameters
 from lance_namespace import (
@@ -60,8 +68,6 @@ from lance_namespace import (
     CreateNamespaceResponse,
     DescribeNamespaceResponse,
     DropNamespaceResponse,
-    ListNamespacesResponse,
-    ListTablesResponse,
     NamespaceExistsRequest,
     TableExistsRequest,
 )
@@ -82,6 +88,31 @@ from ._lancedb import Session
 _MAX_QUERY_K = 2**31 - 1
 
 
+def _create_table_storage_options(
+    storage_options: Optional[Dict[str, str]],
+    data_storage_version: Optional[str],
+    enable_v2_manifest_paths: Optional[bool],
+) -> Optional[Dict[str, str]]:
+    if data_storage_version is None and enable_v2_manifest_paths is None:
+        return storage_options
+    options = dict(storage_options or {})
+    for name, value in (
+        ("data_storage_version", data_storage_version),
+        ("enable_v2_manifest_paths", enable_v2_manifest_paths),
+    ):
+        if value is not None:
+            warnings.warn(
+                f"setting {name} directly on create_table is deprecated. "
+                "Use storage_options instead.",
+                DeprecationWarning,
+                stacklevel=3,
+            )
+            options[f"new_table_{name}"] = (
+                str(value).lower() if isinstance(value, bool) else value
+            )
+    return options
+
+
 def _query_to_namespace_request(
     table_id: List[str],
     query: "Query",
@@ -587,6 +618,9 @@ class LanceNamespaceDBConnection(DBConnection):
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
+        storage_options = _create_table_storage_options(
+            storage_options, data_storage_version, enable_v2_manifest_paths
+        )
         async_table = LOOP.run(
             self._inner.create_table(
                 name,
@@ -825,7 +859,7 @@ class LanceNamespaceDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> NamespaceListing:
         """
         List child namespaces under the given namespace.
 
@@ -842,12 +876,12 @@ class LanceNamespaceDBConnection(DBConnection):
 
         Returns
         -------
-        ListNamespacesResponse
+        NamespaceListing
             Response containing namespace names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return NamespaceListing(
             self._inner.list_namespaces(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
             )
@@ -994,7 +1028,7 @@ class LanceNamespaceDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
+    ) -> TableListing:
         """
         List all tables in this database with pagination support.
 
@@ -1011,12 +1045,12 @@ class LanceNamespaceDBConnection(DBConnection):
 
         Returns
         -------
-        ListTablesResponse
+        TableListing
             Response containing table names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return TableListing(
             self._inner.list_tables(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
             )
@@ -1230,6 +1264,9 @@ class AsyncLanceNamespaceDBConnection:
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
+        storage_options = _create_table_storage_options(
+            storage_options, data_storage_version, enable_v2_manifest_paths
+        )
         table = await self._inner.create_table(
             name,
             data,
@@ -1427,12 +1464,12 @@ class AsyncLanceNamespaceDBConnection:
             namespace_path = []
         await self._inner.drop_all_tables(namespace_path=namespace_path)
 
-    async def list_namespaces(
+    def list_namespaces(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> AsyncNamespaceListing:
         """
         List child namespaces under the given namespace.
 
@@ -1449,12 +1486,12 @@ class AsyncLanceNamespaceDBConnection:
 
         Returns
         -------
-        ListNamespacesResponse
+        NamespaceListing
             Response containing namespace names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return await self._inner.list_namespaces(
+        return self._inner.list_namespaces(
             namespace_path=namespace_path, page_token=page_token, limit=limit
         )
 
@@ -1584,12 +1621,12 @@ class AsyncLanceNamespaceDBConnection:
         except TableNotFoundError:
             return False
 
-    async def list_tables(
+    def list_tables(
         self,
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
+    ) -> AsyncTableListing:
         """
         List all tables in this database with pagination support.
 
@@ -1606,12 +1643,12 @@ class AsyncLanceNamespaceDBConnection:
 
         Returns
         -------
-        ListTablesResponse
+        TableListing
             Response containing table names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return await self._inner.list_tables(
+        return self._inner.list_tables(
             namespace_path=namespace_path, page_token=page_token, limit=limit
         )
 
@@ -1646,7 +1683,7 @@ class AsyncLanceNamespaceDBConnection:
             parameters=parameters,
         )
 
-    async def describe_query(self, query_id: UUID) -> QueryDescription:
+    async def describe_query(self, query_id: Union[UUID, str]) -> QueryDescription:
         """Describe a submitted SQL query when supported."""
         return await self._inner.describe_query(query_id)
 

python/python/lancedb/permutation.py

@@ -909,9 +909,11 @@ class Permutation:
                 f"Cannot rename column `{old_name}` to `{new_name}` because a column "
                 "with that name already exists"
             )
-        new_selection = self.selection.copy()
-        new_selection[new_name] = new_selection[old_name]
-        del new_selection[old_name]
+        # Rebuild the mapping so the renamed column keeps its position
+        new_selection = {
+            (new_name if name == old_name else name): value
+            for name, value in self.selection.items()
+        }
         return self._with_selection(new_selection)
 
     def rename_columns(self, column_map: dict[str, str]) -> "Permutation":

python/python/lancedb/pydantic.py

@@ -127,7 +127,11 @@ def Vector(
                 core_schema.list_schema(
                     min_length=dim,
                     max_length=dim,
-                    items_schema=core_schema.float_schema(),
+                    items_schema=(
+                        core_schema.int_schema()
+                        if pa.types.is_integer(value_type)
+                        else core_schema.float_schema()
+                    ),
                 ),
             )
 
@@ -208,7 +212,11 @@ def MultiVector(
                     items_schema=core_schema.list_schema(
                         min_length=dim,
                         max_length=dim,
-                        items_schema=core_schema.float_schema(),
+                        items_schema=(
+                            core_schema.int_schema()
+                            if pa.types.is_integer(value_type)
+                            else core_schema.float_schema()
+                        ),
                     ),
                 ),
             )
@@ -260,7 +268,9 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
 
 def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
     return [
-        _pydantic_to_field(name, field) for name, field in model.model_fields.items()
+        _pydantic_to_field(name, field)
+        for name, field in model.model_fields.items()
+        if field.exclude is not True
     ]
 
 
@@ -493,6 +503,9 @@ class LanceModel(pydantic.BaseModel):
 
         vec_and_function = []
         for name, field_info in cls.safe_get_fields().items():
+            if field_info.exclude is True:
+                continue
+
             func = get_extras(field_info, "vector_column_for")
             if func is not None:
                 vec_and_function.append([name, func])
@@ -500,6 +513,8 @@ class LanceModel(pydantic.BaseModel):
         configs = []
         for vec, func in vec_and_function:
             for source, field_info in cls.safe_get_fields().items():
+                if field_info.exclude is True:
+                    continue
                 src_func = get_extras(field_info, "source_column_for")
                 if src_func is func:
                     # note we can't use == here since the function is a pydantic
@@ -524,4 +539,20 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
 
 def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]:
     """Convert a Pydantic model to a dictionary."""
-    return model.model_dump()
+    return _enum_members_to_values(model.model_dump())
+
+
+def _enum_members_to_values(value: Any) -> Any:
+    # model_dump() keeps Enum members, and Arrow cannot convert a member of a
+    # plain Enum (one without a str or int mixin). Write its value instead,
+    # which is what the Arrow type of the field is derived from.
+    if isinstance(value, Enum):
+        return value.value
+    if isinstance(value, dict):
+        return {k: _enum_members_to_values(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)) and value:
+        # Skip lists of plain scalars, such as vectors, without walking them.
+        if isinstance(value[0], (int, float, str, bytes)):
+            return value
+        return [_enum_members_to_values(v) for v in value]
+    return value

python/python/lancedb/query.py

@@ -1561,9 +1561,10 @@ class LanceQueryBuilder(ABC):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
-                  LanceRead: uri=..., projection=[vector], ...
+                  LanceRead: uri=..., projection=[vector], num_fragments=..., ...
+        <BLANKLINE>
 
         Parameters
         ----------
@@ -1619,8 +1620,8 @@ class LanceQueryBuilder(ABC):
               GlobalLimitExec: elapsed=..., skip=0, fetch=10,
               metrics=[output_rows=..., elapsed_compute=..., output_bytes=...]
                 FilterExec: elapsed=..., _distance@2 IS NOT NULL, metrics=[...]
-                  SortExec: elapsed=..., TopK(fetch=10), expr=[...],
-                  preserve_partitioning=[...],
+                  SortExec: elapsed=..., TopK(fetch=1), expr=[...],
+                  preserve_partitioning=[...], filter=[...],
                   metrics=[output_rows=..., elapsed_compute=...,
                   output_bytes=..., row_replacements=...]
                     KNNVectorDistance: elapsed=..., metric=l2,
@@ -1633,6 +1634,7 @@ class LanceQueryBuilder(ABC):
                       metrics=[output_rows=..., elapsed_compute=..., output_bytes=...,
                       fragments_scanned=..., ranges_scanned=1, rows_scanned=1,
                       bytes_read=..., iops=..., requests=..., task_wait_time=...]
+        <BLANKLINE>
 
         Parameters
         ----------
@@ -2575,10 +2577,6 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         with_row_ids: bool,
         offset: Optional[int] = None,
     ) -> pa.Table:
-        if norm == "rank":
-            vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
-            fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
-
         # If both result sets are empty (e.g. after hard filtering),
         # return early to avoid errors in reranking or score restoration.
         if vector_results.num_rows == 0 and fts_results.num_rows == 0:
@@ -2605,15 +2603,23 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         original_score_row_ids = None
         # normalize the scores to be between 0 and 1, 0 being most relevant
         # We check whether the results (vector and FTS) are empty, because when
-        # they are, they often are missing the _rowid column, which causes an error
+        # they are, they often are missing the _rowid column, which causes an error.
+        # The original values are captured before any rank conversion so that
+        # they can be restored after reranking.
         if vector_results.num_rows > 0:
             distance_i = vector_results.column_names.index("_distance")
             original_distances = vector_results.column(distance_i)
             original_distance_row_ids = vector_results.column("_rowid")
+            if norm == "rank":
+                vector_results = LanceHybridQueryBuilder._rank(
+                    vector_results, "_distance"
+                )
             vector_results = vector_results.set_column(
                 distance_i,
                 vector_results.field(distance_i),
-                LanceHybridQueryBuilder._normalize_scores(original_distances),
+                LanceHybridQueryBuilder._normalize_scores(
+                    vector_results.column(distance_i)
+                ),
             )
 
         # In fts higher scores represent relevance. Not inverting them here as
@@ -2622,10 +2628,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             score_i = fts_results.column_names.index("_score")
             original_scores = fts_results.column(score_i)
             original_score_row_ids = fts_results.column("_rowid")
+            if norm == "rank":
+                fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
             fts_results = fts_results.set_column(
                 score_i,
                 fts_results.field(score_i),
-                LanceHybridQueryBuilder._normalize_scores(original_scores),
+                LanceHybridQueryBuilder._normalize_scores(fts_results.column(score_i)),
             )
 
         results = reranker.rerank_hybrid(fts_query, vector_results, fts_results)
@@ -2953,9 +2961,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
-                  LanceRead: uri=..., projection=[vector], ...
+                  LanceRead: uri=..., projection=[vector], num_fragments=..., ...
+        <BLANKLINE>
 
         Parameters
         ----------
@@ -3595,9 +3604,10 @@ class AsyncQueryBase(object):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
-                  LanceRead: uri=..., projection=[vector], ...
+                  LanceRead: uri=..., projection=[vector], num_fragments=..., ...
+        <BLANKLINE>
         <BLANKLINE>
 
         Parameters
@@ -4515,9 +4525,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
               LanceRead: uri=..., projection=[text], source=stream(_rowid)
                 GlobalLimitExec: skip=0, fetch=10
                   FilterExec: _distance@2 IS NOT NULL
-                    SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+                    SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                       KNNVectorDistance: metric=l2
-                        LanceRead: uri=..., projection=[vector], ...
+                        LanceRead: uri=..., projection=[vector], num_fragments=..., ...
             ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score, _rowid@0 as _rowid]
               LanceRead: uri=..., projection=[vector, text], source=stream(_rowid)
                 GlobalLimitExec: skip=0, fetch=10
@@ -4871,9 +4881,10 @@ class BaseQueryBuilder(object):
         ProjectionExec: expr=[vector@0 as vector, _distance@2 as _distance]
           GlobalLimitExec: skip=0, fetch=10
             FilterExec: _distance@2 IS NOT NULL
-              SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
+              SortExec: TopK(fetch=1), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
                 KNNVectorDistance: metric=l2
-                  LanceRead: uri=..., projection=[vector], ...
+                  LanceRead: uri=..., projection=[vector], num_fragments=..., ...
+        <BLANKLINE>
         <BLANKLINE>
 
         Parameters

python/python/lancedb/remote/db.py

@@ -36,7 +36,7 @@ from lancedb.remote import ClientConfig, RetryConfig, TimeoutConfig, TlsConfig
 import pyarrow as pa
 
 from ..common import DATA
-from lancedb.listing import Listing
+from lancedb.listing import Listing, NamespaceListing, TableListing
 from ..db import DBConnection, LOOP
 from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
@@ -54,8 +54,6 @@ from lance_namespace import (
     CreateNamespaceResponse,
     DescribeNamespaceResponse,
     DropNamespaceResponse,
-    ListNamespacesResponse,
-    ListTablesResponse,
 )
 from ..pydantic import LanceModel
 from ..table import Table
@@ -268,7 +266,7 @@ class RemoteDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListNamespacesResponse:
+    ) -> NamespaceListing:
         """List immediate child namespace names in the given namespace.
 
         Parameters
@@ -284,12 +282,12 @@ class RemoteDBConnection(DBConnection):
 
         Returns
         -------
-        ListNamespacesResponse
+        NamespaceListing
             Response containing namespace names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return NamespaceListing(
             self._conn.list_namespaces(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
             )
@@ -379,7 +377,7 @@ class RemoteDBConnection(DBConnection):
         namespace_path: Optional[List[str]] = None,
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
-    ) -> ListTablesResponse:
+    ) -> TableListing:
         """List a page of tables in this database.
 
         Parameters
@@ -396,12 +394,12 @@ class RemoteDBConnection(DBConnection):
 
         Returns
         -------
-        ListTablesResponse
+        TableListing
             Response containing table names and optional page_token for pagination.
         """
         if namespace_path is None:
             namespace_path = []
-        return LOOP.run(
+        return TableListing(
             self._conn.list_tables(
                 namespace_path=namespace_path, page_token=page_token, limit=limit
             )
@@ -843,6 +841,20 @@ class RemoteDBConnection(DBConnection):
         job = LOOP.run(self._conn.drop_table_async(name, namespace_path=namespace_path))
         return Job(job if isinstance(job, AsyncJob) else AsyncJob(job))
 
+    @override
+    def drop_all_tables(self, namespace_path: Optional[List[str]] = None):
+        """Drop all tables from the database.
+
+        The remote API does not currently support this operation. Use
+        ``list_tables()`` and ``drop_table()`` to drop tables individually.
+        """
+        LOOP.run(self._conn.drop_all_tables(namespace_path=namespace_path))
+
+    @override
+    def drop_database(self):
+        """Deprecated alias for ``drop_all_tables()``."""
+        LOOP.run(self._conn.drop_database())
+
     @override
     def rename_table(
         self,
@@ -1122,7 +1134,7 @@ class RemoteDBConnection(DBConnection):
         )
 
     @override
-    def describe_query(self, query_id: UUID) -> QueryDescription:
+    def describe_query(self, query_id: Union[UUID, str]) -> QueryDescription:
         """Describe a submitted SQL query by its connection-scoped id."""
         return LOOP.run(
             self._conn.describe_query(

python/python/lancedb/remote/table.py

@@ -7,6 +7,7 @@ import logging
 from functools import cached_property
 import os
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Dict,
@@ -83,6 +84,10 @@ from ..table import (
 from ..types import BaseTokenizerType
 from ..util import infer_vector_column_name
 
+if TYPE_CHECKING:
+    import lance
+    import polars as pl
+
 
 class RemoteTable(Table):
     def __init__(
@@ -244,12 +249,32 @@ class RemoteTable(Table):
     def to_arrow(self, *, blob_mode: BlobMode = "lazy") -> pa.Table:
         """to_arrow() is not supported for remote tables."""
         validate_blob_mode(blob_mode)
-        raise NotImplementedError("to_arrow() is not supported for remote tables.")
+        raise NotImplementedError(
+            "to_arrow() is not supported for remote tables. "
+            "Use table.search().to_arrow() instead."
+        )
 
     def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs):
         """to_pandas() is not supported for remote tables."""
         validate_blob_mode(blob_mode)
-        raise NotImplementedError("to_pandas() is not supported for remote tables.")
+        raise NotImplementedError(
+            "to_pandas() is not supported for remote tables. "
+            "Use table.search().to_pandas() instead."
+        )
+
+    def to_lance(self, **kwargs) -> "lance.LanceDataset":
+        """Direct Lance dataset access is not supported for remote tables.
+
+        Use ``table.search().to_arrow()`` to read query results through the server.
+        """
+        return LOOP.run(self._table.to_lance(**kwargs))
+
+    def to_polars(self, **kwargs) -> "pl.LazyFrame":
+        """Use ``table.search().to_polars()`` to read remote query results."""
+        raise NotImplementedError(
+            "to_polars() is not supported for remote tables. "
+            "Use table.search().to_polars() instead."
+        )
 
     def checkout(self, version: Union[int, str]):
         result = LOOP.run(self._table.checkout(version))
@@ -280,8 +305,12 @@ class RemoteTable(Table):
         """Tokenize a query using the tokenizer configured on an FTS index.
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
-        rebuilt in the client process from index metadata, so the same tokenizer
-        model files must exist locally.
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)
@@ -1152,11 +1181,21 @@ class RemoteTable(Table):
         """
         return LOOP.run(self._table.uri())
 
-    def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
-        return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
+    def take_offsets(
+        self, offsets: list[int], *, with_row_id: bool = False
+    ) -> LanceTakeQueryBuilder:
+        builder = LanceTakeQueryBuilder(self._table.take_offsets(offsets))
+        if with_row_id:
+            builder.with_row_id()
+        return builder
 
-    def take_row_ids(self, row_ids: list[int]) -> LanceTakeQueryBuilder:
-        return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
+    def take_row_ids(
+        self, row_ids: list[int], *, with_row_id: bool = False
+    ) -> LanceTakeQueryBuilder:
+        builder = LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
+        if with_row_id:
+            builder.with_row_id()
+        return builder
 
     def uses_v2_manifest_paths(self) -> bool:
         return LOOP.run(self._table.uses_v2_manifest_paths())
@@ -1184,6 +1223,11 @@ class RemoteTable(Table):
     ) -> "list[Optional[BlobFile]]":
         return LOOP.run(self._table.fetch_blob_files(column, row_ids))
 
+    def open_blob_file(
+        self, descriptor: Union[dict, pa.StructScalar, None]
+    ) -> Optional[BlobFile]:
+        return LOOP.run(self._table.open_blob_file(descriptor))
+
     def head(self, n=5) -> pa.Table:
         """
         Return the first `n` rows of the table.

python/python/lancedb/rerankers/cohere.py

@@ -62,7 +62,7 @@ class CohereReranker(Reranker):
                 "COHERE_API_KEY not set. Either set it in your environment or \
                 pass it as `api_key` argument to the CohereReranker."
             )
-        return cohere.Client(os.environ.get("COHERE_API_KEY") or self.api_key)
+        return cohere.Client(self.api_key or os.environ.get("COHERE_API_KEY"))
 
     def _rerank(self, result_set: pa.Table, query: str):
         result_set = self._handle_empty_results(result_set)

python/python/lancedb/rerankers/openai.py

@@ -21,7 +21,7 @@ class OpenaiReranker(Reranker):
 
     Parameters
     ----------
-    model_name : str, default "gpt-4-turbo-preview"
+    model_name : str, default "gpt-5.6-sol"
         The name of the cross encoder model to use.
     column : str, default "text"
         The name of the column to use as input to the cross encoder model.
@@ -33,7 +33,7 @@ class OpenaiReranker(Reranker):
 
     def __init__(
         self,
-        model_name: str = "gpt-4-turbo-preview",
+        model_name: str = "gpt-5.6-sol",
         column: str = "text",
         return_score="relevance",
         api_key: Optional[str] = None,
@@ -54,7 +54,6 @@ class OpenaiReranker(Reranker):
         response = self._client.chat.completions.create(
             model=self.model_name,
             response_format={"type": "json_object"},
-            temperature=0,
             messages=[
                 {
                     "role": "system",
@@ -70,15 +69,18 @@ class OpenaiReranker(Reranker):
             ],
         )
         results = json.loads(response.choices[0].message.content)["documents"]
-        docs, scores = list(
-            zip(*[(result["content"], result["relevance_score"]) for result in results])
-        )  # tuples
-        # replace the self.column column with the docs
-        result_set = result_set.drop(self.column)
-        result_set = result_set.append_column(
-            self.column, pa.array(docs, type=pa.string())
-        )
-        # add the scores
+        # The model returns the documents in its own order, so match each score
+        # back to its row by content.
+        scores_by_content = {
+            result["content"]: result["relevance_score"] for result in results
+        }
+        missing = sum(doc not in scores_by_content for doc in docs)
+        if missing:
+            raise ValueError(
+                f"The model returned no relevance_score for {missing} of "
+                f"{len(docs)} documents"
+            )
+        scores = [scores_by_content[doc] for doc in docs]
         result_set = result_set.append_column(
             "_relevance_score", pa.array(scores, type=pa.float32())
         )
@@ -129,4 +131,4 @@ class OpenaiReranker(Reranker):
                 "OPENAI_API_KEY not set. Either set it in your environment or \
                 pass it as `api_key` argument to the CohereReranker."
             )
-        return openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY") or self.api_key)
+        return openai.OpenAI(api_key=self.api_key or os.environ.get("OPENAI_API_KEY"))

python/python/lancedb/rerankers/voyageai.py

@@ -64,7 +64,7 @@ class VoyageAIReranker(Reranker):
                 pass it as `api_key` argument to the VoyageAIReranker."
             )
         return voyageai.Client(
-            api_key=os.environ.get("VOYAGE_API_KEY") or self.api_key,
+            api_key=self.api_key or os.environ.get("VOYAGE_API_KEY"),
         )
 
     def _rerank(self, result_set: pa.Table, query: str):

python/python/lancedb/scannable.py

@@ -276,6 +276,14 @@ def _from_iterable(data: Iterator) -> Scannable:
 _registered_modules: set[str] = set()
 
 
+def _hf_batches(data):
+    if data._indices is None:
+        yield from data.data.to_batches()
+    else:
+        for table in data.with_format("arrow").iter(batch_size=1000):
+            yield from table.to_batches()
+
+
 def _register_optional_converters():
     """Register converters for optional dependencies that are already imported."""
 
@@ -323,7 +331,15 @@ def _register_optional_converters():
         def _from_hf_dataset(data: HFDataset) -> Scannable:
             table = data.data.table  # Access underlying Arrow table
             return Scannable(
-                schema=table.schema, num_rows=len(data), reader=table.to_reader
+                schema=table.schema,
+                num_rows=len(data),
+                reader=(
+                    table.to_reader
+                    if data._indices is None
+                    else lambda: pa.RecordBatchReader.from_batches(
+                        table.schema, _hf_batches(data)
+                    )
+                ),
             )
 
         @to_scannable.register(HFDatasetDict)
@@ -335,7 +351,7 @@ def _register_optional_converters():
 
             def gen():
                 for split_name, dataset in data.items():
-                    for batch in dataset.data.to_batches():
+                    for batch in _hf_batches(dataset):
                         split_arr = pa.array(
                             [split_name] * len(batch), type=pa.string()
                         )

python/python/lancedb/schema.py

@@ -4,6 +4,7 @@
 
 """Schema helpers for Lance blob columns."""
 
+import ctypes
 import importlib
 from typing import TYPE_CHECKING, Optional
 
@@ -16,6 +17,7 @@ if TYPE_CHECKING:
 _BLOB_EXTENSION_NAME = "lance.blob.v2"
 _BLOB_V1_KEY = "lance-encoding:blob"
 _ARROW_EXT_NAME_KEY = "ARROW:extension:name"
+_MAX_RUST_USIZE = ctypes.c_size_t(-1).value
 _BLOB_V2_STORAGE_TYPE = pa.struct(
     [
         pa.field("data", pa.large_binary(), nullable=True),
@@ -209,32 +211,44 @@ def blob(
         Maximum size in bytes of a single packed blob sidecar file before
         a new one is started. Must be positive.
 
-    The thresholds are forwarded to ``lance.blob_field``, which validates
-    them and stores them as field metadata, so pylance is required whenever
-    any threshold is set. When all thresholds are omitted the behavior is
-    unchanged.
+    Thresholds are validated and stored as Arrow field metadata, with the
+    same format as ``lance.blob_field``. They do not require pylance.
     """
-    if (
-        inline_size_threshold is None
-        and dedicated_size_threshold is None
-        and pack_file_size_threshold is None
+    metadata = {}
+    for threshold, value, allow_zero, key in (
+        (
+            "inline_size_threshold",
+            inline_size_threshold,
+            True,
+            "lance-encoding:blob-inline-size-threshold",
+        ),
+        (
+            "dedicated_size_threshold",
+            dedicated_size_threshold,
+            False,
+            "lance-encoding:blob-dedicated-size-threshold",
+        ),
+        (
+            "pack_file_size_threshold",
+            pack_file_size_threshold,
+            False,
+            "lance-encoding:blob-pack-file-size-threshold",
+        ),
     ):
-        blob_type = _resolve_blob_type()
-        return pa.field(name, blob_type(), nullable=nullable)
-    try:
-        lance = importlib.import_module("lance")
-    except ModuleNotFoundError as err:
-        raise ValueError(
-            "Setting blob tier thresholds requires pylance to be installed"
-        ) from err
-    _resolve_blob_type()
-    return lance.blob_field(
-        name,
-        nullable=nullable,
-        inline_size_threshold=inline_size_threshold,
-        dedicated_size_threshold=dedicated_size_threshold,
-        pack_file_size_threshold=pack_file_size_threshold,
-    )
+        if value is None:
+            continue
+        if isinstance(value, bool) or not isinstance(value, int):
+            raise TypeError(f"{threshold} must be an int, got {type(value).__name__}")
+        if value < 0 or (value == 0 and not allow_zero):
+            bound = "non-negative" if allow_zero else "positive"
+            raise ValueError(f"{threshold} must be {bound}")
+        if value > _MAX_RUST_USIZE:
+            raise OverflowError(f"{threshold} must fit in a Rust usize")
+        metadata[key] = str(value)
+
+    blob_type = _resolve_blob_type()
+    field = pa.field(name, blob_type(), nullable=nullable)
+    return field.with_metadata(metadata) if metadata else field
 
 
 def vector(dimension: int, value_type: pa.DataType = pa.float32()) -> pa.DataType:

python/python/lancedb/table.py

@@ -15,9 +15,11 @@ from functools import cached_property
 from typing import (
     TYPE_CHECKING,
     Any,
+    AsyncIterator,
     Callable,
     Dict,
     Iterable,
+    Iterator,
     List,
     Literal,
     Optional,
@@ -29,11 +31,12 @@ from typing import (
 )
 from urllib.parse import urlparse
 
-from lancedb.scannable import _register_optional_converters, to_scannable
+from lancedb.scannable import _hf_batches, _register_optional_converters, to_scannable
 
 from . import __version__
 from ._blob import (
     BlobFile,
+    _normalize_blob_descriptor,
     _normalize_blob_row_ids,
     _wrap_blob_files,
     strip_auto_row_ids,
@@ -486,7 +489,7 @@ def _into_pyarrow_reader(
     if _check_for_hugging_face(data):
         if isinstance(data, datasets.Dataset):
             schema = data.features.arrow_schema
-            return pa.RecordBatchReader.from_batches(schema, data.data.to_batches())
+            return pa.RecordBatchReader.from_batches(schema, _hf_batches(data))
         elif isinstance(data, datasets.dataset_dict.DatasetDict):
             schema = _schema_from_hf(data, None)
             if "split" not in schema.names:
@@ -555,7 +558,7 @@ def _into_pyarrow_reader(
     ):
         return data.collect().to_arrow().to_reader()
     elif isinstance(data, Iterable):
-        return _iterator_to_reader(data)
+        return _iterator_to_reader(data, schema)
     else:
         raise TypeError(
             f"Unknown data type {type(data)}. "
@@ -565,10 +568,19 @@ def _into_pyarrow_reader(
         )
 
 
-def _iterator_to_reader(data: Iterable) -> pa.RecordBatchReader:
+def _iterator_to_reader(
+    data: Iterable, schema: Optional[pa.Schema] = None
+) -> pa.RecordBatchReader:
     # Each batch is treated as it's own reader, mainly so we can
     # re-use the _into_pyarrow_reader logic.
-    first = _into_pyarrow_reader(next(data))
+    data = iter(data)
+    try:
+        first_batch = next(data)
+    except StopIteration:
+        if schema is None:
+            raise ValueError("Cannot create table from empty iterable without a schema")
+        return pa.RecordBatchReader.from_batches(schema, [])
+    first = _into_pyarrow_reader(first_batch)
     schema = first.schema
 
     def gen():
@@ -1281,7 +1293,7 @@ def _to_batches_with_split(data):
     with an extra `split` column
     """
     for key, dataset in data.items():
-        for batch in dataset.data.to_batches():
+        for batch in _hf_batches(dataset):
             table = pa.Table.from_batches([batch])
             if "split" not in table.column_names:
                 table = table.append_column(
@@ -1433,13 +1445,13 @@ class Table(ABC):
     @abstractmethod
     def name(self) -> str:
         """The name of this Table"""
-        raise NotImplementedError
+        raise NotImplementedError("name is not supported for this table type")
 
     @property
     @abstractmethod
     def version(self) -> int:
         """The version of this Table"""
-        raise NotImplementedError
+        raise NotImplementedError("version is not supported for this table type")
 
     @property
     @abstractmethod
@@ -1448,7 +1460,7 @@ class Table(ABC):
         of this Table
 
         """
-        raise NotImplementedError
+        raise NotImplementedError("schema is not supported for this table type")
 
     @property
     @abstractmethod
@@ -1477,7 +1489,7 @@ class Table(ABC):
             tags = table.tags.list()
 
         """
-        raise NotImplementedError
+        raise NotImplementedError("tags is not supported for this table type")
 
     @property
     def branches(self) -> "Branches":
@@ -1486,11 +1498,11 @@ class Table(ABC):
         Branches are isolated, writable lines of history forked from another
         branch (or version). Writes on a branch do not affect ``main``.
         """
-        raise NotImplementedError
+        raise NotImplementedError("branches is not supported for this table type")
 
     def current_branch(self) -> Optional[str]:
         """The branch this table handle is scoped to, or ``None`` for ``main``."""
-        raise NotImplementedError
+        raise NotImplementedError("current_branch is not supported for this table type")
 
     def __len__(self) -> int:
         """The number of rows in this Table"""
@@ -1513,7 +1525,7 @@ class Table(ABC):
         filter: str, optional
             A SQL where clause to filter the rows to count.
         """
-        raise NotImplementedError
+        raise NotImplementedError("count_rows is not supported for this table type")
 
     def to_pandas(self, blob_mode: BlobMode = "lazy", **kwargs) -> "pandas.DataFrame":
         """Return the table as a pandas DataFrame.
@@ -1548,20 +1560,26 @@ class Table(ABC):
         -------
         pa.Table
         """
-        raise NotImplementedError
+        raise NotImplementedError("to_arrow is not supported for this table type")
 
     def to_lance(self, **kwargs) -> lance.LanceDataset:
-        """Return the table as a lance.LanceDataset.
+        """Return a local table as a lance.LanceDataset.
+
+        Remote tables cannot be opened as Lance datasets by the client. Use
+        ``table.search().to_arrow()`` to read them through the server.
 
         Returns
         -------
         lance.LanceDataset
         """
-        raise NotImplementedError
+        raise NotImplementedError("to_lance is not supported for this table type")
 
     def to_polars(self, **kwargs) -> "pl.LazyFrame":
         """Return the table as a Polars LazyFrame.
 
+        For remote tables, use ``table.search().to_polars()`` to convert query
+        results to a Polars DataFrame.
+
         Note
         ----
         The Polars streaming engine is not supported because it does not currently
@@ -1572,7 +1590,7 @@ class Table(ABC):
         -------
         polars.LazyFrame
         """
-        raise NotImplementedError
+        raise NotImplementedError("to_polars is not supported for this table type")
 
     # New unified API overload
     @overload
@@ -1677,7 +1695,7 @@ class Table(ABC):
         ...     "l2", vector_column_name="vector"
         ... )
         """
-        raise NotImplementedError
+        raise NotImplementedError("create_index is not supported for this table type")
 
     def create_index_async(
         self,
@@ -1695,7 +1713,9 @@ class Table(ABC):
         be complete when returned; callers must not assume the index exists
         until :meth:`Job.wait` returns.
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "create_index_async is not supported for this table type"
+        )
 
     def drop_index(self, name: str) -> None:
         """
@@ -1715,7 +1735,7 @@ class Table(ABC):
         Use [list_indices][lancedb.table.Table.list_indices] to find the names of
         the indices.
         """
-        raise NotImplementedError
+        raise NotImplementedError("drop_index is not supported for this table type")
 
     def wait_for_index(
         self, index_names: Iterable[str], timeout: timedelta = timedelta(seconds=300)
@@ -1732,14 +1752,14 @@ class Table(ABC):
         timeout: timedelta
             Timeout to wait for asynchronous indexing. The default is 5 minutes.
         """
-        raise NotImplementedError
+        raise NotImplementedError("wait_for_index is not supported for this table type")
 
     @abstractmethod
     def stats(self) -> TableStatistics:
         """
         Retrieve table and fragment statistics.
         """
-        raise NotImplementedError
+        raise NotImplementedError("stats is not supported for this table type")
 
     @abstractmethod
     def create_scalar_index(
@@ -1804,7 +1824,9 @@ class Table(ABC):
         ``my_col = 0 OR not_indexed = 1`` will not be able to use any scalar index on
         ``my_col``.
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "create_scalar_index is not supported for this table type"
+        )
 
     def create_fts_index(
         self,
@@ -1930,7 +1952,9 @@ class Table(ABC):
         ``LANCE_LANGUAGE_MODEL_HOME`` to override the default platform data
         directory under ``lance/language_models``.
         """
-        raise NotImplementedError
+        raise NotImplementedError(
+            "create_fts_index is not supported for this table type"
+        )
 
     @abstractmethod
     def add(
@@ -2004,7 +2028,7 @@ class Table(ABC):
         AddResult
             An object containing the new version number of the table after adding data.
         """
-        raise NotImplementedError
+        raise NotImplementedError("add is not supported for this table type")
 
     def merge_insert(self, on: Union[str, Iterable[str]]) -> LanceMergeInsertBuilder:
         """
@@ -2156,7 +2180,7 @@ class Table(ABC):
             - and also the "_distance" column which is the distance between the query
             vector and the returned vector.
         """
-        raise NotImplementedError
+        raise NotImplementedError("search is not supported for this table type")
 
     @abstractmethod
     def take_offsets(
@@ -2180,11 +2204,14 @@ class Table(ABC):
         ----------
         offsets: list[int]
             The offsets to take.
+        with_row_id: bool, default False
+            If True, include the "_rowid" column in the results.
 
         Returns
         -------
-        pa.RecordBatch
-            A record batch containing the rows at the given offsets.
+        LanceTakeQueryBuilder
+            A query builder that can be executed to get the rows at the given
+            offsets.
         """
 
     def __getitems__(self, offsets: list[int]) -> pa.RecordBatch:
@@ -2265,11 +2292,13 @@ class Table(ABC):
         ----------
         row_ids: list[int]
             The row ids to take.
+        with_row_id: bool, default False
+            If True, include the "_rowid" column in the results.
 
         Returns
         -------
-        AsyncTakeQuery
-            A query object that can be executed to get the rows.
+        LanceTakeQueryBuilder
+            A query builder that can be executed to get the rows.
         """
 
     @abstractmethod
@@ -2349,6 +2378,44 @@ class Table(ABC):
         handles are returned.
         """
 
+    @abstractmethod
+    def open_blob_file(
+        self, descriptor: Union[dict, pa.StructScalar, None]
+    ) -> Optional[BlobFile]:
+        """Open a seekable file from a SQL ``blob_descriptor`` result.
+
+        Accepts the descriptor dictionary or an Arrow struct scalar. Its source
+        pins the table version, branch, column and row identity, so opening does
+        not change this table's checkout. Null descriptors return ``None``.
+        Payload bytes load only when the returned handle is read. An external
+        blob with an unspecified size may require a metadata request on opening.
+
+        ```python
+        rows = db.execute_query(
+            "SELECT blob_descriptor(video) AS video FROM videos"
+        ).read_all()
+        with videos.open_blob_file(rows["video"][0]) as video:
+            header = video.read(4096)
+        ```
+        """
+
+    def open_blob_files(self, descriptors: Iterable) -> Iterator[Optional[BlobFile]]:
+        """Iterate over lazy files from descriptor dictionaries or Arrow scalars.
+
+        Accepts an Arrow array/chunked array or any iterable. Opens each handle
+        as iteration advances, preserving input order and nulls as ``None``.
+        Close each non-null handle after use, normally with a ``with`` block.
+
+        ```python
+        for video in videos.open_blob_files(rows["video"]):
+            if video is not None:
+                with video:
+                    header = video.read(4096)
+        ```
+        """
+        for descriptor in descriptors:
+            yield self.open_blob_file(descriptor)
+
     @abstractmethod
     def _execute_query(
         self,
@@ -2437,7 +2504,7 @@ class Table(ABC):
            x      vector
         0  3  [5.0, 6.0]
         """
-        raise NotImplementedError
+        raise NotImplementedError("delete is not supported for this table type")
 
     @abstractmethod
     def update(
@@ -2504,7 +2571,7 @@ class Table(ABC):
         1  4    [5.0, 6.0]
         2  3  [10.0, 10.0]
         """
-        raise NotImplementedError
+        raise NotImplementedError("update is not supported for this table type")
 
     @abstractmethod
     def cleanup_old_versions(
@@ -2650,7 +2717,11 @@ class Table(ABC):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
 
     @abstractmethod
@@ -3261,11 +3332,21 @@ class LanceTable(Table):
         """Get the current version of the table"""
         return LOOP.run(self._table.version())
 
-    def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
-        return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
+    def take_offsets(
+        self, offsets: list[int], *, with_row_id: bool = False
+    ) -> LanceTakeQueryBuilder:
+        builder = LanceTakeQueryBuilder(self._table.take_offsets(offsets))
+        if with_row_id:
+            builder.with_row_id()
+        return builder
 
-    def take_row_ids(self, row_ids: list[int]) -> LanceTakeQueryBuilder:
-        return LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
+    def take_row_ids(
+        self, row_ids: list[int], *, with_row_id: bool = False
+    ) -> LanceTakeQueryBuilder:
+        builder = LanceTakeQueryBuilder(self._table.take_row_ids(row_ids))
+        if with_row_id:
+            builder.with_row_id()
+        return builder
 
     def blob_columns(self) -> list[str]:
         return LOOP.run(self._table.blob_columns())
@@ -3291,6 +3372,11 @@ class LanceTable(Table):
     ) -> "list[Optional[BlobFile]]":
         return LOOP.run(self._table.fetch_blob_files(column, row_ids))
 
+    def open_blob_file(
+        self, descriptor: Union[dict, pa.StructScalar, None]
+    ) -> Optional[BlobFile]:
+        return LOOP.run(self._table.open_blob_file(descriptor))
+
     @property
     def tags(self) -> Tags:
         """Tag management for the table.
@@ -4912,7 +4998,11 @@ class LanceTable(Table):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)
@@ -5415,6 +5505,11 @@ def has_nan_values(arr: Union[pa.ListArray, pa.ChunkedArray]) -> pa.BooleanArray
     else:
         values_has_nan = pc.is_nan(values)
     values_indices = pc.list_parent_indices(arr)
+    if arr.null_count:
+        # Match flatten(), which excludes child spans under null parent lists.
+        values_indices = pc.filter(
+            values_indices, pc.take(pc.is_valid(arr), values_indices)
+        )
     has_nan_indices = pc.unique(pc.filter(values_indices, values_has_nan))
     indices = pa.array(range(len(arr)), type=pa.uint32())
     return pc.is_in(indices, has_nan_indices)
@@ -5503,6 +5598,7 @@ def _infer_target_schema(
                 field.name,  # preserve original field name
                 target_type,
                 nullable=field.nullable,
+                metadata=field.metadata,
             )
 
             schema = schema.set(i, new_field)
@@ -7401,7 +7497,9 @@ class AsyncTable:
         """
         await self._inner.restore(version)
 
-    def take_offsets(self, offsets: list[int]) -> AsyncTakeQuery:
+    def take_offsets(
+        self, offsets: list[int], *, with_row_id: bool = False
+    ) -> AsyncTakeQuery:
         """
         Take a list of offsets from the table.
 
@@ -7419,15 +7517,23 @@ class AsyncTable:
         ----------
         offsets: list[int]
             The offsets to take.
+        with_row_id: bool, default False
+            If True, include the "_rowid" column in the results.
 
         Returns
         -------
-        pa.RecordBatch
-            A record batch containing the rows at the given offsets.
+        AsyncTakeQuery
+            A query object that can be executed to get the rows at the given
+            offsets.
         """
-        return AsyncTakeQuery(self._inner.take_offsets(offsets), self)
+        query = AsyncTakeQuery(self._inner.take_offsets(offsets), self)
+        if with_row_id:
+            query.with_row_id()
+        return query
 
-    def take_row_ids(self, row_ids: list[int]) -> AsyncTakeQuery:
+    def take_row_ids(
+        self, row_ids: list[int], *, with_row_id: bool = False
+    ) -> AsyncTakeQuery:
         """
         Take a list of row ids from the table.
 
@@ -7448,13 +7554,18 @@ class AsyncTable:
         ----------
         row_ids: list[int]
             The row ids to take.
+        with_row_id: bool, default False
+            If True, include the "_rowid" column in the results.
 
         Returns
         -------
         AsyncTakeQuery
             A query object that can be executed to get the rows.
         """
-        return AsyncTakeQuery(self._inner.take_row_ids(row_ids), self)
+        query = AsyncTakeQuery(self._inner.take_row_ids(row_ids), self)
+        if with_row_id:
+            query.with_row_id()
+        return query
 
     async def blob_columns(self) -> list[str]:
         return await self._inner.blob_columns()
@@ -7485,6 +7596,36 @@ class AsyncTable:
         )
         return _wrap_blob_files(handles)
 
+    async def open_blob_file(
+        self, descriptor: Union[dict, pa.StructScalar, None]
+    ) -> Optional[BlobFile]:
+        """Open a snapshot-bound descriptor as a seekable file.
+
+        See [Table.open_blob_file][lancedb.table.Table.open_blob_file].
+        """
+        descriptor = _normalize_blob_descriptor(descriptor)
+        if descriptor is None:
+            return None
+        return BlobFile(await self._inner.open_blob_file(json.dumps(descriptor)))
+
+    async def open_blob_files(
+        self, descriptors: Iterable
+    ) -> AsyncIterator[Optional[BlobFile]]:
+        """Asynchronously iterate over files in descriptor order, preserving nulls.
+
+        Use ``async for file in table.open_blob_files(descriptors)``. Each
+        non-null file must be closed after use.
+
+        ```python
+        async for video in videos.open_blob_files(rows["video"]):
+            if video is not None:
+                with video:
+                    payload = await video.aread()
+        ```
+        """
+        for descriptor in descriptors:
+            yield await self.open_blob_file(descriptor)
+
     @property
     def tags(self) -> AsyncTags:
         """Tag management for the dataset.
@@ -7614,7 +7755,11 @@ class AsyncTable:
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return await self._inner.tokenize(query, column=column, index_name=index_name)
 

python/python/lancedb/util.py

@@ -426,7 +426,8 @@ def _(value: date):
 
 
 @value_to_sql.register(list)
-def _(value: list):
+@value_to_sql.register(tuple)
+def _(value: Union[list, tuple]):
     return "[" + ", ".join(map(value_to_sql, value)) + "]"
 
 
@@ -451,6 +452,15 @@ def _(value: pa.Scalar):
     return value_to_sql(value.as_py())
 
 
+@value_to_sql.register(pa.MapScalar)
+def _(value: pa.MapScalar):
+    if not value.is_valid:
+        return value_to_sql(None)
+    keys = value_to_sql(list(value.values.field(0)))
+    values = value_to_sql(list(value.values.field(1)))
+    return f"map({keys}, {values})"
+
+
 @value_to_sql.register(np.ndarray)
 def _(value: np.ndarray):
     return value_to_sql(value.tolist())
@@ -471,6 +481,15 @@ def _(value: np.floating):
     return value_to_sql(float(value))
 
 
+@value_to_sql.register(np.datetime64)
+def _(value: np.datetime64):
+    if np.isnat(value):
+        return value_to_sql(None)
+    if np.datetime_data(value.dtype)[0] in ("ps", "fs", "as"):
+        raise ValueError("SQL timestamps cannot preserve sub-nanosecond precision")
+    return value_to_sql(str(np.datetime_as_string(value, unit="ns", timezone="UTC")))
+
+
 def deprecated(func):
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted

python/python/tests/test_blob.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+import ctypes
 import io
 import subprocess
 import sys
@@ -14,6 +15,7 @@ import pytest
 from lance.blob import Blob, BlobType as LanceBlobType
 
 import lancedb
+from lancedb import schema as schema_module
 from lancedb._blob import (
     _set_blob_column,
     blob_v2_projection_sources,
@@ -139,6 +141,41 @@ def test_blob_without_thresholds_has_no_threshold_metadata():
     assert not any(b"threshold" in key for key in metadata)
 
 
+@pytest.mark.parametrize("nullable", [False, True])
+@pytest.mark.parametrize(
+    "thresholds",
+    [
+        {},
+        {"inline_size_threshold": 0},
+        {"inline_size_threshold": 8192},
+        {"dedicated_size_threshold": 1 << 20},
+        {"pack_file_size_threshold": 1 << 26},
+        {
+            "inline_size_threshold": 8192,
+            "dedicated_size_threshold": 1 << 20,
+            "pack_file_size_threshold": 1 << 26,
+        },
+        {
+            "inline_size_threshold": ctypes.c_size_t(-1).value,
+            "dedicated_size_threshold": ctypes.c_size_t(-1).value,
+            "pack_file_size_threshold": ctypes.c_size_t(-1).value,
+        },
+    ],
+)
+def test_blob_thresholds_match_pylance(monkeypatch, nullable, thresholds):
+    expected = lance.blob_field("payload", nullable=nullable, **thresholds)
+    actual = lancedb.blob("payload", nullable=nullable, **thresholds)
+    assert type(actual.type) is LanceBlobType
+    assert actual.equals(expected, check_metadata=True)
+
+    # Both extension classes must encode the same Arrow field on the wire.
+    monkeypatch.setattr(
+        schema_module, "_resolve_blob_type", lambda: schema_module._FallbackBlobType
+    )
+    fallback = lancedb.blob("payload", nullable=nullable, **thresholds)
+    assert pa.schema([fallback]).serialize() == pa.schema([expected]).serialize()
+
+
 def test_blob_thresholds_are_validated():
     with pytest.raises(ValueError, match="inline_size_threshold must be non-negative"):
         lancedb.blob("payload", inline_size_threshold=-1)
@@ -2659,3 +2696,135 @@ def test_create_table_with_python_blob_dict_and_object():
         b"from-dict",
         b"from-blob",
     ]
+
+
+def _openable_descriptors(table, *, by_address):
+    hits = (
+        table.search()
+        .select(["id", "image", "_rowaddr"])
+        .with_row_id(True)
+        .limit(100)
+        .to_arrow(blob_mode="descriptions")
+    )
+    version = table.version
+    committed_at = next(
+        item["timestamp"]
+        for item in table.list_versions()
+        if item["version"] == version
+    )
+    descriptors = {}
+    for row in hits.to_pylist():
+        descriptor = row["image"]
+        if descriptor is not None and descriptor.get("kind") is not None:
+            descriptor = dict(descriptor)
+            descriptor["source"] = {
+                "uri": table.to_lance().uri,
+                "version": version,
+                "committed_at_micros": round(committed_at.timestamp() * 1_000_000),
+                "branch": None,
+                "column": "image",
+                "row_reference": row["_rowaddr" if by_address else "_rowid"],
+                "row_is_address": by_address,
+            }
+        else:
+            descriptor = None
+        descriptors[row["id"]] = descriptor
+    return descriptors
+
+
+@pytest.mark.parametrize("by_address", [False, True])
+def test_open_blob_files_preserve_snapshot_order_nulls_and_empty_values(
+    tmp_path, by_address
+):
+    db = lancedb.connect(str(tmp_path))
+    schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")])
+    table = db.create_table("videos", schema=schema)
+    table.add(
+        [
+            {"id": 1, "image": b"alpha"},
+            {"id": 2, "image": None},
+            {"id": 3, "image": b"beta"},
+            {"id": 4, "image": b""},
+        ]
+    )
+    descriptors = _openable_descriptors(table, by_address=by_address)
+    table.add([{"id": 1, "image": b"replacement"}], mode="overwrite")
+    latest_version = table.version
+    values = pa.array(
+        [descriptors[3], descriptors[2], descriptors[1], descriptors[3], descriptors[4]]
+    )
+    files = table.open_blob_files(
+        pa.chunked_array([values.slice(0, 2), values.slice(2)])
+    )
+    assert iter(files) is files
+    actual = []
+    for file in files:
+        if file is None:
+            actual.append(None)
+        else:
+            with file:
+                actual.append(file.read())
+            assert file.closed
+    assert actual == [b"beta", None, b"alpha", b"beta", b""]
+    with table.open_blob_file(descriptors[1]) as file:
+        file.seek(2)
+        assert file.read_range(0, 2) == b"al"
+        assert file.tell() == 2
+        assert file.read(2) == b"ph"
+    assert table.version == latest_version
+    assert table.open_blob_file(pa.scalar(None, type=values.type)) is None
+    other = db.create_table("other", schema=schema)
+    with pytest.raises(ValueError, match="different table"):
+        other.open_blob_file(descriptors[1])
+    stale = dict(descriptors[1], source=dict(descriptors[1]["source"]))
+    stale["source"]["committed_at_micros"] += 1
+    with pytest.raises(ValueError, match="snapshot has been replaced"):
+        table.open_blob_file(stale)
+    raw = {key: value for key, value in descriptors[1].items() if key != "source"}
+    with pytest.raises(ValueError, match="source"):
+        table.open_blob_file(raw)
+    with pytest.raises(TypeError, match="descriptor"):
+        table.open_blob_file("not a descriptor")
+
+
+def test_open_blob_files_does_not_open_ahead(monkeypatch):
+    table = _blob_table("lazy_descriptor_iteration", [{"id": 1, "image": b"x"}])
+    opened = []
+
+    def open_file(descriptor):
+        opened.append(descriptor)
+        return descriptor
+
+    monkeypatch.setattr(table, "open_blob_file", open_file)
+    files = table.open_blob_files(iter([1, None, 2]))
+    assert opened == []
+    assert next(files) == 1
+    assert opened == [1]
+    assert list(files) == [None, 2]
+    assert opened == [1, None, 2]
+
+
+@pytest.mark.asyncio
+async def test_open_blob_files_async_iterates_snapshot_handles(tmp_path):
+    db = lancedb.connect(str(tmp_path))
+    table = db.create_table(
+        "videos",
+        schema=pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]),
+    )
+    table.add([{"id": 1, "image": b"alpha"}, {"id": 2, "image": None}])
+    descriptors = _openable_descriptors(table, by_address=True)
+    async_db = await lancedb.connect_async(str(tmp_path))
+    videos = await async_db.open_table("videos")
+    file = await videos.open_blob_file(pa.scalar(descriptors[1]))
+    with file:
+        assert file.read(2) == b"al"
+    actual = []
+    async for file in videos.open_blob_files(
+        [descriptors[2], descriptors[1], descriptors[1]]
+    ):
+        if file is None:
+            actual.append(None)
+        else:
+            with file:
+                actual.append(await file.aread())
+    assert actual == [None, b"alpha", b"alpha"]

python/python/tests/test_blob_reader.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import io
+
+import pytest
+
+from lancedb import BlobReader
+
+
+def test_reader_is_lazy_and_seekable():
+    requests = []
+    size = 64 * 1024 * 1024
+
+    def read_range(offset, length):
+        requests.append((offset, length))
+        return bytes((index % 251 for index in range(offset, offset + length)))
+
+    with BlobReader(read_range, size) as reader:
+        assert requests == []
+        assert reader.size() == size
+        reader.seek(-4, io.SEEK_END)
+        assert reader.read(10) == bytes(index % 251 for index in range(size - 4, size))
+        assert reader.read(1) == b""
+        assert requests == [(size - 4, 4)]
+        assert reader.read_range(1, 3) == b"\x01\x02\x03"
+        assert reader.tell() == size
+        reader.seek(2)
+        target = bytearray(3)
+        assert reader.readinto(target) == 3
+        assert target == b"\x02\x03\x04"
+        reader.seek(-1, io.SEEK_CUR)
+        assert reader.tell() == 4
+    with pytest.raises(ValueError, match="closed"):
+        reader.read(1)
+
+
+def test_reader_supports_buffered_io_and_bounds_range_requests():
+    requests = []
+    payload = b"x" * (3 * 1024 * 1024 + 7)
+
+    def read_range(offset, length):
+        requests.append((offset, length))
+        return payload[offset : offset + length]
+
+    with io.BufferedReader(BlobReader(read_range, len(payload))) as reader:
+        assert reader.read() == payload
+    assert requests
+    assert all(length <= 1024 * 1024 for _, length in requests)
+
+
+def test_reader_rejects_invalid_ranges_and_short_reads():
+    reader = BlobReader(lambda offset, length: b"", 10)
+    with pytest.raises(ValueError):
+        reader.seek(-1)
+    with pytest.raises(ValueError):
+        reader.read_range(-1, 1)
+    with pytest.raises(OSError, match="payload length"):
+        reader.read(1)

python/python/tests/test_closed_table_query.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import pytest
+from lancedb.db import AsyncConnection
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("operation", ["count_rows", "query", "head"])
+async def test_closed_table_queries_raise_runtime_error(
+    mem_db_async: AsyncConnection, operation
+):
+    table = await mem_db_async.create_table("closed_query", data=[{"id": 0}])
+    assert (await table.query().to_arrow()).to_pylist() == [{"id": 0}]
+    table.close()
+    table.close()
+    assert not table.is_open()
+
+    with pytest.raises(RuntimeError, match="Table closed_query is closed"):
+        if operation == "query":
+            table.query()
+        else:
+            await getattr(table, operation)()
+
+
+@pytest.mark.asyncio
+async def test_open_table_queries(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table("open_query", data=[{"id": 0}])
+    assert await table.count_rows() == 1
+    assert (await table.query().to_arrow()).to_pylist() == [{"id": 0}]
+    assert (await table.head()).to_pylist() == [{"id": 0}]
+    table.close()

python/python/tests/test_db.py

@@ -86,6 +86,7 @@ def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeyp
 
     assert members["uri"] == str(tmp_path)
     assert members["read_consistency_interval"] is None
+    assert members["session"] is None
     assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})"
     assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
 
@@ -105,6 +106,8 @@ def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkey
 
     assert db.read_consistency_interval == consistency_interval
     assert db_from_inner.read_consistency_interval == consistency_interval
+    assert db.session is None
+    assert db_from_inner.session is None
 
 
 def test_ingest_pd(tmp_path):

python/python/tests/test_duckdb.py

@@ -22,3 +22,19 @@ def test_basic_query(tmp_path):
 
     results = duck_conn.sql("SELECT SUM(y) FROM adapter").fetchall()
     assert results[0][0] == 26
+
+
+def test_query_with_filter(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)  # noqa: F841
+
+    duck_conn = duckdb.connect()
+
+    results = duck_conn.sql("SELECT SUM(y) FROM adapter WHERE x > 2").fetchall()
+    assert results[0][0] == 15
+
+    results = duck_conn.sql("SELECT y FROM adapter WHERE x > 2 ORDER BY y").fetchall()
+    assert results == [(7,), (8,)]

python/python/tests/test_embedding_field_metadata.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import copy
+
+import lancedb
+import pytest
+from lancedb.embeddings.base import TextEmbeddingFunction
+from lancedb.embeddings.registry import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+
+class LocalEmbedding(TextEmbeddingFunction):
+    def ndims(self):
+        return 2
+
+    def generate_embeddings(self, texts):
+        return [[1.0, 0.0] if text == "alpha" else [0.0, 1.0] for text in texts]
+
+
+@pytest.mark.parametrize(
+    "method,tag",
+    [("SourceField", "source_column_for"), ("VectorField", "vector_column_for")],
+)
+@pytest.mark.parametrize("extra", [None, {}, {"x-purpose": "retrieval"}])
+def test_embedding_field_preserves_extra_metadata(method, tag, extra):
+    function = LocalEmbedding.create(max_retries=0)
+    original = copy.deepcopy(extra)
+    field = getattr(function, method)(json_schema_extra=extra)
+    assert extra == original
+    assert field.json_schema_extra[tag] is function
+    for key, value in (extra or {}).items():
+        assert field.json_schema_extra[key] == value
+
+
+@pytest.mark.parametrize(
+    "method,tag",
+    [("SourceField", "source_column_for"), ("VectorField", "vector_column_for")],
+)
+def test_extra_metadata_cannot_replace_embedding_identity(method, tag):
+    function = LocalEmbedding.create(max_retries=0)
+    supplied = {tag: "caller value", "x-purpose": "retrieval"}
+    field = getattr(function, method)(json_schema_extra=supplied)
+    assert supplied[tag] == "caller value"
+    assert field.json_schema_extra[tag] is function
+    assert field.json_schema_extra["x-purpose"] == "retrieval"
+
+
+@pytest.mark.parametrize("method", ["SourceField", "VectorField"])
+def test_embedding_field_preserves_other_field_arguments(method):
+    function = LocalEmbedding.create(max_retries=0)
+    field = getattr(function, method)(title="Custom title", description="Custom field")
+    assert field.title == "Custom title"
+    assert field.description == "Custom field"
+
+
+def test_extra_metadata_preserves_schema_and_local_embedding(tmp_path, monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    registry.register("test-embedding-field-extra")(LocalEmbedding)
+    function = LocalEmbedding.create(max_retries=0)
+
+    class Row(LanceModel):
+        text: str = function.SourceField(json_schema_extra={"x-purpose": "source"})
+        vector: Vector(2) = function.VectorField(
+            json_schema_extra={"x-purpose": "embedding"}
+        )
+
+    json_schema = Row.model_json_schema()
+    assert json_schema["properties"]["text"]["x-purpose"] == "source"
+    assert json_schema["properties"]["vector"]["x-purpose"] == "embedding"
+    configs = Row.parse_embedding_functions()
+    assert len(configs) == 1
+    assert configs[0].function is function
+    assert configs[0].source_column == "text"
+    assert configs[0].vector_column == "vector"
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "rows", schema=Row, data=[{"text": "alpha"}, {"text": "beta"}]
+    )
+    assert table.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert isinstance(reopened.embedding_functions["vector"].function, LocalEmbedding)
+    assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"

python/python/tests/test_embedding_registry.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+from lancedb.embeddings.base import TextEmbeddingFunction
+from lancedb.embeddings.registry import EmbeddingFunctionRegistry, get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+
+class FirstEmbedding(TextEmbeddingFunction):
+    def ndims(self):
+        return 2
+
+    def generate_embeddings(self, texts):
+        return [[1.0, 0.0] if text == "alpha" else [0.0, 1.0] for text in texts]
+
+
+class SecondEmbedding(FirstEmbedding):
+    def generate_embeddings(self, texts):
+        return [[0.0, 1.0] if text == "alpha" else [1.0, 0.0] for text in texts]
+
+
+@pytest.mark.parametrize(
+    "first_alias,second_alias,second_class",
+    [
+        ("shared", "shared", SecondEmbedding),
+        (None, "FirstEmbedding", SecondEmbedding),
+        ("SecondEmbedding", None, SecondEmbedding),
+        (None, None, FirstEmbedding),
+        ("", "", FirstEmbedding),
+    ],
+)
+def test_duplicate_registration_preserves_original(
+    first_alias, second_alias, second_class
+):
+    registry = EmbeddingFunctionRegistry()
+    registry.register(first_alias)(FirstEmbedding)
+    key = first_alias or "FirstEmbedding"
+    original_alias = FirstEmbedding.__embedding_function_registry_alias__
+    second_attributes = dict(second_class.__dict__)
+    with pytest.raises(KeyError, match=f"{key} was already registered"):
+        registry.register(second_alias)(second_class)
+    assert registry.get(key) is FirstEmbedding
+    assert FirstEmbedding.__embedding_function_registry_alias__ == original_alias
+    assert dict(second_class.__dict__) == second_attributes
+
+
+def test_class_name_does_not_collide_with_distinct_alias():
+    registry = EmbeddingFunctionRegistry()
+    registry.register()(FirstEmbedding)
+    same_name = type("FirstEmbedding", (SecondEmbedding,), {})
+    registry.register("separate")(same_name)
+    assert registry.get("FirstEmbedding") is FirstEmbedding
+    assert registry.get("separate") is same_name
+
+
+def test_same_class_explicit_alias_registration_is_idempotent():
+    registry = EmbeddingFunctionRegistry()
+    assert registry.register("shared")(FirstEmbedding) is FirstEmbedding
+    attributes = dict(FirstEmbedding.__dict__)
+    assert registry.register("shared")(FirstEmbedding) is FirstEmbedding
+    assert registry.get("shared") is FirstEmbedding
+    assert dict(FirstEmbedding.__dict__) == attributes
+
+
+def test_alias_collision_does_not_change_persisted_table(tmp_path, monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    alias = "test-alias-collision-persisted-table"
+    registry.register(alias)(FirstEmbedding)
+    function = FirstEmbedding.create(max_retries=0)
+
+    class Row(LanceModel):
+        text: str = function.SourceField()
+        vector: Vector(2) = function.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "rows", schema=Row, data=[{"text": "alpha"}, {"text": "beta"}]
+    )
+    assert table.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    assert registry.register(alias)(FirstEmbedding) is FirstEmbedding
+    with pytest.raises(KeyError, match=f"{alias} was already registered"):
+        registry.register(alias)(SecondEmbedding)
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert isinstance(reopened.embedding_functions["vector"].function, FirstEmbedding)
+    assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"

python/python/tests/test_embeddings.py

@@ -23,6 +23,12 @@ from lancedb.embeddings.utils import retry
 from lancedb.pydantic import LanceModel, Vector
 
 
+@pytest.fixture(autouse=True)
+def isolated_embedding_registry(monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+
+
 def mock_embed_func(input_data):
     return [np.random.randn(128).tolist() for _ in range(len(input_data))]
 
@@ -64,6 +70,37 @@ def test_embedding_function(tmp_path):
     assert np.allclose(actual, expected)
 
 
+def test_embedding_function_registered_without_alias(tmp_path):
+    registry = EmbeddingFunctionRegistry.get_instance()
+
+    @registry.register()
+    class NoAliasEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 2
+
+        def generate_embeddings(self, texts):
+            return [np.array([1.0, 2.0]) for _ in texts]
+
+    assert registry.get("NoAliasEmbeddingFunction") is NoAliasEmbeddingFunction
+
+    func = NoAliasEmbeddingFunction.create()
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(2) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    db.create_table("test", schema=Schema)
+
+    # The class name is written to the table metadata, so the function can be
+    # resolved again when the table is reopened.
+    table = lancedb.connect(tmp_path).open_table("test")
+    table.add([{"text": "hello"}])
+    conf = table.embedding_functions["vector"]
+    assert isinstance(conf.function, NoAliasEmbeddingFunction)
+    assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0]]
+
+
 def test_instructor_ndims_uses_instruction():
     instructor = get_registry().get("instructor").create()
     model = MagicMock()
@@ -596,6 +633,318 @@ def test_openai_no_retry_on_401(mock_sleep):
     assert mock_sleep.call_count == 0
 
 
+class _HttpStatusError(Exception):
+    """Stand-in for provider SDK errors (openai, cohere, ...) with a status code."""
+
+    def __init__(self, status_code, message="error"):
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class _ResponseStatusError(Exception):
+    """Stand-in for requests/httpx errors, which expose `response.status_code`."""
+
+    def __init__(self, status_code):
+        super().__init__(f"HTTP {status_code}")
+        self.response = MagicMock(status_code=status_code)
+
+
+def _run_with_retry(side_effect, **kwargs):
+    """Run a function through the retry helper with a mocked `time.sleep`.
+
+    Returns (mock_func, sleeps, error).
+    """
+    from lancedb.embeddings.utils import retry_with_exponential_backoff
+
+    mock_func = MagicMock(side_effect=side_effect)
+    sleeps = []
+    error = None
+    with patch("time.sleep", side_effect=sleeps.append):
+        try:
+            retry_with_exponential_backoff(mock_func, **kwargs)()
+        except Exception as e:
+            error = e
+    return mock_func, sleeps, error
+
+
+@pytest.mark.parametrize("status", [400, 401, 403, 404, 405, 410, 413, 422])
+@pytest.mark.parametrize("make_error", [_HttpStatusError, _ResponseStatusError])
+def test_retry_does_not_retry_permanent_http_errors(status, make_error):
+    err = make_error(status)
+    mock_func, sleeps, error = _run_with_retry(err, max_retries=7)
+
+    assert error is err
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+@pytest.mark.parametrize("status", [408, 409, 429, 500, 502, 503, 504])
+@pytest.mark.parametrize("make_error", [_HttpStatusError, _ResponseStatusError])
+def test_retry_retries_transient_http_errors(status, make_error):
+    mock_func, sleeps, error = _run_with_retry(
+        [make_error(status), make_error(status), "result"], max_retries=7
+    )
+
+    assert error is None
+    assert mock_func.call_count == 3
+    assert len(sleeps) == 2
+
+
+@pytest.mark.parametrize(
+    "exc",
+    [
+        ConnectionError("connection reset"),
+        TimeoutError("timed out"),
+        OSError("network is unreachable"),
+        ValueError("custom embedding function failure"),
+        RuntimeError("boom"),
+        Exception("plain exception"),
+    ],
+)
+def test_retry_retries_errors_without_http_status(exc):
+    mock_func, sleeps, error = _run_with_retry([exc, exc, "result"], max_retries=7)
+
+    assert error is None
+    assert mock_func.call_count == 3
+    assert len(sleeps) == 2
+
+
+def test_retry_error_status_attributes_of_other_types_are_ignored():
+    # `code` is a string for openai errors ("model_not_found"), and arbitrary
+    # exceptions may carry unrelated numeric attributes. Neither is an HTTP status.
+    class WeirdError(Exception):
+        code = "model_not_found"
+        status_code = None
+
+    class ExitCodeError(Exception):
+        code = 2
+
+    for exc in (WeirdError("a"), ExitCodeError("b")):
+        mock_func, sleeps, error = _run_with_retry(exc, max_retries=2)
+        assert mock_func.call_count == 3
+        assert len(sleeps) == 2
+
+
+def test_retry_botocore_style_errors():
+    # botocore ClientError exposes `response["ResponseMetadata"]["HTTPStatusCode"]`.
+    # Bedrock reports throttling with a 400, which must stay retryable.
+    class ClientError(Exception):
+        def __init__(self, status, code):
+            super().__init__(code)
+            self.response = {
+                "ResponseMetadata": {"HTTPStatusCode": status},
+                "Error": {"Code": code},
+            }
+
+    throttled = ClientError(400, "ThrottlingException")
+    mock_func, sleeps, error = _run_with_retry([throttled, "result"], max_retries=3)
+    assert error is None
+    assert mock_func.call_count == 2
+
+    invalid = ClientError(400, "ValidationException")
+    mock_func, sleeps, error = _run_with_retry(invalid, max_retries=3)
+    assert error is invalid
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+def test_retry_authentication_error_by_name_not_retried():
+    class AuthenticationError(Exception):
+        pass
+
+    mock_func, sleeps, error = _run_with_retry(AuthenticationError("bad key"))
+    assert isinstance(error, AuthenticationError)
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+def test_retry_gives_up_after_max_retries_with_original_error_attached():
+    original = _HttpStatusError(503, "overloaded")
+    mock_func, sleeps, error = _run_with_retry(original, max_retries=3)
+
+    assert mock_func.call_count == 4
+    assert len(sleeps) == 3
+    assert "Maximum number of retries (3) exceeded" in str(error)
+    assert error.args[1] is original
+    assert error.__cause__ is original
+
+
+def test_retry_delay_is_capped_by_max_delay():
+    # Default settings (7 retries, 60s cap): the old implementation could sleep for
+    # up to ~47 minutes in a single retry and hours in total.
+    exc = _HttpStatusError(503)
+    for seed in range(50):
+        import random
+
+        random.seed(seed)
+        _, sleeps, _ = _run_with_retry(exc)
+        assert len(sleeps) == 7
+        assert max(sleeps) <= 60
+        assert sum(sleeps) <= 7 * 60
+
+    # Custom cap
+    _, sleeps, _ = _run_with_retry(exc, max_retries=10, max_delay=5)
+    assert len(sleeps) == 10
+    assert max(sleeps) <= 5
+
+
+def test_retry_delay_without_jitter_is_exact_exponential_then_capped():
+    _, sleeps, _ = _run_with_retry(
+        _HttpStatusError(500),
+        initial_delay=1,
+        exponential_base=2,
+        jitter=False,
+        max_retries=8,
+        max_delay=30,
+    )
+    assert sleeps == [2, 4, 8, 16, 30, 30, 30, 30]
+
+
+def test_retry_jitter_is_bounded():
+    # Each wait is within [delay / 2, delay], where delay grows by exponential_base
+    # per retry (no random factor is compounded across retries).
+    exc = _HttpStatusError(500)
+    for seed in range(50):
+        import random
+
+        random.seed(seed)
+        _, sleeps, _ = _run_with_retry(exc, max_retries=8, max_delay=100)
+        for i, slept in enumerate(sleeps):
+            delay = min(2 ** (i + 1), 100)
+            assert delay / 2 <= slept <= delay
+
+    # Extremes of the random source map to the ends of the range
+    with patch("random.random", return_value=0.0):
+        _, sleeps, _ = _run_with_retry(exc, max_retries=3)
+    assert sleeps == [1, 2, 4]
+    with patch("random.random", return_value=1.0):
+        _, sleeps, _ = _run_with_retry(exc, max_retries=3)
+    assert sleeps == [2, 4, 8]
+
+
+def test_retry_large_max_retries_does_not_overflow():
+    _, sleeps, _ = _run_with_retry(_HttpStatusError(503), max_retries=1500)
+    assert len(sleeps) == 1500
+    assert max(sleeps) <= 60
+
+
+def _register_failing_embedding(name, error):
+    """Register an embedding function that always raises `error`.
+
+    Returns the class and a list that records every call. The function is
+    re-created from table metadata, so the call count has to live outside of it.
+    """
+    calls = []
+
+    @register(name)
+    class FailingEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 4
+
+        def generate_embeddings(self, texts):
+            calls.append(len(texts))
+            raise error
+
+    return FailingEmbeddingFunction, calls
+
+
+@pytest.mark.parametrize("status", [400, 404, 422])
+def test_permanent_embedding_error_is_not_retried(tmp_path, status):
+    """A permanent provider error (e.g. a misspelled model name) must surface
+    immediately from both `table.add` and `table.search` instead of being retried
+    with minutes of backoff."""
+    error = _HttpStatusError(status, "The model `typo-embed` does not exist")
+    cls, calls = _register_failing_embedding(f"failing-embedding-{status}", error)
+    func = cls.create()  # default max_retries=7
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("t", schema=Schema)
+
+    with patch("time.sleep") as mock_sleep:
+        with pytest.raises(Exception, match="The model `typo-embed` does not exist"):
+            table.add([{"text": "hello"}])
+        assert len(calls) == 1
+
+        with pytest.raises(_HttpStatusError, match="does not exist"):
+            table.search("hello").limit(1).to_list()
+        assert len(calls) == 2
+
+    mock_sleep.assert_not_called()
+
+
+def test_transient_embedding_error_is_retried_in_table_add(tmp_path):
+    cls, calls = _register_failing_embedding(
+        "failing-embedding-503", _HttpStatusError(503)
+    )
+    func = cls.create(max_retries=2)
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("t", schema=Schema)
+
+    with patch("time.sleep") as mock_sleep:
+        with pytest.raises(Exception, match="Maximum number of retries"):
+            table.add([{"text": "hello"}])
+
+    assert len(calls) == 3
+    assert mock_sleep.call_count == 2
+    assert all(call.args[0] <= 60 for call in mock_sleep.call_args_list)
+
+
+@patch("time.sleep")
+@pytest.mark.parametrize(
+    "error",
+    [
+        pytest.param("missing_key", id="missing-api-key"),
+        pytest.param(ImportError("No module named 'voyageai'"), id="missing-package"),
+    ],
+)
+def test_no_retry_on_deterministic_errors(mock_sleep, error):
+    from lancedb.embeddings.utils import (
+        api_key_not_found_help,
+        retry_with_exponential_backoff,
+    )
+
+    def func():
+        if error == "missing_key":
+            api_key_not_found_help("voyageai")
+        raise error
+
+    mock_func = MagicMock(side_effect=func)
+    wrapped_func = retry_with_exponential_backoff(mock_func, max_retries=3)
+
+    with pytest.raises((ValueError, ImportError)):
+        wrapped_func()
+
+    assert mock_func.call_count == 1
+    assert mock_sleep.call_count == 0
+
+
+@patch("time.sleep")
+def test_retry_on_wrapped_provider_error(mock_sleep):
+    """Providers such as Bedrock wrap throttling in a plain ValueError."""
+    from lancedb.embeddings.utils import retry_with_exponential_backoff
+
+    mock_func = MagicMock(
+        side_effect=[
+            ValueError("Error raised by boto3 client: ThrottlingException"),
+            [0.1],
+        ]
+    )
+    wrapped_func = retry_with_exponential_backoff(mock_func, max_retries=3)
+
+    assert wrapped_func() == [0.1]
+    assert mock_func.call_count == 2
+    assert mock_sleep.call_count == 1
+
+
 def test_openai_bad_request_partial_batch_recovery():
     # Regression test for issue #1677
 
@@ -708,3 +1057,28 @@ def test_jina_generate_image_input_dict_local_path(tmp_path):
         image_dict = JinaEmbeddings._generate_image_input_dict(image)
         assert "image" in image_dict
         assert isinstance(image_dict["image"], str) and len(image_dict["image"]) > 0
+
+
+def test_jina_embeddings_use_their_own_api_key(monkeypatch):
+    """Each JinaEmbeddings instance must send its own API key."""
+    pytest.importorskip("requests")
+    monkeypatch.delenv("JINA_API_KEY", raising=False)
+    registry = get_registry()
+    registry.set_var("jina_key_a", "key-a")
+    registry.set_var("jina_key_b", "key-b")
+    func_a = registry.get("jina").create(api_key="$var:jina_key_a")
+    func_b = registry.get("jina").create(api_key="$var:jina_key_b")
+
+    sent = []
+
+    def fake_post(session, url, json=None, **kwargs):
+        sent.append(session.headers["Authorization"])
+        response = MagicMock()
+        response.json.return_value = {"data": [{"index": 0, "embedding": [0.0]}]}
+        return response
+
+    with patch("requests.Session.post", fake_post):
+        func_a.compute_source_embeddings(["hello"])
+        func_b.compute_source_embeddings(["hello"])
+
+    assert sent == ["Bearer key-a", "Bearer key-b"]

python/python/tests/test_errors.py

@@ -3,7 +3,28 @@
 
 import pickle
 
+import pytest
+
+from lancedb.db import DBConnection
 from lancedb.remote.errors import HttpError, LanceDBClientError, RetryError
+from lancedb.table import Table
+
+
+@pytest.mark.parametrize(
+    "base, method",
+    [
+        (Table, "to_lance"),
+        (Table, "to_polars"),
+        (DBConnection, "drop_all_tables"),
+        (DBConnection, "drop_database"),
+    ],
+)
+def test_base_unsupported_methods_have_messages(base, method):
+    with pytest.raises(NotImplementedError) as exc_info:
+        getattr(base, method)(None)
+    message = str(exc_info.value)
+    assert method in message
+    assert "not supported" in message
 
 
 def test_pickle_lancedb_client_error():

python/python/tests/test_expr.py

@@ -80,6 +80,67 @@ class TestExprConstruction:
         e = lit(d)
         assert isinstance(e, Expr)
 
+    @pytest.mark.parametrize(
+        "value, decimal_type",
+        [
+            ("1E-7", pa.decimal128(12, 7)),
+            ("-1.23E-7", pa.decimal128(12, 9)),
+            ("1E+3", pa.decimal128(12, 0)),
+            ("-1.23E+5", pa.decimal128(12, 0)),
+            ("0E-7", pa.decimal128(12, 7)),
+            ("-0E+3", pa.decimal128(12, 0)),
+            ("1E-38", pa.decimal128(38, 38)),
+            ("1E+37", pa.decimal128(38, 0)),
+            ("1.234567890123456789012345678901234567E+37", pa.decimal128(38, 0)),
+            ("0E+128", pa.decimal128(1, 0)),
+        ],
+    )
+    def test_lit_decimal_exponent_query(self, tmp_path, value, decimal_type):
+        expected = Decimal(value)
+        table = lancedb.connect(tmp_path).create_table(
+            "decimals",
+            pa.table(
+                {
+                    "id": [1, 2],
+                    # Arrow's Python ingestion rejects 0E+128 even though its
+                    # numeric value is representable. Store canonical zero to
+                    # exercise the expression literal independently of ingestion.
+                    "price": pa.array(
+                        [Decimal(0) if expected.is_zero() else expected, None],
+                        type=decimal_type,
+                    ),
+                }
+            ),
+        )
+        predicate = col("price") == lit(expected)
+        result = table.search().where(predicate).to_arrow()
+        assert result["id"].to_pylist() == [1]
+        assert result["price"].to_pylist() == [expected]
+        # Exercise the SQL serializer and parser as well as the typed query path.
+        sql_result = table.search().where(predicate.to_sql()).to_arrow()
+        assert sql_result["id"].to_pylist() == [1]
+        assert sql_result["price"].to_pylist() == [expected]
+
+    @pytest.mark.parametrize(
+        "value",
+        [
+            "NaN",
+            "sNaN",
+            "Infinity",
+            "-Infinity",
+            "1E-39",
+            "1E+38",
+            "1E+129",
+            "1E+1000000000",
+            "0E+129",
+            "12345678901234567890123456789012345678E+1",
+            "9" * 39,
+        ],
+    )
+    def test_lit_decimal_unrepresentable_raises(self, value):
+        with pytest.raises(ValueError):
+            lit(Decimal(value))
+
 
 class TestExprOperators:
     def test_eq_operator(self):

python/python/tests/test_first_class_function_slice2.py

@@ -23,6 +23,7 @@ import pyarrow as pa
 import pytest
 
 import lancedb
+from lancedb import BlobReader
 from lancedb.functions import (
     PythonRuntimeSpec,
     SecretBinding,
@@ -966,6 +967,60 @@ def test_blob_fields_use_the_scalar_function_semantic_type():
     assert signature.output.arrow_type == "blob_v2"
 
 
+def test_blob_input_access_is_inferred_from_annotations():
+    @udf
+    def read_blob(video: BlobReader, timestamp_s: float) -> bytes:
+        video.seek(int(timestamp_s))
+        return video.read(4)
+
+    @udf
+    def read_bytes(video: bytes) -> int:
+        return len(video)
+
+    reader = read_blob.registration_request.signature
+    assert reader.inputs[0].arrow_type == "blob_v2"
+    assert reader.inputs[0].blob_access == "reader"
+    assert reader.inputs[1].blob_access is None
+    assert reader.output.arrow_type == "binary"
+    eager = read_bytes.registration_request.signature.inputs[0]
+    assert eager.arrow_type == "binary"
+    assert eager.blob_access == "bytes"
+    encoded = json.loads(read_blob.registration_request.to_canonical_json())
+    assert encoded["signature"]["inputs"][0]["blob_access"] == "reader"
+
+
+def test_nullable_blob_reader_and_explicit_blob_bytes():
+    @udf
+    def nullable(video: Optional[BlobReader]) -> int:
+        return 0 if video is None else video.size()
+
+    @udf(
+        input_schema=pa.schema([lancedb.blob("video", nullable=False)]),
+        output_schema=pa.field("size", pa.int64(), nullable=False),
+    )
+    def eager(video: bytes) -> int:
+        return len(video)
+
+    parameter = nullable.registration_request.signature.inputs[0]
+    assert parameter.nullable is True
+    assert parameter.blob_access == "reader"
+    parameter = eager.registration_request.signature.inputs[0]
+    assert parameter.arrow_type == "blob_v2"
+    assert parameter.blob_access == "bytes"
+
+
+def test_blob_reader_rejects_an_incompatible_explicit_schema():
+    def read(video: BlobReader) -> int:
+        return video.size()
+
+    with pytest.raises(TypeError, match="requires a Blob V2 field"):
+        udf(
+            read,
+            input_schema=pa.schema([pa.field("video", pa.int64(), nullable=False)]),
+            output_schema=pa.field("size", pa.int64(), nullable=False),
+        )
+
+
 def test_whole_named_struct_function_can_include_a_blob_result_field():
     @udf(
         input_schema=pa.schema([lancedb.blob("image", nullable=False)]),
@@ -1674,7 +1729,11 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip():
         )
         registration = db.create_function_async(normalize_score)
         assert registration.id == "job-register"
+        assert registration.job_type is None
+        assert registration.status() == "finished"
+        assert registration.job_type == "create_function"
         created = registration.wait()
+        assert registration.job_type == "create_function"
         reopened = db.get_function("normalize_score", version=created.version)
 
     assert created == reopened

python/python/tests/test_fts.py

@@ -391,6 +391,20 @@ def test_custom_stop_words_list(table):
         )
 
 
+def test_list_indices_with_missing_fts_files(table):
+    # Regression test for https://github.com/lancedb/lancedb/issues/4250
+    table.create_fts_index("text")
+    table.create_scalar_index("id")
+
+    fts = next(i for i in table.list_indices() if i.index_type == "FTS")
+    shutil.rmtree(Path(table.uri) / "_indices" / fts.index_uuid)
+
+    indices = {i.name: i for i in table.list_indices()}
+    assert set(indices) == {"text_idx", "id_idx"}
+    assert indices["text_idx"].index_type == "FTS"
+    assert indices["text_idx"].columns == ["text"]
+
+
 def test_search_fts(table):
     table.create_fts_index("text")
     results = table.search("puppy").select(["id", "text"]).limit(5).to_list()

python/python/tests/test_huggingface.py

@@ -129,3 +129,106 @@ def test_generator(tmp_path: Path):
 
     assert len(tbl) == 2
     assert tbl.schema == ds.features.arrow_schema
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("operation", ["create", "add"])
+@pytest.mark.parametrize(
+    "selection",
+    ["normal", "contiguous", "selected", "duplicates", "filtered", "shuffled", "empty"],
+)
+async def test_write_selected_hf_dataset(tmp_path, asynchronous, operation, selection):
+    source = datasets.Dataset.from_dict({"id": list(range(5)), "text": list("abcde")})
+    data = {
+        "normal": source,
+        "contiguous": source.select(range(1, 4)),
+        "selected": source.select([4, 1]),
+        "duplicates": source.select([2, 2, 0]),
+        "filtered": source.filter(lambda row: row["id"] % 2 == 1),
+        "shuffled": source.shuffle(seed=42),
+        "empty": source.select([]),
+    }[selection]
+    expected = data.to_list()
+    if asynchronous:
+        db = await lancedb.connect_async(tmp_path)
+        if operation == "create":
+            table = await db.create_table("selected", data)
+        else:
+            table = await db.create_table("selected", schema=data.features.arrow_schema)
+            await table.add(data)
+        actual = await table.to_arrow()
+    else:
+        db = lancedb.connect(tmp_path)
+        if operation == "create":
+            table = db.create_table("selected", data)
+        else:
+            table = db.create_table("selected", schema=data.features.arrow_schema)
+            table.add(data)
+        actual = table.to_arrow()
+    assert actual.to_pylist() == expected
+    assert actual.schema.equals(data.features.arrow_schema, check_metadata=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("operation", ["create", "add"])
+async def test_write_selected_hf_splits(tmp_path, asynchronous, operation):
+    source = datasets.Dataset.from_dict({"id": list(range(5)), "text": list("abcde")})
+    data = datasets.DatasetDict(
+        {
+            "train": source.select([4, 1]),
+            "test": source.select([3, 0, 2]),
+            "empty": source.select([]),
+        }
+    )
+    expected = [
+        dict(row, split=split)
+        for split, dataset in data.items()
+        for row in dataset.to_list()
+    ]
+    schema = source.features.arrow_schema.append(pa.field("split", pa.string()))
+    if asynchronous:
+        db = await lancedb.connect_async(tmp_path)
+        if operation == "create":
+            table = await db.create_table("splits", data)
+        else:
+            table = await db.create_table("splits", schema=schema)
+            await table.add(data)
+        actual = await table.to_arrow()
+    else:
+        db = lancedb.connect(tmp_path)
+        if operation == "create":
+            table = db.create_table("splits", data)
+        else:
+            table = db.create_table("splits", schema=schema)
+            table.add(data)
+        actual = table.to_arrow()
+    assert actual.to_pylist() == expected
+    assert actual.schema.equals(schema, check_metadata=True)
+
+
+@pytest.mark.parametrize("format_type", [None, "numpy", "pandas", "arrow"])
+@pytest.mark.parametrize("empty", [False, True])
+def test_selected_hf_reader_is_rescannable(format_type, empty):
+    from lancedb.scannable import _register_optional_converters, to_scannable
+
+    source = datasets.Dataset.from_dict(
+        {"id": list(range(2105)), "text": [str(i) for i in range(2105)]}
+    )
+    indices = [] if empty else list(reversed(range(2105))) + [1, 1]
+    data = source.select(indices).with_format(format_type, columns=["id"])
+    original_format = data.format
+    expected = source.select(indices).to_list()
+    _register_optional_converters()
+    scannable = to_scannable(data)
+    assert scannable.rescannable
+    assert scannable.num_rows == len(expected)
+    for _ in range(2):
+        batches = list(scannable.reader())
+        if not empty:
+            assert all(batch.num_rows <= 1000 for batch in batches)
+        actual = pa.Table.from_batches(batches, schema=scannable.schema)
+        assert actual.to_pylist() == expected
+        assert actual.schema.equals(data.features.arrow_schema, check_metadata=True)
+    assert data.format == original_format

python/python/tests/test_listing.py

@@ -88,6 +88,143 @@ async def advance(items, asynchronous):
 KINDS = ["secrets", "views", "jobs", "functions", "materialized_views"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+@pytest.mark.parametrize("asynchronous", [False, True])
+async def test_compatible_name_listing_pages(listing_server, kind, asynchronous):
+    endpoint, requests, responses = listing_server
+    responses.extend(
+        [
+            (200, response_page(kind, ["a", "b"], "next")),
+            (200, response_page(kind, ["c"], None)),
+        ]
+    )
+    db = await connection(endpoint, asynchronous)
+    listing = getattr(db, "list_" + kind)(
+        namespace_path=["team"], page_token="start", limit=2
+    )
+    field = kind
+    assert len(requests) == (0 if asynchronous else 1)
+    if asynchronous:
+        with pytest.raises(RuntimeError, match="Await or iterate"):
+            getattr(listing, field)
+        assert listing.page_token == "start"
+        assert await listing is listing
+    assert len(requests) == 1
+    page_names = getattr(listing, field)
+    assert type(page_names) is list
+    assert page_names == ["a", "b"]
+    page_names.append("changed")
+    assert getattr(listing, field) == ["a", "b"]
+    assert listing.page_token == "next"
+    assert listing.num_page_results() == 2
+    assert await advance(listing, asynchronous) == "a"
+    assert getattr(listing, field) == ["a", "b"]
+    assert await advance(listing, asynchronous) == "b"
+    assert len(requests) == 1
+    assert await advance(listing, asynchronous) == "c"
+    assert getattr(listing, field) == ["c"]
+    assert listing.page_token is None
+    assert len(requests) == 2
+    assert ([item async for item in listing] if asynchronous else list(listing)) == []
+    if asynchronous:
+        assert await listing is listing
+    assert len(requests) == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+async def test_name_listing_skips_empty_page_only_during_iteration(
+    listing_server, kind
+):
+    endpoint, requests, responses = listing_server
+    responses.extend(
+        [
+            (200, response_page(kind, [], "next")),
+            (200, response_page(kind, ["a"], None)),
+        ]
+    )
+    db = await connection(endpoint, True)
+    listing = getattr(db, "list_" + kind)()
+    await listing
+    assert getattr(listing, kind) == []
+    assert listing.page_token == "next"
+    assert len(requests) == 1
+    assert [name async for name in listing] == ["a"]
+    assert len(requests) == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+@pytest.mark.parametrize("asynchronous", [False, True])
+async def test_name_listing_zero_limit_uses_no_request(
+    listing_server, kind, asynchronous
+):
+    endpoint, requests, _ = listing_server
+    db = await connection(endpoint, asynchronous)
+    listing = getattr(db, "list_" + kind)(limit=0)
+    if asynchronous:
+        assert await listing is listing
+        assert [name async for name in listing] == []
+    else:
+        assert list(listing) == []
+    assert getattr(listing, kind) == []
+    assert listing.page_token is None
+    assert requests == []
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+async def test_name_listing_failure_retains_token(listing_server, kind):
+    endpoint, requests, responses = listing_server
+    responses.append((401, {}))
+    db = await connection(endpoint, True)
+    listing = getattr(db, "list_" + kind)(page_token="retry")
+    with pytest.raises(HttpError):
+        await listing
+    assert listing.page_token == "retry"
+    with pytest.raises(HttpError):
+        await listing
+    assert [name async for name in listing] == []
+    assert len(requests) == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+async def test_name_listing_direct_async_iteration(listing_server, kind):
+    endpoint, requests, responses = listing_server
+    responses.extend(
+        [
+            (200, response_page(kind, ["a"], "next")),
+            (200, response_page(kind, ["b"], None)),
+        ]
+    )
+    db = await connection(endpoint, True)
+    listing = getattr(db, "list_" + kind)(limit=1)
+    assert requests == []
+    assert [name async for name in listing] == ["a", "b"]
+    assert getattr(listing, kind) == ["b"]
+    assert len(requests) == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["tables", "namespaces"])
+async def test_name_listing_serializes_concurrent_advances(listing_server, kind):
+    endpoint, requests, responses = listing_server
+    responses.append((200, response_page(kind, ["a", "b"], None)))
+    db = await connection(endpoint, True)
+    listing = getattr(db, "list_" + kind)()
+    results = await asyncio.gather(
+        listing.__anext__(),
+        listing.__anext__(),
+        listing.__anext__(),
+        return_exceptions=True,
+    )
+    assert results[:2] == ["a", "b"]
+    assert isinstance(results[2], StopAsyncIteration)
+    assert len(requests) == 1
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("kind", KINDS)
 @pytest.mark.parametrize("asynchronous", [False, True])

python/python/tests/test_materialized_views.py

@@ -8,6 +8,7 @@ import threading
 
 import lancedb
 import pytest
+from lancedb.db import AsyncConnection
 from lancedb.materialized_view import MaterializedViewDefinition
 from lancedb.remote.db import RemoteDBConnection
 
@@ -56,7 +57,7 @@ def mock_remote_materialized_views():
 
 
 @contextlib.contextmanager
-def mock_remote_materialized_view_create():
+def mock_remote_materialized_view_drop():
     requests = []
 
     class Handler(http.server.BaseHTTPRequestHandler):
@@ -67,7 +68,8 @@ def mock_remote_materialized_view_create():
             length = int(self.headers.get("Content-Length", 0))
             body = json.loads(self.rfile.read(length) or b"{}")
             requests.append((self.path, body))
-            job_id = "mv-drop-123" if self.path.endswith("/drop") else "mv-create-123"
+            assert self.path.endswith("/drop")
+            job_id = "mv-drop-123"
             encoded = json.dumps({"job_id": job_id}).encode()
             self.send_response(202)
             self.send_header("Content-Type", "application/json")
@@ -98,7 +100,7 @@ def test_remote_list_uses_namespace_route():
 
 
 def test_remote_drop_async_returns_server_job():
-    with mock_remote_materialized_view_create() as (host, requests):
+    with mock_remote_materialized_view_drop() as (host, requests):
         db = lancedb.connect(
             "db://dev",
             api_key="fake",
@@ -192,11 +194,13 @@ def test_create_and_refresh_jobs(tmp_path):
         "adults", "people", where="age >= 18", with_no_data=True
     )
     assert create_job.id is None
+    assert create_job.job_type is None
     assert create_job.wait() is None
 
     view = db.open_materialized_view("adults")
     refresh_job = view.refresh_async()
     assert refresh_job.id is None
+    assert refresh_job.job_type is None
     result = refresh_job.wait()
     assert result.mode == "rebuild"
     assert result.rows_written == 2
@@ -337,11 +341,13 @@ async def test_async_create_and_refresh_jobs(tmp_path):
         "adults", "people", with_no_data=True
     )
     assert create_job.id is None
+    assert create_job.job_type is None
     assert await create_job.wait() is None
 
     view = await db.open_materialized_view("adults")
     refresh_job = await view.refresh_async()
     assert refresh_job.id is None
+    assert refresh_job.job_type is None
     result = await refresh_job.wait()
     assert result.mode == "rebuild"
     assert result.rows_written == 1
@@ -567,6 +573,27 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
     assert list(resumed) == []
 
 
+class QueryBackedCreateJob:
+    id = "0198f1b2-c3d4-7e5f-8123-456789abcdef"
+
+    job_type = "query"
+
+    async def wait(self):
+        return None
+
+    async def status(self):
+        return "finished"
+
+
+class CapturingCreateConnection:
+    def __init__(self):
+        self.calls = []
+
+    async def create_materialized_view_async(self, name, source, **options):
+        self.calls.append((name, source, options))
+        return QueryBackedCreateJob()
+
+
 @pytest.mark.parametrize(
     "options",
     [
@@ -576,16 +603,28 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
     ],
 )
 def test_remote_create_forwards_refresh_options_without_interval_default(options):
-    with mock_remote_materialized_view_create() as (host, requests):
-        db = lancedb.connect("db://dev", api_key="fake", host_override=host)
-        job = db.create_materialized_view_async("adults", "people", **options)
-        assert job.id == "mv-create-123"
-    assert requests == [
+    native = CapturingCreateConnection()
+    db = RemoteDBConnection.__new__(RemoteDBConnection)
+    db._conn = AsyncConnection(native)
+    job = db.create_materialized_view_async("adults", "people", **options)
+    assert job.id == QueryBackedCreateJob.id
+    assert job.job_type == "query"
+    assert not hasattr(job, "query_id")
+    assert job.wait() is None
+    assert job.status() == "finished"
+    assert native.calls == [
         (
-            "/v1/materialized_view/adults/create",
+            "adults",
+            "people",
             {
-                "query": 'SELECT * FROM "people"',
-                **options,
+                "projections": None,
+                "filter": None,
+                "limit": None,
+                "with_no_data": False,
+                "auto_refresh_enabled": options.get("auto_refresh_enabled"),
+                "auto_refresh_interval_seconds": options.get(
+                    "auto_refresh_interval_seconds"
+                ),
             },
         )
     ]
@@ -593,18 +632,18 @@ def test_remote_create_forwards_refresh_options_without_interval_default(options
 
 @pytest.mark.asyncio
 async def test_async_remote_create_keeps_interval_unspecified():
-    with mock_remote_materialized_view_create() as (host, requests):
-        db = await lancedb.connect_async("db://dev", api_key="fake", host_override=host)
-        job = await db.create_materialized_view_async(
-            "adults",
-            "people",
-            auto_refresh_enabled=True,
-        )
-        assert job.id == "mv-create-123"
-    assert requests[0][1] == {
-        "query": 'SELECT * FROM "people"',
-        "auto_refresh_enabled": True,
-    }
+    native = CapturingCreateConnection()
+    db = AsyncConnection(native)
+    job = await db.create_materialized_view_async(
+        "adults", "people", auto_refresh_enabled=True
+    )
+    assert job.id == QueryBackedCreateJob.id
+    assert job.job_type == "query"
+    assert not hasattr(job, "query_id")
+    assert await job.wait() is None
+    assert await job.status() == "finished"
+    assert native.calls[0][2]["auto_refresh_enabled"] is True
+    assert native.calls[0][2]["auto_refresh_interval_seconds"] is None
 
 
 @pytest.mark.parametrize(
@@ -625,7 +664,7 @@ def test_embedded_create_refuses_automatic_refresh_before_creating_view(
 
 @pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
 def test_remote_create_rejects_invalid_interval_without_request(seconds):
-    with mock_remote_materialized_view_create() as (host, requests):
+    with mock_remote_materialized_view_drop() as (host, requests):
         db = lancedb.connect("db://dev", api_key="fake", host_override=host)
         with pytest.raises(ValueError, match="positive, finite"):
             db.create_materialized_view_async(

python/python/tests/test_namespace.py

@@ -6,6 +6,7 @@
 import tempfile
 import shutil
 import importlib
+import warnings
 import http.server
 import json
 import threading
@@ -25,6 +26,125 @@ PUSHDOWN_DATA = pa.table(
 )
 
 
+class TestNamespaceCreateTableOptions:
+    @pytest.mark.parametrize("version", ["legacy", "stable"])
+    @pytest.mark.parametrize("empty", [False, True])
+    def test_deprecated_storage_version(self, tmp_path, version, empty):
+        db = lancedb.connect_namespace("dir", {"root": str(tmp_path / "namespace")})
+        data = None if empty else [{"id": 1}]
+        schema = pa.schema([("id", pa.int64())])
+        other_version = "stable" if version == "legacy" else "legacy"
+        options = {"new_table_data_storage_version": other_version}
+        control = lancedb.connect(tmp_path / "control").create_table(
+            "test",
+            data=data,
+            schema=schema,
+            storage_options={"new_table_data_storage_version": version},
+        )
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = db.create_table(
+                "test",
+                data=data,
+                schema=schema,
+                storage_options=options,
+                data_storage_version=version,
+            )
+
+        assert (
+            table.to_lance().data_storage_version
+            == control.to_lance().data_storage_version
+        )
+        assert table.to_arrow().to_pylist() == ([] if empty else data)
+        assert options == {"new_table_data_storage_version": other_version}
+        assert any("data_storage_version" in str(w.message) for w in caught)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("version", ["legacy", "stable"])
+    @pytest.mark.parametrize("empty", [False, True])
+    async def test_deprecated_storage_version_async(self, tmp_path, version, empty):
+        root = str(tmp_path / "namespace")
+        db = lancedb.connect_namespace_async("dir", {"root": root})
+        data = None if empty else [{"id": 1}]
+        schema = pa.schema([("id", pa.int64())])
+        other_version = "stable" if version == "legacy" else "legacy"
+        options = {"new_table_data_storage_version": other_version}
+        control = lancedb.connect(tmp_path / "control").create_table(
+            "test",
+            data=data,
+            schema=schema,
+            storage_options={"new_table_data_storage_version": version},
+        )
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = await db.create_table(
+                "test",
+                data=data,
+                schema=schema,
+                storage_options=options,
+                data_storage_version=version,
+            )
+
+        reopened = lancedb.connect_namespace("dir", {"root": root}).open_table("test")
+        assert (
+            reopened.to_lance().data_storage_version
+            == control.to_lance().data_storage_version
+        )
+        assert (await table.to_arrow()).to_pylist() == ([] if empty else data)
+        assert options == {"new_table_data_storage_version": other_version}
+        assert any("data_storage_version" in str(w.message) for w in caught)
+
+    @pytest.mark.parametrize("enabled", [False, True])
+    @pytest.mark.parametrize("empty", [False, True])
+    def test_deprecated_manifest_paths(self, tmp_path, enabled, empty):
+        db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
+        data = None if empty else [{"id": 1}]
+        options = {"new_table_enable_v2_manifest_paths": str(not enabled).lower()}
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = db.create_table(
+                "test",
+                data=data,
+                schema=pa.schema([("id", pa.int64())]),
+                storage_options=options,
+                enable_v2_manifest_paths=enabled,
+            )
+
+        assert table.uses_v2_manifest_paths() == enabled
+        assert table.to_arrow().to_pylist() == ([] if empty else data)
+        assert db.open_table("test").uses_v2_manifest_paths() == enabled
+        assert options == {
+            "new_table_enable_v2_manifest_paths": str(not enabled).lower()
+        }
+        assert any("enable_v2_manifest_paths" in str(w.message) for w in caught)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("enabled", [False, True])
+    @pytest.mark.parametrize("empty", [False, True])
+    async def test_deprecated_manifest_paths_async(self, tmp_path, enabled, empty):
+        db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
+        data = None if empty else [{"id": 1}]
+        options = {"new_table_enable_v2_manifest_paths": str(not enabled).lower()}
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = await db.create_table(
+                "test",
+                data=data,
+                schema=pa.schema([("id", pa.int64())]),
+                storage_options=options,
+                enable_v2_manifest_paths=enabled,
+            )
+
+        assert await table.uses_v2_manifest_paths() == enabled
+        assert (await table.to_arrow()).to_pylist() == ([] if empty else data)
+        reopened = await db.open_table("test")
+        assert await reopened.uses_v2_manifest_paths() == enabled
+        assert options == {
+            "new_table_enable_v2_manifest_paths": str(not enabled).lower()
+        }
+        assert any("enable_v2_manifest_paths" in str(w.message) for w in caught)
+
+
 def _ipc_file(table: pa.Table = PUSHDOWN_DATA) -> bytes:
     sink = pa.BufferOutputStream()
     with pa.ipc.new_file(sink, table.schema) as writer:

python/python/tests/test_ollama_embeddings.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+
+import numpy as np
+import pyarrow as pa
+import pytest
+from pydantic import ValidationError
+
+from lancedb.embeddings import get_registry
+
+
+@pytest.fixture
+def ollama_embeddings():
+    httpx = pytest.importorskip("httpx")
+    pytest.importorskip("ollama")
+    requests = []
+
+    def respond(request):
+        body = json.loads(request.content)
+        requests.append(body)
+        if body["model"] == "missing-model":
+            return httpx.Response(404, json={"error": "model not found"})
+        return httpx.Response(
+            200,
+            json={
+                "embeddings": [
+                    [float(index), float(len(text))]
+                    for index, text in enumerate(body["input"])
+                ]
+            },
+        )
+
+    function = (
+        get_registry()
+        .get("ollama")
+        .create(
+            name="nomic-embed-text",
+            options={"num_ctx": 4096},
+            keep_alive="10m",
+            ollama_client_kwargs={
+                "transport": httpx.MockTransport(respond),
+                "trust_env": False,
+            },
+        )
+    )
+    yield function, requests
+    function._ollama_client._client.close()
+
+
+@pytest.mark.parametrize(
+    "texts",
+    [
+        pytest.param(["hello", "世界"], id="list"),
+        pytest.param(np.array(["hello", "世界"]), id="numpy-unicode"),
+        pytest.param(np.array(["hello", "世界"], dtype=object), id="numpy-object"),
+        pytest.param(
+            np.array([b"hello", "世界".encode("utf-8")]), id="numpy-utf8-bytes"
+        ),
+        pytest.param([b"hello", "世界".encode("utf-8")], id="list-utf8-bytes"),
+        pytest.param(np.array(["hello", "unused", "世界"])[::2], id="numpy-slice"),
+        pytest.param(pa.array(["hello", "世界"]), id="arrow"),
+        pytest.param(pa.chunked_array([["hello"], ["世界"]]), id="chunked-arrow"),
+    ],
+)
+def test_ollama_source_embeddings_inputs(ollama_embeddings, texts):
+    function, requests = ollama_embeddings
+
+    assert function.compute_source_embeddings(texts) == [[0.0, 5.0], [1.0, 2.0]]
+    assert requests == [
+        {
+            "model": "nomic-embed-text",
+            "input": ["hello", "世界"],
+            "options": {"num_ctx": 4096},
+            "keep_alive": "10m",
+        }
+    ]
+
+
+@pytest.mark.parametrize(
+    "texts",
+    [
+        pytest.param(np.array([["hello"], ["world"]]), id="numpy-two-dimensional"),
+        pytest.param([["hello"], ["world"]], id="list-two-dimensional"),
+        pytest.param(np.array([b"\xff"]), id="numpy-invalid-utf8"),
+        pytest.param([b"\xff"], id="list-invalid-utf8"),
+    ],
+)
+def test_ollama_source_embeddings_rejects_invalid_input_locally(
+    ollama_embeddings, texts
+):
+    function, requests = ollama_embeddings
+
+    with pytest.raises(ValidationError, match="valid string"):
+        function.compute_source_embeddings(texts)
+
+    assert requests == []
+
+
+def test_ollama_generate_embeddings_empty_array(ollama_embeddings):
+    function, requests = ollama_embeddings
+
+    assert function.generate_embeddings(np.array([], dtype=str)) == []
+    assert len(requests) == 1
+    assert requests[0]["input"] == []
+
+
+def test_ollama_query_embeddings(ollama_embeddings):
+    function, requests = ollama_embeddings
+
+    assert function.compute_query_embeddings("hello") == [[0.0, 5.0]]
+    assert len(requests) == 1
+    assert requests[0]["input"] == ["hello"]
+
+
+def test_ollama_source_embeddings_preserves_api_error(ollama_embeddings):
+    ollama = pytest.importorskip("ollama")
+    function, requests = ollama_embeddings
+    function.name = "missing-model"
+
+    with pytest.raises(ollama.ResponseError, match="model not found") as error:
+        function.compute_source_embeddings(np.array(["hello"]))
+
+    assert error.value.status_code == 404
+    assert len(requests) == 1

python/python/tests/test_openai_reranker.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+from types import SimpleNamespace
+
+import pyarrow as pa
+import pytest
+
+from lancedb.rerankers import OpenaiReranker
+
+
+def reranker_replying(documents):
+    """Return an OpenaiReranker whose chat client replies with these documents."""
+    reply = json.dumps({"documents": documents})
+
+    def create(**kwargs):
+        message = SimpleNamespace(content=reply)
+        return SimpleNamespace(choices=[SimpleNamespace(message=message)])
+
+    reranker = OpenaiReranker(api_key="test-key")
+    reranker.__dict__["_client"] = SimpleNamespace(
+        chat=SimpleNamespace(completions=SimpleNamespace(create=create))
+    )
+    return reranker
+
+
+def vector_results():
+    return pa.table(
+        {
+            "id": [1, 2, 3],
+            "text": ["dogs bark", "cats purr", "birds sing"],
+            "_distance": [0.1, 0.2, 0.3],
+        }
+    )
+
+
+def test_scores_stay_with_their_rows_when_the_model_reorders():
+    reranker = reranker_replying(
+        [
+            {"content": "cats purr", "relevance_score": 0.9},
+            {"content": "birds sing", "relevance_score": 0.5},
+            {"content": "dogs bark", "relevance_score": 0.1},
+        ]
+    )
+
+    result = reranker.rerank_vector("which animal purrs?", vector_results())
+
+    assert result.select(["id", "text"]).to_pylist() == [
+        {"id": 2, "text": "cats purr"},
+        {"id": 3, "text": "birds sing"},
+        {"id": 1, "text": "dogs bark"},
+    ]
+    assert result["_relevance_score"].to_pylist() == pytest.approx([0.9, 0.5, 0.1])
+
+
+def test_a_row_the_model_leaves_out_raises():
+    reranker = reranker_replying(
+        [
+            {"content": "cats purr", "relevance_score": 0.9},
+            {"content": "dogs bark", "relevance_score": 0.1},
+        ]
+    )
+
+    with pytest.raises(ValueError, match="1 of 3"):
+        reranker.rerank_vector("which animal purrs?", vector_results())

python/python/tests/test_openai_reranker_credentials.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import pytest
+
+from lancedb.rerankers import OpenaiReranker
+
+
+@pytest.mark.parametrize(
+    "explicit_key, environment_key, expected_key",
+    [
+        ("explicit-test-key", "environment-test-key", "explicit-test-key"),
+        ("explicit-test-key", None, "explicit-test-key"),
+        (None, "environment-test-key", "environment-test-key"),
+    ],
+)
+def test_openai_reranker_api_key_precedence(
+    monkeypatch, explicit_key, environment_key, expected_key
+):
+    pytest.importorskip("openai")
+    if environment_key is None:
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    else:
+        monkeypatch.setenv("OPENAI_API_KEY", environment_key)
+
+    reranker = OpenaiReranker(api_key=explicit_key)
+    client = reranker._client
+    try:
+        assert client.api_key == expected_key
+    finally:
+        client.close()
+
+
+def test_openai_reranker_requires_api_key(monkeypatch):
+    pytest.importorskip("openai")
+    monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+    with pytest.raises(ValueError, match="OPENAI_API_KEY not set"):
+        OpenaiReranker()._client

python/python/tests/test_permutation.py

@@ -811,6 +811,13 @@ def test_rename_column(some_permutation: Permutation):
         some_permutation.rename_column("non_existent", "new_value")
 
 
+def test_rename_column_keeps_position(some_permutation: Permutation):
+    renamed = some_permutation.rename_column("id", "new_id")
+    assert renamed.schema == pa.schema([("new_id", pa.int64()), ("value", pa.int64())])
+    batch = renamed.with_format("arrow").__getitems__([0])
+    assert batch.schema.names == ["new_id", "value"]
+
+
 def test_rename_columns(some_permutation: Permutation):
     assert some_permutation.rename_columns({"value": "new_value"}).schema == pa.schema(
         [("id", pa.int64()), ("new_value", pa.int64())]

python/python/tests/test_pyarrow.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import pyarrow as pa
+import pyarrow.compute as pc
 
 import lancedb
 from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
@@ -48,3 +49,19 @@ def test_dataset_adapter(tmp_path):
     assert adapter.head(10).num_rows == 0
 
     assert adapter.scanner().projected_schema == pa.schema({"x": pa.int64()})
+
+
+def test_dataset_adapter_expression_filter(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)
+    filt = pc.field("x") > 2
+
+    assert adapter.count_rows(filt) == 2
+    assert adapter.to_table(filter=filt) == data.slice(2, 2)
+    assert adapter.head(1, filter=filt) == data.slice(2, 1)
+    # The filter can reference a column that is not projected
+    assert adapter.to_table(columns=["y"], filter=filt) == pa.table({"y": [7, 8]})
+    assert adapter.scanner(filter=filt).count_rows() == 2

python/python/tests/test_pydantic.py

@@ -415,6 +415,67 @@ def test_nullable_vector():
     assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
 
 
+@pytest.mark.parametrize("vector_type", [Vector, MultiVector])
+@pytest.mark.parametrize(
+    "value_type, values",
+    [
+        (pa.int64(), [-(2**53 + 1), 2**53 + 3]),
+        (pa.uint64(), [2**53 + 1, 2**64 - 1]),
+        (pa.int8(), [-128, 127]),
+        (pa.uint8(), [0, 255]),
+        (pa.float32(), [1.25, 2.5]),
+        (pa.float64(), [1.25, 2.5]),
+    ],
+)
+def test_typed_vector_round_trip(tmp_path, vector_type, value_type, values):
+    import lancedb
+
+    annotation = vector_type(2, value_type=value_type)
+    expected = [values, values] if vector_type is MultiVector else values
+
+    class Row(LanceModel):
+        id: int
+        payload: annotation
+
+    row = Row(id=1, payload=expected)
+    table = lancedb.connect(tmp_path).create_table("typed_vectors", [row], schema=Row)
+    table.add([Row(id=2, payload=expected)])
+    assert table.schema == Row.to_arrow_schema()
+    assert table.search().to_arrow().to_pylist() == [
+        {"id": 1, "payload": expected},
+        {"id": 2, "payload": expected},
+    ]
+    assert json.loads(row.model_dump_json())["payload"] == expected
+
+
+@pytest.mark.parametrize("vector_type", [Vector, MultiVector])
+@pytest.mark.parametrize("nullable", [False, True])
+def test_integer_vector_validation(vector_type, nullable):
+    annotation = vector_type(2, value_type=pa.int64(), nullable=nullable)
+
+    class Row(LanceModel):
+        payload: annotation
+
+    def shape(values):
+        return [values] if vector_type is MultiVector else values
+
+    assert Row.to_arrow_schema().field("payload").nullable is nullable
+    assert Row(payload=shape([1.0, -2.0])).model_dump()["payload"] == shape([1, -2])
+    with pytest.raises(pydantic.ValidationError, match="valid integer"):
+        Row(payload=shape([1.5, 2]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=shape([1]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=shape([1, 2, 3]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=None)
+
+    class OptionalRow(LanceModel):
+        payload: Optional[annotation] = None
+
+    assert OptionalRow(payload=None).payload is None
+
+
 def test_bare_vector_raises_clear_error():
     namespace = {
         "__name__": "test_model_without_pyarrow",
@@ -733,3 +794,52 @@ def test_enum_types():
     assert schema.field("priority").type == pa.int64()
     assert schema.field("opt_status").type == pa.dictionary(pa.int32(), pa.utf8())
     assert schema.field("opt_status").nullable
+
+
+def test_plain_enum_values_are_written(mem_db):
+    """Members of an Enum without a str/int mixin are written as their values."""
+
+    class Color(Enum):
+        RED = "red"
+        BLUE = "blue"
+
+    class Priority(Enum):
+        LOW = 1
+        HIGH = 2
+
+    class Tag(LanceModel):
+        color: Color
+
+    class TestModel(LanceModel):
+        color: Color
+        priority: Priority
+        opt_color: Optional[Color] = None
+        colors: List[Color]
+        tag: Tag
+        vector: Vector(2)
+
+    rows = [
+        TestModel(
+            color=Color.RED,
+            priority=Priority.HIGH,
+            opt_color=Color.BLUE,
+            colors=[Color.BLUE, Color.RED],
+            tag=Tag(color=Color.BLUE),
+            vector=[1.0, 2.0],
+        ),
+        TestModel(
+            color=Color.BLUE,
+            priority=Priority.LOW,
+            colors=[],
+            tag=Tag(color=Color.RED),
+            vector=[3.0, 4.0],
+        ),
+    ]
+    tbl = mem_db.create_table("test", schema=TestModel)
+    tbl.add(rows)
+
+    assert tbl.to_arrow().select(["color", "priority", "opt_color"]).to_pylist() == [
+        {"color": "red", "priority": 2, "opt_color": "blue"},
+        {"color": "blue", "priority": 1, "opt_color": None},
+    ]
+    assert tbl.search().to_pydantic(TestModel) == rows

python/python/tests/test_pydantic_excluded_fields.py

@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+from pydantic import BaseModel, Field
+from lancedb.pydantic import LanceModel
+
+
+class Details(BaseModel):
+    label: str
+    transient: str = Field(default="nested default", exclude=True)
+
+
+class Row(LanceModel):
+    id: int
+    details: Details
+    transient: str = Field(default="local default", exclude=True)
+    generated: list[str] = Field(default_factory=lambda: ["local"], exclude=True)
+    retained: str = Field(default="persisted", exclude=False)
+
+
+def rows():
+    return [
+        Row(
+            id=i,
+            details=Details(label=str(i), transient="private"),
+            transient="private",
+            generated=["private"],
+        )
+        for i in (1, 2)
+    ]
+
+
+def assert_schema(schema):
+    assert schema.names == ["id", "details", "retained"]
+    assert [field.name for field in schema.field("details").type] == ["label"]
+    assert Row.field_names() == ["id", "details", "transient", "generated", "retained"]
+
+
+def assert_restored(restored):
+    assert sorted(row.id for row in restored) == [1, 2]
+    for row in restored:
+        assert row.transient == "local default"
+        assert row.generated == ["local"]
+        assert row.details.transient == "nested default"
+        assert row.details.label == str(row.id)
+        assert row.retained == "persisted"
+
+
+@pytest.mark.parametrize("explicit_schema", [False, True])
+def test_excluded_fields_round_trip(tmp_path, explicit_schema):
+    data = rows()
+    kwargs = {"schema": Row} if explicit_schema else {}
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("rows", data=data[:1], **kwargs)
+    table.add(data[1:])
+    assert_schema(table.schema)
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert_restored(reopened.search().to_pydantic(Row))
+    assert data[0].transient == "private"
+    assert data[0].generated == ["private"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("explicit_schema", [False, True])
+async def test_excluded_fields_round_trip_async(tmp_path, explicit_schema):
+    data = rows()
+    kwargs = {"schema": Row} if explicit_schema else {}
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("rows", data=data[:1], **kwargs)
+    await table.add(data[1:])
+    assert_schema(await table.schema())
+    reopened_db = await lancedb.connect_async(tmp_path)
+    reopened = await reopened_db.open_table("rows")
+    assert_restored(await reopened.query().to_pydantic(Row))
+    assert data[0].details.transient == "private"
+
+
+@pytest.mark.parametrize("excluded", ["transient", "source", "vector"])
+def test_excluded_fields_keep_embedding_metadata_consistent(
+    tmp_path, monkeypatch, excluded
+):
+    from lancedb.embeddings.base import TextEmbeddingFunction
+    from lancedb.embeddings.registry import get_registry
+    from lancedb.pydantic import Vector
+
+    calls = []
+
+    class LocalEmbedding(TextEmbeddingFunction):
+        def ndims(self):
+            return 2
+
+        def generate_embeddings(self, texts):
+            calls.extend(texts)
+            return [[1.0, 0.0] for _ in texts]
+
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    registry.register("test-excluded-fields")(LocalEmbedding)
+    function = LocalEmbedding.create(max_retries=0)
+
+    class EmbeddedRow(LanceModel):
+        text: str = function.SourceField(default="alpha", exclude=excluded == "source")
+        vector: Vector(2) = function.VectorField(
+            default_factory=lambda: [1.0, 0.0], exclude=excluded == "vector"
+        )
+        transient: str = Field(default="local", exclude=True)
+
+    configs = EmbeddedRow.parse_embedding_functions()
+    assert len(configs) == (1 if excluded == "transient" else 0)
+    db = lancedb.connect(tmp_path)
+    data = [{"text": "alpha"}] if excluded == "transient" else [EmbeddedRow()]
+    table = db.create_table("embeddings", schema=EmbeddedRow, data=data)
+    assert "transient" not in table.schema.names
+    assert ("text" in table.schema.names) == (excluded != "source")
+    assert ("vector" in table.schema.names) == (excluded != "vector")
+    reopened = lancedb.connect(tmp_path).open_table("embeddings")
+    assert len(reopened.embedding_functions) == len(configs)
+    restored = reopened.search().to_pydantic(EmbeddedRow)
+    assert restored[0].text == "alpha"
+    assert restored[0].vector == [1.0, 0.0]
+    assert restored[0].transient == "local"
+    if excluded == "transient":
+        assert calls == ["alpha"]
+        assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    else:
+        assert calls == []

python/python/tests/test_query.py

@@ -2324,6 +2324,33 @@ def test_take_queries(tmp_path):
     ]
 
 
+def test_take_queries_with_row_id(tmp_path):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("test", pa.table({"idx": range(100)}))
+
+    by_row_id = table.take_row_ids([5, 2, 17], with_row_id=True).to_arrow()
+    assert "_rowid" in by_row_id.column_names
+    by_offset = table.take_offsets([5, 2, 17], with_row_id=True).to_arrow()
+    assert "_rowid" in by_offset.column_names
+
+    # The column is opt-in and must stay out of the default projection.
+    assert "_rowid" not in table.take_row_ids([5, 2, 17]).to_arrow().column_names
+
+
+@pytest.mark.asyncio
+async def test_take_queries_with_row_id_async(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("test", pa.table({"idx": range(100)}))
+
+    by_row_id = await table.take_row_ids([5, 2, 17], with_row_id=True).to_arrow()
+    assert "_rowid" in by_row_id.column_names
+    by_offset = await table.take_offsets([5, 2, 17], with_row_id=True).to_arrow()
+    assert "_rowid" in by_offset.column_names
+
+    default = await table.take_row_ids([5, 2, 17]).to_arrow()
+    assert "_rowid" not in default.column_names
+
+
 def test_take_queries_to_batches(tmp_path):
     # Regression test for the sync take-query path: `to_batches` previously
     # raised ``AttributeError: 'AsyncTakeQuery' object has no attribute

python/python/tests/test_read_consistency_interval.py

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from datetime import timedelta
+
+import lancedb
+import pytest
+from lance_namespace import connect as connect_namespace_client
+from lancedb import _lancedb
+from lancedb.namespace import LanceNamespaceDBConnection
+
+
+@pytest.mark.parametrize("seconds", [-1.0, -0.000001])
+@pytest.mark.parametrize("kind", ["sync", "namespace", "namespace_async", "client"])
+def test_negative_read_consistency_interval(tmp_path, seconds, kind):
+    interval = timedelta(seconds=seconds)
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        if kind == "sync":
+            lancedb.connect(tmp_path, read_consistency_interval=interval)
+        elif kind == "client":
+            client = connect_namespace_client("dir", {"root": str(tmp_path)})
+            LanceNamespaceDBConnection(client, read_consistency_interval=interval)
+        else:
+            connect = (
+                lancedb.connect_namespace
+                if kind == "namespace"
+                else lancedb.connect_namespace_async
+            )
+            connect("dir", {"root": str(tmp_path)}, read_consistency_interval=interval)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("seconds", [-1.0, -0.000001])
+async def test_negative_async_read_consistency_interval(tmp_path, seconds):
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        await lancedb.connect_async(
+            tmp_path, read_consistency_interval=timedelta(seconds=seconds)
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("seconds", [float("nan"), float("inf"), -float("inf"), 1e30])
+@pytest.mark.parametrize("kind", ["connect", "namespace", "client"])
+async def test_binding_rejects_invalid_read_consistency_interval(
+    tmp_path, seconds, kind
+):
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        if kind == "connect":
+            await _lancedb.connect(str(tmp_path), read_consistency_interval=seconds)
+        elif kind == "namespace":
+            _lancedb.connect_namespace(
+                "dir", {"root": str(tmp_path)}, read_consistency_interval=seconds
+            )
+        else:
+            client = connect_namespace_client("dir", {"root": str(tmp_path)})
+            _lancedb.connect_namespace_client(client, read_consistency_interval=seconds)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "interval", [None, timedelta(0), timedelta(microseconds=1), timedelta(seconds=5)]
+)
+async def test_valid_read_consistency_intervals(tmp_path, interval):
+    db = lancedb.connect(tmp_path / "sync", read_consistency_interval=interval)
+    assert db.read_consistency_interval == interval
+    db.close()
+
+    db = await lancedb.connect_async(
+        tmp_path / "async", read_consistency_interval=interval
+    )
+    assert await db.get_read_consistency_interval() == interval
+    db.close()
+
+    db = lancedb.connect_namespace(
+        "dir", {"root": str(tmp_path / "namespace")}, read_consistency_interval=interval
+    )
+    assert db.read_consistency_interval == interval
+    db.close()
+
+    client = connect_namespace_client("dir", {"root": str(tmp_path / "client")})
+    db = LanceNamespaceDBConnection(client, read_consistency_interval=interval)
+    assert db.read_consistency_interval == interval
+    db.close()

python/python/tests/test_remote_db.py

@@ -857,10 +857,116 @@ def test_table_unimplemented_functions():
 
     with mock_lancedb_connection(handler) as db:
         table = db.create_table("test", [{"id": 1}])
-        with pytest.raises(NotImplementedError):
-            table.to_arrow()
-        with pytest.raises(NotImplementedError):
-            table.to_pandas()
+        for method in ["to_arrow", "to_pandas", "to_polars"]:
+            with pytest.raises(NotImplementedError) as exc_info:
+                getattr(table, method)()
+            message = str(exc_info.value)
+            assert method in message
+            assert "not" in message and "supported" in message
+            assert "remote tables" in message
+            assert "cloud" not in message.lower()
+            assert f"search().{method}()" in message
+
+
+@pytest.mark.parametrize("method", ["to_arrow", "to_pandas"])
+@pytest.mark.parametrize("blob_mode", ["lazy", "descriptions", "bytes", "invalid"])
+def test_remote_conversion_blob_mode_errors(method, blob_mode):
+    with blob_remote_table() as table:
+        if blob_mode == "invalid":
+            with pytest.raises(ValueError, match="blob_mode must be one of"):
+                getattr(table, method)(blob_mode=blob_mode)
+        else:
+            with pytest.raises(NotImplementedError) as error:
+                getattr(table, method)(blob_mode=blob_mode)
+            assert f"search().{method}()" in str(error.value)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("version", [None, 1])
+@pytest.mark.parametrize("expose_location", [False, True])
+async def test_remote_to_lance_rejection(
+    tmp_path, version, expose_location, monkeypatch
+):
+    import lance
+
+    location = tmp_path / "test.lance"
+    lance.write_dataset(pa.table({"id": [1]}), location)
+    lance.write_dataset(pa.table({"id": [2]}), location, mode="append")
+    requests = []
+
+    def fail_dataset(*args, **kwargs):
+        raise AssertionError("remote storage must not be opened")
+
+    monkeypatch.setattr(lance, "dataset", fail_dataset)
+
+    def handler(request):
+        requests.append(request.path)
+        assert request.path == "/v1/table/test/describe/"
+        content_len = int(request.headers.get("Content-Length", 0))
+        body = json.loads(request.rfile.read(content_len)) if content_len else {}
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        response = {
+            "version": body.get("version") or 2,
+            "schema": {"fields": []},
+        }
+        if expose_location:
+            response["location"] = str(location)
+        request.wfile.write(json.dumps(response).encode())
+
+    options = {"default_scan_options": {"with_row_id": True}}
+    with mock_lancedb_connection(handler) as db:
+        table = db.open_table("test", version=version)
+        opened_requests = list(requests)
+        with pytest.raises(NotImplementedError) as sync_error:
+            table.to_lance(**options)
+        assert requests == opened_requests
+
+    async with mock_lancedb_connection_async(handler) as db:
+        table = await db.open_table("test", version=version)
+        opened_requests = list(requests)
+        with pytest.raises(NotImplementedError) as async_error:
+            await table.to_lance(**options)
+        assert requests == opened_requests
+
+    assert "to_lance() is not supported for remote tables" in str(sync_error.value)
+    assert "query the server" in str(sync_error.value)
+    assert str(sync_error.value) == str(async_error.value)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "method, kwargs",
+    [
+        ("drop_all_tables", {}),
+        ("drop_all_tables", {"namespace_path": []}),
+        ("drop_all_tables", {"namespace_path": ["team"]}),
+        ("drop_database", {}),
+    ],
+)
+async def test_remote_drop_all_tables_not_supported(method, kwargs):
+    requests = []
+
+    def handler(request):
+        requests.append(request.path)
+        request.send_response(500)
+        request.end_headers()
+
+    with mock_lancedb_connection(handler) as db:
+        with pytest.raises(NotImplementedError) as sync_error:
+            getattr(db, method)(**kwargs)
+
+    async with mock_lancedb_connection_async(handler) as db:
+        with pytest.raises(NotImplementedError) as async_error:
+            await getattr(db, method)(**kwargs)
+
+    assert "Dropping all tables is not currently supported in the remote API" in str(
+        sync_error.value
+    )
+    assert "drop_table" in str(sync_error.value)
+    assert str(sync_error.value) == str(async_error.value)
+    assert requests == []
 
 
 def test_table_to_pandas_not_supported():
@@ -1681,10 +1787,13 @@ def test_remote_python_only_messages(caplog):
             table = db.open_table("test", storage_options={}, index_cache_size=1)
         assert "storage_options is ignored for remote tables" in caplog.text
         assert "index_cache_size is ignored for remote tables" in caplog.text
-        for method in ["to_arrow", "to_pandas"]:
+        for method in ["to_arrow", "to_pandas", "to_polars"]:
             with pytest.raises(NotImplementedError) as error:
                 getattr(table, method)()
-            assert str(error.value) == f"{method}() is not supported for remote tables."
+            assert str(error.value) == (
+                f"{method}() is not supported for remote tables. "
+                f"Use table.search().{method}() instead."
+            )
         for method in ["compact_files", "cleanup_old_versions"]:
             with pytest.warns(UserWarning) as warning:
                 getattr(table, method)()
@@ -1793,6 +1902,79 @@ def test_table_wait_for_index_timeout():
             table.wait_for_index(["id_idx"], timedelta(seconds=1))
 
 
+def _jieba_fts_handler(request):
+    """Serve a table with a ``jieba/default`` FTS index and no other state."""
+    if request.path == "/v1/table/test/describe/":
+        body = {
+            "version": 1,
+            "schema": {
+                "fields": [
+                    {"name": "text", "type": {"type": "string"}, "nullable": True}
+                ]
+            },
+        }
+    elif request.path == "/v1/table/test/index/list/":
+        index_details = {
+            "base_tokenizer": "jieba/default",
+            "language": "English",
+            "with_position": False,
+            "max_token_length": 40,
+            "lower_case": True,
+            "stem": False,
+            "remove_stop_words": False,
+            "ascii_folding": False,
+        }
+        body = {
+            "indexes": [
+                {
+                    "index_name": "text_idx",
+                    "columns": ["text"],
+                    "index_type": "FTS",
+                    "index_details": json.dumps(index_details),
+                }
+            ]
+        }
+    else:
+        request.send_response(404)
+        request.end_headers()
+        return
+    request.send_response(200)
+    request.send_header("Content-Type", "application/json")
+    request.end_headers()
+    request.wfile.write(json.dumps(body).encode())
+
+
+_MODEL_FILES_ERROR = (
+    r"(?s)Invalid directory path.*"
+    r"full text search index 'text_idx'.*"
+    r"tokenize\(\) runs on the client.*'jieba/default'.*"
+    r"LANCE_LANGUAGE_MODEL_HOME.*match the files the server uses"
+)
+
+
+def test_remote_tokenize_model_backed_without_model_files(tmp_path, monkeypatch):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    with mock_lancedb_connection(_jieba_fts_handler) as db:
+        table = db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", column="text")
+
+
+@pytest.mark.asyncio
+async def test_remote_tokenize_model_backed_without_model_files_async(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    async with mock_lancedb_connection_async(_jieba_fts_handler) as db:
+        table = await db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", column="text")
+
+
 def test_stats():
     stats = {
         "total_bytes": 38,
@@ -2152,6 +2334,23 @@ def test_head():
         assert data == pa.table({"id": [1, 2, 3]})
 
 
+def test_remote_take_row_ids_with_row_id():
+    bodies = []
+
+    def handler(body):
+        bodies.append(body)
+        return pa.table({"id": [1, 2], "_rowid": pa.array([0, 1], pa.uint64())})
+
+    with query_test_table(handler) as table:
+        result = table.take_row_ids([0, 1], with_row_id=True).to_arrow()
+        assert "_rowid" in result.column_names
+    assert bodies[-1]["with_row_id"] is True
+
+    with query_test_table(handler) as table:
+        table.take_row_ids([0, 1]).to_arrow()
+    assert "with_row_id" not in bodies[-1]
+
+
 def test_query_sync_minimal():
     def handler(body):
         assert body == {
@@ -3400,14 +3599,17 @@ def blob_descriptor_take_table():
 
 
 @contextlib.contextmanager
-def blob_remote_table(*, server_version=Version("0.5.0")):
+def blob_remote_table(*, server_version=Version("0.5.0"), location=None):
     def handler(request):
         if request.path == "/v1/table/test/describe/":
             request.send_response(200)
             request.send_header("Content-Type", "application/json")
             request.send_header("phalanx-version", str(server_version))
             request.end_headers()
-            request.wfile.write(json.dumps(BLOB_DESCRIBE_RESPONSE).encode())
+            response = dict(BLOB_DESCRIBE_RESPONSE)
+            if location is not None:
+                response["location"] = str(location)
+            request.wfile.write(json.dumps(response).encode())
         elif request.path.startswith("/v1/table/test/blob/image/"):
             path = request.path.partition("?")[0]
             row_id = int(path.split("/")[-2])
@@ -3442,7 +3644,6 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
             if columns == ["id"]:
                 response_table = blob_query_response_table().select(["id"])
             else:
-                assert body["with_row_id"] is True
                 if columns == ["image"]:
                     # Blob handles are sized from a descriptor take.
                     assert body["filter"].startswith("_rowid IN")
@@ -3455,6 +3656,8 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
                     )
                 else:
                     response_table = blob_query_response_table()
+                if not body.get("with_row_id", False):
+                    response_table = response_table.drop(["_rowid"])
             request.send_response(200)
             request.send_header("Content-Type", "application/vnd.apache.arrow.file")
             request.send_header("x-lancedb-version", "1")
@@ -3489,6 +3692,32 @@ def test_remote_blob_columns_and_fetch():
         assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
 
 
+@pytest.mark.parametrize("blob_mode", ["descriptions", "bytes", "lazy"])
+def test_sync_remote_blob_pandas_avoids_storage_access(
+    tmp_path, monkeypatch, blob_mode
+):
+    import lance
+
+    location = tmp_path / "remote.lance"
+    lance.write_dataset(pa.table({"id": [7], "image": [b"server-owned"]}), location)
+
+    def fail_dataset(*args, **kwargs):
+        raise AssertionError("remote storage must not be opened")
+
+    monkeypatch.setattr(lance, "dataset", fail_dataset)
+    with blob_remote_table(location=location) as table:
+        monkeypatch.setattr(table, "to_lance", fail_dataset)
+        query = table.search().select(["id", "image"])
+        if blob_mode == "descriptions":
+            df = query.to_pandas(blob_mode=blob_mode)
+            assert df["id"].tolist() == [1, 2, 3]
+            assert "_rowid" not in df.columns
+            assert df["image"][0]["size"] == 5
+        else:
+            with pytest.raises(NotImplementedError, match="stable table snapshot"):
+                query.to_pandas(blob_mode=blob_mode)
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("use_async", [False, True])
 async def test_remote_fetch_blob_ranges(use_async):

python/python/tests/test_reranker_credentials.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import sys
+import types
+
+import pytest
+
+from lancedb.rerankers import CohereReranker, VoyageAIReranker
+
+RERANKERS = [
+    pytest.param("cohere", "COHERE_API_KEY", CohereReranker, {}, id="cohere"),
+    pytest.param(
+        "voyageai",
+        "VOYAGE_API_KEY",
+        VoyageAIReranker,
+        {"model_name": "rerank-2.5"},
+        id="voyageai",
+    ),
+]
+
+
+def fake_sdk(monkeypatch, module_name):
+    """Replace the SDK module with one whose Client records the API key."""
+    keys = []
+
+    def client(api_key=None):
+        keys.append(api_key)
+        return object()
+
+    module = types.ModuleType(module_name)
+    module.Client = client
+    monkeypatch.setitem(sys.modules, module_name, module)
+    return keys
+
+
+@pytest.mark.parametrize("module_name, env_var, reranker_cls, kwargs", RERANKERS)
+@pytest.mark.parametrize(
+    "explicit_key, environment_key, expected_key",
+    [
+        ("explicit-test-key", "environment-test-key", "explicit-test-key"),
+        ("explicit-test-key", None, "explicit-test-key"),
+        (None, "environment-test-key", "environment-test-key"),
+    ],
+)
+def test_reranker_api_key_precedence(
+    monkeypatch,
+    module_name,
+    env_var,
+    reranker_cls,
+    kwargs,
+    explicit_key,
+    environment_key,
+    expected_key,
+):
+    keys = fake_sdk(monkeypatch, module_name)
+    if environment_key is None:
+        monkeypatch.delenv(env_var, raising=False)
+    else:
+        monkeypatch.setenv(env_var, environment_key)
+
+    reranker_cls(api_key=explicit_key, **kwargs)._client
+    assert keys == [expected_key]
+
+
+@pytest.mark.parametrize("module_name, env_var, reranker_cls, kwargs", RERANKERS)
+def test_reranker_requires_api_key(
+    monkeypatch, module_name, env_var, reranker_cls, kwargs
+):
+    keys = fake_sdk(monkeypatch, module_name)
+    monkeypatch.delenv(env_var, raising=False)
+    with pytest.raises(ValueError, match=f"{env_var} not set"):
+        reranker_cls(**kwargs)._client
+    assert keys == []

python/python/tests/test_rerankers.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import os
+from unittest.mock import MagicMock
 import random
 
 import lancedb
@@ -547,6 +548,24 @@ def test_openai_reranker(tmp_path):
     _run_test_reranker(reranker, table, "single player experience", None, schema)
 
 
+def test_openai_reranker_does_not_send_temperature():
+    reranker = OpenaiReranker(api_key="test-key")
+    create = MagicMock()
+    create.return_value.choices = [
+        MagicMock(
+            message=MagicMock(
+                content='{"documents": [{"content": "a", "relevance_score": 0.5}]}'
+            )
+        )
+    ]
+    reranker.__dict__["_client"] = MagicMock()
+    reranker.__dict__["_client"].chat.completions.create = create
+
+    reranker._rerank(pa.table({"text": ["a"]}), "query")
+
+    assert "temperature" not in create.call_args.kwargs
+
+
 @pytest.mark.skipif(
     os.environ.get("JINA_API_KEY") is None, reason="JINA_API_KEY not set"
 )
@@ -785,6 +804,46 @@ def test_empty_hybrid_result_reranker():
     assert "_rowid" in result.column_names
 
 
+@pytest.mark.parametrize("norm", ["score", "rank"])
+def test_hybrid_return_all_restores_original_scores(norm):
+    from lancedb.query import LanceHybridQueryBuilder
+
+    vector_results = pa.table(
+        {
+            "text": ["a", "b", "c"],
+            "_rowid": pa.array([0, 1, 2], type=pa.uint64()),
+            "_distance": pa.array([0.5, 2.5, 10.0], type=pa.float32()),
+        }
+    )
+    fts_results = pa.table(
+        {
+            "text": ["b", "a"],
+            "_rowid": pa.array([1, 0], type=pa.uint64()),
+            "_score": pa.array([3.25, 1.75], type=pa.float32()),
+        }
+    )
+
+    result = LanceHybridQueryBuilder._combine_hybrid_results(
+        fts_results=fts_results,
+        vector_results=vector_results,
+        norm=norm,
+        fts_query="query",
+        reranker=RRFReranker(return_score="all"),
+        limit=10,
+        with_row_ids=True,
+    )
+
+    # The raw distances and scores are returned, not the normalized values
+    # or ranks used for reranking.
+    rows = {row["_rowid"]: row for row in result.to_pylist()}
+    assert rows[0]["_distance"] == 0.5
+    assert rows[1]["_distance"] == 2.5
+    assert rows[2]["_distance"] == 10.0
+    assert rows[0]["_score"] == 1.75
+    assert rows[1]["_score"] == 3.25
+    assert rows[2]["_score"] is None
+
+
 @pytest.mark.parametrize("return_score", ["relevance", "all"])
 @pytest.mark.parametrize("first_row_id", [1, 2**63])
 @pytest.mark.parametrize("score_type", [pa.float32(), pa.float64()])

python/python/tests/test_session.py

@@ -36,3 +36,13 @@ def test_session_cache_configuration(tmp_path):
     assert final_cache_size > initial_cache_size  # Cache should have grown
     assert final_cache_items >= initial_cache_items  # Items should not decrease
     assert initial_cache_size < index_cache_size + metadata_cache_size
+
+
+def test_connection_session_returns_supplied_session(tmp_path):
+    """Sync connections must expose the session they were opened with."""
+    db = lancedb.connect(tmp_path)
+    assert db.session is None
+
+    session = lancedb.Session()
+    db = lancedb.connect(tmp_path, session=session)
+    assert db.session is session

python/python/tests/test_sql.py

@@ -137,9 +137,23 @@ async def test_async_namespace_connection_rejects_sql(tmp_path):
         await connection.describe_query(NIL_QUERY_ID)
 
 
-def test_describe_query_requires_uuid():
-    with pytest.raises(TypeError, match="UUID"):
-        remote_connection().describe_query(str(NIL_QUERY_ID))
+@pytest.mark.asyncio
+@pytest.mark.parametrize("identity", [NIL_QUERY_ID, str(NIL_QUERY_ID)])
+async def test_describe_query_accepts_uuid_and_job_id_string(identity):
+    calls = []
+
+    class NativeConnection:
+        async def describe_query(self, query_id):
+            calls.append(query_id)
+
+    connection = AsyncConnection(NativeConnection())
+    await connection.describe_query(identity)
+    assert calls == [NIL_QUERY_ID]
+
+
+def test_describe_query_rejects_registry_job_id():
+    with pytest.raises(ValueError, match="UUID"):
+        remote_connection().describe_query("registry-job-id")
 
 
 @pytest.mark.parametrize(

python/python/tests/test_table.py

@@ -59,6 +59,124 @@ def _blob_v2_table(db: DBConnection, name: str):
     return table
 
 
+@pytest.mark.parametrize(
+    "threshold",
+    [
+        "inline_size_threshold",
+        "dedicated_size_threshold",
+        "pack_file_size_threshold",
+    ],
+)
+@pytest.mark.parametrize(
+    "value,error",
+    [
+        (-1, ValueError),
+        (True, TypeError),
+        (False, TypeError),
+        (1.5, TypeError),
+        ("8192", TypeError),
+        (ctypes.c_size_t(-1).value + 1, OverflowError),
+    ],
+)
+def test_blob_thresholds_reject_invalid_values(threshold, value, error):
+    # Also runs in CI with pylance actually uninstalled.
+    with pytest.raises(error, match=f"{threshold} must"):
+        lancedb.blob("blob", **{threshold: value})
+
+
+@pytest.mark.parametrize(
+    "threshold", ["dedicated_size_threshold", "pack_file_size_threshold"]
+)
+def test_blob_thresholds_reject_zero_for_positive_thresholds(threshold):
+    with pytest.raises(ValueError, match=f"{threshold} must be positive"):
+        lancedb.blob("blob", **{threshold: 0})
+
+
+def test_blob_thresholds_accept_integer_boundaries():
+    maximum = ctypes.c_size_t(-1).value
+    field = lancedb.blob(
+        "blob",
+        nullable=False,
+        inline_size_threshold=0,
+        dedicated_size_threshold=maximum,
+        pack_file_size_threshold=maximum,
+    )
+    assert not field.nullable
+    assert field.metadata == {
+        b"lance-encoding:blob-inline-size-threshold": b"0",
+        b"lance-encoding:blob-dedicated-size-threshold": str(maximum).encode(),
+        b"lance-encoding:blob-pack-file-size-threshold": str(maximum).encode(),
+    }
+
+
+def test_blob_thresholds_control_writes_after_reopen(tmp_path):
+    # Keep this in test_table.py: CI runs it without pylance or pandas installed.
+    field = lancedb.blob(
+        "blob",
+        inline_size_threshold=4,
+        dedicated_size_threshold=12,
+        pack_file_size_threshold=8,
+    )
+    metadata = {
+        b"lance-encoding:blob-inline-size-threshold": b"4",
+        b"lance-encoding:blob-dedicated-size-threshold": b"12",
+        b"lance-encoding:blob-pack-file-size-threshold": b"8",
+    }
+    assert field.metadata == metadata
+    schema = pa.schema([pa.field("id", pa.int64()), field])
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("blobs", schema=schema)
+    # Every payload fits in the default inline tier; custom thresholds must
+    # instead send the larger values to packed and dedicated storage.
+    rows = [
+        {"id": 0, "blob": b"a" * 2},
+        {"id": 1, "blob": b"b" * 8},
+        {"id": 2, "blob": b"c" * 16},
+        {"id": 3, "blob": b"d" * 8},
+    ]
+
+    def check_writes(table, expected):
+        assert table.schema.field("blob").metadata.items() >= metadata.items()
+        hits = table.search().with_row_id(True).limit(10).to_arrow()
+        actual = {row["id"]: row for row in hits.to_pylist()}
+        assert actual.keys() == {row["id"] for row in expected}
+        for row in expected:
+            size = len(row["blob"])
+            descriptor = actual[row["id"]]["blob"]
+            assert descriptor["size"] == size
+            assert descriptor["kind"] == (2 if size > 12 else 1 if size > 4 else 0)
+        row_ids = [actual[row["id"]]["_rowid"] for row in expected]
+        assert table.fetch_blobs("blob", row_ids).to_pylist() == [
+            row["blob"] for row in expected
+        ]
+        # Two packed values reach the configured sidecar limit independently.
+        packed = [
+            actual[row["id"]]["blob"] for row in expected if len(row["blob"]) == 8
+        ]
+        assert len({desc["blob_id"] for desc in packed}) == len(packed)
+
+    table.add(rows)
+    check_writes(table, rows)
+    table = lancedb.connect(tmp_path).open_table("blobs")
+    check_writes(table, rows)
+    updates = [
+        {"id": 0, "blob": b"e" * 16},
+        {"id": 1, "blob": b"f" * 2},
+        {"id": 2, "blob": b"g" * 8},
+        {"id": 3, "blob": b"h" * 8},
+        {"id": 4, "blob": b"i" * 16},
+    ]
+    result = (
+        table.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(updates)
+    )
+    assert result.num_updated_rows == 4
+    assert result.num_inserted_rows == 1
+    check_writes(lancedb.connect(tmp_path).open_table("blobs"), updates)
+
+
 async def _blob_v2_table_async(db: AsyncConnection, name: str):
     schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("blob")])
     table = await db.create_table(name, schema=schema)
@@ -1633,6 +1751,25 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
     job.cancel()
 
 
+@pytest.mark.parametrize("method", ["create_index", "create_index_async"])
+@pytest.mark.parametrize("wait_timeout", ["invalid", 1, []])
+def test_create_index_invalid_wait_timeout(mem_db: DBConnection, method, wait_timeout):
+    table = mem_db.create_table("invalid_timeout", [{"id": 1}])
+    with pytest.raises(TypeError):
+        getattr(table, method)("id", config=BTree(), wait_timeout=wait_timeout)
+    assert table.list_indices() == []
+
+
+@pytest.mark.parametrize("method", ["create_index", "create_index_async"])
+@pytest.mark.parametrize("wait_timeout", [None, timedelta(seconds=1)])
+def test_create_index_valid_wait_timeout(mem_db: DBConnection, method, wait_timeout):
+    table = mem_db.create_table("valid_timeout", [{"id": 1}])
+    result = getattr(table, method)("id", config=BTree(), wait_timeout=wait_timeout)
+    if result is not None:
+        result.wait()
+    assert len(table.list_indices()) == 1
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_method(mock_create_index, mem_db: DBConnection):
     table = mem_db.create_table(
@@ -2705,6 +2842,57 @@ def test_update_with_arrow_scalar(mem_db: DBConnection):
     assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0, 3.0, 4.0]]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("async_api", [False, True])
+@pytest.mark.parametrize("entries", [[("o'brien", 4), ("nil", None)], [], None])
+async def test_update_map_arrow_scalar(tmp_path, async_api, entries):
+    map_type = pa.map_(pa.string(), pa.int64())
+    data = pa.table(
+        {
+            "id": [1, 2],
+            "attrs": pa.array([[("original", 1)], entries], type=map_type),
+        }
+    )
+    if async_api:
+        db = await lancedb.connect_async(tmp_path)
+        table = await db.create_table("maps", data)
+        source = await table.query().where("id = 2").to_arrow()
+        result = await table.update({"attrs": source["attrs"][0]}, where="id = 1")
+        actual = await table.query().to_arrow()
+    else:
+        db = lancedb.connect(tmp_path)
+        table = db.create_table("maps", data)
+        source = table.search().where("id = 2").to_arrow()
+        result = table.update(values={"attrs": source["attrs"][0]}, where="id = 1")
+        actual = table.to_arrow()
+    assert result.rows_updated == 1
+    assert actual.schema.field("attrs").type == map_type
+    assert actual.sort_by("id")["attrs"].to_pylist() == [entries, entries]
+
+
+@pytest.mark.parametrize(
+    "map_type, entries",
+    [
+        (pa.map_(pa.int64(), pa.string()), []),
+        (
+            pa.map_(pa.string(), pa.map_(pa.string(), pa.int64())),
+            [("outer", [("inner", 3)]), ("null", None)],
+        ),
+    ],
+)
+def test_update_typed_map_arrow_scalar(mem_db: DBConnection, map_type, entries):
+    table = mem_db.create_table(
+        "maps",
+        pa.table({"id": [1, 2], "attrs": pa.array([None, entries], type=map_type)}),
+    )
+    scalar = table.search().where("id = 2").to_arrow()["attrs"][0]
+    result = table.update(values={"attrs": scalar}, where="id = 1")
+    actual = table.to_arrow().sort_by("id")["attrs"]
+    assert result.rows_updated == 1
+    assert actual.type == map_type
+    assert actual.to_pylist() == [entries, entries]
+
+
 @pytest.mark.parametrize("use_arrow_scalar", [False, True])
 @pytest.mark.parametrize(
     "decimal_type, value",
@@ -4507,6 +4695,58 @@ async def test_alter_columns_rejects_invalid_async(
     assert await table.schema() == initial_schema
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list])
+@pytest.mark.parametrize(
+    "value_type, values, inferred_type",
+    [
+        (pa.float64(), [1.25, 2.5], pa.float32()),
+        (pa.int32(), [0, 255], pa.uint8()),
+        (pa.int32(), [-1, 256], pa.float32()),
+    ],
+)
+async def test_inferred_vector_field_metadata(
+    tmp_db: DBConnection,
+    tmp_db_async: AsyncConnection,
+    asynchronous,
+    list_type,
+    value_type,
+    values,
+    inferred_type,
+):
+    metadata = {
+        "lancedb:description": "Embedding values",
+        "lancedb:tag:model": "test-model",
+    }
+    schema = pa.schema(
+        [
+            pa.field(
+                "vector", list_type(value_type), nullable=False, metadata=metadata
+            ),
+            pa.field("text", pa.string(), metadata={"description": "Source text"}),
+        ],
+        metadata={"description": "Source dataset"},
+    )
+    rows = [{"vector": values, "text": "first"}, {"vector": values, "text": "second"}]
+    data = pa.Table.from_pylist(rows, schema=schema)
+    if asynchronous:
+        await tmp_db_async.create_table("vector_metadata", data=data)
+        table = await tmp_db_async.open_table("vector_metadata")
+        actual_schema = await table.schema()
+        actual_data = await table.to_arrow()
+    else:
+        tmp_db.create_table("vector_metadata", data=data)
+        table = tmp_db.open_table("vector_metadata")
+        actual_schema = table.schema
+        actual_data = table.to_arrow()
+    expected_schema = schema.set(
+        0, schema.field("vector").with_type(pa.list_(inferred_type, 2))
+    )
+    assert actual_schema.equals(expected_schema, check_metadata=True)
+    assert actual_data.to_pylist() == rows
+
+
 def test_update_field_metadata(mem_db: DBConnection):
     data = pa.table({"id": [0, 1], "category": ["a", "b"]})
     table = mem_db.create_table("my_table", data=data)
@@ -4849,6 +5089,57 @@ def test_optimize_warns_on_zero_cleanup_sync(mem_db: DBConnection):
     assert all("concurrent" in str(w.message) for w in seen)
 
 
+def test_optimize_retains_tagged_versions(tmp_db: DBConnection):
+    # A tagged version older than the cleanup cutoff must be retained, not make
+    # optimize() fail part-way (after compaction, before the index step).
+    table = tmp_db.create_table(
+        "test", data=pa.table({"id": list(range(100)), "text": ["a"] * 100})
+    )
+    table.create_index("id", config=BTree())
+    table.tags.create("release-1", table.version)
+    tagged_version = table.version
+    for i in range(3):
+        table.add(pa.table({"id": [1000 + i], "text": ["new"]}))
+    assert table.stats()["fragment_stats"]["num_fragments"] == 4
+    assert table.index_stats("id_idx").num_unindexed_rows == 3
+    sleep(1.1)
+
+    table.optimize(cleanup_older_than=timedelta(seconds=1))
+
+    # The index step ran after the prune step
+    assert table.index_stats("id_idx").num_unindexed_rows == 0
+    # It stays optimizable
+    table.optimize(cleanup_older_than=timedelta(seconds=1))
+    assert table.stats()["fragment_stats"]["num_fragments"] == 1
+
+    # Untagged old versions were pruned, the tagged one was retained
+    versions = [v["version"] for v in table.list_versions()]
+    assert tagged_version in versions
+    for added in range(tagged_version + 1, tagged_version + 4):
+        assert added not in versions
+    assert len(table) == 103
+
+    table.checkout("release-1")
+    assert len(table) == 100
+    assert table.to_arrow().num_rows == 100
+
+
+@pytest.mark.asyncio
+async def test_optimize_retains_tagged_versions_async(tmp_db_async: AsyncConnection):
+    table = await tmp_db_async.create_table("test", data=[{"x": [1]}])
+    await table.tags.create("keep", await table.version())
+    await table.add(data=[{"x": [2]}])
+    await table.add(data=[{"x": [3]}])
+    sleep(1.1)
+
+    stats = await table.optimize(cleanup_older_than=timedelta(seconds=1))
+    assert stats.prune.old_versions_removed >= 1
+
+    await table.checkout("keep")
+    assert await table.count_rows() == 1
+    assert await table.query().to_arrow() == pa.table({"x": [[1]]})
+
+
 def test_replace_field_metadata(tmp_path):
     db = lancedb.connect(tmp_path)
     table = db.create_table("my_table", data=[{"x": 0}])
@@ -4904,6 +5195,81 @@ def test_stats(mem_db: DBConnection):
     assert stats_with_index["fragment_stats"]["num_fragments"] == 1
 
 
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("empty", [False, True])
+@pytest.mark.parametrize("with_schema", [False, True])
+def test_create_table_recordbatch_iterable(
+    mem_db: DBConnection, iterable_type, empty, with_schema
+):
+    schema = pa.schema([pa.field("id", pa.int64())])
+    batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)]
+    data = iterable_type(batches)
+    if empty and not with_schema:
+        with pytest.raises(ValueError, match="empty iterable without a schema"):
+            mem_db.create_table("test", data=data)
+        return
+
+    table = mem_db.create_table(
+        "test", data=data, schema=schema if with_schema else None
+    )
+    assert table.schema == schema
+    assert table.to_arrow().to_pylist() == ([] if empty else [{"id": 1}])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("empty", [False, True])
+@pytest.mark.parametrize("with_schema", [False, True])
+async def test_create_table_recordbatch_iterable_async(
+    mem_db_async: AsyncConnection, iterable_type, empty, with_schema
+):
+    schema = pa.schema([pa.field("id", pa.int64())])
+    batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)]
+    data = iterable_type(batches)
+    if empty and not with_schema:
+        with pytest.raises(ValueError, match="empty iterable without a schema"):
+            await mem_db_async.create_table("test", data=data)
+        return
+
+    table = await mem_db_async.create_table(
+        "test", data=data, schema=schema if with_schema else None
+    )
+    assert await table.schema() == schema
+    assert (await table.to_arrow()).to_pylist() == ([] if empty else [{"id": 1}])
+
+
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
+def test_add_empty_recordbatch_iterable(
+    mem_db: DBConnection, iterable_type, on_bad_vectors
+):
+    table = mem_db.create_table("test", [{"id": 1}])
+    if on_bad_vectors == "error":
+        # The default scannable path still requires an input schema. This
+        # fix does not change its validation policy for schema-less input.
+        with pytest.raises(ValueError, match="empty iterator"):
+            table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    else:
+        # Sanitization has the existing table schema available.
+        table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    assert table.to_arrow().to_pylist() == [{"id": 1}]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
+async def test_add_empty_recordbatch_iterable_async(
+    mem_db_async: AsyncConnection, iterable_type, on_bad_vectors
+):
+    table = await mem_db_async.create_table("test", [{"id": 1}])
+    if on_bad_vectors == "error":
+        with pytest.raises(ValueError, match="empty iterator"):
+            await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    else:
+        await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    assert (await table.to_arrow()).to_pylist() == [{"id": 1}]
+
+
 def test_create_table_empty_list_with_schema(mem_db: DBConnection):
     """Test creating table with empty list data and schema
 

python/python/tests/test_util.py

@@ -21,6 +21,7 @@ from lancedb.table import (
     _infer_target_schema,
     _merge_metadata,
     _sanitize_data,
+    has_nan_values,
     sanitize_create_table,
 )
 import pyarrow as pa
@@ -489,6 +490,20 @@ def test_handle_bad_vectors_nan(on_bad_vectors):
     assert output["vector"].combine_chunks() == expected
 
 
+@pytest.mark.parametrize("dtype", [pa.float16(), pa.float32(), pa.float64()])
+@pytest.mark.parametrize("large", [False, True])
+@pytest.mark.parametrize("chunked", [False, True])
+def test_has_nan_values_skips_hidden_children_of_null_lists(dtype, large, chunked):
+    array_type = pa.LargeListArray if large else pa.ListArray
+    values = pa.array([float("nan"), None, float("nan"), 3.0, 4.0, 5.0], type=dtype)
+    arr = array_type.from_arrays(
+        [0, 2, 4, 6], values, mask=pa.array([True, False, False])
+    )
+    if chunked:
+        arr = pa.chunked_array([arr.slice(0, 2), arr.slice(2, 1)])
+    assert has_nan_values(arr).to_pylist() == [False, True, False]
+
+
 def test_handle_bad_vectors_noop():
     # ChunkedArray should be preserved as-is
     vector = pa.chunked_array(

python/python/tests/test_util_update_values.py

@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from datetime import date, datetime
+
+import lancedb
+import numpy as np
+import pyarrow as pa
+import pytest
+
+
+ARRAY_VALUES = [
+    ((1.0, 2.0), [1.0, 2.0]),
+    ([1.0, 2.0], [1.0, 2.0]),
+    (np.array([1.0, 2.0]), [1.0, 2.0]),
+    (None, None),
+]
+
+
+@pytest.mark.parametrize("value,expected", ARRAY_VALUES)
+def test_update_array_values(tmp_path, value, expected):
+    dtype = pa.list_(pa.float64())
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "values", pa.table({"array": pa.array([[0.0, 0.0]], type=dtype)})
+    )
+    table.update(values={"array": value})
+    assert table.to_arrow()["array"].to_pylist() == [expected]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("value,expected", ARRAY_VALUES)
+async def test_update_array_values_async(tmp_path, value, expected):
+    dtype = pa.list_(pa.float64())
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table(
+        "values", pa.table({"array": pa.array([[0.0, 0.0]], type=dtype)})
+    )
+    await table.update({"array": value})
+    assert (await table.to_arrow())["array"].to_pylist() == [expected]
+
+
+TIME_VALUES = [
+    np.datetime64("2025", "Y"),
+    np.datetime64("2025-01", "M"),
+    np.datetime64("2025-01-02", "D"),
+    np.datetime64("2025-01-02T03", "h"),
+    np.datetime64("2025-01-02T03:04", "m"),
+    np.datetime64("2025-01-02T03:04:05", "s"),
+    np.datetime64("2025-01-02T03:04:05.123", "ms"),
+    np.datetime64("2025-01-02T03:04:05.123456", "us"),
+    np.datetime64("2025-01-02T03:04:05.123456789", "ns"),
+    np.datetime64("NaT", "ns"),
+    datetime(2025, 1, 2, 3, 4, 5, 123456),
+    None,
+]
+
+
+def expected_time(value):
+    if value is None or isinstance(value, np.datetime64) and np.isnat(value):
+        return None
+    return int(np.datetime64(value, "ns").astype(np.int64))
+
+
+@pytest.mark.parametrize("value", TIME_VALUES)
+def test_update_timestamp_values(tmp_path, value):
+    db = lancedb.connect(tmp_path)
+    data = pa.table({"time": pa.array([datetime(2025, 1, 1)], type=pa.timestamp("ns"))})
+    table = db.create_table("times", data)
+    table.update(values={"time": value})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == [
+        expected_time(value)
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("value", TIME_VALUES)
+async def test_update_timestamp_values_async(tmp_path, value):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table({"time": pa.array([datetime(2025, 1, 1)], type=pa.timestamp("ns"))})
+    table = await db.create_table("times", data)
+    await table.update({"time": value})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == [
+        expected_time(value)
+    ]
+
+
+@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], []])
+def test_nested_array_tuple_matches_list(tmp_path, values):
+    db = lancedb.connect(tmp_path)
+    data = pa.table(
+        {"array": pa.array([[[0, 0]]], type=pa.list_(pa.list_(pa.int64())))}
+    )
+    control = db.create_table("list_values", data)
+    candidate = db.create_table("tuple_values", data)
+    tuples = tuple(tuple(value) for value in values)
+    try:
+        control.update(values={"array": values})
+    except ValueError as error:
+        reason = (
+            "Expected a literal value in array" if values else "concat requires input"
+        )
+        assert reason in str(error)
+        with pytest.raises(ValueError, match=reason):
+            candidate.update(values={"array": tuples})
+        assert control.to_arrow().equals(data)
+        assert candidate.to_arrow().equals(data)
+    else:
+        candidate.update(values={"array": tuples})
+        assert candidate.to_arrow().equals(control.to_arrow())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], []])
+async def test_nested_array_tuple_matches_list_async(tmp_path, values):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table(
+        {"array": pa.array([[[0, 0]]], type=pa.list_(pa.list_(pa.int64())))}
+    )
+    control = await db.create_table("list_values", data)
+    candidate = await db.create_table("tuple_values", data)
+    tuples = tuple(tuple(value) for value in values)
+    try:
+        await control.update({"array": values})
+    except ValueError as error:
+        reason = (
+            "Expected a literal value in array" if values else "concat requires input"
+        )
+        assert reason in str(error)
+        with pytest.raises(ValueError, match=reason):
+            await candidate.update({"array": tuples})
+        assert (await control.to_arrow()).equals(data)
+        assert (await candidate.to_arrow()).equals(data)
+    else:
+        await candidate.update({"array": tuples})
+        assert (await candidate.to_arrow()).equals(await control.to_arrow())
+
+
+@pytest.mark.parametrize("unit", ["ps", "fs", "as"])
+def test_subnanosecond_update_does_not_round(tmp_path, unit):
+    db = lancedb.connect(tmp_path)
+    data = pa.table({"time": pa.array([datetime(1970, 1, 1)], type=pa.timestamp("ns"))})
+    table = db.create_table("precise", data)
+    with pytest.raises(ValueError, match="cannot preserve sub-nanosecond precision"):
+        table.update(values={"time": np.datetime64(1, unit)})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == [0]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("unit", ["ps", "fs", "as"])
+async def test_subnanosecond_update_does_not_round_async(tmp_path, unit):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table({"time": pa.array([datetime(1970, 1, 1)], type=pa.timestamp("ns"))})
+    table = await db.create_table("precise", data)
+    with pytest.raises(ValueError, match="cannot preserve sub-nanosecond precision"):
+        await table.update({"time": np.datetime64(1, unit)})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == [0]
+
+
+@pytest.mark.parametrize("unit", ["Y", "M", "W", "D"])
+def test_update_numpy_date_column(tmp_path, unit):
+    value = np.datetime64("2025-01-02", unit)
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "dates", pa.table({"day": pa.array([date(1970, 1, 1)], type=pa.date32())})
+    )
+    table.update(values={"day": value})
+    assert table.to_arrow()["day"].to_pylist() == [value.astype("datetime64[D]").item()]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("unit", ["Y", "M", "W", "D"])
+async def test_update_numpy_date_column_async(tmp_path, unit):
+    value = np.datetime64("2025-01-02", unit)
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table(
+        "dates", pa.table({"day": pa.array([date(1970, 1, 1)], type=pa.date32())})
+    )
+    await table.update({"day": value})
+    assert (await table.to_arrow())["day"].to_pylist() == [
+        value.astype("datetime64[D]").item()
+    ]
+
+
+@pytest.mark.parametrize("timezone", [None, "UTC", "America/New_York", "Asia/Tokyo"])
+@pytest.mark.parametrize(
+    "value",
+    [np.datetime64("2025-01-02T03:04:05.123456789", "ns"), np.datetime64("NaT", "ns")],
+)
+def test_update_numpy_timestamp_matches_arrow_ingestion(tmp_path, timezone, value):
+    dtype = pa.timestamp("ns", tz=timezone)
+    ingested = pa.array(np.array([value]), type=dtype)
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("timestamps", pa.table({"time": ingested}))
+    expected = table.to_arrow()["time"].cast(pa.int64()).to_pylist()
+    table.update(values={"time": value})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("timezone", [None, "UTC", "America/New_York", "Asia/Tokyo"])
+@pytest.mark.parametrize(
+    "value",
+    [np.datetime64("2025-01-02T03:04:05.123456789", "ns"), np.datetime64("NaT", "ns")],
+)
+async def test_update_numpy_timestamp_matches_arrow_ingestion_async(
+    tmp_path, timezone, value
+):
+    dtype = pa.timestamp("ns", tz=timezone)
+    ingested = pa.array(np.array([value]), type=dtype)
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("timestamps", pa.table({"time": ingested}))
+    expected = (await table.to_arrow())["time"].cast(pa.int64()).to_pylist()
+    await table.update({"time": value})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == expected

python/python/tests/test_voyageai_embeddings.py

@@ -6,6 +6,8 @@
 These tests verify model registration and configuration without requiring API calls.
 """
 
+import numpy as np
+import pyarrow as pa
 import pytest
 from unittest.mock import MagicMock, patch
 
@@ -122,3 +124,58 @@ class TestVoyageAIModelRegistration:
         assert "voyage-4" not in func.multimodal_embedding_models
         assert "voyage-4-lite" not in func.multimodal_embedding_models
         assert "voyage-4-large" not in func.multimodal_embedding_models
+
+
+@pytest.mark.parametrize(
+    "inputs",
+    [
+        ["first", "second"],
+        np.array(["first", "second"]),
+        pa.array(["first", "second"]),
+        pa.chunked_array([["first"], ["second"]]),
+    ],
+    ids=["list", "numpy", "arrow", "chunked-arrow"],
+)
+def test_text_source_embeddings_accept_batch_containers(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    client.tokenize.return_value = [[1], [2]]
+    client.embed.return_value.embeddings = [[0.1, 0.2], [0.3, 0.4]]
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    assert func.compute_source_embeddings(inputs) == [[0.1, 0.2], [0.3, 0.4]]
+    client.embed.assert_called_once_with(
+        texts=["first", "second"], model="voyage-3", input_type="document"
+    )
+
+
+@pytest.mark.parametrize("inputs", [[], np.array([], dtype=str)])
+def test_text_source_embeddings_accept_empty_batch(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    assert func.compute_source_embeddings(inputs) == []
+    client.tokenize.assert_not_called()
+    client.embed.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "inputs",
+    [["text", 42], np.array(["text", None]), np.array([["first", "second"]])],
+)
+def test_text_source_embeddings_reject_non_text_batch_entries(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    with pytest.raises(ValueError, match="Each input should be str"):
+        func.compute_source_embeddings(inputs)
+    client.tokenize.assert_not_called()
+    client.embed.assert_not_called()

python/python/tests/test_voyageai_image_inputs.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import base64
+from io import BytesIO
+from unittest.mock import MagicMock
+
+import pytest
+
+from lancedb.embeddings import get_registry
+from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+
+@pytest.mark.parametrize("model_name", ["voyage-multimodal-3", "voyage-multimodal-3.5"])
+@pytest.mark.parametrize("input_kind", ["pil", "bytes", "path"])
+@pytest.mark.parametrize(
+    "mode,color,expected_rgb",
+    [
+        ("RGBA", (64, 128, 192, 128), (64, 128, 192)),
+        ("LA", (96, 128), (96, 96, 96)),
+        ("P", 0, (64, 128, 192)),
+        ("RGB", (64, 128, 192), (64, 128, 192)),
+    ],
+)
+def test_multimodal_source_embeddings_encode_image_modes(
+    model_name, input_kind, mode, color, expected_rgb, tmp_path, monkeypatch
+):
+    Image = pytest.importorskip("PIL.Image")
+    image = Image.new(mode, (8, 6), color)
+    if mode == "P":
+        image.putpalette([64, 128, 192] + [0, 0, 0] * 255)
+        image.info["transparency"] = 0
+    original_pixels = image.tobytes()
+    image_input = image
+    if input_kind != "pil":
+        png = BytesIO()
+        image.save(png, format="PNG")
+        if input_kind == "bytes":
+            image_input = png.getvalue()
+        else:
+            image_input = tmp_path / "image.png"
+            image_input.write_bytes(png.getvalue())
+
+    client = MagicMock()
+    client.multimodal_embed.return_value.embeddings = [[0.1, 0.2]]
+    monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
+    func = get_registry().get("voyageai").create(name=model_name)
+
+    assert func.compute_source_embeddings(image_input) == [[0.1, 0.2]]
+
+    client.multimodal_embed.assert_called_once()
+    request = client.multimodal_embed.call_args.kwargs
+    assert request["model"] == model_name
+    assert request["input_type"] == "document"
+    assert len(request["inputs"]) == 1
+    (content,) = request["inputs"][0]["content"]
+    assert content["type"] == "image_base64"
+    prefix, encoded = content["image_base64"].split(",", 1)
+    assert prefix == "data:image/jpeg;base64"
+    with Image.open(BytesIO(base64.b64decode(encoded, validate=True))) as decoded:
+        assert decoded.format == "JPEG"
+        assert decoded.mode == "RGB"
+        assert decoded.size == (8, 6)
+        assert decoded.getpixel((0, 0)) == pytest.approx(expected_rgb, abs=3)
+    assert image.mode == mode
+    assert image.tobytes() == original_pixels
+    client.embed.assert_not_called()
+
+
+@pytest.mark.parametrize("input_kind", ["bytes", "path"])
+def test_multimodal_source_embeddings_reject_invalid_image(
+    input_kind, tmp_path, monkeypatch
+):
+    PIL = pytest.importorskip("PIL")
+    image_input = b"not an image"
+    if input_kind == "path":
+        image_input = tmp_path / "invalid.png"
+        image_input.write_bytes(b"not an image")
+
+    client = MagicMock()
+    monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
+    func = get_registry().get("voyageai").create(name="voyage-multimodal-3")
+
+    with pytest.raises(PIL.UnidentifiedImageError):
+        func.compute_source_embeddings(image_input)
+
+    client.multimodal_embed.assert_not_called()
+    client.embed.assert_not_called()

python/src/connection.rs

@@ -49,6 +49,18 @@ impl Connection {
     }
 }
 
+fn parse_read_consistency_interval(interval: Option<f64>) -> PyResult<Option<Duration>> {
+    interval
+        .map(|seconds| {
+            Duration::try_from_secs_f64(seconds).map_err(|_| {
+                PyValueError::new_err(
+                    "read_consistency_interval must be finite, non-negative, and within the supported range",
+                )
+            })
+        })
+        .transpose()
+}
+
 fn parse_namespace_client_pushdown_operations(
     operations: Option<Vec<String>>,
 ) -> PyResult<HashSet<NamespaceClientPushdownOperation>> {
@@ -578,29 +590,17 @@ impl Connection {
 
     #[pyo3(signature = (namespace_path=None, page_token=None, limit=None))]
     pub fn list_namespaces(
-        self_: PyRef<'_, Self>,
+        &self,
         namespace_path: Option<Vec<String>>,
         page_token: Option<String>,
         limit: Option<u32>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let py = self_.py();
-        future_into_py(py, async move {
-            use lance_namespace::models::ListNamespacesRequest;
-            let request = ListNamespacesRequest {
-                id: namespace_path,
-                page_token,
-                limit: limit.map(|l| l as i32),
-                ..Default::default()
-            };
-            let response = inner.list_namespaces(request).await.infer_error()?;
-            Python::attach(|py| -> PyResult<Py<PyDict>> {
-                let dict = PyDict::new(py);
-                dict.set_item("namespaces", response.namespaces)?;
-                dict.set_item("page_token", response.page_token)?;
-                Ok(dict.unbind())
-            })
-        })
+    ) -> PyResult<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_namespaces(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, limit),
+            ),
+        ))
     }
 
     #[pyo3(signature = (namespace_path, mode=None, properties=None))]
@@ -713,29 +713,17 @@ impl Connection {
 
     #[pyo3(signature = (namespace_path=None, page_token=None, limit=None))]
     pub fn list_tables(
-        self_: PyRef<'_, Self>,
+        &self,
         namespace_path: Option<Vec<String>>,
         page_token: Option<String>,
         limit: Option<u32>,
-    ) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.get_inner()?.clone();
-        let py = self_.py();
-        future_into_py(py, async move {
-            use lance_namespace::models::ListTablesRequest;
-            let request = ListTablesRequest {
-                id: namespace_path,
-                page_token,
-                limit: limit.map(|l| l as i32),
-                ..Default::default()
-            };
-            let response = inner.list_tables(request).await.infer_error()?;
-            Python::attach(|py| -> PyResult<Py<PyDict>> {
-                let dict = PyDict::new(py);
-                dict.set_item("tables", response.tables)?;
-                dict.set_item("page_token", response.page_token)?;
-                Ok(dict.unbind())
-            })
-        })
+    ) -> PyResult<crate::listing::NameListing> {
+        Ok(crate::listing::NameListing::new(
+            self.get_inner()?.list_tables(
+                &namespace_path.unwrap_or_default(),
+                crate::listing::options(page_token, limit),
+            ),
+        ))
     }
 
     /// Get the configuration for constructing an equivalent namespace client.
@@ -1077,6 +1065,7 @@ pub fn connect(
     namespace_client_properties: Option<HashMap<String, String>>,
     oauth_config: Option<crate::oauth::PyOAuthConfig>,
 ) -> PyResult<Bound<'_, PyAny>> {
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     future_into_py(py, async move {
         let mut builder = lancedb::connect(&uri);
         if let Some(api_key) = api_key {
@@ -1095,7 +1084,6 @@ pub fn connect(
         #[cfg(not(feature = "remote"))]
         let _ = sql_host_override;
         if let Some(read_consistency_interval) = read_consistency_interval {
-            let read_consistency_interval = Duration::from_secs_f64(read_consistency_interval);
             builder = builder.read_consistency_interval(read_consistency_interval);
         }
         if let Some(storage_options) = storage_options {
@@ -1144,7 +1132,7 @@ pub fn connect_namespace_client(
     namespace_client_impl: Option<String>,
     namespace_client_properties: Option<HashMap<String, String>>,
 ) -> PyResult<Connection> {
-    let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     let namespace_client_pushdown_operations =
         parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
     let ns_properties = namespace_client_properties.unwrap_or_default();
@@ -1201,7 +1189,7 @@ pub fn connect_namespace(
     session: Option<crate::session::Session>,
     namespace_client_pushdown_operations: Option<Vec<String>>,
 ) -> PyResult<Connection> {
-    let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     let namespace_client_pushdown_operations =
         parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
 

python/src/expr.rs

@@ -240,22 +240,60 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
 
 fn parse_decimal(s: &str) -> PyResult<(i128, u8, i8)> {
     let s = s.trim();
-    let dot_pos = s.find('.');
-    let scale = if let Some(pos) = dot_pos {
-        (s.len() - pos - 1) as i8
-    } else {
-        0
+    let (mantissa, exponent) = match s.split_once(['e', 'E']) {
+        Some((mantissa, exponent)) => (
+            mantissa,
+            exponent.parse::<i64>().map_err(|e| {
+                PyValueError::new_err(format!("failed to parse decimal exponent: {}", e))
+            })?,
+        ),
+        None => (s, 0),
     };
+    let fractional_digits = mantissa.find('.').map_or(0, |pos| mantissa.len() - pos - 1);
+    // Keep the coefficient intact instead of expanding an exponent into zeros.
+    // Arrow Decimal128 has at most 38 digits and an i8 scale (at most 38).
+    let mut scale = i64::try_from(fractional_digits)
+        .ok()
+        .and_then(|fractional_digits| fractional_digits.checked_sub(exponent))
+        .and_then(|scale| i8::try_from(scale).ok())
+        .filter(|scale| *scale <= 38)
+        .ok_or_else(|| PyValueError::new_err("decimal scale is outside Decimal128 range"))?;
 
-    let digits = s.replace('.', "");
-    let val = digits
+    let digits = mantissa.replace('.', "");
+    let mut val = digits
         .parse::<i128>()
         .map_err(|e| PyValueError::new_err(format!("failed to parse decimal digits: {}", e)))?;
 
-    // Precision is total number of digits
-    let precision = digits.trim_start_matches('-').len() as u8;
+    // Positive scale cannot exceed precision, including for tiny values such as 1E-38.
+    let mut precision = digits
+        .trim_start_matches('-')
+        .len()
+        .max(usize::try_from(scale).unwrap_or(0));
+    if scale < 0 {
+        // Arrow cannot cast SQL text to a decimal with negative scale. Normalize
+        // positive exponents within Decimal128's bounded coefficient range.
+        if val != 0 {
+            let shift = -i16::from(scale) as u32;
+            precision += shift as usize;
+            if precision > 38 {
+                return Err(PyValueError::new_err(
+                    "decimal precision exceeds Decimal128 maximum of 38",
+                ));
+            }
+            val = 10_i128
+                .checked_pow(shift)
+                .and_then(|factor| val.checked_mul(factor))
+                .ok_or_else(|| PyValueError::new_err("decimal coefficient exceeds i128 range"))?;
+        }
+        scale = 0;
+    }
+    if precision > 38 {
+        return Err(PyValueError::new_err(
+            "decimal precision exceeds Decimal128 maximum of 38",
+        ));
+    }
 
-    Ok((val, precision, scale))
+    Ok((val, precision as u8, scale))
 }
 
 /// Call an arbitrary registered SQL function by name.

python/src/job.rs

@@ -81,6 +81,11 @@ impl Job {
         self.inner.id().map(str::to_string)
     }
 
+    #[getter]
+    pub fn job_type(&self) -> Option<String> {
+        self.inner.job_type()
+    }
+
     pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner.clone();
         future_into_py(

python/src/listing.rs

@@ -3,10 +3,11 @@
 
 use crate::error::PythonErrorExt;
 use crate::runtime::future_into_py;
-use futures::{StreamExt, future::poll_fn};
+use futures::{Future, StreamExt, future::poll_fn};
 use lancedb::listing::{Listing, ListingOptions};
 use pyo3::exceptions::PyStopAsyncIteration;
-use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pymethods};
+use pyo3::types::{PyDict, PyDictMethods};
+use pyo3::{Bound, Py, PyAny, PyRef, PyResult, Python, pyclass, pymethods};
 use std::sync::{Arc, Mutex};
 
 pub fn options(page_token: Option<String>, page_limit: Option<u32>) -> ListingOptions {
@@ -52,6 +53,31 @@ impl NameListing {
             Ok(value)
         })
     }
+
+    fn next_page(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+        let inner = self_.inner.clone();
+        let next_lock = self_.next_lock.clone();
+        future_into_py(self_.py(), async move {
+            let _guard = next_lock.lock().await;
+            let page = futures::future::poll_fn(|cx| {
+                let mut inner = inner.lock().unwrap();
+                let future = inner.next_page();
+                futures::pin_mut!(future);
+                future.poll(cx)
+            })
+            .await
+            .infer_error()?;
+            Python::attach(|py| -> PyResult<Option<Py<PyDict>>> {
+                page.map(|page| {
+                    let dict = PyDict::new(py);
+                    dict.set_item("items", page.items)?;
+                    dict.set_item("page_token", page.page_token)?;
+                    Ok(dict.unbind())
+                })
+                .transpose()
+            })
+        })
+    }
 }
 
 #[pyclass]

python/src/table.rs

@@ -24,7 +24,7 @@ use arrow::{
     ffi_stream::ArrowArrayStreamReader,
     pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
 };
-use lancedb::blob::{BlobFile, BlobRangeRequest};
+use lancedb::blob::{BlobDescriptor, BlobFile, BlobRangeRequest};
 use lancedb::connection::LanceFileVersion;
 use lancedb::index::scalar::FtsIndexBuilder;
 use lancedb::table::{
@@ -1266,7 +1266,9 @@ impl Table {
         train: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let index = extract_index_params(&index)?;
-        let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
+        let timeout = wait_timeout
+            .map(|t| t.extract::<std::time::Duration>())
+            .transpose()?;
         let mut op = self_
             .inner_ref()?
             .create_index_with_timeout(&[column], index, timeout);
@@ -1297,7 +1299,9 @@ impl Table {
         train: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let index = extract_index_params(&index)?;
-        let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
+        let timeout = wait_timeout
+            .map(|t| t.extract::<std::time::Duration>())
+            .transpose()?;
         let mut op = self_
             .inner_ref()?
             .create_index_with_timeout(&[column], index, timeout);
@@ -1560,8 +1564,8 @@ impl Table {
         })
     }
 
-    pub fn query(&self) -> Query {
-        Query::new(self.inner_ref().unwrap().query())
+    pub fn query(&self) -> PyResult<Query> {
+        Ok(Query::new(self.inner_ref()?.query()))
     }
 
     #[getter]
@@ -1679,6 +1683,23 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (descriptor))]
+    pub fn open_blob_file<'py>(
+        self_: PyRef<'py, Self>,
+        descriptor: &str,
+    ) -> PyResult<Bound<'py, PyAny>> {
+        let descriptor: BlobDescriptor = serde_json::from_str(descriptor)
+            .map_err(|error| PyValueError::new_err(format!("invalid blob descriptor: {error}")))?;
+        let inner = self_.inner_ref()?.clone();
+        future_into_py(self_.py(), async move {
+            let file = inner.open_blob_file(&descriptor).await.infer_error()?;
+            Ok(PyBlobFile {
+                inner: Arc::new(file),
+                closed: AtomicBool::new(false),
+            })
+        })
+    }
+
     /// Optimize the on-disk data by compacting and pruning old data, for better performance.
     #[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None, compaction_options=None))]
     pub fn optimize<'py>(

python/uv.lock

@@ -2006,8 +2006,8 @@ requires-dist = [
     { name = "pyarrow", marker = "extra == 'tests'", specifier = "<25" },
     { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" },
     { name = "pydantic", specifier = ">=2.7.4,<3" },
-    { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" },
-    { name = "pylance", marker = "extra == 'tests'", specifier = "==14.0.0b12" },
+    { name = "pylance", marker = "extra == 'pylance'", specifier = ">=15.0.0b1" },
+    { name = "pylance", marker = "extra == 'tests'", specifier = "==15.0.0b1" },
     { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" },
     { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" },
     { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" },
@@ -3938,7 +3938,7 @@ crypto = [
 
 [[package]]
 name = "pylance"
-version = "14.0.0b12"
+version = "15.0.0b1"
 source = { registry = "https://pypi.fury.io/lance-format/" }
 dependencies = [
     { name = "lance-namespace" },
@@ -3948,12 +3948,12 @@ dependencies = [
     { name = "pyarrow" },
 ]
 wheels = [
-    { url = "https://pypi.fury.io/lance-format/-/ver_1Ixasc/pylance-14.0.0b12-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:84350a3c5c1eea3398f4c4fad7cfe85b831d831a5e43b5ca2a783dbdb0e11969" },
-    { url = "https://pypi.fury.io/lance-format/-/ver_sCEkG/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76b386c182a6a8e7b86fbef28f84d1dd39e4626ac8681a0cf36f40a45dd7cb87" },
-    { url = "https://pypi.fury.io/lance-format/-/ver_1vdT1C/pylance-14.0.0b12-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6aa548c7937fcdfa8f1943a2d59041a42f8b500199880dbb509c7195fa253381" },
-    { url = "https://pypi.fury.io/lance-format/-/ver_2fXu1V/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:79f82d6a6c50f37aab338470e3733436e6a27fbb5972d29fcf453ece56d3b3c6" },
-    { url = "https://pypi.fury.io/lance-format/-/ver_24UavN/pylance-14.0.0b12-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:de0305999257002785f60169e6926896584d66f1da050bb5afd1d48c6e3d9bb9" },
-    { url = "https://pypi.fury.io/lance-format/-/ver_XNynX/pylance-14.0.0b12-cp310-abi3-win_amd64.whl", hash = "sha256:4d0cdb689e898ce71e8e434d00a3b97639cea8fb97da21f1c921543be363268f" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_23L1Nd/pylance-15.0.0b1-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:19605d3daca1224cc0db079054be5d4eec403dcf286bf4617430bfc259ab0a52" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_1Ho3RK/pylance-15.0.0b1-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be22b0b6f1e1d250ffaa82b449e982c817db33103c483d1005020285310623cf" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_1QmyF0/pylance-15.0.0b1-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5b2761d60b1b9db112058213f1f7b2de82b08cbe65888fab08b0a56e702482e" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_1in2Ph/pylance-15.0.0b1-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:294a0fedae312ed32d1b66cddbf7e69770ec9b4c82a873af60c619ed88450718" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_21xpLH/pylance-15.0.0b1-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ff20a16703d4d3f3bc675b25ce1871ba1d54137929dc46305bffdd07f7b35a6b" },
+    { url = "https://pypi.fury.io/lance-format/-/ver_AAGzF/pylance-15.0.0b1-cp310-abi3-win_amd64.whl", hash = "sha256:3b134e24a5becfc940ca134cf78aa3bae8b80ec40f8927a1b856273ab76213f2" },
 ]
 
 [[package]]