chore: merge main into repair branch for #3350

Preserve upstream namespace warning tests and existing fork regression imports. Rebuild the Python extension and validate 609 focused tests, Ruff, Pyright, and Rust formatting.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.41.0-beta.6"
+version = "0.41.0-beta.13"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -22,6 +22,8 @@ bytes.workspace = true
 lancedb.workspace = true
 datafusion-common.workspace = true
 lance-core.workspace = true
+# The workspace path dependency includes the patched crate in Maturin sources.
+lance-arrow.workspace = true
 lance-namespace.workspace = true
 lance-namespace-impls.workspace = true
 lance-io.workspace = true

python/README.md

@@ -8,6 +8,8 @@ A Python library for [LanceDB](https://github.com/lancedb/lancedb).
 pip install lancedb
 ```
 
+If no wheel is published for your platform, pip builds `lancedb` from the source distribution. That build needs a [Rust toolchain](https://rustup.rs/) and [protoc](https://grpc.io/docs/protoc-installation/) (the Protocol Buffers compiler) on your `PATH`, and can take a while.
+
 ### Pre-Haswell x86_64 hosts: `lancedb-compat`
 
 The default `lancedb` wheel targets `x86-64-haswell` (AVX2 + FMA + F16C) for full performance on modern hardware. Pre-Haswell hosts — Intel Sandy Bridge / Ivy Bridge / Westmere; AMD Bulldozer / Piledriver / Steamroller — don't have AVX2 and crash with `Illegal instruction` at `import lancedb`.

python/python/lancedb/__init__.py

@@ -15,7 +15,7 @@ from ._lancedb import FtsToken
 from ._lancedb import FunctionErrorFragment as FunctionErrorFragment
 from ._lancedb import FunctionErrorRecord as FunctionErrorRecord
 from ._lancedb import FunctionErrors as FunctionErrors
-from ._lancedb import LsmWriteSpec
+from ._lancedb import RTEWriteSpec
 from ._lancedb import tokenize as _tokenize
 from .common import URI, sanitize_uri
 from urllib.parse import urlparse
@@ -610,7 +610,7 @@ __all__ = [
     "SqlQuery",
     "LanceDBConnection",
     "LanceNamespaceDBConnection",
-    "LsmWriteSpec",
+    "RTEWriteSpec",
     "RemoteDBConnection",
     "Session",
     "Table",

python/python/lancedb/_lancedb.pyi

@@ -270,6 +270,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Table: ...
     async def create_materialized_view_async(
         self,
@@ -279,6 +281,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job: ...
     def list_materialized_views(
         self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
@@ -501,10 +505,17 @@ class Table:
     async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
     async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
     async def add_computed_columns(
-        self, columns: list[tuple[str, str]]
+        self,
+        columns: list[tuple[str, str]],
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[int] = None,
     ) -> AddColumnsResult: ...
     async def add_function_columns(
-        self, application_json: str, output_name: Optional[str]
+        self,
+        application_json: str,
+        output_name: Optional[str],
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[int] = None,
     ) -> AddColumnsResult: ...
     async def refresh_column(self, column: str) -> RefreshColumnResult: ...
     async def refresh_column_async(self, column: str) -> Job: ...
@@ -538,17 +549,17 @@ class Table:
     async def initial_storage_options(self) -> Optional[Dict[str, str]]: ...
     async def latest_storage_options(self) -> Optional[Dict[str, str]]: ...
     async def set_unenforced_primary_key(self, columns: List[str]) -> None: ...
-    async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
-    async def unset_lsm_write_spec(self) -> None: ...
-    async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
+    async def enable_rte(self, spec: RTEWriteSpec) -> None: ...
+    async def disable_rte(self) -> None: ...
+    async def get_rte_spec(self) -> Optional[RTEWriteSpec]: ...
     # WAL-PK-FUSION: delete both.
     def hybrid_pk_fusion_learned(self) -> bool: ...
     def note_hybrid_pk_fusion(self) -> None: ...
-    async def checkpoint_lsm(self) -> None: ...
-    async def flush_lsm(self) -> None: ...
-    async def compact_lsm(self) -> None: ...
-    async def get_lsm_stats(self, include_generation_rows: bool) -> Optional[dict]: ...
-    async def close_lsm_writers(self) -> None: ...
+    async def checkpoint_rte(self) -> None: ...
+    async def flush_rte(self) -> None: ...
+    async def compact_rte(self) -> None: ...
+    async def get_rte_stats(self, include_sstable_rows: bool) -> Optional[dict]: ...
+    async def close_rte_writers(self) -> None: ...
     @property
     def tags(self) -> Tags: ...
     @property
@@ -661,7 +672,7 @@ class Query:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
-    def use_lsm(self, enable: bool): ...
+    def use_rte(self, enable: bool): ...
     def nearest_to(self, query_vec: pa.Array) -> VectorQuery: ...
     def nearest_to_text(self, query: dict) -> FTSQuery: ...
     def order_by(self, ordering: Optional[List[ColumnOrdering]]): ...
@@ -678,7 +689,7 @@ class Query:
 class TakeQuery:
     def select(self, columns: List[str]): ...
     def with_row_id(self): ...
-    def use_lsm(self, enable: bool): ...
+    def use_rte(self, enable: bool): ...
     async def output_schema(self) -> pa.Schema: ...
     async def execute(self) -> RecordBatchStream: ...
     async def explain_plan(self, verbose: Optional[bool]) -> str: ...
@@ -697,7 +708,7 @@ class FTSQuery:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
-    def use_lsm(self, enable: bool): ...
+    def use_rte(self, enable: bool): ...
     def get_query(self) -> str: ...
     def add_query_vector(self, query_vec: pa.Array) -> None: ...
     def nearest_to(self, query_vec: pa.Array) -> HybridQuery: ...
@@ -725,7 +736,7 @@ class VectorQuery:
     def column(self, column: str): ...
     def distance_type(self, distance_type: str): ...
     def postfilter(self): ...
-    def use_lsm(self, enable: bool): ...
+    def use_rte(self, enable: bool): ...
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
     def search_effort(self, search_effort: Optional[float]): ...
@@ -750,7 +761,7 @@ class HybridQuery:
     def fast_search(self): ...
     def with_row_id(self): ...
     def postfilter(self): ...
-    def use_lsm(self, enable: bool): ...
+    def use_rte(self, enable: bool): ...
     def distance_type(self, distance_type: str): ...
     def refine_factor(self, refine_factor: int): ...
     def nprobes(self, nprobes: int): ...
@@ -778,7 +789,7 @@ class PyQueryRequest:
     select_source_columns: Optional[Dict[str, str]]
     fast_search: Optional[bool]
     with_row_id: Optional[bool]
-    use_lsm: Optional[bool]
+    use_rte: Optional[bool]
     column: Optional[str]
     query_vector: Optional[List[pa.Array]]
     nprobes: Optional[int]
@@ -839,22 +850,22 @@ class FtsToken:
     text: str
     position: int
 
-class LsmWriteSpec:
-    """Specification selecting Lance's MemWAL LSM-style write path for
-    `merge_insert`."""
+class RTEWriteSpec:
+    """Specification enabling the realtime engine (RTE), which routes
+    `merge_insert` through Lance's MemWAL LSM-style write path."""
 
     @staticmethod
-    def bucket(column: str, num_buckets: int) -> "LsmWriteSpec": ...
+    def bucket(column: str, num_buckets: int) -> "RTEWriteSpec": ...
     @staticmethod
-    def identity(column: str) -> "LsmWriteSpec": ...
+    def identity(column: str) -> "RTEWriteSpec": ...
     @staticmethod
-    def unsharded() -> "LsmWriteSpec": ...
-    def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "LsmWriteSpec":
-        """Set which indexes the MemWAL keeps up to date. None resolves every
-        index on the table at install, failing if one cannot be maintained;
-        a list is verbatim, empty means none."""
+    def unsharded() -> "RTEWriteSpec": ...
+    def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "RTEWriteSpec":
+        """Set which indexes the RTE keeps up to date. None is every index on
+        the table, now and later, skipping any it cannot maintain; a list is
+        verbatim, empty means none."""
         ...
-    def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "LsmWriteSpec":
+    def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "RTEWriteSpec":
         """Return a copy of this spec recording the given default
         `ShardWriter` configuration in the MemWAL index."""
         ...
@@ -868,7 +879,7 @@ class LsmWriteSpec:
     def num_buckets(self) -> Optional[int]: ...
     @property
     def maintained_indexes(self) -> Optional[List[str]]:
-        """Indexes the MemWAL keeps up to date, or None for every supported one."""
+        """Indexes the RTE keeps up to date, or None for every supported one."""
         ...
     @property
     def writer_config_defaults(self) -> Dict[str, str]: ...
@@ -940,6 +951,15 @@ class AsyncPermutationBuilder:
         counts: Optional[List[int]] = None,
         fixed: Optional[int] = None,
     ) -> "AsyncPermutationBuilder": ...
+    def bootstrap_sample(
+        self,
+        num_samples: int,
+        *,
+        ratios: Optional[List[float]] = None,
+        counts: Optional[List[int]] = None,
+        fixed: Optional[int] = None,
+        seed: Optional[int] = None,
+    ) -> "AsyncPermutationBuilder": ...
     def split_calculated(self, calculation: str) -> "AsyncPermutationBuilder": ...
     def shuffle(
         self, seed: Optional[int], clump_size: Optional[int]

python/python/lancedb/_wal_hybrid.py

@@ -1,14 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-"""WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables.
+"""WAL-PK-FUSION: hybrid search's primary-key fallback for RTE tables.
 
-Hybrid search joins its vector and full-text legs on `_rowid`, which a MemWAL
+Hybrid search joins its vector and full-text legs on `_rowid`, which an RTE
 table cannot supply: the fresh tier has no stable row id, so the server rejects
 `with_row_id` before it plans. When that happens the legs are joined on a
 surrogate `_rowid` built from the table's primary key instead.
 
-This is a stopgap until MemWAL supports `_rowid`, and everything it needs lives
+This is a stopgap until RTE supports `_rowid`, and everything it needs lives
 here so it can be removed in one pass:
 
 1. Delete this file and `tests/test_wal_hybrid.py`.
@@ -69,21 +69,21 @@ def _pk_columns(schema: pa.Schema) -> List[str]:
 # builds it from the same words -- so this substring survives either path and
 # the wrapping each adds. It is a coupling to a message rather than a code,
 # because the refusal arrives as a generic 400; if the server ever rewords it
-# the fallback stops firing and hybrid on a MemWAL table errors visibly again,
+# the fallback stops firing and hybrid on an RTE table errors visibly again,
 # which is loud rather than silently wrong.
 _WAL_ROW_ID_REFUSAL = "does not support with_row_id"
 
 WAL_ROW_ID_UNSUPPORTED = (
-    "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has "
+    "hybrid search on an RTE table cannot return _rowid: the fresh tier has "
     "no stable row id, and the ids the fusion joins on are synthesized from "
-    "the primary key. Set use_lsm(False) to read the base table only (results "
-    "will exclude un-compacted MemWAL data)"
+    "the primary key. Set use_rte(False) to read the base table only (results "
+    "will exclude un-compacted RTE data)"
 )
 
 
 def _is_wal_row_id_refusal(error: BaseException) -> bool:
     """Whether `error` is the server declining `_rowid` because the table is
-    MemWAL-backed, as opposed to any other failure the query might hit."""
+    RTE-backed, as opposed to any other failure the query might hit."""
     return _WAL_ROW_ID_REFUSAL in str(error)
 
 
@@ -112,7 +112,7 @@ def _stamp_surrogate_row_ids(
     """Give each table a `_rowid` column derived from its primary key.
 
     The fusion only ever compares that column for equality — to deduplicate, to
-    group by in a reranker, and to restore the original scores — so on a MemWAL
+    group by in a reranker, and to restore the original scores — so on an RTE
     table, where the server has no stable row id to hand out, any injective
     mapping of the primary key does the same job. Ids are assigned in first-seen
     order across `tables`, so passing them in fusion order (vector, then FTS)
@@ -124,7 +124,7 @@ def _stamp_surrogate_row_ids(
     """
     if not pk_columns:
         raise ValueError(
-            "hybrid search on a MemWAL table needs an unenforced primary key to "
+            "hybrid search on an RTE table needs an unenforced primary key to "
             "deduplicate on, and this table declares none"
         )
 
@@ -151,13 +151,13 @@ def _stamp_surrogate_row_ids(
     return stamped
 
 
-def pk_fusion_learned(table, use_lsm: Optional[bool]) -> bool:
+def pk_fusion_learned(table, use_rte: Optional[bool]) -> bool:
     """Whether a previous hybrid query on `table` was refused `_rowid`, so this
     one should go straight to the primary key.
 
     Learned from a refusal, never probed, so asking is free.
     """
-    if table is None or use_lsm is False:
+    if table is None or use_rte is False:
         return False
     # Only an unambiguous True forks to the key: a table type that answers the
     # question loosely keeps the `_rowid` path.
@@ -165,7 +165,7 @@ def pk_fusion_learned(table, use_lsm: Optional[bool]) -> bool:
 
 
 class PkFusion:
-    """How the legs of a hybrid query on a MemWAL table are joined: on a
+    """How the legs of a hybrid query on an RTE table are joined: on a
     surrogate `_rowid` built from the primary key."""
 
     def __init__(self, pk_columns: List[str]):
@@ -190,16 +190,16 @@ class PkFusion:
         # has turned `_with_row_id` on — including `return_score="all"`.
         if blob_v2_projection_sources(schema, projection):
             raise NotImplementedError(
-                "hybrid search cannot project a blob column on a MemWAL table: "
+                "hybrid search cannot project a blob column on an RTE table: "
                 "fetching blobs needs a real _rowid, and the fresh tier has no "
-                "stable one. Set use_lsm(False) to read the base table only "
-                "(results will exclude un-compacted MemWAL data)"
+                "stable one. Set use_rte(False) to read the base table only "
+                "(results will exclude un-compacted RTE data)"
             )
         return cls(_pk_columns(schema))
 
     def inject(self, columns):
         """`columns` with the key columns it is missing, to project in place of
-        the `_rowid` a MemWAL table would refuse."""
+        the `_rowid` an RTE table would refuse."""
         columns, self.injected = _with_pk_columns(columns, self.pk_columns)
         return columns
 
@@ -226,7 +226,7 @@ class PkFusion:
 
 
 def _check_refusal(error: Exception, *, fused: bool, caller_requested_row_id: bool):
-    """Re-raise `error` unless it is a MemWAL `_rowid` refusal worth retrying on
+    """Re-raise `error` unless it is an RTE `_rowid` refusal worth retrying on
     the primary key."""
     if fused or not _is_wal_row_id_refusal(error):
         raise error
@@ -248,7 +248,7 @@ def with_pk_fallback(
     `_rowid`.
 
     `_rowid` is asked for first rather than paying a round trip up front to
-    find out: only a MemWAL table refuses, and the refusal has to be handled
+    find out: only an RTE table refuses, and the refusal has to be handled
     anyway -- a write spec installed elsewhere can arrive between any two
     queries. It costs nothing server-side: it is raised before the query is
     planned. `fused` reports whether the attempt already used the key.

python/python/lancedb/db.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
+import asyncio
 from datetime import timedelta
 from pathlib import Path
 import sys
@@ -607,6 +608,8 @@ class DBConnection(EnforceOverrides):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
 
@@ -631,6 +634,13 @@ class DBConnection(EnforceOverrides):
             Cap the view at this many rows, in materialization order.
         with_no_data: bool, default False
             Skip the initial refresh and leave the backing table empty.
+        auto_refresh_enabled: bool, optional
+            Enable or disable automatic refresh on Cloud and Enterprise.
+            When omitted, the server determines whether automatic refresh is enabled.
+            Embedded databases do not support automatic refresh.
+        auto_refresh_interval_seconds: float, optional
+            Positive, finite refresh interval in seconds on Cloud and Enterprise.
+            When omitted, the server determines when to refresh.
 
         Returns
         -------
@@ -649,6 +659,8 @@ class DBConnection(EnforceOverrides):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         """Submit materialized-view creation and return its job.
 
@@ -1288,6 +1300,10 @@ class LanceDBConnection(DBConnection):
         self.storage_options = storage_options
         self._manifest_enabled = manifest_enabled
         self._namespace_client_properties = namespace_client_properties
+        # The native connection does not expose the Python Session wrapper.
+        # Keep the object supplied by the caller so this property stays
+        # synchronous for debugger inspection.
+        self._session = session
         if _inner is not None:
             self._conn = _inner
             # Native-derived wrappers resolve this in their async reconstruction
@@ -1354,7 +1370,12 @@ class LanceDBConnection(DBConnection):
 
     @property
     def session(self) -> Optional[Session]:
-        return self._conn.session
+        """Return the session supplied when this connection was opened.
+
+        None when the caller did not pass a session, including connections
+        reconstructed with :meth:`from_inner`.
+        """
+        return self._session
 
     @property
     def uri(self) -> str:
@@ -1706,6 +1727,8 @@ class LanceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
         See
@@ -1734,6 +1757,8 @@ class LanceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return MaterializedView(self.open_table(name))
@@ -1748,6 +1773,8 @@ class LanceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._conn.create_materialized_view_async(
@@ -1757,6 +1784,8 @@ class LanceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)
@@ -2633,8 +2662,16 @@ class AsyncConnection(object):
         if fill_value is None:
             fill_value = 0.0
 
-        data, schema = sanitize_create_table(
-            data, schema, metadata, on_bad_vectors, fill_value
+        # Input preparation may advance a user-provided iterator.  Keep that work
+        # off the background event loop so an iterator can use the synchronous
+        # LanceDB API without blocking the loop that API needs to make progress.
+        data, schema = await asyncio.to_thread(
+            sanitize_create_table,
+            data,
+            schema,
+            metadata,
+            on_bad_vectors,
+            fill_value,
         )
         validate_schema(schema)
 
@@ -2767,6 +2804,8 @@ class AsyncConnection(object):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncMaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
         See
@@ -2782,6 +2821,8 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             **native_options,
         )
         return AsyncMaterializedView(AsyncTable(inner))
@@ -2795,6 +2836,8 @@ class AsyncConnection(object):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncJob[None]:
         """Submit materialized-view creation and return its job.
 
@@ -2810,6 +2853,8 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             **native_options,
         )
         return AsyncJob(inner)

python/python/lancedb/embeddings/__init__.py

@@ -3,7 +3,12 @@
 
 
 # ruff: noqa: F401
-from .base import EmbeddingFunction, EmbeddingFunctionConfig, TextEmbeddingFunction
+from .base import (
+    EmbeddingFunction,
+    EmbeddingFunctionConfig,
+    TextEmbeddingFunction,
+    UntrustedEmbeddingConfigError,
+)
 from .bedrock import BedRockText
 from .cohere import CohereEmbeddingFunction
 from .gemini_text import GeminiText
@@ -29,6 +34,7 @@ __all__ = [
     "EmbeddingFunction",
     "EmbeddingFunctionConfig",
     "TextEmbeddingFunction",
+    "UntrustedEmbeddingConfigError",
     "EmbeddingFunctionRegistry",
     "get_registry",
     "register",

python/python/lancedb/embeddings/base.py

@@ -3,7 +3,7 @@
 
 from abc import ABC, abstractmethod
 import copy
-from typing import List, Union
+from typing import List, Optional, Tuple, Union
 
 from lancedb.util import add_note
 import numpy as np
@@ -13,6 +13,18 @@ from pydantic import BaseModel, Field, PrivateAttr
 from .utils import TEXT, retry_with_exponential_backoff
 
 
+class UntrustedEmbeddingConfigError(PermissionError):
+    """
+    Raised when using an embedding function restored from table metadata whose
+    settings this process has not approved.
+
+    See
+    [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code]
+    and
+    [allow_endpoint][lancedb.embeddings.EmbeddingFunctionRegistry.allow_endpoint].
+    """
+
+
 class EmbeddingFunction(BaseModel, ABC):
     """
     An ABC for embedding functions.
@@ -26,9 +38,11 @@ class EmbeddingFunction(BaseModel, ABC):
     3. ndims() which returns the number of dimensions of the vector column
     """
 
-    max_retries: int = (
-        7  # Setting 0 disables retires. Maybe this should not be enabled by default,
-    )
+    # Maximum number of retries for transient errors (rate limits, 5xx responses,
+    # connection errors, ...), with exponential backoff capped at 60 seconds between
+    # attempts. Permanent errors (HTTP 4xx other than 408/409/429) are never retried.
+    # Setting 0 disables retries.
+    max_retries: int = 7
     _ndims: int = PrivateAttr()
     _original_args: dict = PrivateAttr()
 
@@ -37,11 +51,29 @@ class EmbeddingFunction(BaseModel, ABC):
         """
         Create an instance of the embedding function
         """
+        from .registry import EmbeddingFunctionRegistry
+
         resolved_kwargs = cls.__resolveVariables(kwargs)
         instance = cls(**resolved_kwargs)
-        instance._original_args = kwargs
+        instance._original_args = dict(kwargs)
+        if (
+            instance.remote_code_model() is not None
+            and "trust_remote_code" in cls.model_fields
+        ):
+            # Record the flag even when left at its default, so a missing flag
+            # in table metadata only ever means a table written before this.
+            instance._original_args.setdefault(
+                "trust_remote_code", instance.trust_remote_code
+            )
+        # The caller wrote this config, so this process trusts it. Without this,
+        # a table created with it could not be reopened in the same process.
+        EmbeddingFunctionRegistry.get_instance()._trust(instance)
         return instance
 
+    @classmethod
+    def _resolve_variables(cls, args: dict) -> dict:
+        return cls.__resolveVariables(args)
+
     @classmethod
     def __resolveVariables(cls, args: dict) -> dict:
         """
@@ -92,6 +124,38 @@ class EmbeddingFunction(BaseModel, ABC):
         """
         return []
 
+    def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]:
+        """
+        Return the model whose repository code this function runs when its
+        `trust_remote_code` field is enabled, as a `(name, revision)` pair.
+
+        Return None if the function never runs code from a model repository.
+        When the function is restored from table metadata, `trust_remote_code`
+        is only enabled for models allowed with
+        [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code].
+        """
+        return None
+
+    @classmethod
+    def _legacy_trust_remote_code_default(cls) -> bool:
+        """
+        The `trust_remote_code` to use for table metadata that omits it. Only
+        tables written before the flag was always recorded omit it, so this is
+        the default from that time.
+        """
+        field = cls.model_fields.get("trust_remote_code")
+        return bool(field.default) if field is not None else False
+
+    def endpoints(self) -> List[str]:
+        """
+        Return the non-default URLs this function sends data or credentials to.
+
+        When the function is restored from table metadata, each of these must be
+        allowed with
+        [allow_endpoint][lancedb.embeddings.EmbeddingFunctionRegistry.allow_endpoint].
+        """
+        return []
+
     @abstractmethod
     def compute_query_embeddings(self, *args, **kwargs) -> list[Union[np.array, None]]:
         """
@@ -177,7 +241,9 @@ class EmbeddingFunction(BaseModel, ABC):
         Creates a pydantic Field that can automatically annotate
         the source column for this embedding function
         """
-        return Field(json_schema_extra={"source_column_for": self}, **kwargs)
+        json_schema_extra = dict(kwargs.pop("json_schema_extra", None) or {})
+        json_schema_extra["source_column_for"] = self
+        return Field(json_schema_extra=json_schema_extra, **kwargs)
 
     def VectorField(self, **kwargs):
         """
@@ -188,7 +254,9 @@ class EmbeddingFunction(BaseModel, ABC):
         static type checkers. LanceDB will infer the fixed vector dimension from
         this embedding function.
         """
-        return Field(json_schema_extra={"vector_column_for": self}, **kwargs)
+        json_schema_extra = dict(kwargs.pop("json_schema_extra", None) or {})
+        json_schema_extra["vector_column_for"] = self
+        return Field(json_schema_extra=json_schema_extra, **kwargs)
 
     def __eq__(self, __value: object) -> bool:
         if not hasattr(__value, "__dict__"):
@@ -233,3 +301,35 @@ class TextEmbeddingFunction(EmbeddingFunction):
     ) -> list[Union[np.array, None]]:
         """Generate the embeddings for the given texts"""
         pass
+
+
+class UntrustedEmbeddingFunction(EmbeddingFunction):
+    """
+    Stands in for an embedding function restored from table metadata whose
+    settings this process has not approved.
+
+    The table stays usable for anything that doesn't compute embeddings, while
+    computing embeddings raises
+    [UntrustedEmbeddingConfigError][lancedb.embeddings.UntrustedEmbeddingConfigError].
+    The real function is never constructed, since some load their model in
+    `__init__`.
+    """
+
+    function_name: str
+    reasons: Tuple[str, ...]
+
+    def _error(self) -> UntrustedEmbeddingConfigError:
+        details = "\n".join(f"  - {reason}" for reason in self.reasons)
+        return UntrustedEmbeddingConfigError(
+            f"Embedding function '{self.function_name}' was restored from table "
+            "metadata with settings this process has not approved:\n" + details
+        )
+
+    def compute_query_embeddings(self, *args, **kwargs):
+        raise self._error()
+
+    def compute_source_embeddings(self, *args, **kwargs):
+        raise self._error()
+
+    def ndims(self) -> int:
+        raise self._error()

python/python/lancedb/embeddings/jinaai.py

@@ -5,9 +5,10 @@
 import os
 import io
 import base64
+from functools import cached_property
 from urllib.parse import urlparse
 from pathlib import Path
-from typing import TYPE_CHECKING, ClassVar, List, Union, Optional, Any, Dict
+from typing import TYPE_CHECKING, List, Union, Optional, Any, Dict
 
 import numpy as np
 import pyarrow as pa
@@ -51,7 +52,6 @@ class JinaEmbeddings(EmbeddingFunction):
 
     name: str = "jina-clip-v1"
     api_key: Optional[str] = None
-    _session: ClassVar = None
 
     def ndims(self):
         # TODO: fix hardcoding
@@ -206,8 +206,7 @@ class JinaEmbeddings(EmbeddingFunction):
         texts: list[str] or np.ndarray (of str)
             The texts to embed
         """
-        self._init_client()
-        resp = JinaEmbeddings._session.post(  # type: ignore
+        resp = self._session.post(
             API_URL, json={"input": input, "model": self.name}
         ).json()
         if "data" not in resp:
@@ -220,14 +219,15 @@ class JinaEmbeddings(EmbeddingFunction):
 
         return [result["embedding"] for result in sorted_embeddings]
 
-    def _init_client(self):
+    @cached_property
+    def _session(self):
         import requests
 
-        if JinaEmbeddings._session is None:
-            if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
-                api_key_not_found_help("jina")
-            api_key = self.api_key or os.environ.get("JINA_API_KEY")
-            JinaEmbeddings._session = requests.Session()
-            JinaEmbeddings._session.headers.update(
-                {"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}
-            )
+        if self.api_key is None and os.environ.get("JINA_API_KEY") is None:
+            api_key_not_found_help("jina")
+        api_key = self.api_key or os.environ.get("JINA_API_KEY")
+        session = requests.Session()
+        session.headers.update(
+            {"Authorization": f"Bearer {api_key}", "Accept-Encoding": "identity"}
+        )
+        return session

python/python/lancedb/embeddings/ollama.py

@@ -14,6 +14,9 @@ if TYPE_CHECKING:
     import ollama
 
 
+DEFAULT_OLLAMA_HOST = "http://localhost:11434"
+
+
 @register("ollama")
 class OllamaEmbeddings(TextEmbeddingFunction):
     """
@@ -24,11 +27,25 @@ class OllamaEmbeddings(TextEmbeddingFunction):
     """
 
     name: str = "nomic-embed-text"
-    host: str = "http://localhost:11434"
+    host: str = DEFAULT_OLLAMA_HOST
     options: Optional[dict] = None  # type = ollama.Options
     keep_alive: Optional[Union[float, str]] = None
     ollama_client_kwargs: Optional[dict] = {}
 
+    def endpoints(self) -> List[str]:
+        urls = []
+        if self.host.rstrip("/") != DEFAULT_OLLAMA_HOST:
+            urls.append(self.host)
+        # These kwargs go to the underlying httpx client, where a proxy would
+        # see every request.
+        for key in ("proxy", "proxies"):
+            proxy = (self.ollama_client_kwargs or {}).get(key)
+            if isinstance(proxy, str):
+                urls.append(proxy)
+            elif isinstance(proxy, dict):
+                urls.extend(v for v in proxy.values() if isinstance(v, str))
+        return urls
+
     def ndims(self) -> int:
         return len(self.generate_embeddings(["foo"])[0])
 
@@ -53,6 +70,8 @@ class OllamaEmbeddings(TextEmbeddingFunction):
             The texts to embed
         """
         # TODO retry, rate limit, token limit
+        if isinstance(texts, np.ndarray):
+            texts = texts.tolist()
         embeddings = self._compute_embedding(texts)
         return list(embeddings)
 

python/python/lancedb/embeddings/openai.py

@@ -58,6 +58,9 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
     def sensitive_keys():
         return ["api_key"]
 
+    def endpoints(self) -> List[str]:
+        return [self.base_url] if self.base_url else []
+
     @staticmethod
     def model_names():
         return [
@@ -98,24 +101,35 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
                 valid_indices.append(idx)
 
         # TODO retry, rate limit, token limit
-        try:
+        def embed_batch(texts_batch, indices_batch):
             kwargs = {
-                "input": valid_texts,
+                "input": texts_batch,
                 "model": self.name,
             }
             if self.name != "text-embedding-ada-002":
                 kwargs["dimensions"] = self.dim
 
             rs = self._openai_client.embeddings.create(**kwargs)
-            valid_embeddings = {
-                idx: v.embedding for v, idx in zip(rs.data, valid_indices)
-            }
+            return {idx: v.embedding for v, idx in zip(rs.data, indices_batch)}
+
+        def embed_one_or_log(idx, text):
+            # Retry a single rejected input
+            try:
+                return embed_batch([text], [idx])
+            except openai.BadRequestError:
+                logging.exception("Bad request for input at index %d: %r", idx, text)
+                return {}
+
+        try:
+            valid_embeddings = embed_batch(valid_texts, valid_indices)
         except openai.AuthenticationError:
             logging.error("Authentication failed: Invalid API key provided")
             raise
         except openai.BadRequestError:
-            logging.exception("Bad request: %s", texts)
-            return [None] * len(texts)
+            # Retry each input individually
+            valid_embeddings = {}
+            for idx, text in zip(valid_indices, valid_texts):
+                valid_embeddings.update(embed_one_or_log(idx, text))
         except Exception:
             logging.exception("OpenAI embeddings error")
             raise

python/python/lancedb/embeddings/registry.py

@@ -2,9 +2,15 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import json
-from typing import Dict, Optional, Type
+from typing import Dict, Optional, Set, Tuple, Type
 
-from .base import EmbeddingFunction, EmbeddingFunctionConfig
+from pydantic import TypeAdapter
+
+from .base import (
+    EmbeddingFunction,
+    EmbeddingFunctionConfig,
+    UntrustedEmbeddingFunction,
+)
 
 
 class EmbeddingFunctionRegistry:
@@ -42,6 +48,9 @@ class EmbeddingFunctionRegistry:
     def __init__(self):
         self._functions = {}
         self._variables = {}
+        # Revision is a str, None for the default branch, or _ANY_REVISION.
+        self._allowed_remote_code: Set[Tuple[str, object]] = set()
+        self._allowed_endpoints: Set[str] = set()
 
     def register(self, alias: Optional[str] = None):
         """
@@ -60,11 +69,15 @@ class EmbeddingFunctionRegistry:
         def decorator(cls):
             if not issubclass(cls, EmbeddingFunction):
                 raise TypeError("Must be a subclass of EmbeddingFunction")
-            if cls.__name__ in self._functions:
-                raise KeyError(f"{cls.__name__} was already registered")
             key = alias or cls.__name__
+            if key in self._functions and (
+                not alias or self._functions[key] is not cls
+            ):
+                raise KeyError(f"{key} was already registered")
             self._functions[key] = cls
-            cls.__embedding_function_registry_alias__ = alias
+            # Store the name the class is registered under, so the table
+            # metadata written for it can be looked up again when reading.
+            cls.__embedding_function_registry_alias__ = key
             return cls
 
         return decorator
@@ -122,20 +135,129 @@ class EmbeddingFunctionRegistry:
             obj["vector_column"]: EmbeddingFunctionConfig(
                 vector_column=obj["vector_column"],
                 source_column=obj["source_column"],
-                function=self.get(obj["name"]).create(**obj["model"]),
+                function=self._restore(obj["name"], obj["model"]),
             )
             for obj in raw_list
         }
 
+    def _restore(self, name: str, args: dict) -> EmbeddingFunction:
+        """
+        Create an embedding function from table metadata. Anyone who can write
+        the table controls `args`, so settings that run model code or send data
+        elsewhere must already be trusted by this process.
+        """
+        cls = self.get(name)
+        resolved = cls._resolve_variables(args)
+        # Inspect the config without running __init__, which may load a model.
+        probe = cls.model_construct(**resolved)
+
+        reasons = []
+        model = probe.remote_code_model()
+        if model is not None:
+            allowed = self._remote_code_allowed(*model)
+            explicit = "trust_remote_code" in resolved
+            if explicit:
+                requested = TypeAdapter(bool).validate_python(
+                    resolved["trust_remote_code"]
+                )
+            else:
+                requested = cls._legacy_trust_remote_code_default()
+            # Always set the flag, since a subclass may default it to True.
+            resolved["trust_remote_code"] = requested and allowed
+            # A legacy table that omits the flag is downgraded rather than
+            # blocked: most sentence-transformers models don't need remote code,
+            # and those tables relied on the old True default without asking.
+            if requested and not allowed and explicit:
+                model_name, revision = model
+                reasons.append(
+                    f"runs code from model '{model_name}' at revision "
+                    f"{revision!r}. If you trust it, call "
+                    f"get_registry().allow_remote_code({model_name!r}, "
+                    f"revision={revision!r})"
+                )
+        reasons.extend(
+            f"sends data to endpoint '{url}'. If you trust it, call "
+            f"get_registry().allow_endpoint({url!r})"
+            for url in probe.endpoints()
+            if _normalize_url(url) not in self._allowed_endpoints
+        )
+
+        if reasons:
+            func = UntrustedEmbeddingFunction(
+                function_name=name, reasons=tuple(reasons)
+            )
+        else:
+            func = cls(**resolved)
+        func._original_args = args
+        return func
+
+    def allow_remote_code(self, model: str, revision: Optional[str] = None) -> None:
+        """
+        Trust the code in a model repository when an embedding function that
+        loads it is restored from table metadata.
+
+        Table metadata cannot enable `trust_remote_code` by itself, since anyone
+        who can write to the table could then run code in this process. Embedding
+        functions created in this process with `trust_remote_code=True` are
+        allowed automatically.
+
+        Parameters
+        ----------
+        model : str
+            The model name, such as `"nomic-ai/nomic-embed-text-v1.5"`.
+        revision : Optional[str]
+            The revision the table's embedding function must pin. Pinning a
+            commit hash is recommended, since the code at a branch or tag can
+            change after you review it. If None, any revision is trusted,
+            including pull request refs that anyone can push to the repository.
+        """
+        self._allowed_remote_code.add(
+            (model, _ANY_REVISION if revision is None else revision)
+        )
+
+    def allow_endpoint(self, url: str) -> None:
+        """
+        Trust a custom endpoint, such as an OpenAI `base_url` or Ollama `host`,
+        when an embedding function that uses it is restored from table metadata.
+
+        Table metadata cannot pick an endpoint by itself, since it could send
+        your data and API keys to a server you don't control. Embedding functions
+        created in this process are allowed to use their endpoints automatically.
+
+        Parameters
+        ----------
+        url : str
+            The endpoint URL. Trailing slashes are ignored.
+        """
+        self._allowed_endpoints.add(_normalize_url(url))
+
+    def _remote_code_allowed(self, model: str, revision: Optional[str]) -> bool:
+        return (model, _ANY_REVISION) in self._allowed_remote_code or (
+            model,
+            revision,
+        ) in self._allowed_remote_code
+
+    def _trust(self, func: EmbeddingFunction) -> None:
+        model = func.remote_code_model()
+        if model is not None and getattr(func, "trust_remote_code", False):
+            # Only the revision this code selected, where None means the default
+            # branch. Other revisions, such as PR refs, may hold other code.
+            self._allowed_remote_code.add(model)
+        for url in func.endpoints():
+            self.allow_endpoint(url)
+
     def function_to_metadata(self, conf: "EmbeddingFunctionConfig"):
         """
         Convert the given embedding function and source / vector column configs
         into a config dictionary that can be serialized into arrow metadata
         """
         func = conf.function
-        name = getattr(
-            func, "__embedding_function_registry_alias__", func.__class__.__name__
-        )
+        if isinstance(func, UntrustedEmbeddingFunction):
+            name = func.function_name
+        else:
+            name = getattr(
+                func, "__embedding_function_registry_alias__", func.__class__.__name__
+            )
         json_data = func.safe_model_dump()
         return {
             "name": name,
@@ -180,6 +302,13 @@ class EmbeddingFunctionRegistry:
         return self._variables[name]
 
 
+_ANY_REVISION = object()
+
+
+def _normalize_url(url: str) -> str:
+    return url.rstrip("/")
+
+
 # Global instance
 __REGISTRY__ = EmbeddingFunctionRegistry()
 

python/python/lancedb/embeddings/sentence_transformers.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 
-from typing import List, Union
+from typing import List, Optional, Tuple, Union
 
 import numpy as np
 
@@ -27,19 +27,32 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
         The device to use for the model
     normalize : bool, default True
         Whether to normalize the embeddings
-    trust_remote_code : bool, default True
-        Whether to trust the remote code
+    trust_remote_code : bool, default False
+        Whether to run code from the model's repository. Only enable this for
+        repositories you trust. When restored from table metadata, this is only
+        enabled for models allowed with
+        [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code].
+    revision : Optional[str], default None
+        The model revision to load, such as a commit hash.
     """
 
     name: str = "all-MiniLM-L6-v2"
     device: str = "cpu"
     normalize: bool = True
-    trust_remote_code: bool = True
+    trust_remote_code: bool = False
+    revision: Optional[str] = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self._ndims = None
 
+    def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]:
+        return (self.name, self.revision)
+
+    @classmethod
+    def _legacy_trust_remote_code_default(cls) -> bool:
+        return True
+
     @property
     def embedding_model(self):
         """
@@ -84,5 +97,8 @@ class SentenceTransformerEmbeddings(TextEmbeddingFunction):
             "sentence_transformers", "sentence-transformers"
         )
         return sentence_transformers.SentenceTransformer(
-            self.name, device=self.device, trust_remote_code=self.trust_remote_code
+            self.name,
+            device=self.device,
+            trust_remote_code=self.trust_remote_code,
+            revision=self.revision,
         )

python/python/lancedb/embeddings/transformers.py

@@ -3,7 +3,7 @@
 
 
 from functools import cached_property
-from typing import List, Any
+from typing import List, Any, Optional, Tuple
 
 import numpy as np
 
@@ -34,7 +34,11 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         Whether or not to allow for custom models defined on the HuggingFace
         Hub in their own modeling files. This option should only be set to True
         for repositories you trust and in which you have read the code, as it
-        will execute code present on the Hub on your local machine.
+        will execute code present on the Hub on your local machine. When
+        restored from table metadata, this is only enabled for models allowed with
+        [allow_remote_code][lancedb.embeddings.EmbeddingFunctionRegistry.allow_remote_code].
+    revision : Optional[str]
+        The model revision to load, such as a commit hash. Default is None.
 
     to download package, run :
         `pip install transformers`
@@ -45,6 +49,7 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
     name: str = "colbert-ir/colbertv2.0"
     device: str = "cpu"
     trust_remote_code: bool = False
+    revision: Optional[str] = None
     _tokenizer: Any = PrivateAttr()
     _model: Any = PrivateAttr()
 
@@ -52,12 +57,17 @@ class TransformersEmbeddingFunction(EmbeddingFunction):
         super().__init__(*args, **kwargs)
         self._ndims = None
         transformers = attempt_import_or_raise("transformers")
-        self._tokenizer = transformers.AutoTokenizer.from_pretrained(self.name)
+        self._tokenizer = transformers.AutoTokenizer.from_pretrained(
+            self.name, revision=self.revision
+        )
         self._model = transformers.AutoModel.from_pretrained(
-            self.name, trust_remote_code=self.trust_remote_code
+            self.name, trust_remote_code=self.trust_remote_code, revision=self.revision
         )
         self._model.to(self.device)
 
+    def remote_code_model(self) -> Optional[Tuple[str, Optional[str]]]:
+        return (self.name, self.revision)
+
     model_config = ConfigDict(ignored_types=(cached_property,))
 
     def ndims(self):

python/python/lancedb/embeddings/utils.py

@@ -241,15 +241,30 @@ def retry_with_exponential_backoff(
     exponential_base: float = 2,
     jitter: bool = True,
     max_retries: int = 7,
+    max_delay: float = 60,
 ):
     """Retry a function with exponential backoff.
 
+    Only errors that may succeed when repeated are retried. Errors that carry an
+    HTTP status code are retried when the status is 408, 409, 429 or >= 500; any
+    other 4xx status (for example 400, 401, 403, 404 or 422) is a permanent failure
+    that is raised immediately. Errors without a recognisable status code (for
+    example connection errors, timeouts, or exceptions raised by a custom embedding
+    function) are retried.
+
+    The delay before the n-th retry is ``initial_delay * exponential_base ** n``,
+    never more than ``max_delay``. With ``jitter`` enabled, a random amount of up to
+    half of that delay is removed ("equal jitter"), so the actual wait is always in
+    the range ``[delay / 2, delay]``.
+
     Args:
         func (function): The function to be retried.
         initial_delay (float): Initial delay in seconds (default is 1).
         exponential_base (float): The base for exponential backoff (default is 2).
         jitter (bool): Whether to add jitter to the delay (default is True).
         max_retries (int): Maximum number of retries (default is 7).
+        max_delay (float): Maximum delay between two attempts, in seconds
+            (default is 60).
 
     Returns:
         function: The decorated function.
@@ -265,13 +280,12 @@ def retry_with_exponential_backoff(
             try:
                 return func(*args, **kwargs)
 
-            # Currently retrying on all exceptions as there is no way to know the
-            # format of the error msgs used by different APIs. We'll log the error
-            # and say that it is assumed that if this portion errors out, it's due
-            # to rate limit but the user should check the error message to be sure.
+            # Errors are retried unless they are known to be permanent (see
+            # `_is_non_retryable_error`). Custom embedding functions can raise
+            # arbitrary exceptions, so unknown errors are assumed to be transient.
             except Exception as e:  # noqa: PERF203
-                # Don't retry on authentication errors (e.g., OpenAI 401)
-                # These are permanent failures that won't be fixed by retrying
+                # Don't retry permanent failures (e.g., OpenAI 401 or 404). They
+                # won't be fixed by retrying and would only delay the real error.
                 if _is_non_retryable_error(e):
                     raise
 
@@ -280,42 +294,107 @@ def retry_with_exponential_backoff(
                 if num_retries > max_retries:
                     raise Exception(
                         f"Maximum number of retries ({max_retries}) exceeded.", e
-                    )
+                    ) from e
 
-                delay *= exponential_base * (1 + jitter * random.random())
+                delay = min(delay * exponential_base, max_delay)
+                sleep_for = delay
+                if jitter:
+                    sleep_for = delay / 2 + random.random() * delay / 2
                 logging.warning(
                     "Error occurred: %s \n Retrying in %s seconds (retry %s of %s) \n",
                     e,
-                    delay,
+                    sleep_for,
                     num_retries,
                     max_retries,
                 )
-                time.sleep(delay)
+                time.sleep(sleep_for)
 
     return wrapper
 
 
+# HTTP statuses in the 4xx range that are worth retrying: request timeout,
+# conflict (the OpenAI SDK treats it as transient) and rate limiting.
+_RETRYABLE_CLIENT_STATUSES = (408, 409, 429)
+
+# AWS error codes that are reported with a 4xx status but are transient.
+_RETRYABLE_AWS_ERROR_CODES = (
+    "Throttl",
+    "TooManyRequests",
+    "RequestLimitExceeded",
+    "ProvisionedThroughputExceeded",
+    "ModelNotReady",
+    "ModelTimeout",
+)
+
+
+def _get_http_status(error: Exception):
+    """Best-effort extraction of an HTTP status code from an exception.
+
+    Provider SDKs expose it in different ways: ``status_code`` (openai, cohere),
+    ``http_status`` (voyageai), ``response.status_code`` (requests, httpx),
+    ``code`` (google-api-core, urllib) or ``response["ResponseMetadata"]``
+    (botocore). Returns None if no status can be found.
+    """
+    candidates = [
+        getattr(error, "status_code", None),
+        getattr(error, "http_status", None),
+    ]
+    response = getattr(error, "response", None)
+    if isinstance(response, dict):
+        candidates.append(response.get("ResponseMetadata", {}).get("HTTPStatusCode"))
+    else:
+        candidates.append(getattr(response, "status_code", None))
+    candidates.append(getattr(error, "code", None))
+
+    for status in candidates:
+        if isinstance(status, int) and not isinstance(status, bool):
+            if 100 <= status <= 599:
+                return status
+    return None
+
+
 def _is_non_retryable_error(error: Exception) -> bool:
     """Check if an error should not be retried.
 
+    An error is non-retryable if it is an authentication error or if it carries an
+    HTTP client-error status (4xx) other than 408, 409 and 429. Errors without a
+    recognisable status are treated as retryable.
+
     Args:
         error: The exception to check
 
     Returns:
         True if the error should not be retried, False otherwise
     """
+    from .base import UntrustedEmbeddingConfigError
+
+    if isinstance(error, UntrustedEmbeddingConfigError):
+        return True
+
     # Check for OpenAI authentication errors
     error_type = type(error).__name__
     if error_type == "AuthenticationError":
         return True
 
-    # Check for other common non-retryable HTTP status codes
-    # 401 Unauthorized, 403 Forbidden
-    if hasattr(error, "status_code"):
-        if error.status_code in (401, 403):
-            return True
+    # A missing API key or a missing optional dependency fails the same way on
+    # every attempt
+    if isinstance(error, (MissingApiKeyError, ImportError)):
+        return True
 
-    return False
+    status = _get_http_status(error)
+    if status is None or not 400 <= status < 500:
+        return False
+    if status in _RETRYABLE_CLIENT_STATUSES:
+        return False
+
+    # AWS reports throttling with a 400 status; the error code tells them apart.
+    response = getattr(error, "response", None)
+    if isinstance(response, dict):
+        aws_code = str(response.get("Error", {}).get("Code", ""))
+        if aws_code.startswith(_RETRYABLE_AWS_ERROR_CODES):
+            return False
+
+    return True
 
 
 def url_retrieve(url: str):
@@ -332,9 +411,15 @@ def url_retrieve(url: str):
         raise ConnectionError("could not download {} due to {}".format(url, err))
 
 
+class MissingApiKeyError(ValueError):
+    """Raised when an embedding provider's API key is not set."""
+
+
 def api_key_not_found_help(provider):
     logging.error("Could not find API key for %s", provider)
-    raise ValueError(f"Please set the {provider.upper()}_API_KEY environment variable.")
+    raise MissingApiKeyError(
+        f"Please set the {provider.upper()}_API_KEY environment variable."
+    )
 
 
 def is_flash_attn_2_available():

python/python/lancedb/embeddings/voyageai.py

@@ -75,7 +75,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
             content = {"type": "text", "text": input_data}
     elif isinstance(input_data, PIL_Image.Image):
         buffered = BytesIO()
-        input_data.save(buffered, format="JPEG")
+        input_data.convert("RGB").save(buffered, format="JPEG")
         img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
         content = {
             "type": "image_base64",
@@ -84,7 +84,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
     elif isinstance(input_data, bytes):
         img = PIL_Image.open(BytesIO(input_data))
         buffered = BytesIO()
-        img.save(buffered, format="JPEG")
+        img.convert("RGB").save(buffered, format="JPEG")
         img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
         content = {
             "type": "image_base64",
@@ -103,7 +103,7 @@ def transform_input(input_data: Union[str, bytes, Path]):
         else:
             img = PIL_Image.open(input_data)
             buffered = BytesIO()
-            img.save(buffered, format="JPEG")
+            img.convert("RGB").save(buffered, format="JPEG")
             img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
             content = {
                 "type": "image_base64",
@@ -145,6 +145,8 @@ def sanitize_text_input(inputs: TEXT) -> List[str]:
     """
     if isinstance(inputs, str):
         inputs = [inputs]
+    elif isinstance(inputs, (list, np.ndarray)):
+        inputs = list(inputs)
     elif isinstance(inputs, pa.Array):
         inputs = inputs.to_pylist()
     elif isinstance(inputs, pa.ChunkedArray):

python/python/lancedb/embeddings/watsonx.py

@@ -87,6 +87,9 @@ class WatsonxEmbeddings(TextEmbeddingFunction):
     url: Optional[str] = None
     params: Optional[Dict] = None
 
+    def endpoints(self) -> List[str]:
+        return [self.url] if self.url else []
+
     @staticmethod
     def sensitive_keys():
         return ["api_key"]

python/python/lancedb/functions.py

@@ -697,6 +697,7 @@ _GRAMMAR_PRIMITIVES = (
     (pa.string(), "utf8"),
     (pa.large_string(), "large_utf8"),
     (pa.binary(), "binary"),
+    (pa.large_binary(), "large_binary"),
     (pa.date32(), "date32"),
     (pa.date64(), "date64"),
 )
@@ -846,11 +847,7 @@ def _exact_blob_storage_type(field: pa.Field) -> dict[str, Any]:
             {
                 "name": child.name,
                 "nullable": child.nullable,
-                "type": (
-                    {"type": "large_binary"}
-                    if pa.types.is_large_binary(child.type)
-                    else _exact_arrow_type(child.type)
-                ),
+                "type": _exact_arrow_type(child.type),
             }
             for child in storage
         ],

python/python/lancedb/integrations/pyarrow.py

@@ -52,6 +52,9 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
             raise NotImplementedError("memory_pool not supported")
 
     def count_rows(self):
+        if isinstance(self.filter, pa.compute.Expression):
+            # stream the batches so memory does not grow with the matching rows
+            return sum(batch.num_rows for batch in self.to_reader())
         return self.table.count_rows(self.filter)
 
     def from_batches(self, **kwargs):
@@ -83,6 +86,8 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
         return self.to_reader().read_all()
 
     def to_reader(self, *, limit: Optional[int] = None):
+        if isinstance(self.filter, pa.compute.Expression):
+            return self._to_reader_with_expression(limit=limit)
         query = self.table.search()
         # Disable the builtin limit
         if limit is None:
@@ -98,13 +103,30 @@ class PyarrowScannerAdapter(pa.dataset.Scanner):
             query = query.where(self.filter, prefilter=True)
         return query.to_batches(batch_size=self.batch_size)
 
+    def _to_reader_with_expression(self, *, limit: Optional[int] = None):
+        # LanceDB filters are SQL strings, but engines such as DuckDB push down
+        # pyarrow compute expressions.  Scan the table and apply the filter and
+        # projection with pyarrow, since the filter may reference columns that
+        # are not part of the projection.
+        if limit is not None and limit <= 0:
+            raise ValueError("limit must be positive")
+        query = self.table.search().limit(self.table.count_rows())
+        scanner = pa.dataset.Scanner.from_batches(
+            query.to_batches(batch_size=self.batch_size),
+            columns=self.columns,
+            filter=self.filter,
+        )
+        if limit is None:
+            return scanner.to_reader()
+        return scanner.head(limit).to_reader()
+
 
 class PyarrowDatasetAdapter(pa.dataset.Dataset):
     def __init__(self, table: Table):
         self.table = table
 
     def count_rows(self, filter: Optional[Filter] = None):
-        return self.table.count_rows(filter)
+        return self.scanner(filter=filter).count_rows()
 
     def get_fragments(self, filter: Optional[Filter] = None):
         raise NotImplementedError

python/python/lancedb/merge.py

@@ -37,7 +37,7 @@ class LanceMergeInsertBuilder(object):
         self._when_not_matched_by_source_condition_expr = None
         self._timeout = None
         self._use_index = True
-        self._use_lsm = None
+        self._use_rte = None
         self._validate_single_shard = None
 
     def when_matched_update_all(
@@ -115,22 +115,22 @@ class LanceMergeInsertBuilder(object):
         self._use_index = use_index
         return self
 
-    def use_lsm(self, enable: bool) -> LanceMergeInsertBuilder:
+    def use_rte(self, enable: bool) -> LanceMergeInsertBuilder:
         """
-        Control MemWAL routing for this merge.
+        Control RTE routing for this merge.
 
-        By default (unset), a `merge_insert` on a table with an LSM write spec is
+        By default (unset), a `merge_insert` on a table with an RTE write spec is
         routed through Lance's MemWAL shard writer, and a table without one uses
         the standard path.
 
         Parameters
         ----------
         enable: bool
-            ``True`` forces MemWAL routing and errors if the table has no LSM
+            ``True`` forces RTE routing and errors if the table has no RTE
             write spec. ``False`` forces the standard write path even when a spec
             is set.
         """
-        self._use_lsm = enable
+        self._use_rte = enable
         return self
 
     def validate_single_shard(
@@ -139,13 +139,13 @@ class LanceMergeInsertBuilder(object):
         """
         Controls how an LSM merge checks that its input targets a single shard.
 
-        When a table has an LSM write spec, every row in a `merge_insert` call
+        When a table has an RTE write spec, every row in a `merge_insert` call
         must route to the same shard. When `True` (the default), every row is
         inspected to verify this. When `False`, only the first row is inspected
         and the shard it routes to is used for the whole input — a faster path
         for callers that have already pre-sharded their input.
 
-        Has no effect on tables without an LSM write spec.
+        Has no effect on tables without an RTE write spec.
 
         Parameters
         ----------

python/python/lancedb/namespace.py

@@ -11,6 +11,7 @@ through a namespace abstraction.
 from __future__ import annotations
 
 import sys
+import warnings
 from typing import Any, Dict, Iterable, List, Optional, Union
 from uuid import UUID
 
@@ -82,6 +83,31 @@ from ._lancedb import Session
 _MAX_QUERY_K = 2**31 - 1
 
 
+def _create_table_storage_options(
+    storage_options: Optional[Dict[str, str]],
+    data_storage_version: Optional[str],
+    enable_v2_manifest_paths: Optional[bool],
+) -> Optional[Dict[str, str]]:
+    if data_storage_version is None and enable_v2_manifest_paths is None:
+        return storage_options
+    options = dict(storage_options or {})
+    for name, value in (
+        ("data_storage_version", data_storage_version),
+        ("enable_v2_manifest_paths", enable_v2_manifest_paths),
+    ):
+        if value is not None:
+            warnings.warn(
+                f"setting {name} directly on create_table is deprecated. "
+                "Use storage_options instead.",
+                DeprecationWarning,
+                stacklevel=3,
+            )
+            options[f"new_table_{name}"] = (
+                str(value).lower() if isinstance(value, bool) else value
+            )
+    return options
+
+
 def _query_to_namespace_request(
     table_id: List[str],
     query: "Query",
@@ -597,6 +623,9 @@ class LanceNamespaceDBConnection(DBConnection):
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
+        storage_options = _create_table_storage_options(
+            storage_options, data_storage_version, enable_v2_manifest_paths
+        )
         async_table = LOOP.run(
             self._inner.create_table(
                 name,
@@ -678,6 +707,8 @@ class LanceNamespaceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> "MaterializedView":
         """Define a materialized view over a table in the root namespace.
         See
@@ -693,6 +724,8 @@ class LanceNamespaceDBConnection(DBConnection):
                         where=where,
                         limit=limit,
                         with_no_data=with_no_data,
+                        auto_refresh_enabled=auto_refresh_enabled,
+                        auto_refresh_interval_seconds=auto_refresh_interval_seconds,
                     )
                 ).name
             )
@@ -708,6 +741,8 @@ class LanceNamespaceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._inner.create_materialized_view_async(
@@ -717,6 +752,8 @@ class LanceNamespaceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)
@@ -1238,6 +1275,9 @@ class AsyncLanceNamespaceDBConnection:
         if mode.lower() not in ["create", "overwrite"]:
             raise ValueError("mode must be either 'create' or 'overwrite'")
         validate_table_name(name)
+        storage_options = _create_table_storage_options(
+            storage_options, data_storage_version, enable_v2_manifest_paths
+        )
         table = await self._inner.create_table(
             name,
             data,
@@ -1304,6 +1344,8 @@ class AsyncLanceNamespaceDBConnection:
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> "AsyncMaterializedView":
         """Define a materialized view over a table in the root namespace."""
         view = await self._inner.create_materialized_view(
@@ -1313,6 +1355,8 @@ class AsyncLanceNamespaceDBConnection:
             where=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
         )
         # Reopen through the namespace so the view's table carries the
         # namespace client and pushdown configuration a bare inner table lacks.
@@ -1327,6 +1371,8 @@ class AsyncLanceNamespaceDBConnection:
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncJob[None]:
         """Submit materialized-view creation and return its job."""
         return await self._inner.create_materialized_view_async(
@@ -1336,6 +1382,8 @@ class AsyncLanceNamespaceDBConnection:
             where=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
         )
 
     async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":

python/python/lancedb/permutation.py

@@ -48,7 +48,7 @@ class PermutationBuilder:
         By default, the permutation builder will create a single split that contains all
         rows in the same order as the base table.
 
-        Tables with an LSM write spec are rejected: unflushed rows have no row id.
+        Tables with an RTE write spec are rejected: unflushed rows have no row id.
         """
         self._async = async_permutation_builder(table)
 
@@ -169,6 +169,50 @@ class PermutationBuilder:
         )
         return self
 
+    def bootstrap_sample(
+        self,
+        num_samples: int,
+        *,
+        ratios: Optional[list[float]] = None,
+        counts: Optional[list[int]] = None,
+        fixed: Optional[int] = None,
+        seed: Optional[int] = None,
+        split_names: Optional[list[str]] = None,
+    ) -> "PermutationBuilder":
+        """
+        Sample rows uniformly **with replacement**, independently per split.
+
+        Unlike the other split strategies, the number of output rows is
+        ``num_samples``, not the number of (optionally filtered) input rows: the
+        same row can appear zero, one, or many times, and ``num_samples`` may be
+        larger or smaller than the input row count. Useful for bootstrap
+        resampling, oversampling rare classes, or any training workload that
+        wants a fixed-size stream of random draws rather than a single
+        deterministic pass over every row. Combine with :meth:`filter` to
+        restrict the eligible population.
+
+        One of ratios, counts, or fixed must be provided, with the same meaning
+        as in :meth:`split_random`, except they divide ``num_samples`` across
+        splits rather than the input row count: if fixed is 4, each split gets
+        ``num_samples // 4`` samples.
+
+        Each split draws its own sample using a seed derived independently from
+        ``(seed, split_index)``, so a split's content does not change if the samples are
+        later redistributed across a different number of splits.
+
+        The optional split_names can be provided to name the splits.  If not provided,
+        the splits can only be referenced by their index.
+        """
+        self._async.bootstrap_sample(
+            num_samples,
+            ratios=ratios,
+            counts=counts,
+            fixed=fixed,
+            seed=seed,
+            split_names=split_names,
+        )
+        return self
+
     def split_calculated(
         self, calculation: str, split_names: Optional[list[str]] = None
     ) -> "PermutationBuilder":

python/python/lancedb/pydantic.py

@@ -127,7 +127,11 @@ def Vector(
                 core_schema.list_schema(
                     min_length=dim,
                     max_length=dim,
-                    items_schema=core_schema.float_schema(),
+                    items_schema=(
+                        core_schema.int_schema()
+                        if pa.types.is_integer(value_type)
+                        else core_schema.float_schema()
+                    ),
                 ),
             )
 
@@ -208,7 +212,11 @@ def MultiVector(
                     items_schema=core_schema.list_schema(
                         min_length=dim,
                         max_length=dim,
-                        items_schema=core_schema.float_schema(),
+                        items_schema=(
+                            core_schema.int_schema()
+                            if pa.types.is_integer(value_type)
+                            else core_schema.float_schema()
+                        ),
                     ),
                 ),
             )
@@ -260,7 +268,9 @@ def _py_type_to_arrow_type(py_type: Type[Any], field: FieldInfo) -> pa.DataType:
 
 def _pydantic_model_to_fields(model: pydantic.BaseModel) -> List[pa.Field]:
     return [
-        _pydantic_to_field(name, field) for name, field in model.model_fields.items()
+        _pydantic_to_field(name, field)
+        for name, field in model.model_fields.items()
+        if field.exclude is not True
     ]
 
 
@@ -493,6 +503,9 @@ class LanceModel(pydantic.BaseModel):
 
         vec_and_function = []
         for name, field_info in cls.safe_get_fields().items():
+            if field_info.exclude is True:
+                continue
+
             func = get_extras(field_info, "vector_column_for")
             if func is not None:
                 vec_and_function.append([name, func])
@@ -500,6 +513,8 @@ class LanceModel(pydantic.BaseModel):
         configs = []
         for vec, func in vec_and_function:
             for source, field_info in cls.safe_get_fields().items():
+                if field_info.exclude is True:
+                    continue
                 src_func = get_extras(field_info, "source_column_for")
                 if src_func is func:
                     # note we can't use == here since the function is a pydantic
@@ -524,4 +539,20 @@ def get_extras(field_info: FieldInfo, key: str) -> Any:
 
 def model_to_dict(model: pydantic.BaseModel) -> dict[str, Any]:
     """Convert a Pydantic model to a dictionary."""
-    return model.model_dump()
+    return _enum_members_to_values(model.model_dump())
+
+
+def _enum_members_to_values(value: Any) -> Any:
+    # model_dump() keeps Enum members, and Arrow cannot convert a member of a
+    # plain Enum (one without a str or int mixin). Write its value instead,
+    # which is what the Arrow type of the field is derived from.
+    if isinstance(value, Enum):
+        return value.value
+    if isinstance(value, dict):
+        return {k: _enum_members_to_values(v) for k, v in value.items()}
+    if isinstance(value, (list, tuple)) and value:
+        # Skip lists of plain scalars, such as vectors, without walking them.
+        if isinstance(value[0], (int, float, str, bytes)):
+            return value
+        return [_enum_members_to_values(v) for v in value]
+    return value

python/python/lancedb/query.py

@@ -82,6 +82,7 @@ if TYPE_CHECKING:
 
 T = TypeVar("T", bound="LanceModel")
 AnalyzePlanDistributedMetrics = Literal["aggregate", "per_worker", "full"]
+_SYNC_VECTOR_QUERY_TYPES = (list, tuple, np.ndarray, pa.Array, pa.ChunkedArray)
 
 # Number of rows a hybrid query returns when no limit was set on it. This
 # mirrors the default the Rust query builder applies to its sub-queries.
@@ -128,6 +129,11 @@ def _blob_bytes_without_snapshot_error() -> NotImplementedError:
     )
 
 
+def _is_remote_query_table(table: Any) -> bool:
+    async_table = getattr(table, "_table", table)
+    return not async_table._inner._is_native()
+
+
 def _query_needs_lance_scanner(builder: Any) -> bool:
     """Whether the query sets options that to_arrow ignores."""
     to_query_object = getattr(builder, "to_query_object", None)
@@ -347,6 +353,28 @@ def _finish_plain_scan_pandas(
     return df
 
 
+def _finish_remote_pandas(
+    tbl: pa.Table,
+    *,
+    blob_mode: BlobMode,
+    blob_paths: Iterable[str],
+    flatten: Optional[Union[int, bool]],
+    **kwargs,
+) -> pd.DataFrame:
+    # A live remote table can be replaced between query and blob fetch.
+    # Row ids and version numbers do not identify the producing table.
+    if not blob_column_paths(tbl.schema):
+        return flatten_columns(tbl, flatten).to_pandas(**kwargs)
+    if blob_mode == "descriptions":
+        tbl = strip_auto_row_ids(tbl, blob_paths)
+        return flatten_columns(tbl, flatten).to_pandas(**kwargs)
+    raise NotImplementedError(
+        f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize "
+        "blob columns without a stable table snapshot; "
+        "use blob_mode='descriptions'"
+    )
+
+
 async def _finish_plain_scan_pandas_async(
     scanner: _LanceScanner,
     *,
@@ -860,10 +888,10 @@ class Query(pydantic.BaseModel):
     # if true, will only search the indexed data
     fast_search: Optional[bool] = None
 
-    # MemWAL LSM read routing: None auto-routes when the table carries a write
+    # RTE LSM read routing: None auto-routes when the table carries a write
     # spec, True forces the LSM scanner (errors without a spec), False reads the
     # base table only
-    use_lsm: Optional[bool] = None
+    use_rte: Optional[bool] = None
 
     # size of the nearest neighbor list maintained during HNSW search
     ef: Optional[int] = None
@@ -883,9 +911,9 @@ class Query(pydantic.BaseModel):
         query.full_text_query = req.full_text_search
         query.columns = req.select
         query.with_row_id = req.with_row_id
-        # use_lsm is a genuine tri-state (None / True / False); preserve it as-is
+        # use_rte is a genuine tri-state (None / True / False); preserve it as-is
         # so a round-tripped query keeps an explicit False.
-        query.use_lsm = req.use_lsm
+        query.use_rte = req.use_rte
         query.vector_column = req.column
         query.vector = req.query_vector
         query.distance_type = req.distance_type
@@ -997,10 +1025,12 @@ class LanceQueryBuilder(ABC):
                 fast_search=fast_search,
             )
 
-        if isinstance(query, list):
+        if isinstance(query, (list, tuple)):
             query = np.array(query, dtype=np.float32)
         elif isinstance(query, np.ndarray):
             query = query.astype(np.float32)
+        elif isinstance(query, (pa.Array, pa.ChunkedArray)):
+            query = np.array(query.to_pylist(), dtype=np.float32)
         else:
             raise TypeError(f"Unsupported query type: {type(query)}")
 
@@ -1022,7 +1052,7 @@ class LanceQueryBuilder(ABC):
             query = cls._query_to_vector(table, query, vector_column_name)
             return query, query_type
         elif query_type == "auto":
-            if isinstance(query, (list, np.ndarray)):
+            if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
                 return query, "vector"
             else:
                 conf = table.embedding_functions.get(vector_column_name)
@@ -1038,7 +1068,7 @@ class LanceQueryBuilder(ABC):
 
     @classmethod
     def _query_to_vector(cls, table, query, vector_column_name):
-        if isinstance(query, (list, np.ndarray)):
+        if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
             return query
         conf = table.embedding_functions.get(vector_column_name)
         if conf is not None:
@@ -1058,7 +1088,7 @@ class LanceQueryBuilder(ABC):
         self._with_row_address = None
         self._fragments = None
         self._fragment_ids = None
-        self._use_lsm = None
+        self._use_rte = None
         self._vector = None
         self._text = None
         self._ef = None
@@ -1107,34 +1137,50 @@ class LanceQueryBuilder(ABC):
             The maximum time to wait for the query to complete.
             If None, wait indefinitely.
         blob_mode: str, default "lazy"
-            Controls how blob columns are returned. ``"lazy"`` returns
-            :class:`~lancedb._blob.BlobFile` handles and needs a plain scan.
-            ``"descriptions"`` and ``"bytes"`` work for every query shape. A
-            plain scan without ``timeout`` reads the local dataset through
-            Lance's scanner, which also returns legacy blob columns as bytes.
-            Other shapes use :meth:`to_arrow` with the same ``blob_mode``.
+            Controls how blob columns are returned. Eligible local plain scans
+            use Lance's scanner, which returns lazy blob handles in ``"lazy"``
+            mode and also supports legacy blob columns. Other local blob v2
+            queries return descriptors in ``"lazy"`` and ``"descriptions"``
+            modes, or payloads in ``"bytes"`` mode, through :meth:`to_arrow`.
+            Remote blob queries support only ``"descriptions"`` until a stable
+            table snapshot can bind the query and blob fetches.
         **kwargs
             Forwarded to pyarrow.Table.to_pandas after query execution and
             optional flattening.
         """
         validate_blob_mode(blob_mode)
+        if _is_remote_query_table(self._table):
+            blob_sources = self._validated_blob_sources(blob_mode)
+            return _finish_remote_pandas(
+                self.to_arrow(timeout=timeout),
+                blob_mode=blob_mode,
+                blob_paths=blob_sources,
+                flatten=flatten,
+                **kwargs,
+            )
+
         output_schema = getattr(self, "output_schema", None)
         if output_schema is not None and schema_has_blob_field(output_schema()):
             if (flatten is None or blob_mode != "lazy") and timeout is None:
                 df = self._plain_scan_to_pandas(blob_mode, flatten=flatten, **kwargs)
                 if df is not None:
                     return df
-            if blob_mode == "lazy" or _query_needs_lance_scanner(self):
+            if _query_needs_lance_scanner(self):
                 raise _unsupported_blob_pandas_error(
                     "this query shape cannot use Lance native pandas conversion"
                 )
 
-        tbl = flatten_columns(self._to_arrow_in_mode(timeout, blob_mode), flatten)
-        if blob_mode == "lazy" and schema_has_blob_field(tbl.schema):
+        arrow_mode = "descriptions" if blob_mode == "lazy" else blob_mode
+        tbl = self._to_arrow_in_mode(timeout, arrow_mode)
+        if (
+            blob_mode == "lazy"
+            and schema_has_blob_field(tbl.schema)
+            and not blob_v2_projection_sources(self._table.schema, self._columns)
+        ):
             raise _unsupported_blob_pandas_error(
-                "this query shape cannot use Lance native pandas conversion"
+                "the query has no fetchable blob v2 source"
             )
-        return tbl.to_pandas(**kwargs)
+        return flatten_columns(tbl, flatten).to_pandas(**kwargs)
 
     @abstractmethod
     def to_arrow(
@@ -1477,20 +1523,20 @@ class LanceQueryBuilder(ABC):
         self._fragment_ids = fragment_ids
         return self
 
-    def use_lsm(self, enable: bool) -> Self:
-        """Control MemWAL LSM read routing for this query.
+    def use_rte(self, enable: bool) -> Self:
+        """Control RTE LSM read routing for this query.
 
-        By default (unset), a query against a table with an LSM write spec is
+        By default (unset), a query against a table with an RTE write spec is
         routed through the LSM scanner so it also returns data written via the
         ``merge_insert`` LSM path that has not yet been compacted into the base
-        table (active/frozen memtables + flushed generations); a table without a
+        table (active/frozen memtables + SSTables); a table without a
         spec reads the base table.
 
         Parameters
         ----------
         enable : bool
-            ``True`` forces the LSM scanner and errors if the table has no LSM
-            write spec. ``False`` bypasses the MemWAL and reads the base table
+            ``True`` forces the LSM scanner and errors if the table has no RTE
+            write spec. ``False`` bypasses the RTE and reads the base table
             only, even when a spec is present.
 
         Returns
@@ -1498,7 +1544,7 @@ class LanceQueryBuilder(ABC):
         LanceQueryBuilder
             The LanceQueryBuilder object.
         """
-        self._use_lsm = enable
+        self._use_rte = enable
         return self
 
     def explain_plan(self, verbose: Optional[bool] = False) -> str:
@@ -1835,7 +1881,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
         probing may continue when too few candidates are found. Any explicit
         effort, including 0.5, overrides nprobes in either call order. Explicit
         minimum and maximum bounds still apply. This is not a recall guarantee
-        and does not change refinement or HNSW ef. MemWAL LSM queries do not
+        and does not change refinement or HNSW ef. RTE LSM queries do not
         yet support this option.
 
         Examples
@@ -2012,7 +2058,7 @@ class LanceVectorQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
-            use_lsm=self._use_lsm,
+            use_rte=self._use_rte,
             offset=self._offset,
             fast_search=self._fast_search,
             ef=self._ef,
@@ -2237,7 +2283,7 @@ class LanceFtsQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
-            use_lsm=self._use_lsm,
+            use_rte=self._use_rte,
             full_text_query=FullTextSearchQuery(
                 query=self._query_with_phrase_semantics(), columns=self._fts_columns
             ),
@@ -2319,7 +2365,7 @@ class LanceEmptyQueryBuilder(LanceQueryBuilder):
             with_row_address=self._with_row_address,
             fragments=self._fragments,
             fragment_ids=self._fragment_ids,
-            use_lsm=self._use_lsm,
+            use_rte=self._use_rte,
             offset=self._offset,
             order_by=self._order_by,
         )
@@ -2395,7 +2441,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         if query is not None and (vector is not None or text is not None):
             raise ValueError(
                 "You can either provide a string query in search() method "
-                "or set `vector()` and `text()` explicitly for hybrid search."
+                "or set `vector()` and `text()` explicitly for hybrid search. "
                 "But not both."
             )
 
@@ -2415,7 +2461,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         """Set whether to return row ids.
 
         Recorded separately from `_with_row_id`, which `rerank(return_score=
-        "all")` also sets for its own use — on a MemWAL table only a caller who
+        "all")` also sets for its own use — on an RTE table only a caller who
         asked is refused, and the reranker still falls back to the primary key.
         """
         self._caller_requested_row_id = with_row_id
@@ -2529,10 +2575,6 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         with_row_ids: bool,
         offset: Optional[int] = None,
     ) -> pa.Table:
-        if norm == "rank":
-            vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
-            fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
-
         # If both result sets are empty (e.g. after hard filtering),
         # return early to avoid errors in reranking or score restoration.
         if vector_results.num_rows == 0 and fts_results.num_rows == 0:
@@ -2559,15 +2601,23 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         original_score_row_ids = None
         # normalize the scores to be between 0 and 1, 0 being most relevant
         # We check whether the results (vector and FTS) are empty, because when
-        # they are, they often are missing the _rowid column, which causes an error
+        # they are, they often are missing the _rowid column, which causes an error.
+        # The original values are captured before any rank conversion so that
+        # they can be restored after reranking.
         if vector_results.num_rows > 0:
             distance_i = vector_results.column_names.index("_distance")
             original_distances = vector_results.column(distance_i)
             original_distance_row_ids = vector_results.column("_rowid")
+            if norm == "rank":
+                vector_results = LanceHybridQueryBuilder._rank(
+                    vector_results, "_distance"
+                )
             vector_results = vector_results.set_column(
                 distance_i,
                 vector_results.field(distance_i),
-                LanceHybridQueryBuilder._normalize_scores(original_distances),
+                LanceHybridQueryBuilder._normalize_scores(
+                    vector_results.column(distance_i)
+                ),
             )
 
         # In fts higher scores represent relevance. Not inverting them here as
@@ -2576,10 +2626,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             score_i = fts_results.column_names.index("_score")
             original_scores = fts_results.column(score_i)
             original_score_row_ids = fts_results.column("_rowid")
+            if norm == "rank":
+                fts_results = LanceHybridQueryBuilder._rank(fts_results, "_score")
             fts_results = fts_results.set_column(
                 score_i,
                 fts_results.field(score_i),
-                LanceHybridQueryBuilder._normalize_scores(original_scores),
+                LanceHybridQueryBuilder._normalize_scores(fts_results.column(score_i)),
             )
 
         results = reranker.rerank_hybrid(fts_query, vector_results, fts_results)
@@ -2685,7 +2737,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         if reranker.score == "all":
             # WAL-PK-FUSION: without the fallback, `self.with_row_id(True)`.
             # Not through `with_row_id`, which records caller intent: the
-            # reranker's need for row ids must not refuse a MemWAL table.
+            # reranker's need for row ids must not refuse an RTE table.
             self._with_row_id = True
 
         return self
@@ -2723,7 +2775,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         probing may continue when too few candidates are found. Any explicit
         effort, including 0.5, overrides nprobes in either call order. Explicit
         minimum and maximum bounds still apply. This is not a recall guarantee
-        and does not change refinement or HNSW ef. MemWAL LSM queries do not
+        and does not change refinement or HNSW ef. RTE LSM queries do not
         yet support this option.
 
         Examples
@@ -2997,7 +3049,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         # WAL-PK-FUSION: without the fallback, select `self._columns` as is.
         self._pk_fusion = None
         columns = self._columns
-        if _wal_hybrid.pk_fusion_learned(self._table, self._use_lsm):
+        if _wal_hybrid.pk_fusion_learned(self._table, self._use_rte):
             self._pk_fusion = _wal_hybrid.PkFusion.for_query(
                 self._table.schema,
                 self._columns,
@@ -3014,9 +3066,9 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         if self._with_row_id and self._pk_fusion is None:
             self._vector_query.with_row_id(True)
             self._fts_query.with_row_id(True)
-        if self._use_lsm is not None:
-            self._vector_query.use_lsm(self._use_lsm)
-            self._fts_query.use_lsm(self._use_lsm)
+        if self._use_rte is not None:
+            self._vector_query.use_rte(self._use_rte)
+            self._fts_query.use_rte(self._use_rte)
         if self._phrase_query:
             self._fts_query.phrase_query(True)
         if self._distance_type:
@@ -3364,13 +3416,13 @@ class AsyncQueryBase(object):
             If not specified, no timeout is applied. If the query does not
             complete within the specified time, an error will be raised.
         blob_mode: str, default "lazy"
-            Controls how blob columns are returned. ``"lazy"`` returns
-            :class:`~lancedb._blob.BlobFile` handles and needs a plain scan.
-            ``"descriptions"`` and ``"bytes"`` work for every query shape. A
-            plain scan without ``timeout`` reads the local dataset through
-            Lance's scanner, which also returns legacy blob columns as bytes.
-            Other shapes use :meth:`to_arrow` with the same ``blob_mode``.
-            Remote queries support ``"descriptions"`` only.
+            Controls how blob columns are returned. Eligible local plain scans
+            use Lance's scanner, which returns lazy blob handles in ``"lazy"``
+            mode and also supports legacy blob columns. Other local blob v2
+            queries return descriptors in ``"lazy"`` and ``"descriptions"``
+            modes, or payloads in ``"bytes"`` mode, through :meth:`to_arrow`.
+            Remote blob queries support only ``"descriptions"`` until a stable
+            table snapshot can bind the query and blob fetches.
         **kwargs
             Forwarded to pyarrow.Table.to_pandas after query execution and
             optional flattening.
@@ -3391,19 +3443,22 @@ class AsyncQueryBase(object):
                 )
                 if df is not None:
                     return df
-            if blob_mode == "lazy" or _query_needs_lance_scanner(self):
+            if _query_needs_lance_scanner(self):
                 raise _unsupported_blob_pandas_error(
                     "this query shape cannot use Lance native pandas conversion"
                 )
 
-        tbl = flatten_columns(
-            await self.to_arrow(timeout=timeout, blob_mode=blob_mode), flatten
-        )
-        if blob_mode == "lazy" and schema_has_blob_field(tbl.schema):
+        arrow_mode = "descriptions" if blob_mode == "lazy" else blob_mode
+        tbl = await self.to_arrow(timeout=timeout, blob_mode=arrow_mode)
+        if (
+            blob_mode == "lazy"
+            and schema_has_blob_field(tbl.schema)
+            and not await self._validated_blob_sources("descriptions")
+        ):
             raise _unsupported_blob_pandas_error(
-                "this query shape cannot use Lance native pandas conversion"
+                "the query has no fetchable blob v2 source"
             )
-        return tbl.to_pandas(**kwargs)
+        return flatten_columns(tbl, flatten).to_pandas(**kwargs)
 
     async def _remote_to_pandas(
         self,
@@ -3413,21 +3468,13 @@ class AsyncQueryBase(object):
         blob_mode: BlobMode,
         **kwargs,
     ) -> "pd.DataFrame":
-        # A live remote table can be replaced between query and blob fetch.
-        # Row ids and version numbers do not identify the producing table.
         tbl = await self.to_arrow(timeout=timeout)
-        projected_blob_paths = set(blob_column_paths(tbl.schema))
-        if not projected_blob_paths:
-            return flatten_columns(tbl, flatten).to_pandas(**kwargs)
-
-        if blob_mode == "descriptions":
-            tbl = strip_auto_row_ids(tbl, self._blob_paths)
-            return flatten_columns(tbl, flatten).to_pandas(**kwargs)
-
-        raise NotImplementedError(
-            f"remote to_pandas(blob_mode={blob_mode!r}) cannot safely materialize "
-            "blob columns without a stable table snapshot; "
-            "use blob_mode='descriptions'"
+        return _finish_remote_pandas(
+            tbl,
+            blob_mode=blob_mode,
+            blob_paths=self._blob_paths,
+            flatten=flatten,
+            **kwargs,
         )
 
     async def _plain_scan_to_pandas(
@@ -3685,25 +3732,25 @@ class AsyncStandardQuery(AsyncQueryBase):
         self._inner.fast_search()
         return self
 
-    def use_lsm(self, enable: bool) -> Self:
+    def use_rte(self, enable: bool) -> Self:
         """
-        Control MemWAL LSM read routing for this query.
+        Control RTE LSM read routing for this query.
 
-        By default (unset), a query against a table with an LSM write spec (see
-        [AsyncTable.set_lsm_write_spec][lancedb.table.AsyncTable.set_lsm_write_spec])
+        By default (unset), a query against a table with an RTE write spec (see
+        [AsyncTable.enable_rte][lancedb.table.AsyncTable.enable_rte])
         is routed through the LSM scanner so it also returns data written via the
         ``merge_insert`` LSM path that has not yet been compacted into the base
-        table (the active/frozen in-memory memtables and the flushed generations),
+        table (the active/frozen in-memory memtables and the SSTables),
         deduplicated by primary key; a table without a spec reads the base table.
 
         Parameters
         ----------
         enable : bool
-            ``True`` forces the LSM scanner and errors if the table has no LSM
-            write spec. ``False`` bypasses the MemWAL and reads the base table
+            ``True`` forces the LSM scanner and errors if the table has no RTE
+            write spec. ``False`` bypasses the RTE and reads the base table
             only, even when a spec is present.
         """
-        self._inner.use_lsm(enable)
+        self._inner.use_rte(enable)
         return self
 
     def postfilter(self) -> Self:
@@ -4026,7 +4073,7 @@ class AsyncVectorQueryBase:
         minimum and maximum bounds remain effective. Zero starts at the caller
         minimum and one searches all partitions within the maximum. Later
         probing may continue to obtain enough candidates. This is not a recall
-        guarantee and does not change refinement or HNSW ef. MemWAL LSM queries
+        guarantee and does not change refinement or HNSW ef. RTE LSM queries
         do not yet support this option.
 
         Examples
@@ -4350,7 +4397,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         # WAL-PK-FUSION: without the fallback, keep only the two `with_row_id`
         # calls.
         self._pk_fusion = None
-        if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_lsm):
+        if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_rte):
             self._pk_fusion = _wal_hybrid.PkFusion.for_query(
                 await self._table.schema(),
                 _query_request_projection(fts_req),
@@ -4539,13 +4586,13 @@ class AsyncTakeQuery(AsyncQueryBase):
     def __init__(self, inner: LanceTakeQuery, table: Optional["AsyncTable"] = None):
         super().__init__(inner, table)
 
-    def use_lsm(self, enable: bool) -> "AsyncTakeQuery":
-        """Control MemWAL LSM read routing for this take query.
+    def use_rte(self, enable: bool) -> "AsyncTakeQuery":
+        """Control RTE LSM read routing for this take query.
 
-        ``False`` bypasses the MemWAL and reads the base table only — the escape
+        ``False`` bypasses the RTE and reads the base table only — the escape
         hatch, since take-by-row-id/offset is not supported on the LSM scanner.
         """
-        self._inner.use_lsm(enable)
+        self._inner.use_rte(enable)
         return self
 
     async def _plain_scan_to_pandas(
@@ -4606,14 +4653,14 @@ class BaseQueryBuilder(object):
         self._inner.with_row_id()
         return self
 
-    def use_lsm(self, enable: bool) -> Self:
+    def use_rte(self, enable: bool) -> Self:
         """
-        Control MemWAL LSM read routing for this query.
+        Control RTE LSM read routing for this query.
 
-        ``False`` bypasses the MemWAL and reads the base table only, the escape
+        ``False`` bypasses the RTE and reads the base table only, the escape
         hatch for shapes the LSM scanner cannot honor (e.g. take-by-row-id).
         """
-        self._inner.use_lsm(enable)
+        self._inner.use_rte(enable)
         return self
 
     def with_row_address(self, with_row_address: bool = True) -> Self:

python/python/lancedb/remote/db.py

@@ -723,6 +723,8 @@ class RemoteDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         from .table import RemoteTable
 
@@ -734,6 +736,8 @@ class RemoteDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return MaterializedView(
@@ -755,6 +759,8 @@ class RemoteDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._conn.create_materialized_view_async(
@@ -764,6 +770,8 @@ class RemoteDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)

python/python/lancedb/remote/header.py

@@ -141,7 +141,7 @@ class OAuthProvider(HeaderProvider):
 
                 # Set expiration if provided
                 expires_in = token_data.get("expires_in")
-                if expires_in:
+                if expires_in is not None:
                     self._token_expires_at = time.time() + expires_in
                 else:
                     # Token doesn't expire or expiration unknown

python/python/lancedb/remote/table.py

@@ -31,7 +31,7 @@ from lancedb._lancedb import (
     DropColumnsResult,
     FtsToken,
     IndexConfig,
-    LsmWriteSpec,
+    RTEWriteSpec,
     MergeResult,
     UpdateResult,
 )
@@ -279,8 +279,12 @@ class RemoteTable(Table):
         """Tokenize a query using the tokenizer configured on an FTS index.
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
-        rebuilt in the client process from index metadata, so the same tokenizer
-        model files must exist locally.
+        rebuilt in the client process from index metadata. For remote tables,
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)
@@ -1001,8 +1005,17 @@ class RemoteTable(Table):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+        return LOOP.run(
+            self._table.add_columns(
+                transforms,
+                computed=computed,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
+            )
+        )
 
     def refresh_column(self, column: str):
         return LOOP.run(self._table.refresh_column(column))
@@ -1034,20 +1047,20 @@ class RemoteTable(Table):
         return LOOP.run(self._table.drop_columns(columns))
 
     def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
-        """Not supported for remote tables."""
+        """Set the unenforced primary key for this table to a single column."""
         return LOOP.run(self._table.set_unenforced_primary_key(columns))
 
-    def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
-        """Install an LsmWriteSpec."""
-        return LOOP.run(self._table.set_lsm_write_spec(spec))
+    def enable_rte(self, spec: "RTEWriteSpec") -> None:
+        """Install an RTEWriteSpec."""
+        return LOOP.run(self._table.enable_rte(spec))
 
-    def unset_lsm_write_spec(self) -> None:
-        """Remove the LsmWriteSpec."""
-        return LOOP.run(self._table.unset_lsm_write_spec())
+    def disable_rte(self) -> None:
+        """Remove the RTEWriteSpec."""
+        return LOOP.run(self._table.disable_rte())
 
-    def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
-        """Read the installed LsmWriteSpec, or ``None``."""
-        return LOOP.run(self._table.get_lsm_write_spec())
+    def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
+        """Read the installed RTEWriteSpec, or ``None``."""
+        return LOOP.run(self._table.get_rte_spec())
 
     # WAL-PK-FUSION: delete both hooks.
     def _hybrid_pk_fusion_learned(self) -> bool:
@@ -1057,31 +1070,31 @@ class RemoteTable(Table):
     def _note_hybrid_pk_fusion(self) -> None:
         self._table._note_hybrid_pk_fusion()
 
-    def checkpoint_lsm(self) -> None:
+    def checkpoint_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
-        return LOOP.run(self._table.checkpoint_lsm())
+        [`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte]."""
+        return LOOP.run(self._table.checkpoint_rte())
 
-    def flush_lsm(self) -> None:
+    def flush_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
-        return LOOP.run(self._table.flush_lsm())
+        [`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte]."""
+        return LOOP.run(self._table.flush_rte())
 
-    def compact_lsm(self) -> None:
+    def compact_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
-        return LOOP.run(self._table.compact_lsm())
+        [`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte]."""
+        return LOOP.run(self._table.compact_rte())
 
-    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
+    def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
         """Synchronous version of
-        [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
+        [`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats]."""
         return LOOP.run(
-            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
+            self._table.get_rte_stats(include_sstable_rows=include_sstable_rows)
         )
 
-    def close_lsm_writers(self) -> None:
+    def close_rte_writers(self) -> None:
         """No-op for remote tables (no local shard writers)."""
-        return LOOP.run(self._table.close_lsm_writers())
+        return LOOP.run(self._table.close_rte_writers())
 
     def drop_index(self, index_name: str):
         return LOOP.run(self._table.drop_index(index_name))

python/python/lancedb/rerankers/base.py

@@ -241,6 +241,7 @@ class Reranker(ABC):
         This op is slower than just keeping relevance score but can be useful
         for debugging.
         """
+        combined_schema = pa.unify_schemas([vector_results.schema, fts_results.schema])
         # add nulls to fts results for _distance
         if "_distance" not in fts_results.column_names:
             fts_results = fts_results.append_column(
@@ -265,7 +266,9 @@ class Reranker(ABC):
             else:
                 vector_results_dict[key] = value
 
-        combined = pa.Table.from_pylist(list(vector_results_dict.values()))
+        combined = pa.Table.from_pylist(
+            list(vector_results_dict.values()), schema=combined_schema
+        )
         return combined
 
     def _keep_relevance_score(self, combined_results: pa.Table):

python/python/lancedb/rerankers/cohere.py

@@ -62,7 +62,7 @@ class CohereReranker(Reranker):
                 "COHERE_API_KEY not set. Either set it in your environment or \
                 pass it as `api_key` argument to the CohereReranker."
             )
-        return cohere.Client(os.environ.get("COHERE_API_KEY") or self.api_key)
+        return cohere.Client(self.api_key or os.environ.get("COHERE_API_KEY"))
 
     def _rerank(self, result_set: pa.Table, query: str):
         result_set = self._handle_empty_results(result_set)

python/python/lancedb/rerankers/openai.py

@@ -70,15 +70,18 @@ class OpenaiReranker(Reranker):
             ],
         )
         results = json.loads(response.choices[0].message.content)["documents"]
-        docs, scores = list(
-            zip(*[(result["content"], result["relevance_score"]) for result in results])
-        )  # tuples
-        # replace the self.column column with the docs
-        result_set = result_set.drop(self.column)
-        result_set = result_set.append_column(
-            self.column, pa.array(docs, type=pa.string())
-        )
-        # add the scores
+        # The model returns the documents in its own order, so match each score
+        # back to its row by content.
+        scores_by_content = {
+            result["content"]: result["relevance_score"] for result in results
+        }
+        missing = sum(doc not in scores_by_content for doc in docs)
+        if missing:
+            raise ValueError(
+                f"The model returned no relevance_score for {missing} of "
+                f"{len(docs)} documents"
+            )
+        scores = [scores_by_content[doc] for doc in docs]
         result_set = result_set.append_column(
             "_relevance_score", pa.array(scores, type=pa.float32())
         )

python/python/lancedb/rerankers/voyageai.py

@@ -32,7 +32,7 @@ class VoyageAIReranker(Reranker):
     return_score : str, default "relevance"
         options are "relevance" or "all". Only "relevance" is supported for now.
     api_key : str, default None
-        The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+        The API key to use. If None, will use the VOYAGE_API_KEY environment variable.
     truncation : Optional[bool], default None
     """
 
@@ -64,7 +64,7 @@ class VoyageAIReranker(Reranker):
                 pass it as `api_key` argument to the VoyageAIReranker."
             )
         return voyageai.Client(
-            api_key=os.environ.get("VOYAGE_API_KEY") or self.api_key,
+            api_key=self.api_key or os.environ.get("VOYAGE_API_KEY"),
         )
 
     def _rerank(self, result_set: pa.Table, query: str):

python/python/lancedb/scannable.py

@@ -276,6 +276,14 @@ def _from_iterable(data: Iterator) -> Scannable:
 _registered_modules: set[str] = set()
 
 
+def _hf_batches(data):
+    if data._indices is None:
+        yield from data.data.to_batches()
+    else:
+        for table in data.with_format("arrow").iter(batch_size=1000):
+            yield from table.to_batches()
+
+
 def _register_optional_converters():
     """Register converters for optional dependencies that are already imported."""
 
@@ -323,7 +331,15 @@ def _register_optional_converters():
         def _from_hf_dataset(data: HFDataset) -> Scannable:
             table = data.data.table  # Access underlying Arrow table
             return Scannable(
-                schema=table.schema, num_rows=len(data), reader=table.to_reader
+                schema=table.schema,
+                num_rows=len(data),
+                reader=(
+                    table.to_reader
+                    if data._indices is None
+                    else lambda: pa.RecordBatchReader.from_batches(
+                        table.schema, _hf_batches(data)
+                    )
+                ),
             )
 
         @to_scannable.register(HFDatasetDict)
@@ -335,7 +351,7 @@ def _register_optional_converters():
 
             def gen():
                 for split_name, dataset in data.items():
-                    for batch in dataset.data.to_batches():
+                    for batch in _hf_batches(dataset):
                         split_arr = pa.array(
                             [split_name] * len(batch), type=pa.string()
                         )

python/python/lancedb/streaming.py

@@ -19,6 +19,7 @@ above.
 """
 
 import ctypes
+import hashlib
 import heapq
 import logging
 import os
@@ -39,6 +40,7 @@ from torch.utils.data import DataLoader, IterableDataset, get_worker_info
 
 from .permutation import (
     Permutation,
+    PermutationBuilder,
     Transforms,
     permutation_builder,
     _drop_base_version,
@@ -244,6 +246,20 @@ class StreamingDataset(IterableDataset):
         default) rows are shuffled using ``shuffle_seed`` and ``epoch``.
         When ``False`` rows are divided into splits sequentially in storage
         order, which can be useful for deterministic debugging or evaluation.
+        Ignored when ``split_strategy`` is provided.
+    split_strategy:
+        Escape hatch for any ``PermutationBuilder`` split strategy this class
+        doesn't otherwise expose a dedicated parameter for -- ``split_hash``,
+        ``split_calculated``, ``bootstrap_sample`` (uniform sampling with
+        replacement), or any strategy added later.  A callable that receives a
+        fresh ``PermutationBuilder`` for ``table`` and returns a configured one;
+        it is called and ``.execute()``'d directly, taking over permutation
+        construction entirely.  When provided, ``shuffle``, ``shuffle_seed``,
+        ``epoch``, and ``shuffle_clump_size`` are ignored -- the callable is
+        solely responsible for how rows are split and ordered.  For example,
+        ``split_strategy=lambda b: b.bootstrap_sample(10_000, fixed=num_splits,
+        seed=42)`` draws 10,000 samples with replacement, divided evenly across
+        ``num_splits`` splits.
     shuffle_seed:
         Base seed for the random permutation.  Combined with ``epoch`` so
         each epoch produces a different ordering.  Pass ``None`` to generate
@@ -261,10 +277,15 @@ class StreamingDataset(IterableDataset):
         storage) at the cost of higher memory usage per split buffer.  Defaults
         to ``DEFAULT_READ_BATCH_SIZE`` (64).
     io_queue_depth:
-        Number of I/O batches to keep in flight per split.  Higher values
-        overlap storage latency with transform and training compute at the cost
-        of more memory and threads.  Must be greater than zero.  Defaults to
-        ``DEFAULT_PREFETCH_BATCHES`` (4).
+        Number of fetched batches to keep ahead of the transform stage per
+        split: this bounds the combined count of in-flight fetches and
+        already-fetched batches still waiting for a transform slot.  Higher
+        values overlap storage latency with transform and training compute
+        at the cost of more memory and threads, but I/O stops running ahead
+        once this many batches are buffered, applying backpressure when the
+        transform stage (or ``transform_queue_depth``) is the bottleneck.
+        Must be greater than zero.  Defaults to ``DEFAULT_PREFETCH_BATCHES``
+        (4).
     columns:
         Optional list of column names to read.  When set, only those columns
         are fetched from storage; all others are omitted.  ``None`` (the
@@ -388,6 +409,9 @@ class StreamingDataset(IterableDataset):
         *,
         num_splits: Optional[int] = None,
         shuffle: bool = True,
+        split_strategy: Optional[
+            Callable[[PermutationBuilder], PermutationBuilder]
+        ] = None,
         shuffle_seed: Optional[int] = 0,
         epoch: int = 0,
         rank: int = 0,
@@ -578,13 +602,70 @@ class StreamingDataset(IterableDataset):
         builder = permutation_builder(table)
         if filter is not None:
             builder = builder.filter(filter)
-        if shuffle:
+        if split_strategy is not None:
+            # Escape hatch: lets any PermutationBuilder split strategy drive this
+            # dataset (split_hash, split_calculated, bootstrap_sample, or any future
+            # strategy) without StreamingDataset needing a bespoke parameter for each
+            # one. shuffle/shuffle_seed/epoch/shuffle_clump_size are ignored in this
+            # path -- the caller's split_strategy is solely responsible for ordering.
+            self._perm_table = split_strategy(builder).execute()
+            # _rank_splits below assumes split ids are exactly 0..num_splits-1 with no
+            # gaps (every other path guarantees this by always building with
+            # fixed=num_splits) -- a split_strategy callable has no such guarantee, and
+            # a mismatch would otherwise silently drop rows (ids >= num_splits) or leave
+            # some ranks reading an empty split (missing ids < num_splits).
+            split_value_counts = (
+                self._perm_table.to_arrow().column("split_id").value_counts()
+            )
+            counts_by_split = dict(
+                zip(
+                    split_value_counts.field("values").to_pylist(),
+                    split_value_counts.field("counts").to_pylist(),
+                )
+            )
+            expected_splits = set(range(num_splits))
+            if set(counts_by_split) != expected_splits:
+                raise ValueError(
+                    f"split_strategy produced split ids {sorted(counts_by_split)}, "
+                    f"but num_splits={num_splits} requires exactly "
+                    f"{{0, ..., {num_splits - 1}}}. Pass num_splits matching the "
+                    "strategy's own split count (e.g. `fixed=num_splits` on "
+                    "bootstrap_sample/split_random/etc.)."
+                )
+            # The round-robin iteration below (and in _iter_owned) assumes every
+            # split has the same row count -- it stops as soon as any one owned
+            # split is exhausted (see its own comment: "all splits exhaust
+            # simultaneously"). Every other path guarantees this by always building
+            # with fixed=num_splits; a split_strategy using ratios/counts/split_hash
+            # weights has no such guarantee, and unequal sizes would otherwise
+            # silently drop rows from the larger splits -- or, with world_size > 1
+            # where each rank owns different splits, end ranks after different row
+            # counts and desync or hang a distributed run's end-of-epoch collectives.
+            if len(set(counts_by_split.values())) > 1:
+                raise ValueError(
+                    f"split_strategy produced unequal split sizes "
+                    f"{counts_by_split} (split id -> row count). Every split must "
+                    "have the same row count. Use `fixed=num_splits` rather than "
+                    "`ratios`/`counts`/weighted `split_hash`, which can produce "
+                    "unequal sizes, or split unevenly-sized groups into separate "
+                    "StreamingDatasets instead."
+                )
+        elif shuffle:
             perm_seed = shuffle_seed + epoch * _EPOCH_PRIME
             self._perm_table = builder.split_random(
                 fixed=num_splits, seed=perm_seed, clump_size=shuffle_clump_size
             ).execute()
         else:
             self._perm_table = builder.split_sequential(fixed=num_splits).execute()
+        # Only the split_strategy path needs a permutation fingerprint in the
+        # checkpoint (see `_checkpoint_snapshot`): split_random/split_sequential are
+        # already fully pinned by shuffle_seed/num_splits, and a fingerprint over raw
+        # row ids would spuriously reject an otherwise-valid resume after those row
+        # ids get renumbered by compaction (row ids are not stable by default).
+        self._uses_split_strategy = split_strategy is not None
+        # Lazily computed and cached by `_perm_fingerprint` -- `_perm_table` never
+        # changes after construction, so this is correct to compute once.
+        self._perm_fingerprint_cache: Optional[str] = None
 
         if self._blocks_per_epoch == "auto":
             self._blocks_per_epoch = self._estimate_blocks_per_epoch()
@@ -836,8 +917,17 @@ class StreamingDataset(IterableDataset):
             io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
 
         def _fill_io(i: int) -> None:
+            # Budget covers both in-flight fetches (io_pending) and batches
+            # that have already landed but are still waiting for a transform
+            # slot (raw_batches).  Without raw_batches in the count, a slow
+            # transform stage never applies backpressure to Stage 1: each
+            # completed fetch is drained into raw_batches and immediately
+            # replaced by a new one, so I/O keeps racing arbitrarily far
+            # ahead of the transform stage instead of merely overlapping
+            # storage latency with it.
             while (
-                len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i]
+                len(io_pending[i]) + len(raw_batches[i]) < io_queue_depth
+                and fetch_head[i] < split_sizes[i]
             ):
                 _submit_io(i)
 
@@ -1392,11 +1482,51 @@ class StreamingDataset(IterableDataset):
                 )
         return state
 
+    @property
+    def _perm_fingerprint(self) -> str:
+        """Cheap identity check for the built permutation, computed once and cached
+        (``_perm_table`` never changes after construction).
+
+        Only meaningful -- and only recorded in a checkpoint -- for the
+        ``split_strategy`` path (see ``_checkpoint_snapshot``): ``num_splits``/
+        ``shuffle_seed`` alone cannot detect a `load_state_dict` resuming saved
+        per-split progress against a *different* permutation than the one that
+        produced it, since a ``split_strategy`` callable ignores ``shuffle_seed``
+        entirely -- e.g. a ``bootstrap_sample`` with no seed (or any changed
+        callable) can rebuild a completely different permutation on the next run
+        while ``shuffle_seed`` keeps its same default value and the existing check
+        passes anyway. ``split_random``/``split_sequential`` are already fully
+        pinned by ``shuffle_seed``/``num_splits`` and must *not* additionally be
+        fingerprinted: this hashes physical row ids, which are not stable across
+        compaction by default, so checking it on those paths would reject an
+        otherwise-valid resume after the table gets compacted between save and
+        load even though the seeded permutation itself is unchanged.
+        """
+        if self._perm_fingerprint_cache is None:
+            arrow_tbl = self._perm_table.to_arrow()
+            hasher = hashlib.sha256()
+            hasher.update(str(arrow_tbl.num_rows).encode())
+            for name in ("row_id", "split_id"):
+                column = arrow_tbl.column(name).combine_chunks()
+                hasher.update(column.to_numpy(zero_copy_only=False).tobytes())
+            self._perm_fingerprint_cache = hasher.hexdigest()
+        return self._perm_fingerprint_cache
+
     def _checkpoint_snapshot(self) -> dict:
+        # Only recorded for split_strategy: split_random/split_sequential are
+        # already fully pinned by shuffle_seed/num_splits below, and row ids are not
+        # stable across compaction by default, so fingerprinting them unconditionally
+        # would reject an otherwise-valid resume after the table got compacted.
+        perm_fingerprint = (
+            {"perm_fingerprint": self._perm_fingerprint}
+            if (self._uses_split_strategy)
+            else {}
+        )
         if self._pack_sequences is not None:
             return {
                 "shuffle_seed": self._shuffle_seed,
                 "num_splits": self._num_splits,
+                **perm_fingerprint,
                 "epoch": self._epoch,
                 "pack_sequences": self._pack_sequences,
                 "eos_id": self._eos_id,
@@ -1417,6 +1547,7 @@ class StreamingDataset(IterableDataset):
         return {
             "shuffle_seed": self._shuffle_seed,
             "num_splits": self._num_splits,
+            **perm_fingerprint,
             "epoch": self._epoch,
             "samples_consumed_per_split": samples,
             "positions_consumed_per_split": positions,
@@ -1546,8 +1677,16 @@ class StreamingDataset(IterableDataset):
 
         Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ
         from the checkpoint, since a different split structure or shuffle order
-        makes mid-epoch resumption meaningless.  Packed checkpoints
-        pin ``pack_sequences``, ``eos_id``, ``pad_id``,
+        makes mid-epoch resumption meaningless. Also raises if the checkpoint
+        was built with ``split_strategy`` and records a permutation fingerprint
+        that does not match the current dataset's permutation -- this catches a
+        changed or non-deterministically-reseeded ``split_strategy`` that
+        ``num_splits``/``shuffle_seed`` alone cannot detect, since ``shuffle_seed``
+        is unused on that path. Checkpoints from ``split_random``/
+        ``split_sequential`` never carry this fingerprint (row ids are not stable
+        across compaction, so it would reject an otherwise-valid resume), and
+        continue to rely on ``shuffle_seed``/``num_splits`` alone as before.
+        Packed checkpoints pin ``pack_sequences``, ``eos_id``, ``pad_id``,
         ``blocks_per_epoch``, and ``epoch``.
         """
         if state["num_splits"] != self._num_splits:
@@ -1560,6 +1699,19 @@ class StreamingDataset(IterableDataset):
                 f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, "
                 f"current dataset has {self._shuffle_seed}"
             )
+        checkpoint_fingerprint = state.get("perm_fingerprint")
+        if (
+            checkpoint_fingerprint is not None
+            and checkpoint_fingerprint != self._perm_fingerprint
+        ):
+            raise ValueError(
+                "permutation mismatch: this checkpoint was saved against a "
+                "different permutation (e.g. a split_strategy that resamples "
+                "differently between runs, such as bootstrap_sample with no "
+                "seed, or a changed callable) than the current dataset built. "
+                "Resuming would apply saved per-split positions to the wrong "
+                "rows."
+            )
         self._consumer_checkpoint_requires_uniform = False
 
         if "pack_buffers" in state or self._pack_sequences is not None:
@@ -1661,6 +1813,10 @@ class StreamingDataset(IterableDataset):
         first = states[0]
         packed = "pack_buffers" in first
         config_keys = ["shuffle_seed", "num_splits", "epoch"]
+        if "perm_fingerprint" in first:
+            # Absent only in checkpoints saved before this key existed; skip rather
+            # than KeyError so old checkpoints keep merging.
+            config_keys.append("perm_fingerprint")
         if packed:
             config_keys.extend(
                 ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"]
@@ -1670,6 +1826,8 @@ class StreamingDataset(IterableDataset):
             if ("pack_buffers" in state) != packed:
                 raise ValueError("cannot merge packed and unpacked state dicts")
             for key in config_keys:
+                if key not in state:
+                    continue
                 if state[key] != first[key]:
                     raise ValueError(
                         f"{key} mismatch across state dicts: "

python/python/lancedb/table.py

@@ -30,7 +30,7 @@ from typing import (
 )
 from urllib.parse import urlparse
 
-from lancedb.scannable import _register_optional_converters, to_scannable
+from lancedb.scannable import _hf_batches, _register_optional_converters, to_scannable
 
 from . import __version__
 from ._blob import (
@@ -121,6 +121,33 @@ def _should_push_down_query_table(
     return namespace_client is not None and "QueryTable" in pushdown_operations
 
 
+_CUDA_UNAVAILABLE_MESSAGE = (
+    "CUDA acceleration is unavailable in this Python environment. Install a "
+    "CUDA-enabled PyTorch build and verify that the CUDA device is accessible, or "
+    "omit `accelerator` (or set `accelerator=None`) to create the index on CPU."
+)
+
+
+def _validate_index_accelerator(accelerator: Any) -> None:
+    if not isinstance(accelerator, str) or accelerator.split(":", 1)[0] != "cuda":
+        return
+
+    try:
+        import torch
+
+        cuda_available = torch.cuda.is_available()
+    except (ImportError, RuntimeError, AssertionError) as err:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
+
+    if not cuda_available:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE)
+
+    try:
+        torch.cuda.init()
+    except (RuntimeError, AssertionError) as err:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
+
+
 def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
     """Return a Polars frame unchanged while blocking predicate pushdown."""
     return frame
@@ -225,7 +252,7 @@ if TYPE_CHECKING:
         DeleteResult,
         DropColumnsResult,
         FtsToken,
-        LsmWriteSpec,
+        RTEWriteSpec,
         MergeResult,
         UpdateResult,
     )
@@ -343,10 +370,19 @@ def _blob_input_to_arrow(data: Any, schema: Optional[pa.Schema]) -> Optional[pa.
             if isinstance(field.type, pa.ExtensionType)
             else field.type
         )
-        storage = pa.array(
-            [_blob_value_to_storage(value) for value in values[field.name]],
-            type=storage_type,
-        )
+        storage_values = [_blob_value_to_storage(value) for value in values[field.name]]
+        missing_range_fields = {"position", "size"} - set(storage_type.names)
+        # Arrow silently ignores dictionary keys absent from the storage type.
+        if missing_range_fields and any(
+            value is not None
+            and any(value.get(name) is not None for name in missing_range_fields)
+            for value in storage_values
+        ):
+            raise ValueError(
+                f"Blob column '{field.name}' storage schema cannot store byte ranges. "
+                "Create the column with lancedb.blob() to use position and size."
+            )
+        storage = pa.array(storage_values, type=storage_type)
         column = (
             pa.ExtensionArray.from_storage(field.type, storage)
             if isinstance(field.type, pa.ExtensionType)
@@ -366,7 +402,7 @@ def _into_pyarrow_reader(
     if _check_for_hugging_face(data):
         if isinstance(data, datasets.Dataset):
             schema = data.features.arrow_schema
-            return pa.RecordBatchReader.from_batches(schema, data.data.to_batches())
+            return pa.RecordBatchReader.from_batches(schema, _hf_batches(data))
         elif isinstance(data, datasets.dataset_dict.DatasetDict):
             schema = _schema_from_hf(data, None)
             if "split" not in schema.names:
@@ -435,7 +471,7 @@ def _into_pyarrow_reader(
     ):
         return data.collect().to_arrow().to_reader()
     elif isinstance(data, Iterable):
-        return _iterator_to_reader(data)
+        return _iterator_to_reader(data, schema)
     else:
         raise TypeError(
             f"Unknown data type {type(data)}. "
@@ -445,10 +481,19 @@ def _into_pyarrow_reader(
         )
 
 
-def _iterator_to_reader(data: Iterable) -> pa.RecordBatchReader:
+def _iterator_to_reader(
+    data: Iterable, schema: Optional[pa.Schema] = None
+) -> pa.RecordBatchReader:
     # Each batch is treated as it's own reader, mainly so we can
     # re-use the _into_pyarrow_reader logic.
-    first = _into_pyarrow_reader(next(data))
+    data = iter(data)
+    try:
+        first_batch = next(data)
+    except StopIteration:
+        if schema is None:
+            raise ValueError("Cannot create table from empty iterable without a schema")
+        return pa.RecordBatchReader.from_batches(schema, [])
+    first = _into_pyarrow_reader(first_batch)
     schema = first.schema
 
     def gen():
@@ -581,7 +626,7 @@ def _cast_to_target_schema(
 def _preserve_blob_v2_fields(
     inferred_schema: pa.Schema, existing_schema: pa.Schema
 ) -> pa.Schema:
-    """Keep existing blob fields when an overwrite infers a replacement schema."""
+    """Keep inferred blob semantics without overriding explicit replacement types."""
     existing_fields = {field.name: field for field in existing_schema}
     fields = [
         _preserve_blob_v2_field(field, existing_fields[field.name])
@@ -593,6 +638,9 @@ def _preserve_blob_v2_fields(
 
 
 def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field:
+    if is_blob_v2_field(inferred):
+        # An explicitly declared blob layout is part of the replacement schema.
+        return inferred
     if is_blob_v2_field(existing):
         return existing
 
@@ -630,12 +678,10 @@ def _preserve_blob_v2_field(inferred: pa.Field, existing: pa.Field) -> pa.Field:
 def _coerce_blob_write_columns(
     batch: pa.RecordBatch, target_schema: pa.Schema
 ) -> pa.RecordBatch:
-    """Materialize blob storage structs before the stream leaves Python.
+    """Materialize Python bytes and URIs as blob structs before Arrow casting.
 
-    merge_insert requires its source reader to already match the table's
-    physical schema. Unlike add and insert, it does not pass through
-    LanceDB's Rust blob coercion, so preserving binary input here would
-    reach Lance as binary and fail the schema check.
+    Existing blob structs retain their input layout. Rust normalizes them to
+    the stored schema and rejects populated ranges on two-field targets.
     """
     columns = []
     fields = []
@@ -966,6 +1012,17 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field:
                 "(Utf8, LargeUtf8, Utf8View), or a Struct with a "
                 "'data' or 'uri' child"
             )
+        if pa.types.is_struct(source_type) or (
+            isinstance(source_type, pa.ExtensionType)
+            and (
+                not isinstance(target_field.type, pa.ExtensionType)
+                or type(source_type) is type(target_field.type)
+            )
+        ):
+            # Keep the source layout until Rust can fill missing range fields or
+            # validate every batch before dropping all-null range fields. Arrow
+            # casting here would reject differing layouts before that validation.
+            return field.with_metadata(target_field.metadata)
     # LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but
     # casting the input to that storage type here merely relabels the raw JSON bytes as
     # JSONB. Lance must see arrow.json so it can perform the JSONB encoding.
@@ -1149,7 +1206,7 @@ def _to_batches_with_split(data):
     with an extra `split` column
     """
     for key, dataset in data.items():
-        for batch in dataset.data.to_batches():
+        for batch in _hf_batches(dataset):
             table = pa.Table.from_batches([batch])
             if "split" not in table.column_names:
                 table = table.append_column(
@@ -1825,8 +1882,8 @@ class Table(ABC):
             - pyarrow.Table or pyarrow.RecordBatch
         mode: str
             The mode to use when writing the data. Valid values are
-            "append" and "overwrite". Overwrite infers a replacement schema
-            but preserves existing blob v2 fields present in the new data.
+            "append" and "overwrite". Overwrite adopts explicitly declared blob
+            layouts, while inferred bytes retain the existing blob v2 fields.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
             One of "error", "drop", "fill".
@@ -2192,8 +2249,9 @@ class Table(ABC):
         One missing or deleted row ID rejects the entire batch; no partial
         results are returned.
 
-        Row IDs can be obtained from a query with ``with_row_id(True)``. This
-        API is currently supported only by local tables.
+        Row IDs can be obtained from a query with ``with_row_id(True)``.
+        Remote tables read the requested ranges through version-pinned blob
+        handles using HTTP byte-range requests.
         """
 
     @abstractmethod
@@ -2511,7 +2569,11 @@ class Table(ABC):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
 
     @abstractmethod
@@ -2541,6 +2603,8 @@ class Table(ABC):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ):
         """
         Add new columns with defined values.
@@ -2585,6 +2649,15 @@ class Table(ABC):
             [`refresh_column_async`][lancedb.table.Table.refresh_column_async].
             Cannot be combined with ``transforms``.
 
+        auto_refresh_enabled: bool, optional
+            Enable automatic refresh for each declared computed output. Defaults
+            to True. Local tables store this option; scheduling requires a server.
+        auto_refresh_interval_seconds: int, optional
+            Positive minimum interval after a successful refresh, in seconds.
+            When omitted, the server determines when to refresh. Applies to every
+            computed output in this call.
+            Scheduling options require computed or Function columns.
+
         Returns
         -------
         AddColumnsResult
@@ -3855,7 +3928,9 @@ class LanceTable(Table):
 
             # Handle accelerator through pylance
             if accelerator is not None:
-                self.to_lance().create_index(
+                dataset = self.to_lance()
+                _validate_index_accelerator(accelerator)
+                dataset.create_index(
                     column=column,
                     index_type=index_type,
                     metric=metric,
@@ -3880,6 +3955,8 @@ class LanceTable(Table):
                 acc = getattr(config, "accelerator", None)
                 if acc is not None:
                     # Dispatch to pylance for GPU acceleration
+                    dataset = self.to_lance()
+                    _validate_index_accelerator(acc)
                     index_type_map = {
                         "IvfFlat": "IVF_FLAT",
                         "IvfSq": "IVF_SQ",
@@ -3891,7 +3968,7 @@ class LanceTable(Table):
                     cfg_type = type(config).__name__
                     lance_index_type = index_type_map.get(cfg_type, "IVF_PQ")
 
-                    self.to_lance().create_index(
+                    dataset.create_index(
                         column=column,
                         index_type=lance_index_type,
                         metric=getattr(config, "distance_type", "l2"),
@@ -4389,8 +4466,8 @@ class LanceTable(Table):
             The data to insert into the table.
         mode: str
             The mode to use when writing the data. Valid values are
-            "append" and "overwrite". Overwrite infers a replacement schema
-            but preserves existing blob v2 fields present in the new data.
+            "append" and "overwrite". Overwrite adopts explicitly declared blob
+            layouts, while inferred bytes retain the existing blob v2 fields.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
             One of "error", "drop", "fill", "null".
@@ -5041,7 +5118,11 @@ class LanceTable(Table):
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return LOOP.run(
             self._table.tokenize(query, column=column, index_name=index_name)
@@ -5073,8 +5154,17 @@ class LanceTable(Table):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+        return LOOP.run(
+            self._table.add_columns(
+                transforms,
+                computed=computed,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
+            )
+        )
 
     def refresh_column(self, column: str) -> "RefreshColumnResult":
         """Fill a computed column's unfilled rows and recompute those whose
@@ -5119,47 +5209,47 @@ class LanceTable(Table):
         [`AsyncTable.set_unenforced_primary_key`][lancedb.AsyncTable.set_unenforced_primary_key]."""
         return LOOP.run(self._table.set_unenforced_primary_key(columns))
 
-    def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
-        """Install an LsmWriteSpec. See
-        [`AsyncTable.set_lsm_write_spec`][lancedb.AsyncTable.set_lsm_write_spec]."""
-        return LOOP.run(self._table.set_lsm_write_spec(spec))
+    def enable_rte(self, spec: "RTEWriteSpec") -> None:
+        """Install an RTEWriteSpec. See
+        [`AsyncTable.enable_rte`][lancedb.AsyncTable.enable_rte]."""
+        return LOOP.run(self._table.enable_rte(spec))
 
-    def unset_lsm_write_spec(self) -> None:
-        """Remove the LsmWriteSpec. See
-        [`AsyncTable.unset_lsm_write_spec`][lancedb.AsyncTable.unset_lsm_write_spec]."""
-        return LOOP.run(self._table.unset_lsm_write_spec())
+    def disable_rte(self) -> None:
+        """Remove the RTEWriteSpec. See
+        [`AsyncTable.disable_rte`][lancedb.AsyncTable.disable_rte]."""
+        return LOOP.run(self._table.disable_rte())
 
-    def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
-        """Read the installed LsmWriteSpec, or ``None``. See
-        [`AsyncTable.get_lsm_write_spec`][lancedb.AsyncTable.get_lsm_write_spec]."""
-        return LOOP.run(self._table.get_lsm_write_spec())
+    def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
+        """Read the installed RTEWriteSpec, or ``None``. See
+        [`AsyncTable.get_rte_spec`][lancedb.AsyncTable.get_rte_spec]."""
+        return LOOP.run(self._table.get_rte_spec())
 
-    def checkpoint_lsm(self) -> None:
+    def checkpoint_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
-        return LOOP.run(self._table.checkpoint_lsm())
+        [`AsyncTable.checkpoint_rte`][lancedb.AsyncTable.checkpoint_rte]."""
+        return LOOP.run(self._table.checkpoint_rte())
 
-    def flush_lsm(self) -> None:
+    def flush_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
-        return LOOP.run(self._table.flush_lsm())
+        [`AsyncTable.flush_rte`][lancedb.AsyncTable.flush_rte]."""
+        return LOOP.run(self._table.flush_rte())
 
-    def compact_lsm(self) -> None:
+    def compact_rte(self) -> None:
         """Synchronous version of
-        [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
-        return LOOP.run(self._table.compact_lsm())
+        [`AsyncTable.compact_rte`][lancedb.AsyncTable.compact_rte]."""
+        return LOOP.run(self._table.compact_rte())
 
-    def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
+    def get_rte_stats(self, *, include_sstable_rows: bool = False) -> Optional[dict]:
         """Synchronous version of
-        [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
+        [`AsyncTable.get_rte_stats`][lancedb.AsyncTable.get_rte_stats]."""
         return LOOP.run(
-            self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
+            self._table.get_rte_stats(include_sstable_rows=include_sstable_rows)
         )
 
-    def close_lsm_writers(self) -> None:
-        """Close cached MemWAL shard writers. See
-        [`AsyncTable.close_lsm_writers`][lancedb.AsyncTable.close_lsm_writers]."""
-        return LOOP.run(self._table.close_lsm_writers())
+    def close_rte_writers(self) -> None:
+        """Close cached RTE shard writers. See
+        [`AsyncTable.close_rte_writers`][lancedb.AsyncTable.close_rte_writers]."""
+        return LOOP.run(self._table.close_rte_writers())
 
     def uses_v2_manifest_paths(self) -> bool:
         """
@@ -5535,6 +5625,11 @@ def has_nan_values(arr: Union[pa.ListArray, pa.ChunkedArray]) -> pa.BooleanArray
     else:
         values_has_nan = pc.is_nan(values)
     values_indices = pc.list_parent_indices(arr)
+    if arr.null_count:
+        # Match flatten(), which excludes child spans under null parent lists.
+        values_indices = pc.filter(
+            values_indices, pc.take(pc.is_valid(arr), values_indices)
+        )
     has_nan_indices = pc.unique(pc.filter(values_indices, values_has_nan))
     indices = pa.array(range(len(arr)), type=pa.uint32())
     return pc.is_in(indices, has_nan_indices)
@@ -5585,6 +5680,12 @@ def _infer_target_schema(
             # Use the most common length of the list as the dimensions
             dim = _modal_list_size(peeked.column(i))
 
+            if dim is None:
+                # Every list entry is null (e.g. embeddings not computed yet),
+                # so the dimension is unknowable. Leave the column as a
+                # variable-length list instead of guessing a fixed size.
+                continue
+
             # Determine target type based on value type
             if pa.types.is_floating(field.type.value_type):
                 target_type = pa.list_(pa.float32(), dim)
@@ -5617,6 +5718,7 @@ def _infer_target_schema(
                 field.name,  # preserve original field name
                 target_type,
                 nullable=field.nullable,
+                metadata=field.metadata,
             )
 
             schema = schema.set(i, new_field)
@@ -5624,9 +5726,14 @@ def _infer_target_schema(
     return schema, reader
 
 
-def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> int:
-    # Use the most common length of the list as the dimensions
-    return pc.mode(pc.list_value_length(arr))[0].as_py()["mode"]
+def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> Optional[int]:
+    # Use the most common length of the list as the dimensions. pyarrow's
+    # `mode` skips nulls, so an all-null column yields an empty result; return
+    # None in that case so the caller can keep a variable-length list.
+    modes = pc.mode(pc.list_value_length(arr))
+    if len(modes) == 0:
+        return None
+    return modes[0].as_py()["mode"]
 
 
 def _infer_vector_dim(arr: Union[pa.Array, pa.ChunkedArray]) -> Optional[int]:
@@ -5817,56 +5924,56 @@ class AsyncTable:
             columns = list(columns)
         await self._inner.set_unenforced_primary_key(columns)
 
-    async def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
-        """Install an LsmWriteSpec on this table.
+    async def enable_rte(self, spec: "RTEWriteSpec") -> None:
+        """Install an RTEWriteSpec on this table.
 
         The spec selects Lance's MemWAL LSM-style write path for future
-        `merge_insert` calls. ``LsmWriteSpec`` chooses one of three sharding
+        `merge_insert` calls. ``RTEWriteSpec`` chooses one of three sharding
         strategies:
 
-        - ``LsmWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by
+        - ``RTEWriteSpec.bucket(column, num_buckets)`` — hash-bucket writes by
           the single-column unenforced primary key.
-        - ``LsmWriteSpec.identity(column)`` — shard by the raw value of a
+        - ``RTEWriteSpec.identity(column)`` — shard by the raw value of a
           scalar column.
-        - ``LsmWriteSpec.unsharded()`` — route every write to a single shard.
+        - ``RTEWriteSpec.unsharded()`` — route every write to a single shard.
 
         All variants require the table to have an unenforced primary key set
         via [`set_unenforced_primary_key`]; bucket sharding additionally
         requires it to be the single column being bucketed.
 
-        By default the MemWAL maintains every index on the table, resolved
-        here — a snapshot, so an index created afterwards needs the spec unset
-        and set again. This fails if one cannot be maintained; name the set
-        with ``with_maintained_indexes`` to install anyway. That pins an exact
-        set (a still-building index is rejected, not omitted); ``[]`` maintains
-        none.
+        By default the RTE maintains every index on the table, re-read as the
+        table changes, so an index created later is maintained too; one of a
+        kind the RTE cannot mirror is skipped. Name the set with
+        ``with_maintained_indexes`` to pin it instead: each name must already
+        exist and be maintainable (a still-building index is rejected, not
+        omitted); ``[]`` maintains none.
 
         Parameters
         ----------
-        spec : LsmWriteSpec
+        spec : RTEWriteSpec
             The sharding spec to install.
 
         Examples
         --------
-        >>> from lancedb import LsmWriteSpec
+        >>> from lancedb import RTEWriteSpec
         >>> # table.set_unenforced_primary_key("id")
-        >>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16))
+        >>> # table.enable_rte(RTEWriteSpec.bucket("id", 16))
         """
-        await self._inner.set_lsm_write_spec(spec)
+        await self._inner.enable_rte(spec)
 
-    async def unset_lsm_write_spec(self) -> None:
-        """Remove the LsmWriteSpec from this table.
+    async def disable_rte(self) -> None:
+        """Remove the RTEWriteSpec from this table.
 
         Reverts to the standard `merge_insert` write path. Errors if no spec
         is currently set.
         """
-        await self._inner.unset_lsm_write_spec()
+        await self._inner.disable_rte()
 
-    async def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
-        """Read the LsmWriteSpec currently installed on this table.
+    async def get_rte_spec(self) -> Optional["RTEWriteSpec"]:
+        """Read the RTEWriteSpec currently installed on this table.
 
-        Returns ``None`` when the LSM write path is not enabled at all — no
-        spec has been set, or one was removed with `unset_lsm_write_spec`.
+        Returns ``None`` when the RTE write path is not enabled at all — no
+        spec has been set, or one was removed with `disable_rte`.
         That is a different answer from a spec whose ``maintained_indexes``
         is ``None``, which is an installed spec selecting indexes
         automatically.
@@ -5876,7 +5983,7 @@ class AsyncTable:
         or gains later, ``[]`` maintains none, and a non-empty list maintains
         exactly those. All three round-trip.
         """
-        return await self._inner.get_lsm_write_spec()
+        return await self._inner.get_rte_spec()
 
     # WAL-PK-FUSION: delete both hooks.
     def _hybrid_pk_fusion_learned(self) -> bool:
@@ -5887,19 +5994,19 @@ class AsyncTable:
         """See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table]."""
         self._inner.note_hybrid_pk_fusion()
 
-    async def checkpoint_lsm(self) -> None:
-        """Converge this table's LSM write path into its base table.
+    async def checkpoint_rte(self) -> None:
+        """Converge this table's RTE write path into its base table.
 
-        One flush, sealing every memtable into L0, then compaction triggers
-        until every generation that existed at that moment has reached base.
-        The loop runs client-side, reading progress from ``get_lsm_stats``.
+        One flush, freezing every memtable into an SSTable, then compaction triggers
+        until every SSTable that existed at that moment has reached base.
+        The loop runs client-side, reading progress from ``get_rte_stats``.
 
-        Best-effort: generations created *while* it runs are deliberately not
+        Best-effort: SSTables created *while* it runs are deliberately not
         waited on, which is what lets it terminate on a table taking writes.
         Idempotent and safe on a cadence.
 
         There is no deadline, and the caller owns that. It returns when the
-        target generations are gone, raises on a terminal server fault, and
+        target SSTables are gone, raises on a terminal server fault, and
         otherwise waits however long the server takes. A slow table and a
         stuck one are the same picture from the client: the compactor pool is
         shared across every table on the node, so a checkpoint queued behind
@@ -5907,56 +6014,56 @@ class AsyncTable:
         ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
         costs nothing.
         """
-        await self._inner.checkpoint_lsm()
+        await self._inner.checkpoint_rte()
 
-    async def flush_lsm(self) -> None:
-        """Seal every bucket's active memtable into L0.
+    async def flush_rte(self) -> None:
+        """Freeze every table shard's active memtable into an SSTable.
 
-        Does not touch the base table — moving L0 into base is
-        `compact_lsm`. On a node that has not claimed this table, this claims
+        Does not touch the base table — compacting SSTables into base is
+        `compact_rte`. On a node that has not claimed this table, this claims
         it and replays its WAL log first.
         """
-        await self._inner.flush_lsm()
+        await self._inner.flush_rte()
 
-    async def compact_lsm(self) -> None:
-        """Trigger a background L0 to base compaction pass per bucket.
+    async def compact_rte(self) -> None:
+        """Trigger a background SSTable compaction pass per table shard.
 
         Returns once the passes are dispatched, not once they finish: watch
-        ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
-        until the current L0 has reached base.
+        ``get_rte_stats`` for progress, or use ``checkpoint_rte`` to loop
+        until the current SSTables have reached base.
         """
-        await self._inner.compact_lsm()
+        await self._inner.compact_rte()
 
-    async def get_lsm_stats(
-        self, *, include_generation_rows: bool = False
+    async def get_rte_stats(
+        self, *, include_sstable_rows: bool = False
     ) -> Optional[dict]:
-        """Read live per-bucket LSM state.
+        """Read live per-table-shard RTE state.
 
-        Answers "how far behind is my fresh tier", "which bucket is hot", and
+        Answers "how far behind is my fresh tier", "which table shard is hot", and
         "why is my fresh-tier vector search brute-force". Mutates no table
         state, though on a node that has not claimed this table it claims it,
         exactly as a read would.
 
-        Returns ``None`` only when the LSM write path is not enabled.
+        Returns ``None`` only when the RTE write path is not enabled.
 
         Parameters
         ----------
-        include_generation_rows
-            Report a row count per L0 generation. Off by default: each count
-            opens an uncached Lance dataset, and ``checkpoint_lsm`` polls this
+        include_sstable_rows
+            Report a row count per SSTable. Off by default: each count
+            opens an uncached Lance dataset, and ``checkpoint_rte`` polls this
             needing only generation numbers.
         """
-        return await self._inner.get_lsm_stats(include_generation_rows)
+        return await self._inner.get_rte_stats(include_sstable_rows)
 
-    async def close_lsm_writers(self) -> None:
-        """Drain and close any cached MemWAL shard writers for this table.
+    async def close_rte_writers(self) -> None:
+        """Drain and close any cached RTE shard writers for this table.
 
-        When an LSM write spec is installed, `merge_insert` opens MemWAL shard
+        When an RTE write spec is installed, `merge_insert` opens RTE shard
         writers and caches them for reuse across calls. This closes them,
         flushing pending data; writers reopen lazily on the next
         `merge_insert`. It is a no-op when no writers are cached.
         """
-        await self._inner.close_lsm_writers()
+        await self._inner.close_rte_writers()
 
     @property
     def name(self) -> str:
@@ -6427,8 +6534,8 @@ class AsyncTable:
             - pyarrow.Table or pyarrow.RecordBatch
         mode: str
             The mode to use when writing the data. Valid values are
-            "append" and "overwrite". Overwrite infers a replacement schema
-            but preserves existing blob v2 fields present in the new data.
+            "append" and "overwrite". Overwrite adopts explicitly declared blob
+            layouts, while inferred bytes retain the existing blob v2 fields.
         on_bad_vectors: str, default "error"
             What to do if any of the vectors are not the same size or contains NaNs.
             One of "error", "drop", "fill", "null".
@@ -6463,8 +6570,12 @@ class AsyncTable:
         if mode == "overwrite":
             # For overwrite, apply the same preprocessing as create_table
             # so vector columns are inferred as FixedSizeList.
-            data, _ = sanitize_create_table(
-                data, None, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            data, _ = await asyncio.to_thread(
+                sanitize_create_table,
+                data,
+                None,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
             )
             if blob_v2_column_paths(schema):
                 # Plain bytes cannot be inferred as blob v2. Keep matching blob
@@ -6474,7 +6585,8 @@ class AsyncTable:
         elif on_bad_vectors != "error" or (
             schema.metadata is not None and b"embedding_functions" in schema.metadata
         ):
-            data = _sanitize_data(
+            data = await asyncio.to_thread(
+                _sanitize_data,
                 data,
                 schema,
                 metadata=schema.metadata,
@@ -6483,7 +6595,10 @@ class AsyncTable:
                 allow_subschema=True,
             )
         _register_optional_converters()
-        data = to_scannable(data)
+        # Converting an iterator peeks at its first item.  A synchronous query in
+        # that iterator schedules work on LOOP, so peeking on LOOP's own thread
+        # would deadlock waiting for itself.
+        data = await asyncio.to_thread(to_scannable, data)
         progress, owns = _normalize_progress(progress)
         try:
             return await self._inner.add(
@@ -6845,8 +6960,8 @@ class AsyncTable:
             take_query = self.take_offsets(query.take_offsets)
             if query.columns:
                 take_query = take_query.select(query.columns)
-            if query.use_lsm is not None:
-                take_query = take_query.use_lsm(query.use_lsm)
+            if query.use_rte is not None:
+                take_query = take_query.use_rte(query.use_rte)
             if query.with_row_id:
                 take_query = take_query.with_row_id()
             return take_query
@@ -6862,8 +6977,8 @@ class AsyncTable:
             async_query = async_query.where(query.filter)
         if query.fast_search:
             async_query = async_query.fast_search()
-        if query.use_lsm is not None:
-            async_query = async_query.use_lsm(query.use_lsm)
+        if query.use_rte is not None:
+            async_query = async_query.use_rte(query.use_rte)
         if query.with_row_id:
             async_query = async_query.with_row_id()
         if query.order_by:
@@ -6989,7 +7104,7 @@ class AsyncTable:
                 when_not_matched_by_source_condition_expr=merge._when_not_matched_by_source_condition_expr,
                 timeout=merge._timeout,
                 use_index=merge._use_index,
-                use_lsm=merge._use_lsm,
+                use_rte=merge._use_rte,
                 validate_single_shard=merge._validate_single_shard,
             ),
         )
@@ -7122,6 +7237,8 @@ class AsyncTable:
         | None = None,
         *,
         computed: dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
         """
         Add new columns with defined values.
@@ -7161,12 +7278,46 @@ class AsyncTable:
             On LanceDB Cloud and Enterprise the expression is planned by
             the server. Cannot be combined with ``transforms``.
 
+        auto_refresh_enabled: bool, optional
+            Enable automatic refresh for each computed output declared by this
+            call. Defaults to True. Local tables store the setting; automatic
+            scheduling requires Cloud or Enterprise.
+        auto_refresh_interval_seconds: int, optional
+            Positive minimum seconds after a successful refresh before another
+            automatic refresh is eligible. When omitted, the server determines when
+            to refresh. Unchanged inputs do
+            not force recomputation. Applies to every declared computed output.
+            These options cannot be used with one-time transforms or ordinary
+            all-null columns.
+
+        Examples
+        --------
+        >>> await table.add_columns(  # doctest: +SKIP
+        ...     computed={"total": "price * quantity"},
+        ...     auto_refresh_enabled=True,
+        ...     auto_refresh_interval_seconds=300,
+        ... )
+
         Returns
         -------
         AddColumnsResult
             version: the new version number of the table after adding columns.
 
         """
+        if auto_refresh_enabled is not None and type(auto_refresh_enabled) is not bool:
+            raise ValueError("auto_refresh_enabled must be a boolean")
+        if auto_refresh_interval_seconds is not None and (
+            type(auto_refresh_interval_seconds) is not int
+            or not 0 < auto_refresh_interval_seconds <= (2**64 - 1) // 1000
+        ):
+            raise ValueError(
+                "auto_refresh_interval_seconds must be a positive integer "
+                "no greater than u64::MAX / 1000"
+            )
+        options_supplied = (
+            auto_refresh_enabled is not None
+            or auto_refresh_interval_seconds is not None
+        )
         function_application = None
         function_output_name = None
         if isinstance(transforms, FunctionApplication):
@@ -7190,7 +7341,10 @@ class AsyncTable:
                 )
             function_application._ensure_declarable()
             return await self._inner.add_function_columns(
-                function_application.to_canonical_json(), function_output_name
+                function_application.to_canonical_json(),
+                function_output_name,
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
             )
 
         if isinstance(transforms, pa.Field):
@@ -7204,7 +7358,15 @@ class AsyncTable:
                 raise ValueError(
                     "add_columns cannot take both transforms and computed columns"
                 )
-            return await self._inner.add_computed_columns(list(computed.items()))
+            return await self._inner.add_computed_columns(
+                list(computed.items()),
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
+            )
+        if options_supplied:
+            raise ValueError(
+                "automatic refresh options require computed or Function columns"
+            )
         if transforms is None:
             raise ValueError("add_columns requires transforms or computed columns")
         if isinstance(transforms, pa.Schema):
@@ -7657,7 +7819,11 @@ class AsyncTable:
 
         Model-backed tokenizers such as ``jieba/*`` and ``lindera/*`` are
         rebuilt in the client process from index metadata. For remote tables,
-        this means the same tokenizer model files must also exist locally.
+        this means the same tokenizer model files must also exist locally
+        (under ``LANCE_LANGUAGE_MODEL_HOME``) and must match the files the
+        server uses. A mismatch cannot be detected: the returned tokens may
+        differ from what the server's index produces. If the model files are
+        missing, an error naming the tokenizer is raised.
         """
         return await self._inner.tokenize(query, column=column, index_name=index_name)
 

python/python/lancedb/util.py

@@ -34,20 +34,20 @@ def safe_import_adlfs():
 adlfs = safe_import_adlfs()
 
 
-def get_uri_scheme(uri: str) -> str:
+def get_uri_scheme(uri: Union[str, pathlib.Path]) -> str:
     """
     Get the scheme of a URI. If the URI does not have a scheme, assume it is a file URI.
 
     Parameters
     ----------
-    uri : str
+    uri : str or pathlib.Path
         The URI to parse.
 
     Returns
     -------
     str: The scheme of the URI.
     """
-    parsed = urlparse(uri)
+    parsed = urlparse(str(uri))
     scheme = parsed.scheme
     if not scheme:
         scheme = "file"
@@ -61,25 +61,26 @@ def get_uri_scheme(uri: str) -> str:
     return scheme
 
 
-def get_uri_location(uri: str) -> str:
+def get_uri_location(uri: Union[str, pathlib.Path]) -> str:
     """
     Get the location of a URI. If the parameter is not a url, assumes it is just a path
 
     Parameters
     ----------
-    uri : str
+    uri : str or pathlib.Path
         The URI to parse.
 
     Returns
     -------
     str: Location part of the URL, without scheme
     """
-    parsed = urlparse(uri)
+    uri_str = str(uri)
+    parsed = urlparse(uri_str)
     if len(parsed.scheme) == 1:
         # Windows drive names are parsed as the scheme
         # e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
         # So we add special handling here for schemes that are a single character
-        return uri
+        return uri_str
 
     if not parsed.netloc:
         return parsed.path
@@ -236,6 +237,19 @@ def inf_vector_column_query(schema: pa.Schema, dim: Optional[int] = None) -> str
     vector_col_names = []
     for field in schema:
         vector_col_names.extend(_iter_vector_columns(field, [], dim))
+
+    if len(vector_col_names) == 0 and dim is not None:
+        # The query's dimension matched no column. If the schema is
+        # unambiguous without that filter, the single vector column is still
+        # the right one to search: a dimension mismatch is a query-execution
+        # error, not a missing-column error.
+        unfiltered = []
+        for field in schema:
+            unfiltered.extend(_iter_vector_columns(field, []))
+        if len(unfiltered) == 1:
+            return unfiltered[0]
+        vector_col_names = unfiltered
+
     if len(vector_col_names) > 1:
         raise ValueError(
             "Schema has more than one vector column. "
@@ -296,7 +310,16 @@ def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
         if query.ndim == 0:
             return None
         return query.shape[-1]
-    if isinstance(query, list) and query:
+    if isinstance(query, (pa.Array, pa.ChunkedArray)):
+        if len(query) == 0:
+            return None
+        if pa.types.is_fixed_size_list(query.type):
+            return query.type.list_size
+        if pa.types.is_list(query.type) or pa.types.is_large_list(query.type):
+            first = query[0].as_py()
+            return len(first) if first is not None else None
+        return len(query)
+    if isinstance(query, (list, tuple)) and query:
         first = query[0]
         if isinstance(first, (list, tuple, np.ndarray)):
             return len(first)
@@ -403,7 +426,8 @@ def _(value: date):
 
 
 @value_to_sql.register(list)
-def _(value: list):
+@value_to_sql.register(tuple)
+def _(value: Union[list, tuple]):
     return "[" + ", ".join(map(value_to_sql, value)) + "]"
 
 
@@ -448,6 +472,15 @@ def _(value: np.floating):
     return value_to_sql(float(value))
 
 
+@value_to_sql.register(np.datetime64)
+def _(value: np.datetime64):
+    if np.isnat(value):
+        return value_to_sql(None)
+    if np.datetime_data(value.dtype)[0] in ("ps", "fs", "as"):
+        raise ValueError("SQL timestamps cannot preserve sub-nanosecond precision")
+    return value_to_sql(str(np.datetime_as_string(value, unit="ns", timezone="UTC")))
+
+
 def deprecated(func):
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted

python/python/tests/test_blob.py

@@ -63,6 +63,25 @@ def _blob_array(name, values):
     return pa.ExtensionArray.from_storage(blob_type, storage)
 
 
+def _blob_layout_data(rows, with_ranges, *, as_extension=True):
+    field = lancedb.blob("image")
+    storage_type = field.type.storage_type
+    if not with_ranges or not as_extension:
+        storage_type = pa.struct(list(storage_type)[: (4 if with_ranges else 2)])
+        field = pa.field(
+            "image",
+            storage_type,
+            metadata={"ARROW:extension:name": "lance.blob.v2"},
+        )
+    images = pa.array([row["image"] for row in rows], type=storage_type)
+    if isinstance(field.type, pa.ExtensionType):
+        images = pa.ExtensionArray.from_storage(field.type, images)
+    return pa.Table.from_arrays(
+        [pa.array([row["id"] for row in rows], type=pa.int64()), images],
+        schema=pa.schema([pa.field("id", pa.int64()), field]),
+    )
+
+
 def _row_ids_by_id(table):
     hits = table.search().with_row_id(True).limit(1000).to_arrow()
     assert "_rowid" in hits.column_names
@@ -998,6 +1017,133 @@ def test_merge_insert_writes_python_bytes():
     assert blobs.to_pylist() == [b"updated", b"inserted"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"])
+@pytest.mark.parametrize("target_with_ranges", [False, True])
+async def test_merge_insert_normalizes_blob_layouts(
+    use_async, input_type, target_with_ranges
+):
+    data = _blob_layout_data(
+        [{"id": 1, "image": {"data": b"before"}}], target_with_ranges
+    )
+    table = lancedb.connect("memory:///").create_table("merge_layouts", data=data)
+    original_schema = table.schema
+    updates = _blob_layout_data(
+        [
+            {"id": 1, "image": {"data": b"updated"}},
+            {"id": 2, "image": {"data": b"inserted"}},
+            {"id": 3, "image": None},
+        ],
+        not target_with_ranges,
+        as_extension=input_type != "struct",
+    )
+    if input_type == "reader":
+        updates = updates.to_reader(max_chunksize=1)
+    writer = table._table if use_async else table
+    result = (
+        writer.merge_insert("id")
+        .when_matched_update_all()
+        .when_not_matched_insert_all()
+        .execute(updates)
+    )
+    if use_async:
+        result = await result
+
+    assert result.num_updated_rows == 1
+    assert result.num_inserted_rows == 2
+    assert table.schema == original_schema
+    by_id = _row_ids_by_id(table)
+    assert table.fetch_blobs("image", [by_id[i] for i in (1, 2, 3)]).to_pylist() == [
+        b"updated",
+        b"inserted",
+        None,
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize(
+    "byte_range", [{"position": 0}, {"size": 6}, {"position": 4, "size": 6}]
+)
+async def test_merge_insert_legacy_blob_rejects_ranges_atomically(
+    use_async, byte_range
+):
+    data = _blob_layout_data([{"id": 1, "image": {"data": b"before"}}], False)
+    table = lancedb.connect("memory:///").create_table("reject_ranges", data=data)
+    original_schema, original_version = table.schema, table.version
+    updates = _blob_layout_data(
+        [
+            {"id": 1, "image": {"data": b"updated"}},
+            {"id": 2, "image": {"uri": "file:///payload", **byte_range}},
+        ],
+        True,
+    ).to_reader(max_chunksize=1)
+    writer = table._table if use_async else table
+    with pytest.raises(RuntimeError, match="cannot store byte ranges"):
+        result = (
+            writer.merge_insert("id")
+            .when_matched_update_all()
+            .when_not_matched_insert_all()
+            .execute(updates)
+        )
+        if use_async:
+            await result
+
+    assert table.version == original_version
+    assert table.schema == original_schema
+    assert table.count_rows() == 1
+    assert table.fetch_blobs("image", [_row_ids_by_id(table)[1]]).to_pylist() == [
+        b"before"
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize("input_type", ["extension", "struct", "reader"])
+@pytest.mark.parametrize("source_with_ranges", [False, True])
+async def test_add_overwrite_adopts_explicit_blob_layout(
+    tmp_path, use_async, input_type, source_with_ranges
+):
+    data = _blob_layout_data(
+        [{"id": 1, "image": {"data": b"before"}}], not source_with_ranges
+    )
+    table = lancedb.connect("memory:///").create_table("overwrite_layouts", data=data)
+    payload = b"0123456789abcdef"
+    path = tmp_path / "payload.bin"
+    path.write_bytes(payload)
+    value = (
+        {"uri": path.as_uri(), "position": 4, "size": 6}
+        if source_with_ranges
+        else {"data": b"updated"}
+    )
+    updates = _blob_layout_data(
+        [{"id": 2, "image": value}, {"id": 3, "image": None}],
+        source_with_ranges,
+        as_extension=input_type != "struct",
+    )
+    if input_type == "reader":
+        updates = updates.to_reader(max_chunksize=1)
+    writer = table._table if use_async else table
+    result = writer.add(
+        updates, mode="overwrite", allow_external_blob_outside_bases=True
+    )
+    if use_async:
+        await result
+
+    blob_type = table.schema.field("image").type
+    if isinstance(blob_type, pa.ExtensionType):
+        blob_type = blob_type.storage_type
+    assert blob_type.num_fields == (4 if source_with_ranges else 2)
+    assert table.blob_columns() == ["image"]
+    assert table.count_rows() == 2
+    by_id = _row_ids_by_id(table)
+    assert table.fetch_blobs("image", [by_id[2], by_id[3]]).to_pylist() == [
+        payload[4:10] if source_with_ranges else b"updated",
+        None,
+    ]
+
+
 @pytest.mark.parametrize("write", ["merge_insert", "create_table"])
 def test_uri_string_blob_write_reaches_external_base_validation(tmp_path, write):
     blob_path = tmp_path / "payload.bin"
@@ -1632,7 +1778,7 @@ def test_fetch_blob_files_lazy_read():
     assert handles[0].read() == payload
     handles[0].close()
     assert handles[0].closed
-    with pytest.raises(RuntimeError, match="already closed"):
+    with pytest.raises(ValueError, match="read of closed file"):
         handles[0].read()
 
 
@@ -1681,8 +1827,8 @@ def test_blob_file_finalizer_and_close_on_runtime_worker():
         explicit_handle.close()  # Close remains idempotent after the worker call.
         try:
             explicit_handle.read()
-        except RuntimeError as error:
-            assert "already closed" in str(error)
+        except ValueError as error:
+            assert "read of closed file" in str(error)
         else:
             raise AssertionError("a closed blob file was readable")
         print("worker close completed")
@@ -2305,6 +2451,90 @@ def test_add_external_uri_string_round_trips_with_flag(tmp_path):
     assert blobs[0].as_py() == payload
 
 
+@pytest.mark.parametrize("as_pandas", [False, True])
+@pytest.mark.parametrize("as_blob", [False, True])
+@pytest.mark.parametrize("minimal_schema", [False, True])
+@pytest.mark.parametrize("position", [0, 4])
+def test_add_external_blob_range_matches_storage_schema(
+    tmp_path, as_pandas, as_blob, minimal_schema, position
+):
+    payload = b"0123456789abcdef"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+    field = (
+        pa.field("image", pa.large_binary(), metadata={"lance-encoding:blob": "true"})
+        if minimal_schema
+        else lancedb.blob("image")
+    )
+    db = lancedb.connect(tmp_path / "db")
+    table = db.create_table(
+        "ranges", schema=pa.schema([pa.field("id", pa.int64()), field])
+    )
+    value = (
+        Blob.from_uri(blob_path.as_uri(), position=position, size=6)
+        if as_blob
+        else {"uri": blob_path.as_uri(), "position": position, "size": 6}
+    )
+    rows = [{"id": 1, "image": b"inline"}, {"id": 2, "image": value}]
+    data = pd.DataFrame(rows) if as_pandas else rows
+
+    if minimal_schema:
+        with pytest.raises(ValueError, match="Blob column 'image'.*storage schema"):
+            table.add(data, allow_external_blob_outside_bases=True)
+        assert table.count_rows() == 0
+    else:
+        table.add(data, allow_external_blob_outside_bases=True)
+        by_id = _row_ids_by_id(table)
+        assert table.fetch_blobs("image", [by_id[1], by_id[2]]).to_pylist() == [
+            b"inline",
+            payload[position : position + 6],
+        ]
+
+
+@pytest.mark.parametrize("as_pandas", [False, True])
+def test_add_minimal_blob_schema_preserves_whole_file_and_inline_values(
+    tmp_path, as_pandas
+):
+    payload = b"whole-file"
+    blob_path = tmp_path / "payload.bin"
+    blob_path.write_bytes(payload)
+    uri = blob_path.as_uri()
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field(
+                "image", pa.large_binary(), metadata={"lance-encoding:blob": "true"}
+            ),
+        ]
+    )
+    table = lancedb.connect(tmp_path / "db").create_table("minimal", schema=schema)
+    values = [
+        None,
+        b"inline",
+        Blob.from_bytes(b"from-blob"),
+        uri,
+        Blob.from_uri(uri),
+        {"uri": uri, "position": None, "size": None},
+    ]
+    rows = [{"id": i, "image": value} for i, value in enumerate(values)]
+    table.add(
+        pd.DataFrame(rows) if as_pandas else rows,
+        allow_external_blob_outside_bases=True,
+    )
+
+    by_id = _row_ids_by_id(table)
+    assert table.fetch_blobs(
+        "image", [by_id[i] for i in range(len(values))]
+    ).to_pylist() == [
+        None,
+        b"inline",
+        b"from-blob",
+        payload,
+        payload,
+        payload,
+    ]
+
+
 @pytest.mark.parametrize("as_pandas", [False, True])
 def test_add_bytes_and_uri_in_one_batch_preserves_external_uri(tmp_path, as_pandas):
     payload = b"external-payload"

python/python/tests/test_closed_table_query.py

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import pytest
+from lancedb.db import AsyncConnection
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("operation", ["count_rows", "query", "head"])
+async def test_closed_table_queries_raise_runtime_error(
+    mem_db_async: AsyncConnection, operation
+):
+    table = await mem_db_async.create_table("closed_query", data=[{"id": 0}])
+    assert (await table.query().to_arrow()).to_pylist() == [{"id": 0}]
+    table.close()
+    table.close()
+    assert not table.is_open()
+
+    with pytest.raises(RuntimeError, match="Table closed_query is closed"):
+        if operation == "query":
+            table.query()
+        else:
+            await getattr(table, operation)()
+
+
+@pytest.mark.asyncio
+async def test_open_table_queries(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table("open_query", data=[{"id": 0}])
+    assert await table.count_rows() == 1
+    assert (await table.query().to_arrow()).to_pylist() == [{"id": 0}]
+    assert (await table.head()).to_pylist() == [{"id": 0}]
+    table.close()

python/python/tests/test_db.py

@@ -87,6 +87,7 @@ def test_sync_debugger_inspection_does_not_use_background_loop(tmp_path, monkeyp
 
     assert members["uri"] == str(tmp_path)
     assert members["read_consistency_interval"] is None
+    assert members["session"] is None
     assert repr(db) == f"LanceDBConnection(uri={str(tmp_path)!r})"
     assert repr(table) == f"LanceTable(name='test', _conn={db!r})"
 
@@ -106,6 +107,8 @@ def test_read_consistency_interval_does_not_use_background_loop(tmp_path, monkey
 
     assert db.read_consistency_interval == consistency_interval
     assert db_from_inner.read_consistency_interval == consistency_interval
+    assert db.session is None
+    assert db_from_inner.session is None
 
 
 def test_serialize_preserves_zero_read_consistency_interval(tmp_path):

python/python/tests/test_duckdb.py

@@ -22,3 +22,19 @@ def test_basic_query(tmp_path):
 
     results = duck_conn.sql("SELECT SUM(y) FROM adapter").fetchall()
     assert results[0][0] == 26
+
+
+def test_query_with_filter(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)  # noqa: F841
+
+    duck_conn = duckdb.connect()
+
+    results = duck_conn.sql("SELECT SUM(y) FROM adapter WHERE x > 2").fetchall()
+    assert results[0][0] == 15
+
+    results = duck_conn.sql("SELECT y FROM adapter WHERE x > 2 ORDER BY y").fetchall()
+    assert results == [(7,), (8,)]

python/python/tests/test_elastic_dataloader.py

@@ -2036,6 +2036,92 @@ def test_transform_queue_depth_bounds_cooked_rows(lance_table):
     )
 
 
+def test_raw_queue_depth_bounded_when_transform_is_the_bottleneck(tmp_path):
+    """io_queue_depth bounds raw_batches even when transform is the bottleneck.
+
+    Regression test for a bug where Stage 1 (I/O) applied no backpressure at
+    all: _fill_io only capped io_pending (in-flight fetches) at
+    io_queue_depth, but every completed fetch was drained into raw_batches
+    unconditionally (_drain_io) and immediately replaced by a new fetch
+    (_fill_io again), so I/O kept racing arbitrarily far ahead of a slow
+    transform stage -- raw_batches had no cap of its own.
+    transform_queue_depth did not help either: its admission check in
+    _try_submit_tx only counts cooked rows and in-flight transform futures
+    (`len(cooked[i]) + len(tx_pending[i]) * batch_size`); it never looks at
+    len(raw_batches[i]), so it cannot throttle Stage 1.
+
+    The fix makes _fill_io count both io_pending and raw_batches against the
+    io_queue_depth budget, so raw_queue_depth should never exceed
+    io_queue_depth * read_batch_size per split, regardless of how slow the
+    transform stage is.
+
+    I/O is forced to run synchronously (same technique as
+    test_transform_queue_depth_does_not_admit_at_capacity_minus_one) so the
+    assertion is deterministic regardless of machine load, rather than
+    relying on a real-time race between I/O and transform threads.  Only the
+    transform runs on a real background thread (with a short sleep) so the
+    consumer genuinely blocks waiting for it, letting Stage 1 attempt to run
+    ahead between blocking waits.
+    """
+    import concurrent.futures as cf
+    from concurrent.futures import ThreadPoolExecutor
+    from unittest.mock import patch
+    import pyarrow as pa
+    import time
+
+    n_rows = 200
+    read_batch_size = 10
+    io_queue_depth = 2
+    transform_queue_depth = 2
+    max_allowed = io_queue_depth * read_batch_size
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("t", pa.table({"id": list(range(n_rows))}))
+
+    def slow_transform(batch: pa.RecordBatch) -> list:
+        time.sleep(0.005)
+        return batch.column("id").to_pylist()
+
+    original_submit = ThreadPoolExecutor.submit
+
+    def synchronous_io_submit(self, fn, *args, **kwargs):
+        if getattr(fn, "__name__", "") == "_io_call":
+            fut = cf.Future()
+            try:
+                fut.set_result(fn(*args, **kwargs))
+            except Exception as exc:
+                fut.set_exception(exc)
+            return fut
+        return original_submit(self, fn, *args, **kwargs)
+
+    with patch.object(ThreadPoolExecutor, "submit", synchronous_io_submit):
+        ds = StreamingDataset(
+            table,
+            num_splits=1,
+            shuffle_seed=SHUFFLE_SEED,
+            read_batch_size=read_batch_size,
+            io_queue_depth=io_queue_depth,
+            transform_queue_depth=transform_queue_depth,
+            transform_parallelism=1,
+            transform=slow_transform,
+        )
+
+        peak_raw = 0
+        ids = []
+        for item in ds:
+            ids.append(item)  # slow_transform yields bare ids, not dicts
+            if ds.raw_queue_depth > peak_raw:
+                peak_raw = ds.raw_queue_depth
+
+    assert peak_raw <= max_allowed, (
+        f"raw_queue_depth peaked at {peak_raw}, expected <= {max_allowed} "
+        "(io_queue_depth * read_batch_size)"
+    )
+    # The backpressure fix must not come at the cost of dropping or
+    # duplicating rows.
+    assert sorted(ids) == list(range(n_rows))
+
+
 def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path):
     """Admission requires a full read_batch_size of free space, not just one slot.
 
@@ -3173,6 +3259,22 @@ def test_doc_example_checkpoint(lance_table):
     )
 
 
+def test_streaming_dataset_over_table_with_schema_metadata(tmp_path):
+    """StreamingDataset builds its permutation at construction; base table schema
+    metadata (e.g. Hugging Face's `source-column-name-map`) must not break it."""
+    schema = pa.schema(
+        [pa.field("id", pa.int64())],
+        metadata={b"source-column-name-map": b"{}", b"huggingface": b"{}"},
+    )
+    table = lancedb.connect(tmp_path).create_table(
+        "data", pa.table({"id": list(range(NUM_ROWS))}, schema=schema)
+    )
+
+    ds = StreamingDataset(table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+
+    assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
+
+
 # ---------------------------------------------------------------------------
 # Remote tables (LanceDB Cloud / Enterprise)
 # ---------------------------------------------------------------------------
@@ -3195,3 +3297,230 @@ def test_streaming_dataset_over_remote_table():
     assert len(server.scans) == 1, "the permutation is built with one row-id scan"
     assert server.takes, "rows must be fetched with row-id takes"
     assert_server_safe_row_id_requests(server)
+
+
+# ---------------------------------------------------------------------------
+# split_strategy escape hatch
+# ---------------------------------------------------------------------------
+
+
+def test_split_strategy_bootstrap_sample(lance_table):
+    """split_strategy lets StreamingDataset use bootstrap_sample directly, producing
+    more rows than the base table has (sampling with replacement)."""
+    num_samples = 4 * NUM_ROWS
+    ds = StreamingDataset(
+        lance_table,
+        split_strategy=lambda b: b.bootstrap_sample(
+            num_samples, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+        ),
+        num_splits=NUM_SPLITS,
+    )
+    ids = [row["id"] for row in ds]
+
+    assert len(ids) == num_samples
+    # 4x oversampling a small table must produce at least one repeat.
+    assert len(set(ids)) < len(ids)
+    assert set(ids).issubset(set(range(NUM_ROWS)))
+
+
+def test_split_strategy_matches_direct_permutation_builder_use(lance_table):
+    """StreamingDataset(split_strategy=...), read single-process (no DataLoader
+    workers, so this one process owns every split and round-robins across all of
+    them), must yield exactly what calling the same strategy directly on a
+    PermutationBuilder would read, interleaved one row per split per cycle.
+
+    Reads each split via __getitems__ (explicit indices), not plain iteration:
+    Permutation's sequential __iter__ returns empty for a permutation with repeated
+    row_ids (confirmed directly -- len() is correct, __getitems__ returns the right
+    rows, but `for row in perm` yields nothing). That's a pre-existing limitation of
+    Permutation's iterator, unrelated to Bootstrap sampling's correctness: it doesn't
+    affect StreamingDataset or this benchmark, since both only ever read via
+    explicit-index batched fetches (take_offsets/__getitems__), never the plain
+    iterator -- flagged separately, not fixed here (out of scope for this change).
+    """
+    from lancedb.permutation import Permutation, permutation_builder
+
+    def strategy(builder):
+        return builder.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=1)
+
+    direct_perm_tbl = strategy(permutation_builder(lance_table)).execute()
+    expected_by_split = []
+    for split in range(NUM_SPLITS):
+        perm = Permutation.from_tables(lance_table, direct_perm_tbl, split=split)
+        rows = perm.__getitems__(list(range(len(perm))))
+        expected_by_split.append([row["id"] for row in rows])
+    samples_per_split = len(expected_by_split[0])
+    expected = [
+        expected_by_split[split][cycle]
+        for cycle in range(samples_per_split)
+        for split in range(NUM_SPLITS)
+    ]
+
+    ds = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+    actual = [row["id"] for row in ds]
+    assert actual == expected
+
+
+def test_split_strategy_is_reproducible_across_instances(lance_table):
+    def strategy(builder):
+        return builder.bootstrap_sample(3 * NUM_ROWS, fixed=NUM_SPLITS, seed=99)
+
+    ds1 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+    ds2 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+
+    assert [row["id"] for row in ds1] == [row["id"] for row in ds2]
+
+
+def test_split_strategy_num_splits_mismatch_raises(lance_table):
+    """StreamingDataset(split_strategy=...) must reject a strategy whose split
+    count doesn't match num_splits. `_rank_splits` assumes split ids are exactly
+    0..num_splits-1 with no gaps; a mismatch would otherwise silently drop rows
+    (split ids >= num_splits) or leave some ranks reading an empty split (missing
+    ids < num_splits)."""
+    with pytest.raises(ValueError, match="num_splits"):
+        StreamingDataset(
+            lance_table,
+            split_strategy=lambda b: b.bootstrap_sample(
+                2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+            ),
+            num_splits=NUM_SPLITS + 1,  # the strategy above always builds NUM_SPLITS
+        )
+
+
+def test_split_strategy_unequal_split_sizes_raises(lance_table):
+    """StreamingDataset(split_strategy=...) must reject a strategy whose splits
+    all exist (ids 0..num_splits-1) but aren't the same size. The round-robin
+    iteration (see _iter_owned) assumes every owned split has the same row count
+    and stops as soon as any one is exhausted -- `ratios`/`counts` (unlike
+    `fixed`) can produce unequal split sizes, which would otherwise silently
+    drop rows from the larger splits."""
+    with pytest.raises(ValueError, match="unequal split sizes"):
+        StreamingDataset(
+            lance_table,
+            split_strategy=lambda b: b.bootstrap_sample(
+                100, ratios=[0.7, 0.3], seed=SHUFFLE_SEED
+            ),
+            num_splits=2,
+        )
+
+
+def test_split_strategy_permutation_mismatch_on_resume_raises(lance_table):
+    """load_state_dict must reject resuming against a different permutation than
+    the one that produced the checkpoint. num_splits/shuffle_seed alone can't catch
+    this for split_strategy, since that path ignores shuffle_seed entirely: a
+    bootstrap_sample reseeded differently resamples completely differently while
+    shuffle_seed keeps whatever value the caller passed either way, so the
+    pre-existing checks would pass despite the mismatch."""
+
+    def strategy(seed):
+        return lambda b: b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=seed)
+
+    ds1 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy(1),
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    it = iter(ds1)
+    for _ in range(NUM_SPLITS):  # one full cycle: every split advances uniformly
+        next(it)
+    checkpoint = ds1.state_dict()
+
+    ds2 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy(2),  # different seed -> a different permutation
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,  # unchanged -- irrelevant on this path
+    )
+    with pytest.raises(ValueError, match="permutation mismatch"):
+        ds2.load_state_dict(checkpoint)
+
+
+def test_split_strategy_resume_with_matching_permutation_succeeds(lance_table):
+    """The common case: resuming against the same split_strategy (same seed) must
+    work exactly like the non-split_strategy checkpoint/resume path, continuing
+    from exactly the same position with no gaps or repeats."""
+
+    def strategy(b):
+        return b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=7)
+
+    ds1 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    it = iter(ds1)
+    for _ in range(NUM_SPLITS):
+        next(it)
+    checkpoint = ds1.state_dict()
+    remaining_original = [row["id"] for row in it]
+
+    ds2 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    ds2.load_state_dict(checkpoint)  # must not raise
+    remaining_resumed = [row["id"] for row in ds2]
+
+    assert remaining_original == remaining_resumed
+
+
+def test_default_checkpoint_omits_perm_fingerprint(lance_table):
+    """split_random/split_sequential checkpoints must not carry a
+    perm_fingerprint -- see test_default_checkpoint_resumes_after_compaction for
+    why checking one on this path would be wrong, not just unnecessary."""
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    assert "perm_fingerprint" not in ds.state_dict()
+
+
+def test_split_strategy_checkpoint_includes_perm_fingerprint(lance_table):
+    ds = StreamingDataset(
+        lance_table,
+        split_strategy=lambda b: b.bootstrap_sample(
+            2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+        ),
+        num_splits=NUM_SPLITS,
+    )
+    assert "perm_fingerprint" in ds.state_dict()
+
+
+def test_default_checkpoint_resumes_after_compaction(lance_table):
+    """split_random/split_sequential checkpoints must resume normally even after
+    the table is compacted between save and load. Lance row ids are not stable
+    across compaction by default (confirmed empirically: a 2-fragment table's
+    row ids differ before and after `optimize()`), so fingerprinting raw row ids
+    on this path -- instead of relying on shuffle_seed/num_splits alone, as it
+    does -- would reject this resume even though the seeded permutation itself
+    is unchanged."""
+    lance_table.add(pa.table({"id": [NUM_ROWS]}))  # a second fragment to compact
+
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    it = iter(ds)
+    for _ in range(NUM_SPLITS):
+        next(it)
+    checkpoint = ds.state_dict()
+    assert "perm_fingerprint" not in checkpoint
+
+    before_compaction = set(
+        lance_table.to_lance()
+        .to_table(columns=[], with_row_id=True)["_rowid"]
+        .to_pylist()
+    )
+    lance_table.optimize()
+    after_compaction = set(
+        lance_table.to_lance()
+        .to_table(columns=[], with_row_id=True)["_rowid"]
+        .to_pylist()
+    )
+    assert before_compaction != after_compaction, (
+        "test setup didn't actually renumber row ids -- fix the fixture, not "
+        "the assertion below"
+    )
+
+    ds_resumed = StreamingDataset(
+        lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
+    )
+    ds_resumed.load_state_dict(checkpoint)  # must not raise

python/python/tests/test_embedding_field_metadata.py

@@ -0,0 +1,84 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import copy
+
+import lancedb
+import pytest
+from lancedb.embeddings.base import TextEmbeddingFunction
+from lancedb.embeddings.registry import get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+
+class LocalEmbedding(TextEmbeddingFunction):
+    def ndims(self):
+        return 2
+
+    def generate_embeddings(self, texts):
+        return [[1.0, 0.0] if text == "alpha" else [0.0, 1.0] for text in texts]
+
+
+@pytest.mark.parametrize(
+    "method,tag",
+    [("SourceField", "source_column_for"), ("VectorField", "vector_column_for")],
+)
+@pytest.mark.parametrize("extra", [None, {}, {"x-purpose": "retrieval"}])
+def test_embedding_field_preserves_extra_metadata(method, tag, extra):
+    function = LocalEmbedding.create(max_retries=0)
+    original = copy.deepcopy(extra)
+    field = getattr(function, method)(json_schema_extra=extra)
+    assert extra == original
+    assert field.json_schema_extra[tag] is function
+    for key, value in (extra or {}).items():
+        assert field.json_schema_extra[key] == value
+
+
+@pytest.mark.parametrize(
+    "method,tag",
+    [("SourceField", "source_column_for"), ("VectorField", "vector_column_for")],
+)
+def test_extra_metadata_cannot_replace_embedding_identity(method, tag):
+    function = LocalEmbedding.create(max_retries=0)
+    supplied = {tag: "caller value", "x-purpose": "retrieval"}
+    field = getattr(function, method)(json_schema_extra=supplied)
+    assert supplied[tag] == "caller value"
+    assert field.json_schema_extra[tag] is function
+    assert field.json_schema_extra["x-purpose"] == "retrieval"
+
+
+@pytest.mark.parametrize("method", ["SourceField", "VectorField"])
+def test_embedding_field_preserves_other_field_arguments(method):
+    function = LocalEmbedding.create(max_retries=0)
+    field = getattr(function, method)(title="Custom title", description="Custom field")
+    assert field.title == "Custom title"
+    assert field.description == "Custom field"
+
+
+def test_extra_metadata_preserves_schema_and_local_embedding(tmp_path, monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    registry.register("test-embedding-field-extra")(LocalEmbedding)
+    function = LocalEmbedding.create(max_retries=0)
+
+    class Row(LanceModel):
+        text: str = function.SourceField(json_schema_extra={"x-purpose": "source"})
+        vector: Vector(2) = function.VectorField(
+            json_schema_extra={"x-purpose": "embedding"}
+        )
+
+    json_schema = Row.model_json_schema()
+    assert json_schema["properties"]["text"]["x-purpose"] == "source"
+    assert json_schema["properties"]["vector"]["x-purpose"] == "embedding"
+    configs = Row.parse_embedding_functions()
+    assert len(configs) == 1
+    assert configs[0].function is function
+    assert configs[0].source_column == "text"
+    assert configs[0].vector_column == "vector"
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "rows", schema=Row, data=[{"text": "alpha"}, {"text": "beta"}]
+    )
+    assert table.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert isinstance(reopened.embedding_functions["vector"].function, LocalEmbedding)
+    assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"

python/python/tests/test_embedding_registry.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+from lancedb.embeddings.base import TextEmbeddingFunction
+from lancedb.embeddings.registry import EmbeddingFunctionRegistry, get_registry
+from lancedb.pydantic import LanceModel, Vector
+
+
+class FirstEmbedding(TextEmbeddingFunction):
+    def ndims(self):
+        return 2
+
+    def generate_embeddings(self, texts):
+        return [[1.0, 0.0] if text == "alpha" else [0.0, 1.0] for text in texts]
+
+
+class SecondEmbedding(FirstEmbedding):
+    def generate_embeddings(self, texts):
+        return [[0.0, 1.0] if text == "alpha" else [1.0, 0.0] for text in texts]
+
+
+@pytest.mark.parametrize(
+    "first_alias,second_alias,second_class",
+    [
+        ("shared", "shared", SecondEmbedding),
+        (None, "FirstEmbedding", SecondEmbedding),
+        ("SecondEmbedding", None, SecondEmbedding),
+        (None, None, FirstEmbedding),
+        ("", "", FirstEmbedding),
+    ],
+)
+def test_duplicate_registration_preserves_original(
+    first_alias, second_alias, second_class
+):
+    registry = EmbeddingFunctionRegistry()
+    registry.register(first_alias)(FirstEmbedding)
+    key = first_alias or "FirstEmbedding"
+    original_alias = FirstEmbedding.__embedding_function_registry_alias__
+    second_attributes = dict(second_class.__dict__)
+    with pytest.raises(KeyError, match=f"{key} was already registered"):
+        registry.register(second_alias)(second_class)
+    assert registry.get(key) is FirstEmbedding
+    assert FirstEmbedding.__embedding_function_registry_alias__ == original_alias
+    assert dict(second_class.__dict__) == second_attributes
+
+
+def test_class_name_does_not_collide_with_distinct_alias():
+    registry = EmbeddingFunctionRegistry()
+    registry.register()(FirstEmbedding)
+    same_name = type("FirstEmbedding", (SecondEmbedding,), {})
+    registry.register("separate")(same_name)
+    assert registry.get("FirstEmbedding") is FirstEmbedding
+    assert registry.get("separate") is same_name
+
+
+def test_same_class_explicit_alias_registration_is_idempotent():
+    registry = EmbeddingFunctionRegistry()
+    assert registry.register("shared")(FirstEmbedding) is FirstEmbedding
+    attributes = dict(FirstEmbedding.__dict__)
+    assert registry.register("shared")(FirstEmbedding) is FirstEmbedding
+    assert registry.get("shared") is FirstEmbedding
+    assert dict(FirstEmbedding.__dict__) == attributes
+
+
+def test_alias_collision_does_not_change_persisted_table(tmp_path, monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    alias = "test-alias-collision-persisted-table"
+    registry.register(alias)(FirstEmbedding)
+    function = FirstEmbedding.create(max_retries=0)
+
+    class Row(LanceModel):
+        text: str = function.SourceField()
+        vector: Vector(2) = function.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "rows", schema=Row, data=[{"text": "alpha"}, {"text": "beta"}]
+    )
+    assert table.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    assert registry.register(alias)(FirstEmbedding) is FirstEmbedding
+    with pytest.raises(KeyError, match=f"{alias} was already registered"):
+        registry.register(alias)(SecondEmbedding)
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert isinstance(reopened.embedding_functions["vector"].function, FirstEmbedding)
+    assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"

python/python/tests/test_embeddings.py

@@ -23,6 +23,12 @@ from lancedb.embeddings.utils import retry
 from lancedb.pydantic import LanceModel, Vector
 
 
+@pytest.fixture(autouse=True)
+def isolated_embedding_registry(monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+
+
 def mock_embed_func(input_data):
     return [np.random.randn(128).tolist() for _ in range(len(input_data))]
 
@@ -64,6 +70,37 @@ def test_embedding_function(tmp_path):
     assert np.allclose(actual, expected)
 
 
+def test_embedding_function_registered_without_alias(tmp_path):
+    registry = EmbeddingFunctionRegistry.get_instance()
+
+    @registry.register()
+    class NoAliasEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 2
+
+        def generate_embeddings(self, texts):
+            return [np.array([1.0, 2.0]) for _ in texts]
+
+    assert registry.get("NoAliasEmbeddingFunction") is NoAliasEmbeddingFunction
+
+    func = NoAliasEmbeddingFunction.create()
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(2) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    db.create_table("test", schema=Schema)
+
+    # The class name is written to the table metadata, so the function can be
+    # resolved again when the table is reopened.
+    table = lancedb.connect(tmp_path).open_table("test")
+    table.add([{"text": "hello"}])
+    conf = table.embedding_functions["vector"]
+    assert isinstance(conf.function, NoAliasEmbeddingFunction)
+    assert table.to_arrow()["vector"].to_pylist() == [[1.0, 2.0]]
+
+
 def test_instructor_ndims_uses_instruction():
     instructor = get_registry().get("instructor").create()
     model = MagicMock()
@@ -596,6 +633,363 @@ def test_openai_no_retry_on_401(mock_sleep):
     assert mock_sleep.call_count == 0
 
 
+class _HttpStatusError(Exception):
+    """Stand-in for provider SDK errors (openai, cohere, ...) with a status code."""
+
+    def __init__(self, status_code, message="error"):
+        super().__init__(message)
+        self.status_code = status_code
+
+
+class _ResponseStatusError(Exception):
+    """Stand-in for requests/httpx errors, which expose `response.status_code`."""
+
+    def __init__(self, status_code):
+        super().__init__(f"HTTP {status_code}")
+        self.response = MagicMock(status_code=status_code)
+
+
+def _run_with_retry(side_effect, **kwargs):
+    """Run a function through the retry helper with a mocked `time.sleep`.
+
+    Returns (mock_func, sleeps, error).
+    """
+    from lancedb.embeddings.utils import retry_with_exponential_backoff
+
+    mock_func = MagicMock(side_effect=side_effect)
+    sleeps = []
+    error = None
+    with patch("time.sleep", side_effect=sleeps.append):
+        try:
+            retry_with_exponential_backoff(mock_func, **kwargs)()
+        except Exception as e:
+            error = e
+    return mock_func, sleeps, error
+
+
+@pytest.mark.parametrize("status", [400, 401, 403, 404, 405, 410, 413, 422])
+@pytest.mark.parametrize("make_error", [_HttpStatusError, _ResponseStatusError])
+def test_retry_does_not_retry_permanent_http_errors(status, make_error):
+    err = make_error(status)
+    mock_func, sleeps, error = _run_with_retry(err, max_retries=7)
+
+    assert error is err
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+@pytest.mark.parametrize("status", [408, 409, 429, 500, 502, 503, 504])
+@pytest.mark.parametrize("make_error", [_HttpStatusError, _ResponseStatusError])
+def test_retry_retries_transient_http_errors(status, make_error):
+    mock_func, sleeps, error = _run_with_retry(
+        [make_error(status), make_error(status), "result"], max_retries=7
+    )
+
+    assert error is None
+    assert mock_func.call_count == 3
+    assert len(sleeps) == 2
+
+
+@pytest.mark.parametrize(
+    "exc",
+    [
+        ConnectionError("connection reset"),
+        TimeoutError("timed out"),
+        OSError("network is unreachable"),
+        ValueError("custom embedding function failure"),
+        RuntimeError("boom"),
+        Exception("plain exception"),
+    ],
+)
+def test_retry_retries_errors_without_http_status(exc):
+    mock_func, sleeps, error = _run_with_retry([exc, exc, "result"], max_retries=7)
+
+    assert error is None
+    assert mock_func.call_count == 3
+    assert len(sleeps) == 2
+
+
+def test_retry_error_status_attributes_of_other_types_are_ignored():
+    # `code` is a string for openai errors ("model_not_found"), and arbitrary
+    # exceptions may carry unrelated numeric attributes. Neither is an HTTP status.
+    class WeirdError(Exception):
+        code = "model_not_found"
+        status_code = None
+
+    class ExitCodeError(Exception):
+        code = 2
+
+    for exc in (WeirdError("a"), ExitCodeError("b")):
+        mock_func, sleeps, error = _run_with_retry(exc, max_retries=2)
+        assert mock_func.call_count == 3
+        assert len(sleeps) == 2
+
+
+def test_retry_botocore_style_errors():
+    # botocore ClientError exposes `response["ResponseMetadata"]["HTTPStatusCode"]`.
+    # Bedrock reports throttling with a 400, which must stay retryable.
+    class ClientError(Exception):
+        def __init__(self, status, code):
+            super().__init__(code)
+            self.response = {
+                "ResponseMetadata": {"HTTPStatusCode": status},
+                "Error": {"Code": code},
+            }
+
+    throttled = ClientError(400, "ThrottlingException")
+    mock_func, sleeps, error = _run_with_retry([throttled, "result"], max_retries=3)
+    assert error is None
+    assert mock_func.call_count == 2
+
+    invalid = ClientError(400, "ValidationException")
+    mock_func, sleeps, error = _run_with_retry(invalid, max_retries=3)
+    assert error is invalid
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+def test_retry_authentication_error_by_name_not_retried():
+    class AuthenticationError(Exception):
+        pass
+
+    mock_func, sleeps, error = _run_with_retry(AuthenticationError("bad key"))
+    assert isinstance(error, AuthenticationError)
+    assert mock_func.call_count == 1
+    assert sleeps == []
+
+
+def test_retry_gives_up_after_max_retries_with_original_error_attached():
+    original = _HttpStatusError(503, "overloaded")
+    mock_func, sleeps, error = _run_with_retry(original, max_retries=3)
+
+    assert mock_func.call_count == 4
+    assert len(sleeps) == 3
+    assert "Maximum number of retries (3) exceeded" in str(error)
+    assert error.args[1] is original
+    assert error.__cause__ is original
+
+
+def test_retry_delay_is_capped_by_max_delay():
+    # Default settings (7 retries, 60s cap): the old implementation could sleep for
+    # up to ~47 minutes in a single retry and hours in total.
+    exc = _HttpStatusError(503)
+    for seed in range(50):
+        import random
+
+        random.seed(seed)
+        _, sleeps, _ = _run_with_retry(exc)
+        assert len(sleeps) == 7
+        assert max(sleeps) <= 60
+        assert sum(sleeps) <= 7 * 60
+
+    # Custom cap
+    _, sleeps, _ = _run_with_retry(exc, max_retries=10, max_delay=5)
+    assert len(sleeps) == 10
+    assert max(sleeps) <= 5
+
+
+def test_retry_delay_without_jitter_is_exact_exponential_then_capped():
+    _, sleeps, _ = _run_with_retry(
+        _HttpStatusError(500),
+        initial_delay=1,
+        exponential_base=2,
+        jitter=False,
+        max_retries=8,
+        max_delay=30,
+    )
+    assert sleeps == [2, 4, 8, 16, 30, 30, 30, 30]
+
+
+def test_retry_jitter_is_bounded():
+    # Each wait is within [delay / 2, delay], where delay grows by exponential_base
+    # per retry (no random factor is compounded across retries).
+    exc = _HttpStatusError(500)
+    for seed in range(50):
+        import random
+
+        random.seed(seed)
+        _, sleeps, _ = _run_with_retry(exc, max_retries=8, max_delay=100)
+        for i, slept in enumerate(sleeps):
+            delay = min(2 ** (i + 1), 100)
+            assert delay / 2 <= slept <= delay
+
+    # Extremes of the random source map to the ends of the range
+    with patch("random.random", return_value=0.0):
+        _, sleeps, _ = _run_with_retry(exc, max_retries=3)
+    assert sleeps == [1, 2, 4]
+    with patch("random.random", return_value=1.0):
+        _, sleeps, _ = _run_with_retry(exc, max_retries=3)
+    assert sleeps == [2, 4, 8]
+
+
+def test_retry_large_max_retries_does_not_overflow():
+    _, sleeps, _ = _run_with_retry(_HttpStatusError(503), max_retries=1500)
+    assert len(sleeps) == 1500
+    assert max(sleeps) <= 60
+
+
+def _register_failing_embedding(name, error):
+    """Register an embedding function that always raises `error`.
+
+    Returns the class and a list that records every call. The function is
+    re-created from table metadata, so the call count has to live outside of it.
+    """
+    calls = []
+
+    @register(name)
+    class FailingEmbeddingFunction(TextEmbeddingFunction):
+        def ndims(self):
+            return 4
+
+        def generate_embeddings(self, texts):
+            calls.append(len(texts))
+            raise error
+
+    return FailingEmbeddingFunction, calls
+
+
+@pytest.mark.parametrize("status", [400, 404, 422])
+def test_permanent_embedding_error_is_not_retried(tmp_path, status):
+    """A permanent provider error (e.g. a misspelled model name) must surface
+    immediately from both `table.add` and `table.search` instead of being retried
+    with minutes of backoff."""
+    error = _HttpStatusError(status, "The model `typo-embed` does not exist")
+    cls, calls = _register_failing_embedding(f"failing-embedding-{status}", error)
+    func = cls.create()  # default max_retries=7
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("t", schema=Schema)
+
+    with patch("time.sleep") as mock_sleep:
+        with pytest.raises(Exception, match="The model `typo-embed` does not exist"):
+            table.add([{"text": "hello"}])
+        assert len(calls) == 1
+
+        with pytest.raises(_HttpStatusError, match="does not exist"):
+            table.search("hello").limit(1).to_list()
+        assert len(calls) == 2
+
+    mock_sleep.assert_not_called()
+
+
+def test_transient_embedding_error_is_retried_in_table_add(tmp_path):
+    cls, calls = _register_failing_embedding(
+        "failing-embedding-503", _HttpStatusError(503)
+    )
+    func = cls.create(max_retries=2)
+
+    class Schema(LanceModel):
+        text: str = func.SourceField()
+        vector: Vector(func.ndims()) = func.VectorField()
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("t", schema=Schema)
+
+    with patch("time.sleep") as mock_sleep:
+        with pytest.raises(Exception, match="Maximum number of retries"):
+            table.add([{"text": "hello"}])
+
+    assert len(calls) == 3
+    assert mock_sleep.call_count == 2
+    assert all(call.args[0] <= 60 for call in mock_sleep.call_args_list)
+
+
+@patch("time.sleep")
+@pytest.mark.parametrize(
+    "error",
+    [
+        pytest.param("missing_key", id="missing-api-key"),
+        pytest.param(ImportError("No module named 'voyageai'"), id="missing-package"),
+    ],
+)
+def test_no_retry_on_deterministic_errors(mock_sleep, error):
+    from lancedb.embeddings.utils import (
+        api_key_not_found_help,
+        retry_with_exponential_backoff,
+    )
+
+    def func():
+        if error == "missing_key":
+            api_key_not_found_help("voyageai")
+        raise error
+
+    mock_func = MagicMock(side_effect=func)
+    wrapped_func = retry_with_exponential_backoff(mock_func, max_retries=3)
+
+    with pytest.raises((ValueError, ImportError)):
+        wrapped_func()
+
+    assert mock_func.call_count == 1
+    assert mock_sleep.call_count == 0
+
+
+@patch("time.sleep")
+def test_retry_on_wrapped_provider_error(mock_sleep):
+    """Providers such as Bedrock wrap throttling in a plain ValueError."""
+    from lancedb.embeddings.utils import retry_with_exponential_backoff
+
+    mock_func = MagicMock(
+        side_effect=[
+            ValueError("Error raised by boto3 client: ThrottlingException"),
+            [0.1],
+        ]
+    )
+    wrapped_func = retry_with_exponential_backoff(mock_func, max_retries=3)
+
+    assert wrapped_func() == [0.1]
+    assert mock_func.call_count == 2
+    assert mock_sleep.call_count == 1
+
+
+def test_openai_bad_request_partial_batch_recovery():
+    # Regression test for issue #1677
+
+    # Fake exception types, avoids requiring the optional openai/httpx deps
+    class FakeBadRequestError(Exception):
+        pass
+
+    class FakeAuthenticationError(Exception):
+        pass
+
+    fake_openai_module = MagicMock()
+    fake_openai_module.BadRequestError = FakeBadRequestError
+    fake_openai_module.AuthenticationError = FakeAuthenticationError
+
+    registry = get_registry()
+    model = registry.get("openai").create(name="text-embedding-ada-002")
+
+    def fake_create(input, model=None, **kwargs):
+        # Full batch always rejected
+        if len(input) > 1:
+            raise FakeBadRequestError("bad request")
+        if input[0] == "bad text":
+            raise FakeBadRequestError("bad request")
+        embedding = MagicMock()
+        embedding.embedding = [0.1, 0.2, 0.3]
+        result = MagicMock()
+        result.data = [embedding]
+        return result
+
+    mock_client = MagicMock()
+    mock_client.embeddings.create.side_effect = fake_create
+
+    with (
+        patch(
+            "lancedb.embeddings.openai.attempt_import_or_raise",
+            return_value=fake_openai_module,
+        ),
+        patch.object(type(model), "_openai_client", new_callable=lambda: mock_client),
+    ):
+        embeddings = model.generate_embeddings(["good text", "bad text"])
+
+    assert embeddings[0] == [0.1, 0.2, 0.3]
+    assert embeddings[1] is None
+
+
 def test_ollama_embeddings_pickle():
     """OllamaEmbeddings must pickle even after the cached client is created."""
     registry = get_registry()
@@ -663,3 +1057,28 @@ def test_jina_generate_image_input_dict_local_path(tmp_path):
         image_dict = JinaEmbeddings._generate_image_input_dict(image)
         assert "image" in image_dict
         assert isinstance(image_dict["image"], str) and len(image_dict["image"]) > 0
+
+
+def test_jina_embeddings_use_their_own_api_key(monkeypatch):
+    """Each JinaEmbeddings instance must send its own API key."""
+    pytest.importorskip("requests")
+    monkeypatch.delenv("JINA_API_KEY", raising=False)
+    registry = get_registry()
+    registry.set_var("jina_key_a", "key-a")
+    registry.set_var("jina_key_b", "key-b")
+    func_a = registry.get("jina").create(api_key="$var:jina_key_a")
+    func_b = registry.get("jina").create(api_key="$var:jina_key_b")
+
+    sent = []
+
+    def fake_post(session, url, json=None, **kwargs):
+        sent.append(session.headers["Authorization"])
+        response = MagicMock()
+        response.json.return_value = {"data": [{"index": 0, "embedding": [0.0]}]}
+        return response
+
+    with patch("requests.Session.post", fake_post):
+        func_a.compute_source_embeddings(["hello"])
+        func_b.compute_source_embeddings(["hello"])
+
+    assert sent == ["Bearer key-a", "Bearer key-b"]

python/python/tests/test_embeddings_trust.py

@@ -0,0 +1,291 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""
+Restoring an embedding function from table metadata must not let the table grant
+itself trust: remote model code and custom endpoints need approval from the
+process that opens the table.
+"""
+
+import json
+from unittest.mock import patch
+
+import lancedb
+import pyarrow as pa
+import pytest
+from lancedb.embeddings import (
+    EmbeddingFunctionConfig,
+    OllamaEmbeddings,
+    OpenAIEmbeddings,
+    SentenceTransformerEmbeddings,
+    UntrustedEmbeddingConfigError,
+    get_registry,
+)
+from lancedb.embeddings.base import UntrustedEmbeddingFunction
+
+
+@pytest.fixture(autouse=True)
+def fresh_trust(monkeypatch):
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_allowed_remote_code", set())
+    monkeypatch.setattr(registry, "_allowed_endpoints", set())
+
+
+def metadata_for(name: str, model: dict) -> dict:
+    entry = {
+        "name": name,
+        "model": model,
+        "source_column": "text",
+        "vector_column": "vector",
+    }
+    return {b"embedding_functions": json.dumps([entry]).encode()}
+
+
+def restore(name: str, model: dict):
+    return get_registry().parse_functions(metadata_for(name, model))["vector"].function
+
+
+def test_sentence_transformers_does_not_trust_remote_code_by_default():
+    assert SentenceTransformerEmbeddings.create().trust_remote_code is False
+
+
+@pytest.mark.parametrize("name", ["sentence-transformers", "huggingface", "colbert"])
+def test_restored_remote_code_requires_allowlist(name):
+    model = {"name": "attacker/model", "trust_remote_code": True}
+    func = restore(name, model)
+
+    with pytest.raises(UntrustedEmbeddingConfigError, match="attacker/model"):
+        func.compute_query_embeddings("hello")
+    with pytest.raises(UntrustedEmbeddingConfigError):
+        func.ndims()
+
+
+def test_blocked_function_does_not_construct_model():
+    # huggingface loads its model in __init__, so the check has to happen first.
+    with patch(
+        "lancedb.embeddings.transformers.attempt_import_or_raise"
+    ) as attempt_import:
+        restore("huggingface", {"name": "attacker/model", "trust_remote_code": True})
+    attempt_import.assert_not_called()
+
+
+def test_allowlisted_model_restores_with_remote_code():
+    get_registry().allow_remote_code("org/model")
+    func = restore(
+        "sentence-transformers", {"name": "org/model", "trust_remote_code": True}
+    )
+    assert isinstance(func, SentenceTransformerEmbeddings)
+    assert func.trust_remote_code is True
+
+
+def test_allowlist_does_not_cover_other_models():
+    get_registry().allow_remote_code("org/model")
+    func = restore(
+        "sentence-transformers", {"name": "org/other", "trust_remote_code": True}
+    )
+    with pytest.raises(UntrustedEmbeddingConfigError, match="org/other"):
+        func.compute_query_embeddings("hello")
+
+
+def test_pinned_revision_must_match():
+    get_registry().allow_remote_code("org/model", revision="abc123")
+
+    pinned = restore(
+        "sentence-transformers",
+        {"name": "org/model", "revision": "abc123", "trust_remote_code": True},
+    )
+    assert pinned.trust_remote_code is True
+
+    for model in [
+        {"name": "org/model", "revision": "def456", "trust_remote_code": True},
+        {"name": "org/model", "trust_remote_code": True},
+    ]:
+        func = restore("sentence-transformers", model)
+        with pytest.raises(UntrustedEmbeddingConfigError, match="org/model"):
+            func.compute_query_embeddings("hello")
+
+
+def test_restored_trust_flag_follows_allowlist_when_absent():
+    # Tables written before the default changed don't record trust_remote_code.
+    assert restore(
+        "sentence-transformers", {"name": "org/model"}
+    ).trust_remote_code is (False)
+    get_registry().allow_remote_code("org/model")
+    assert restore("sentence-transformers", {"name": "org/model"}).trust_remote_code
+
+
+def test_created_function_records_trust_flag():
+    # Otherwise a new table using the default would look like a pre-fix table,
+    # where a missing flag meant True.
+    conf = EmbeddingFunctionConfig(
+        source_column="text",
+        vector_column="vector",
+        function=SentenceTransformerEmbeddings.create(name="org/model"),
+    )
+    assert get_registry().function_to_metadata(conf)["model"] == {
+        "name": "org/model",
+        "trust_remote_code": False,
+    }
+
+    get_registry().allow_remote_code("org/model")
+    metadata = get_registry().get_table_metadata([conf])
+    restored = get_registry().parse_functions(metadata)["vector"].function
+    assert restored.trust_remote_code is False
+
+
+def test_huggingface_missing_flag_keeps_false_default():
+    # huggingface always defaulted to False, so approval must not flip it.
+    get_registry().allow_remote_code("org/model")
+    with patch("lancedb.embeddings.transformers.attempt_import_or_raise"):
+        func = restore("huggingface", {"name": "org/model"})
+    assert func.trust_remote_code is False
+
+
+def test_restored_explicit_false_stays_false():
+    get_registry().allow_remote_code("org/model")
+    func = restore(
+        "sentence-transformers", {"name": "org/model", "trust_remote_code": False}
+    )
+    assert func.trust_remote_code is False
+
+
+def test_create_trusts_its_own_config(tmp_path):
+    func = SentenceTransformerEmbeddings.create(
+        name="org/model", trust_remote_code=True
+    )
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema(
+        [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 4))]
+    )
+    db.create_table(
+        "t",
+        schema=schema,
+        embedding_functions=[
+            EmbeddingFunctionConfig(
+                source_column="text", vector_column="vector", function=func
+            )
+        ],
+    )
+
+    restored = db.open_table("t").embedding_functions["vector"].function
+    assert isinstance(restored, SentenceTransformerEmbeddings)
+    assert restored.trust_remote_code is True
+
+
+def test_create_trusts_only_the_revision_it_selected():
+    # Anyone can push a PR ref to a Hugging Face repo, so trusting a model's
+    # default branch must not extend to its other revisions.
+    SentenceTransformerEmbeddings.create(name="org/model", trust_remote_code=True)
+    func = restore(
+        "sentence-transformers",
+        {"name": "org/model", "revision": "refs/pr/123", "trust_remote_code": True},
+    )
+    assert isinstance(func, UntrustedEmbeddingFunction)
+
+    same = restore(
+        "sentence-transformers", {"name": "org/model", "trust_remote_code": True}
+    )
+    assert same.trust_remote_code is True
+
+
+def test_explicit_unpinned_approval_covers_any_revision():
+    get_registry().allow_remote_code("org/model")
+    func = restore(
+        "sentence-transformers",
+        {"name": "org/model", "revision": "abc123", "trust_remote_code": True},
+    )
+    assert func.trust_remote_code is True
+
+
+def test_missing_flag_does_not_inherit_unapproved_trust_default():
+    class LegacySentenceTransformer(SentenceTransformerEmbeddings):
+        trust_remote_code: bool = True
+
+    registry = get_registry()
+    registry.register("legacy-trust-default")(LegacySentenceTransformer)
+    try:
+        func = restore("legacy-trust-default", {"name": "attacker/model"})
+        assert not func.trust_remote_code
+    finally:
+        del registry._functions["legacy-trust-default"]
+
+
+def test_trusting_one_model_does_not_trust_a_swapped_name():
+    SentenceTransformerEmbeddings.create(name="org/model", trust_remote_code=True)
+    func = restore(
+        "sentence-transformers", {"name": "attacker/model", "trust_remote_code": True}
+    )
+    with pytest.raises(UntrustedEmbeddingConfigError):
+        func.compute_query_embeddings("hello")
+
+
+ATTACKER_URL = "https://attacker.example/v1"
+
+
+@pytest.mark.parametrize(
+    "name,model",
+    [
+        ("openai", {"base_url": ATTACKER_URL}),
+        ("ollama", {"host": ATTACKER_URL}),
+        ("ollama", {"ollama_client_kwargs": {"proxy": ATTACKER_URL}}),
+        ("watsonx", {"url": ATTACKER_URL}),
+    ],
+)
+def test_restored_custom_endpoint_requires_allowlist(name, model):
+    func = restore(name, model)
+    with pytest.raises(UntrustedEmbeddingConfigError, match="attacker.example"):
+        func.compute_query_embeddings("hello")
+
+    get_registry().allow_endpoint(ATTACKER_URL)
+    assert isinstance(restore(name, model), get_registry().get(name))
+
+
+def test_default_endpoints_need_no_allowlist():
+    assert isinstance(restore("openai", {}), OpenAIEmbeddings)
+    assert isinstance(
+        restore("ollama", {"host": "http://localhost:11434"}), OllamaEmbeddings
+    )
+
+
+def test_endpoint_allowlist_ignores_trailing_slash():
+    get_registry().allow_endpoint("http://my-host:11434/")
+    assert isinstance(
+        restore("ollama", {"host": "http://my-host:11434"}), OllamaEmbeddings
+    )
+
+
+def test_create_trusts_its_own_endpoint():
+    OpenAIEmbeddings.create(base_url="http://my-proxy/v1")
+    assert isinstance(
+        restore("openai", {"base_url": "http://my-proxy/v1"}), OpenAIEmbeddings
+    )
+
+
+def test_blocked_function_round_trips_metadata():
+    model = {"name": "attacker/model", "trust_remote_code": True}
+    func = restore("sentence-transformers", model)
+    conf = EmbeddingFunctionConfig(
+        source_column="text", vector_column="vector", function=func
+    )
+    entry = get_registry().function_to_metadata(conf)
+    assert entry["name"] == "sentence-transformers"
+    assert entry["model"] == model
+
+
+def test_table_with_blocked_function_still_opens(tmp_path):
+    db = lancedb.connect(tmp_path)
+    schema = pa.schema(
+        [pa.field("text", pa.string()), pa.field("vector", pa.list_(pa.float32(), 2))],
+        metadata=metadata_for(
+            "sentence-transformers",
+            {"name": "attacker/model", "trust_remote_code": True},
+        ),
+    )
+    db.create_table("t", schema=schema)
+    tbl = db.open_table("t")
+
+    tbl.add([{"text": "hi", "vector": [1.0, 2.0]}])
+    assert tbl.search([1.0, 2.0]).limit(1).to_list()[0]["text"] == "hi"
+
+    with pytest.raises(UntrustedEmbeddingConfigError, match="allow_remote_code"):
+        tbl.search("hi").limit(1).to_list()

python/python/tests/test_expr.py

@@ -80,6 +80,67 @@ class TestExprConstruction:
         e = lit(d)
         assert isinstance(e, Expr)
 
+    @pytest.mark.parametrize(
+        "value, decimal_type",
+        [
+            ("1E-7", pa.decimal128(12, 7)),
+            ("-1.23E-7", pa.decimal128(12, 9)),
+            ("1E+3", pa.decimal128(12, 0)),
+            ("-1.23E+5", pa.decimal128(12, 0)),
+            ("0E-7", pa.decimal128(12, 7)),
+            ("-0E+3", pa.decimal128(12, 0)),
+            ("1E-38", pa.decimal128(38, 38)),
+            ("1E+37", pa.decimal128(38, 0)),
+            ("1.234567890123456789012345678901234567E+37", pa.decimal128(38, 0)),
+            ("0E+128", pa.decimal128(1, 0)),
+        ],
+    )
+    def test_lit_decimal_exponent_query(self, tmp_path, value, decimal_type):
+        expected = Decimal(value)
+        table = lancedb.connect(tmp_path).create_table(
+            "decimals",
+            pa.table(
+                {
+                    "id": [1, 2],
+                    # Arrow's Python ingestion rejects 0E+128 even though its
+                    # numeric value is representable. Store canonical zero to
+                    # exercise the expression literal independently of ingestion.
+                    "price": pa.array(
+                        [Decimal(0) if expected.is_zero() else expected, None],
+                        type=decimal_type,
+                    ),
+                }
+            ),
+        )
+        predicate = col("price") == lit(expected)
+        result = table.search().where(predicate).to_arrow()
+        assert result["id"].to_pylist() == [1]
+        assert result["price"].to_pylist() == [expected]
+        # Exercise the SQL serializer and parser as well as the typed query path.
+        sql_result = table.search().where(predicate.to_sql()).to_arrow()
+        assert sql_result["id"].to_pylist() == [1]
+        assert sql_result["price"].to_pylist() == [expected]
+
+    @pytest.mark.parametrize(
+        "value",
+        [
+            "NaN",
+            "sNaN",
+            "Infinity",
+            "-Infinity",
+            "1E-39",
+            "1E+38",
+            "1E+129",
+            "1E+1000000000",
+            "0E+129",
+            "12345678901234567890123456789012345678E+1",
+            "9" * 39,
+        ],
+    )
+    def test_lit_decimal_unrepresentable_raises(self, value):
+        with pytest.raises(ValueError):
+            lit(Decimal(value))
+
 
 class TestExprOperators:
     def test_eq_operator(self):

python/python/tests/test_first_class_function_slice1.py

@@ -398,8 +398,21 @@ class _FunctionDeclarationInner:
     def __init__(self):
         self.calls = []
 
-    async def add_function_columns(self, application_json, output_name):
-        self.calls.append((json.loads(application_json), output_name))
+    async def add_function_columns(
+        self,
+        application_json,
+        output_name,
+        auto_refresh_enabled,
+        auto_refresh_interval_seconds,
+    ):
+        self.calls.append(
+            (
+                json.loads(application_json),
+                output_name,
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
+            )
+        )
         return "declared"
 
 
@@ -420,14 +433,18 @@ async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomica
     )
     assert result == "declared"
     assert inner.calls[-1][1] == "features"
+    assert inner.calls[-1][2:] == (None, None)
 
     bare = application._copy(update={"columns": {}}).rename(
         columns={"normalized_text": "search_text"}
     )
-    result = await table.add_columns(bare)
+    result = await table.add_columns(
+        bare, auto_refresh_enabled=False, auto_refresh_interval_seconds=300
+    )
     assert result == "declared"
     assert inner.calls[-1][1] is None
     assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
+    assert inner.calls[-1][2:] == (False, 300)
 
 
 @pytest.mark.asyncio

python/python/tests/test_first_class_function_slice2.py

@@ -622,7 +622,6 @@ def test_canonical_arrow_type_prefers_the_compact_grammar():
     for outside in [
         pa.timestamp("us"),
         pa.decimal128(10, 2),
-        pa.large_binary(),
         pa.binary(4),
         pa.duration("s"),
         pa.list_(pa.float32(), 0),
@@ -1280,6 +1279,28 @@ def test_explicit_large_utf8_schemas_use_the_canonical_function_name():
     }
 
 
+def test_large_binary_schemas_use_the_canonical_function_name():
+    payload = pa.field("data", pa.large_binary(), nullable=True)
+
+    @udf(
+        input_schema=pa.schema([payload]),
+        output_schema=pa.field("result", pa.large_binary(), nullable=False),
+    )
+    def preserve(data):
+        return data
+
+    signature = preserve.registration_request.signature
+    assert signature.inputs[0].arrow_type == "large_binary"
+    assert signature.output.arrow_type == "large_binary"
+    cells = pa.large_list(pa.field("item", pa.struct([payload]), nullable=True))
+    assert json.loads(_canonical_arrow_type(cells))["fields"][0]["type"] == {
+        "type": "struct",
+        "fields": [
+            {"name": "data", "nullable": True, "type": {"type": "large_binary"}}
+        ],
+    }
+
+
 def test_nested_struct_output_uses_canonical_exact_json():
     token = pa.struct(
         [

python/python/tests/test_header_provider.py

@@ -114,6 +114,30 @@ class TestOAuthProvider:
             "x-lancedb-credential-type": "oidc",
         }
 
+    def test_zero_lifetime_token_is_refreshed(self):
+        """A token that expires immediately must not be cached indefinitely."""
+        call_count = 0
+
+        def fetcher():
+            nonlocal call_count
+            call_count += 1
+            return {
+                "access_token": f"token{call_count}",
+                "expires_in": 0,
+            }
+
+        provider = OAuthProvider(fetcher, refresh_buffer_seconds=0)
+
+        assert provider.get_headers() == {
+            "Authorization": "Bearer token1",
+            "x-lancedb-credential-type": "oidc",
+        }
+        assert provider.get_headers() == {
+            "Authorization": "Bearer token2",
+            "x-lancedb-credential-type": "oidc",
+        }
+        assert call_count == 2
+
     def test_missing_access_token(self):
         """Test error handling when access_token is missing."""
 

python/python/tests/test_huggingface.py

@@ -129,3 +129,106 @@ def test_generator(tmp_path: Path):
 
     assert len(tbl) == 2
     assert tbl.schema == ds.features.arrow_schema
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("operation", ["create", "add"])
+@pytest.mark.parametrize(
+    "selection",
+    ["normal", "contiguous", "selected", "duplicates", "filtered", "shuffled", "empty"],
+)
+async def test_write_selected_hf_dataset(tmp_path, asynchronous, operation, selection):
+    source = datasets.Dataset.from_dict({"id": list(range(5)), "text": list("abcde")})
+    data = {
+        "normal": source,
+        "contiguous": source.select(range(1, 4)),
+        "selected": source.select([4, 1]),
+        "duplicates": source.select([2, 2, 0]),
+        "filtered": source.filter(lambda row: row["id"] % 2 == 1),
+        "shuffled": source.shuffle(seed=42),
+        "empty": source.select([]),
+    }[selection]
+    expected = data.to_list()
+    if asynchronous:
+        db = await lancedb.connect_async(tmp_path)
+        if operation == "create":
+            table = await db.create_table("selected", data)
+        else:
+            table = await db.create_table("selected", schema=data.features.arrow_schema)
+            await table.add(data)
+        actual = await table.to_arrow()
+    else:
+        db = lancedb.connect(tmp_path)
+        if operation == "create":
+            table = db.create_table("selected", data)
+        else:
+            table = db.create_table("selected", schema=data.features.arrow_schema)
+            table.add(data)
+        actual = table.to_arrow()
+    assert actual.to_pylist() == expected
+    assert actual.schema.equals(data.features.arrow_schema, check_metadata=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("operation", ["create", "add"])
+async def test_write_selected_hf_splits(tmp_path, asynchronous, operation):
+    source = datasets.Dataset.from_dict({"id": list(range(5)), "text": list("abcde")})
+    data = datasets.DatasetDict(
+        {
+            "train": source.select([4, 1]),
+            "test": source.select([3, 0, 2]),
+            "empty": source.select([]),
+        }
+    )
+    expected = [
+        dict(row, split=split)
+        for split, dataset in data.items()
+        for row in dataset.to_list()
+    ]
+    schema = source.features.arrow_schema.append(pa.field("split", pa.string()))
+    if asynchronous:
+        db = await lancedb.connect_async(tmp_path)
+        if operation == "create":
+            table = await db.create_table("splits", data)
+        else:
+            table = await db.create_table("splits", schema=schema)
+            await table.add(data)
+        actual = await table.to_arrow()
+    else:
+        db = lancedb.connect(tmp_path)
+        if operation == "create":
+            table = db.create_table("splits", data)
+        else:
+            table = db.create_table("splits", schema=schema)
+            table.add(data)
+        actual = table.to_arrow()
+    assert actual.to_pylist() == expected
+    assert actual.schema.equals(schema, check_metadata=True)
+
+
+@pytest.mark.parametrize("format_type", [None, "numpy", "pandas", "arrow"])
+@pytest.mark.parametrize("empty", [False, True])
+def test_selected_hf_reader_is_rescannable(format_type, empty):
+    from lancedb.scannable import _register_optional_converters, to_scannable
+
+    source = datasets.Dataset.from_dict(
+        {"id": list(range(2105)), "text": [str(i) for i in range(2105)]}
+    )
+    indices = [] if empty else list(reversed(range(2105))) + [1, 1]
+    data = source.select(indices).with_format(format_type, columns=["id"])
+    original_format = data.format
+    expected = source.select(indices).to_list()
+    _register_optional_converters()
+    scannable = to_scannable(data)
+    assert scannable.rescannable
+    assert scannable.num_rows == len(expected)
+    for _ in range(2):
+        batches = list(scannable.reader())
+        if not empty:
+            assert all(batch.num_rows <= 1000 for batch in batches)
+        actual = pa.Table.from_batches(batches, schema=scannable.schema)
+        assert actual.to_pylist() == expected
+        assert actual.schema.equals(data.features.arrow_schema, check_metadata=True)
+    assert data.format == original_format

python/python/tests/test_hybrid_query.py

@@ -536,3 +536,15 @@ def test_normalize_scores():
             assert pc.equal(result, expected), (
                 f"Expected {expected} but got {result} for invert={invert}"
             )
+
+
+def test_hybrid_query_rejects_query_with_vector(sync_table: Table):
+    # Pins the full message: the space between "hybrid search." and "But" is
+    # load-bearing (see the _validate_query literal concatenation).
+    with pytest.raises(ValueError) as excinfo:
+        sync_table.search("dog", query_type="hybrid").vector([0.5, -0.5]).to_arrow()
+    assert str(excinfo.value) == (
+        "You can either provide a string query in search() method "
+        "or set `vector()` and `text()` explicitly for hybrid search. "
+        "But not both."
+    )

python/python/tests/test_index.py

@@ -430,6 +430,28 @@ async def test_create_ivfrq_index(some_table: AsyncTable):
     assert indices[0].name == "vector_idx"
 
 
+@pytest.mark.asyncio
+async def test_read_ivfrq_index_created_with_lance(db_async, some_table: AsyncTable):
+    pytest.importorskip("lance")
+    dataset = await some_table.to_lance()
+    dataset.create_index(
+        "vector",
+        index_type="IVF_RQ",
+        metric="cosine",
+        num_partitions=1,
+    )
+
+    # Open a fresh LanceDB handle so the index is discovered from disk.
+    table = await db_async.open_table("some_table")
+    indices = await table.list_indices()
+    assert len(indices) == 1
+    assert indices[0].index_type == "IvfRq"
+    assert indices[0].columns == ["vector"]
+
+    results = await table.query().nearest_to(list(range(DIM))).limit(8).to_arrow()
+    assert len(results) == 8
+
+
 @pytest.mark.asyncio
 async def test_create_hnswpq_index(some_table: AsyncTable):
     await some_table.create_index("vector", config=HnswPq(num_partitions=10))

python/python/tests/test_materialized_views.py

@@ -129,8 +129,21 @@ def test_sync_remote_create_uses_public_async_connection():
             where=None,
             limit=None,
             with_no_data=False,
+            auto_refresh_enabled=None,
+            auto_refresh_interval_seconds=None,
         ):
-            calls.append((name, source, select, where, limit, with_no_data))
+            calls.append(
+                (
+                    name,
+                    source,
+                    select,
+                    where,
+                    limit,
+                    with_no_data,
+                    auto_refresh_enabled,
+                    auto_refresh_interval_seconds,
+                )
+            )
             return StubAsyncMaterializedView()
 
         async def drop_materialized_view(self, name, *, namespace_path=None):
@@ -148,9 +161,11 @@ def test_sync_remote_create_uses_public_async_connection():
         where="age >= 18",
         limit=10,
         with_no_data=True,
+        auto_refresh_enabled=True,
+        auto_refresh_interval_seconds=0.5,
     )
     assert view.name == "adults"
-    assert calls == [("adults", "people", ["name"], "age >= 18", 10, True)]
+    assert calls == [("adults", "people", ["name"], "age >= 18", 10, True, True, 0.5)]
 
     db.drop_materialized_view("adults", namespace_path=["analytics"])
     assert calls[-1] == ("drop", "adults", ["analytics"])
@@ -550,3 +565,89 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
     assert sorted([first, *resumed]) == ["a_view", "z_view"]
     assert resumed.page_token() is None
     assert list(resumed) == []
+
+
+@pytest.mark.parametrize(
+    "options",
+    [
+        {},
+        {"auto_refresh_enabled": True},
+        {"auto_refresh_enabled": False, "auto_refresh_interval_seconds": 0.5},
+    ],
+)
+def test_remote_create_forwards_refresh_options_without_interval_default(options):
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = lancedb.connect("db://dev", api_key="fake", host_override=host)
+        job = db.create_materialized_view_async("adults", "people", **options)
+        assert job.id == "mv-create-123"
+    assert requests == [
+        (
+            "/v1/materialized_view/adults/create",
+            {
+                "query": 'SELECT * FROM "people"',
+                **options,
+            },
+        )
+    ]
+
+
+@pytest.mark.asyncio
+async def test_async_remote_create_keeps_interval_unspecified():
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = await lancedb.connect_async("db://dev", api_key="fake", host_override=host)
+        job = await db.create_materialized_view_async(
+            "adults",
+            "people",
+            auto_refresh_enabled=True,
+        )
+        assert job.id == "mv-create-123"
+    assert requests[0][1] == {
+        "query": 'SELECT * FROM "people"',
+        "auto_refresh_enabled": True,
+    }
+
+
+@pytest.mark.parametrize(
+    "options",
+    [
+        {"auto_refresh_enabled": True},
+        {"auto_refresh_interval_seconds": 300},
+    ],
+)
+def test_embedded_create_refuses_automatic_refresh_before_creating_view(
+    tmp_path, options
+):
+    db = make_db(tmp_path)
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        db.create_materialized_view("adults", "people", **options)
+    assert "adults" not in db.table_names()
+
+
+@pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
+def test_remote_create_rejects_invalid_interval_without_request(seconds):
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = lancedb.connect("db://dev", api_key="fake", host_override=host)
+        with pytest.raises(ValueError, match="positive, finite"):
+            db.create_materialized_view_async(
+                "adults", "people", auto_refresh_interval_seconds=seconds
+            )
+    assert requests == []
+
+
+def test_namespace_forwards_automatic_refresh_request(tmp_path):
+    db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
+
+
+@pytest.mark.asyncio
+async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
+    db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        await db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
+
+
+def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False)
+    assert view.table.count_rows() == 3

python/python/tests/test_merge_insert_lsm.py

@@ -1,14 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-"""Tests for the MemWAL LSM ``merge_insert`` dispatch."""
+"""Tests for the RTE LSM ``merge_insert`` dispatch."""
 
 from datetime import timedelta
 
 import lancedb
 import pyarrow as pa
 import pytest
-from lancedb._lancedb import LsmWriteSpec
+from lancedb._lancedb import RTEWriteSpec
 from lancedb.index import FTS, IvfPq
 
 SCHEMA = pa.schema(
@@ -49,12 +49,12 @@ def _region_reader(rows):
 
 
 def _bucket_table(tmp_path):
-    """A table with ``id`` as the primary key and a single-bucket LSM spec."""
+    """A table with ``id`` as the primary key and a single-bucket RTE spec."""
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
     # num_buckets = 1: every row routes to the single bucket.
-    table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1))
+    table.enable_rte(RTEWriteSpec.bucket("id", 1))
     return table
 
 
@@ -67,7 +67,7 @@ def test_lsm_merge_insert_bucket(tmp_path):
         .when_not_matched_insert_all()
         .execute(_reader([3, 4, 5]))
     )
-    # LSM path: rows go to the MemWAL, so only num_rows is populated.
+    # LSM path: rows go to the RTE, so only num_rows is populated.
     assert result.num_rows == 3
     assert result.version == 0
     assert result.num_inserted_rows == 0
@@ -78,7 +78,7 @@ def test_lsm_merge_insert_unsharded(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
     result = (
         table.merge_insert("id")
         .when_matched_update_all()
@@ -92,7 +92,7 @@ def test_lsm_merge_insert_identity(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _region_reader([(1, "us"), (2, "us")]))
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.identity("region"))
+    table.enable_rte(RTEWriteSpec.identity("region"))
     # All rows share one identity value, so they route to one shard.
     result = (
         table.merge_insert([])
@@ -103,31 +103,31 @@ def test_lsm_merge_insert_identity(tmp_path):
     assert result.num_rows == 2
 
 
-def test_lsm_merge_insert_use_lsm_false(tmp_path):
+def test_lsm_merge_insert_use_rte_false(tmp_path):
     table = _bucket_table(tmp_path)  # rows id = 1, 2, 3
-    # use_lsm(False) opts out: the standard path runs and commits even with a spec.
+    # use_rte(False) opts out: the standard path runs and commits even with a spec.
     result = (
         table.merge_insert("id")
         .when_not_matched_insert_all()
-        .use_lsm(False)
+        .use_rte(False)
         .execute(_reader([3, 4, 5]))
     )
     assert result.num_inserted_rows == 2
     assert table.count_rows() == 5
 
 
-def test_lsm_merge_insert_use_lsm_true_without_spec_errors(tmp_path):
-    # A table with a primary key but no LSM write spec installed.
+def test_lsm_merge_insert_use_rte_true_without_spec_errors(tmp_path):
+    # A table with a primary key but no RTE write spec installed.
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
-    # use_lsm(True) demands MemWAL routing; without a spec it errors.
-    with pytest.raises(Exception, match="use_lsm"):
+    # use_rte(True) demands RTE routing; without a spec it errors.
+    with pytest.raises(Exception, match="use_rte"):
         (
             table.merge_insert("id")
             .when_matched_update_all()
             .when_not_matched_insert_all()
-            .use_lsm(True)
+            .use_rte(True)
             .execute(_reader([3, 4, 5]))
         )
 
@@ -145,7 +145,7 @@ def test_lsm_merge_insert_validate_single_shard_off(tmp_path):
 
 
 def test_lsm_merge_insert_no_spec_uses_standard_path(tmp_path):
-    # A table with a primary key but no LSM write spec installed.
+    # A table with a primary key but no RTE write spec installed.
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
@@ -186,7 +186,7 @@ def test_lsm_close_writers(tmp_path):
         .when_not_matched_insert_all()
         .execute(_reader([7, 8]))
     )
-    table.close_lsm_writers()
+    table.close_rte_writers()
     # The writer reopens lazily on the next merge_insert.
     result = (
         table.merge_insert([])
@@ -204,14 +204,14 @@ async def test_async_lsm_merge_insert(tmp_path):
     )
     table = await db.create_table("t", _reader([1, 2, 3]))
     await table.set_unenforced_primary_key("id")
-    await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1))
+    await table.enable_rte(RTEWriteSpec.bucket("id", 1))
 
     builder = (
         table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all()
     )
     result = await builder.execute(_reader([3, 4, 5]))
     assert result.num_rows == 3
-    await table.close_lsm_writers()
+    await table.close_rte_writers()
 
 
 def _lsm_upsert(table, ids):
@@ -228,7 +228,7 @@ def test_lsm_read_sees_active_memtable(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))  # base ids 1,2,3
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
 
     _lsm_upsert(table, [4, 5])  # active memtable only, not committed to base
 
@@ -236,8 +236,8 @@ def test_lsm_read_sees_active_memtable(tmp_path):
     lsm = table.search().to_arrow()
     assert sorted(lsm["id"].to_pylist()) == [1, 2, 3, 4, 5]
 
-    # use_lsm(False) bypasses the MemWAL and reads the base table only.
-    base_only = table.search().use_lsm(False).to_arrow()
+    # use_rte(False) bypasses the RTE and reads the base table only.
+    base_only = table.search().use_rte(False).to_arrow()
     assert sorted(base_only["id"].to_pylist()) == [1, 2, 3]
 
 
@@ -245,7 +245,7 @@ def test_lsm_read_dedup_newest_wins(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))  # id 2 -> value 1
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
 
     _lsm_upsert(table, [2, 3, 4])  # ids 2,3,4 -> values 0,1,2
 
@@ -258,31 +258,31 @@ def test_lsm_read_dedup_newest_wins(tmp_path):
 def test_lsm_read_without_spec_reads_base(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
-    table.set_unenforced_primary_key("id")  # no LSM write spec
+    table.set_unenforced_primary_key("id")  # no RTE write spec
 
-    # No spec: default read and use_lsm(False) both read the base table, no error.
+    # No spec: default read and use_rte(False) both read the base table, no error.
     assert sorted(table.search().to_arrow()["id"].to_pylist()) == [1, 2, 3]
-    assert sorted(table.search().use_lsm(False).to_arrow()["id"].to_pylist()) == [
+    assert sorted(table.search().use_rte(False).to_arrow()["id"].to_pylist()) == [
         1,
         2,
         3,
     ]
 
 
-def test_lsm_read_unsupported_shape_errors_without_use_lsm_false(tmp_path):
+def test_lsm_read_unsupported_shape_errors_without_use_rte_false(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
     _lsm_upsert(table, [4])
 
-    # with_row_id is unsupported by the LSM scanner; on a MemWAL table the default
+    # with_row_id is unsupported by the LSM scanner; on an RTE table the default
     # (auto-routed) read hard-errors instead of silently reading a stale base.
     with pytest.raises(Exception):
         table.search().with_row_id(True).to_arrow()
 
-    # use_lsm(False) is the escape hatch: it reads the base table only.
-    base = table.search().with_row_id(True).use_lsm(False).to_arrow()
+    # use_rte(False) is the escape hatch: it reads the base table only.
+    base = table.search().with_row_id(True).use_rte(False).to_arrow()
     assert sorted(base["id"].to_pylist()) == [1, 2, 3]
 
 
@@ -293,7 +293,7 @@ async def test_async_lsm_read(tmp_path):
     )
     table = await db.create_table("t", _reader([1, 2, 3]))
     await table.set_unenforced_primary_key("id")
-    await table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    await table.enable_rte(RTEWriteSpec.unsharded())
 
     builder = (
         table.merge_insert([]).when_matched_update_all().when_not_matched_insert_all()
@@ -331,7 +331,7 @@ def _vector_reader(rows):
 def _vector_table(tmp_path):
     """Base table whose vector column is indexed so its rows are visible to the LSM
     vector scanner (the base arm uses ``fast_search`` — indexed data only), plus an
-    unsharded LSM spec that maintains that index for the memtable.
+    unsharded RTE spec that maintains that index for the memtable.
 
     Rows 1,2 are category ``a``, row 3 is ``b``, and 4..60 are filler ``c`` that
     give the tiny IVF index enough data to train.
@@ -353,9 +353,7 @@ def _vector_table(tmp_path):
         "vector", config=IvfPq(num_partitions=1, num_sub_vectors=2, num_bits=4)
     )
     index_name = table.list_indices()[0].name
-    table.set_lsm_write_spec(
-        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
-    )
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
     return table
 
 
@@ -379,9 +377,9 @@ def test_lsm_read_vector_sees_memtable(tmp_path):
     assert {1, 2, 3} <= ids  # indexed base rows
     assert 1000 in ids  # in-flight memtable row
 
-    # use_lsm(False) bypasses the MemWAL, so the in-flight row is not visible.
+    # use_rte(False) bypasses the RTE, so the in-flight row is not visible.
     base_ids = set(
-        table.search(query).use_lsm(False).limit(100).to_arrow()["id"].to_pylist()
+        table.search(query).use_rte(False).limit(100).to_arrow()["id"].to_pylist()
     )
     assert {1, 2, 3} <= base_ids
     assert 1000 not in base_ids
@@ -453,9 +451,7 @@ def test_lsm_read_fts_sees_memtable(tmp_path):
     # Native FTS index (tantivy is not compatible with the LSM memtable index).
     table.create_index("text", config=FTS())
     index_name = table.list_indices()[0].name
-    table.set_lsm_write_spec(
-        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
-    )
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
 
     # in-flight doc 4 lands in the memtable's maintained FTS index.
     (
@@ -498,8 +494,8 @@ def test_lsm_read_vector_unsupported_knobs_error(tmp_path):
     with pytest.raises(Exception, match="use_index"):
         table.search(query).bypass_vector_index().to_arrow()
 
-    # use_lsm(False) is the escape hatch: the base-only standard path honors them.
-    base = table.search(query).distance_range(0.0, 100.0).use_lsm(False).to_arrow()
+    # use_rte(False) is the escape hatch: the base-only standard path honors them.
+    base = table.search(query).distance_range(0.0, 100.0).use_rte(False).to_arrow()
     assert 1000 not in set(base["id"].to_pylist())
 
 
@@ -544,7 +540,7 @@ def test_lsm_read_fts_unmaintained_index_errors(tmp_path):
     table.create_index("text", config=FTS())
     # No maintained indexes: the active memtable FTS arm cannot serve un-compacted
     # docs, so the search would silently omit them — reject instead.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
     with pytest.raises(Exception, match="maintained"):
         table.search("fox", query_type="fts", fts_columns="text").to_arrow()
 
@@ -553,7 +549,7 @@ def test_lsm_read_time_travel_errors(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
     pinned = table.version
     table.add(_reader([4, 5]))  # standard add commits a newer version
     table.checkout(pinned)  # detached head at the historical version
@@ -562,8 +558,8 @@ def test_lsm_read_time_travel_errors(tmp_path):
     # historical version is rejected.
     with pytest.raises(Exception, match="time-travel"):
         table.search().to_arrow()
-    # use_lsm(False) reads the base table at the pinned version.
-    base = table.search().use_lsm(False).to_arrow()
+    # use_rte(False) reads the base table at the pinned version.
+    base = table.search().use_rte(False).to_arrow()
     assert sorted(base["id"].to_pylist()) == [1, 2, 3]
 
 
@@ -571,14 +567,14 @@ def test_lsm_read_take_row_ids_errors(tmp_path):
     db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
     table = db.create_table("t", _reader([1, 2, 3]))
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
+    table.enable_rte(RTEWriteSpec.unsharded())
     _lsm_upsert(table, [4])
     # take-by-row-id auto-routes through the LSM scanner, which has no stable _rowid,
     # so it hard-errors instead of failing with an opaque column-not-found error.
     with pytest.raises(Exception, match="row id"):
         table.take_row_ids([0, 1]).to_arrow()
-    # use_lsm(False) is the escape hatch: it reads the base table.
-    base = table.take_row_ids([0, 1]).use_lsm(False).to_arrow()
+    # use_rte(False) is the escape hatch: it reads the base table.
+    base = table.take_row_ids([0, 1]).use_rte(False).to_arrow()
     assert base.num_rows == 2
 
 
@@ -588,9 +584,7 @@ def test_lsm_read_fts_postfilter_errors(tmp_path):
     table.set_unenforced_primary_key("id")
     table.create_index("text", config=FTS())
     index_name = table.list_indices()[0].name
-    table.set_lsm_write_spec(
-        LsmWriteSpec.unsharded().with_maintained_indexes([index_name])
-    )
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([index_name]))
     # The LSM scanner always prefilters; postfilter on FTS changes result semantics,
     # so it hard-errors (previously only the vector arm rejected it).
     with pytest.raises(Exception, match="postfilter"):
@@ -608,9 +602,7 @@ def test_lsm_read_fts_multiple_same_type_indexes_errors(tmp_path):
     table.set_unenforced_primary_key("id")
     table.create_index("text", config=FTS(), name="fts_a")
     table.create_index("text", config=FTS(), name="fts_b", replace=False)
-    table.set_lsm_write_spec(
-        LsmWriteSpec.unsharded().with_maintained_indexes(["fts_a"])
-    )
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes(["fts_a"]))
     # Two FTS indexes on the column: the base planner's chosen index is ambiguous, so
     # the scanner can't pick a catch-up watermark and rejects rather than risk
     # dropping rows the actually-used index has not caught up to.
@@ -631,7 +623,7 @@ def test_lsm_read_vector_unmaintained_index_errors(tmp_path):
     )
     # Spec with NO maintained indexes: the base vector index's catch-up is untracked,
     # so the scanner rejects rather than risk dropping compacted-but-unindexed rows.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
     with pytest.raises(Exception, match="maintained"):
         table.search([1.0] * VECTOR_DIM).to_arrow()
 
@@ -644,7 +636,7 @@ def test_lsm_read_fts_optimized_index_not_rejected(tmp_path):
     table.add(_fts_reader([(i, "lazy fox") for i in range(6, 11)]))
     table.optimize()  # may split the FTS index into multiple physical segments
     name = table.list_indices()[0].name
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([name]))
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([name]))
     # Multiple physical segments of one logical index must not be miscounted as
     # multiple indexes and rejected.
     ids = set(

python/python/tests/test_namespace.py

@@ -9,6 +9,7 @@ import importlib
 import multiprocessing as mp
 import sys
 from datetime import timedelta
+import warnings
 import http.server
 import json
 import threading
@@ -29,6 +30,125 @@ PUSHDOWN_DATA = pa.table(
 )
 
 
+class TestNamespaceCreateTableOptions:
+    @pytest.mark.parametrize("version", ["legacy", "stable"])
+    @pytest.mark.parametrize("empty", [False, True])
+    def test_deprecated_storage_version(self, tmp_path, version, empty):
+        db = lancedb.connect_namespace("dir", {"root": str(tmp_path / "namespace")})
+        data = None if empty else [{"id": 1}]
+        schema = pa.schema([("id", pa.int64())])
+        other_version = "stable" if version == "legacy" else "legacy"
+        options = {"new_table_data_storage_version": other_version}
+        control = lancedb.connect(tmp_path / "control").create_table(
+            "test",
+            data=data,
+            schema=schema,
+            storage_options={"new_table_data_storage_version": version},
+        )
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = db.create_table(
+                "test",
+                data=data,
+                schema=schema,
+                storage_options=options,
+                data_storage_version=version,
+            )
+
+        assert (
+            table.to_lance().data_storage_version
+            == control.to_lance().data_storage_version
+        )
+        assert table.to_arrow().to_pylist() == ([] if empty else data)
+        assert options == {"new_table_data_storage_version": other_version}
+        assert any("data_storage_version" in str(w.message) for w in caught)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("version", ["legacy", "stable"])
+    @pytest.mark.parametrize("empty", [False, True])
+    async def test_deprecated_storage_version_async(self, tmp_path, version, empty):
+        root = str(tmp_path / "namespace")
+        db = lancedb.connect_namespace_async("dir", {"root": root})
+        data = None if empty else [{"id": 1}]
+        schema = pa.schema([("id", pa.int64())])
+        other_version = "stable" if version == "legacy" else "legacy"
+        options = {"new_table_data_storage_version": other_version}
+        control = lancedb.connect(tmp_path / "control").create_table(
+            "test",
+            data=data,
+            schema=schema,
+            storage_options={"new_table_data_storage_version": version},
+        )
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = await db.create_table(
+                "test",
+                data=data,
+                schema=schema,
+                storage_options=options,
+                data_storage_version=version,
+            )
+
+        reopened = lancedb.connect_namespace("dir", {"root": root}).open_table("test")
+        assert (
+            reopened.to_lance().data_storage_version
+            == control.to_lance().data_storage_version
+        )
+        assert (await table.to_arrow()).to_pylist() == ([] if empty else data)
+        assert options == {"new_table_data_storage_version": other_version}
+        assert any("data_storage_version" in str(w.message) for w in caught)
+
+    @pytest.mark.parametrize("enabled", [False, True])
+    @pytest.mark.parametrize("empty", [False, True])
+    def test_deprecated_manifest_paths(self, tmp_path, enabled, empty):
+        db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
+        data = None if empty else [{"id": 1}]
+        options = {"new_table_enable_v2_manifest_paths": str(not enabled).lower()}
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = db.create_table(
+                "test",
+                data=data,
+                schema=pa.schema([("id", pa.int64())]),
+                storage_options=options,
+                enable_v2_manifest_paths=enabled,
+            )
+
+        assert table.uses_v2_manifest_paths() == enabled
+        assert table.to_arrow().to_pylist() == ([] if empty else data)
+        assert db.open_table("test").uses_v2_manifest_paths() == enabled
+        assert options == {
+            "new_table_enable_v2_manifest_paths": str(not enabled).lower()
+        }
+        assert any("enable_v2_manifest_paths" in str(w.message) for w in caught)
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("enabled", [False, True])
+    @pytest.mark.parametrize("empty", [False, True])
+    async def test_deprecated_manifest_paths_async(self, tmp_path, enabled, empty):
+        db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
+        data = None if empty else [{"id": 1}]
+        options = {"new_table_enable_v2_manifest_paths": str(not enabled).lower()}
+        with warnings.catch_warnings(record=True) as caught:
+            warnings.simplefilter("always", DeprecationWarning)
+            table = await db.create_table(
+                "test",
+                data=data,
+                schema=pa.schema([("id", pa.int64())]),
+                storage_options=options,
+                enable_v2_manifest_paths=enabled,
+            )
+
+        assert await table.uses_v2_manifest_paths() == enabled
+        assert (await table.to_arrow()).to_pylist() == ([] if empty else data)
+        reopened = await db.open_table("test")
+        assert await reopened.uses_v2_manifest_paths() == enabled
+        assert options == {
+            "new_table_enable_v2_manifest_paths": str(not enabled).lower()
+        }
+        assert any("enable_v2_manifest_paths" in str(w.message) for w in caught)
+
+
 def _ipc_file(table: pa.Table = PUSHDOWN_DATA) -> bytes:
     sink = pa.BufferOutputStream()
     with pa.ipc.new_file(sink, table.schema) as writer:

python/python/tests/test_ollama_embeddings.py

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+
+import httpx
+import numpy as np
+import pyarrow as pa
+import pytest
+from pydantic import ValidationError
+
+from lancedb.embeddings import get_registry
+
+
+@pytest.fixture
+def ollama_embeddings():
+    pytest.importorskip("ollama")
+    requests = []
+
+    def respond(request):
+        body = json.loads(request.content)
+        requests.append(body)
+        if body["model"] == "missing-model":
+            return httpx.Response(404, json={"error": "model not found"})
+        return httpx.Response(
+            200,
+            json={
+                "embeddings": [
+                    [float(index), float(len(text))]
+                    for index, text in enumerate(body["input"])
+                ]
+            },
+        )
+
+    function = (
+        get_registry()
+        .get("ollama")
+        .create(
+            name="nomic-embed-text",
+            options={"num_ctx": 4096},
+            keep_alive="10m",
+            ollama_client_kwargs={
+                "transport": httpx.MockTransport(respond),
+                "trust_env": False,
+            },
+        )
+    )
+    yield function, requests
+    function._ollama_client._client.close()
+
+
+@pytest.mark.parametrize(
+    "texts",
+    [
+        pytest.param(["hello", "世界"], id="list"),
+        pytest.param(np.array(["hello", "世界"]), id="numpy-unicode"),
+        pytest.param(np.array(["hello", "世界"], dtype=object), id="numpy-object"),
+        pytest.param(
+            np.array([b"hello", "世界".encode("utf-8")]), id="numpy-utf8-bytes"
+        ),
+        pytest.param([b"hello", "世界".encode("utf-8")], id="list-utf8-bytes"),
+        pytest.param(np.array(["hello", "unused", "世界"])[::2], id="numpy-slice"),
+        pytest.param(pa.array(["hello", "世界"]), id="arrow"),
+        pytest.param(pa.chunked_array([["hello"], ["世界"]]), id="chunked-arrow"),
+    ],
+)
+def test_ollama_source_embeddings_inputs(ollama_embeddings, texts):
+    function, requests = ollama_embeddings
+
+    assert function.compute_source_embeddings(texts) == [[0.0, 5.0], [1.0, 2.0]]
+    assert requests == [
+        {
+            "model": "nomic-embed-text",
+            "input": ["hello", "世界"],
+            "options": {"num_ctx": 4096},
+            "keep_alive": "10m",
+        }
+    ]
+
+
+@pytest.mark.parametrize(
+    "texts",
+    [
+        pytest.param(np.array([["hello"], ["world"]]), id="numpy-two-dimensional"),
+        pytest.param([["hello"], ["world"]], id="list-two-dimensional"),
+        pytest.param(np.array([b"\xff"]), id="numpy-invalid-utf8"),
+        pytest.param([b"\xff"], id="list-invalid-utf8"),
+    ],
+)
+def test_ollama_source_embeddings_rejects_invalid_input_locally(
+    ollama_embeddings, texts
+):
+    function, requests = ollama_embeddings
+
+    with pytest.raises(ValidationError, match="valid string"):
+        function.compute_source_embeddings(texts)
+
+    assert requests == []
+
+
+def test_ollama_generate_embeddings_empty_array(ollama_embeddings):
+    function, requests = ollama_embeddings
+
+    assert function.generate_embeddings(np.array([], dtype=str)) == []
+    assert len(requests) == 1
+    assert requests[0]["input"] == []
+
+
+def test_ollama_query_embeddings(ollama_embeddings):
+    function, requests = ollama_embeddings
+
+    assert function.compute_query_embeddings("hello") == [[0.0, 5.0]]
+    assert len(requests) == 1
+    assert requests[0]["input"] == ["hello"]
+
+
+def test_ollama_source_embeddings_preserves_api_error(ollama_embeddings):
+    ollama = pytest.importorskip("ollama")
+    function, requests = ollama_embeddings
+    function.name = "missing-model"
+
+    with pytest.raises(ollama.ResponseError, match="model not found") as error:
+        function.compute_source_embeddings(np.array(["hello"]))
+
+    assert error.value.status_code == 404
+    assert len(requests) == 1

python/python/tests/test_openai_reranker.py

@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import json
+from types import SimpleNamespace
+
+import pyarrow as pa
+import pytest
+
+from lancedb.rerankers import OpenaiReranker
+
+
+def reranker_replying(documents):
+    """Return an OpenaiReranker whose chat client replies with these documents."""
+    reply = json.dumps({"documents": documents})
+
+    def create(**kwargs):
+        message = SimpleNamespace(content=reply)
+        return SimpleNamespace(choices=[SimpleNamespace(message=message)])
+
+    reranker = OpenaiReranker(api_key="test-key")
+    reranker.__dict__["_client"] = SimpleNamespace(
+        chat=SimpleNamespace(completions=SimpleNamespace(create=create))
+    )
+    return reranker
+
+
+def vector_results():
+    return pa.table(
+        {
+            "id": [1, 2, 3],
+            "text": ["dogs bark", "cats purr", "birds sing"],
+            "_distance": [0.1, 0.2, 0.3],
+        }
+    )
+
+
+def test_scores_stay_with_their_rows_when_the_model_reorders():
+    reranker = reranker_replying(
+        [
+            {"content": "cats purr", "relevance_score": 0.9},
+            {"content": "birds sing", "relevance_score": 0.5},
+            {"content": "dogs bark", "relevance_score": 0.1},
+        ]
+    )
+
+    result = reranker.rerank_vector("which animal purrs?", vector_results())
+
+    assert result.select(["id", "text"]).to_pylist() == [
+        {"id": 2, "text": "cats purr"},
+        {"id": 3, "text": "birds sing"},
+        {"id": 1, "text": "dogs bark"},
+    ]
+    assert result["_relevance_score"].to_pylist() == pytest.approx([0.9, 0.5, 0.1])
+
+
+def test_a_row_the_model_leaves_out_raises():
+    reranker = reranker_replying(
+        [
+            {"content": "cats purr", "relevance_score": 0.9},
+            {"content": "dogs bark", "relevance_score": 0.1},
+        ]
+    )
+
+    with pytest.raises(ValueError, match="1 of 3"):
+        reranker.rerank_vector("which animal purrs?", vector_results())

python/python/tests/test_permutation.py

@@ -115,6 +115,35 @@ def test_split_random_fixed(mem_db):
         assert split_ids.count(split_id) == 25
 
 
+def test_permutation_over_table_with_schema_metadata(tmp_path):
+    """Schema-level metadata on the base table must not break the build.
+
+    Hugging Face datasets carry keys like `huggingface` or `source-column-name-map`.
+    """
+    schema = pa.schema(
+        [pa.field("x", pa.int64())],
+        # `split_names` collides with a permutation config key; the config must win.
+        metadata={b"huggingface": b"{}", b"split_names": b'["base"]'},
+    )
+    tbl = connect(tmp_path).create_table(
+        "t", pa.table({"x": list(range(100))}, schema=schema)
+    )
+    permutation_tbl = (
+        permutation_builder(tbl)
+        .split_random(fixed=2, seed=1, split_names=["train", "test"])
+        .execute()
+    )
+
+    permutations = Permutations(tbl, permutation_tbl)
+    assert permutations.split_names == ["train", "test"]
+    train = Permutation.from_tables(tbl, permutation_tbl, "train")
+    test = Permutation.from_tables(tbl, permutation_tbl, "test")
+    assert train.num_rows + test.num_rows == 100
+    train_x = [row["x"] for batch in train.iter(16) for row in batch]
+    test_x = [row["x"] for batch in test.iter(16) for row in batch]
+    assert sorted(train_x + test_x) == list(range(100))
+
+
 def test_split_random_with_seed(mem_db):
     """Test that seeded random splits are reproducible."""
     tbl = mem_db.create_table("test_table", pa.table({"x": range(50), "y": range(50)}))
@@ -132,6 +161,76 @@ def test_split_random_with_seed(mem_db):
     assert data1["split_id"] == data2["split_id"]
 
 
+def test_bootstrap_sample_fixed(mem_db):
+    """Output row count is num_samples, not the input row count, and splits divide
+    it evenly."""
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
+    permutation_tbl = (
+        permutation_builder(tbl).bootstrap_sample(1000, fixed=4, seed=42).execute()
+    )
+
+    assert permutation_tbl.count_rows() == 1000
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    split_ids = data["split_id"]
+    assert set(split_ids) == {0, 1, 2, 3}
+    for split_id in range(4):
+        assert split_ids.count(split_id) == 250
+
+    # Every sampled row_id must still be one of the 20 real rows.
+    assert set(data["row_id"]).issubset(set(range(20)))
+
+
+def test_bootstrap_sample_draws_with_replacement(mem_db):
+    """Sampling more rows than exist must produce repeats."""
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(5)}))
+    permutation_tbl = (
+        permutation_builder(tbl).bootstrap_sample(500, fixed=1, seed=1).execute()
+    )
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    assert len(data["row_id"]) == 500
+    # Only 5 distinct rows exist, so with 500 draws every one of them must repeat.
+    for row_id in range(5):
+        assert data["row_id"].count(row_id) > 1
+
+
+def test_bootstrap_sample_with_seed_is_reproducible(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(50)}))
+
+    perm1 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
+    perm2 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
+
+    data1 = perm1.search(None).to_arrow().to_pydict()
+    data2 = perm2.search(None).to_arrow().to_pydict()
+    assert data1["row_id"] == data2["row_id"]
+    assert data1["split_id"] == data2["split_id"]
+
+
+def test_bootstrap_sample_respects_filter(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
+    permutation_tbl = (
+        permutation_builder(tbl)
+        .filter("x >= 15")
+        .bootstrap_sample(100, fixed=1, seed=7)
+        .execute()
+    )
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    assert len(data["row_id"]) == 100
+    assert set(data["row_id"]).issubset(set(range(15, 20)))
+
+
+def test_bootstrap_sample_requires_exactly_one_size_arg(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(10)}))
+    with pytest.raises(ValueError):
+        permutation_builder(tbl).bootstrap_sample(10).execute()
+    with pytest.raises(ValueError):
+        permutation_builder(tbl).bootstrap_sample(
+            10, ratios=[0.5, 0.5], fixed=2
+        ).execute()
+
+
 def test_split_hash(mem_db):
     """Test hash-based splitting."""
     tbl = mem_db.create_table(

python/python/tests/test_pyarrow.py

@@ -2,6 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import pyarrow as pa
+import pyarrow.compute as pc
 
 import lancedb
 from lancedb.integrations.pyarrow import PyarrowDatasetAdapter
@@ -48,3 +49,19 @@ def test_dataset_adapter(tmp_path):
     assert adapter.head(10).num_rows == 0
 
     assert adapter.scanner().projected_schema == pa.schema({"x": pa.int64()})
+
+
+def test_dataset_adapter_expression_filter(tmp_path):
+    data = pa.table({"x": [1, 2, 3, 4], "y": [5, 6, 7, 8]})
+    conn = lancedb.connect(tmp_path)
+    tbl = conn.create_table("test", data)
+
+    adapter = PyarrowDatasetAdapter(tbl)
+    filt = pc.field("x") > 2
+
+    assert adapter.count_rows(filt) == 2
+    assert adapter.to_table(filter=filt) == data.slice(2, 2)
+    assert adapter.head(1, filter=filt) == data.slice(2, 1)
+    # The filter can reference a column that is not projected
+    assert adapter.to_table(columns=["y"], filter=filt) == pa.table({"y": [7, 8]})
+    assert adapter.scanner(filter=filt).count_rows() == 2

python/python/tests/test_pydantic.py

@@ -415,6 +415,67 @@ def test_nullable_vector():
     assert schema == pa.schema([pa.field("vec", pa.list_(pa.float32(), 16), True)])
 
 
+@pytest.mark.parametrize("vector_type", [Vector, MultiVector])
+@pytest.mark.parametrize(
+    "value_type, values",
+    [
+        (pa.int64(), [-(2**53 + 1), 2**53 + 3]),
+        (pa.uint64(), [2**53 + 1, 2**64 - 1]),
+        (pa.int8(), [-128, 127]),
+        (pa.uint8(), [0, 255]),
+        (pa.float32(), [1.25, 2.5]),
+        (pa.float64(), [1.25, 2.5]),
+    ],
+)
+def test_typed_vector_round_trip(tmp_path, vector_type, value_type, values):
+    import lancedb
+
+    annotation = vector_type(2, value_type=value_type)
+    expected = [values, values] if vector_type is MultiVector else values
+
+    class Row(LanceModel):
+        id: int
+        payload: annotation
+
+    row = Row(id=1, payload=expected)
+    table = lancedb.connect(tmp_path).create_table("typed_vectors", [row], schema=Row)
+    table.add([Row(id=2, payload=expected)])
+    assert table.schema == Row.to_arrow_schema()
+    assert table.search().to_arrow().to_pylist() == [
+        {"id": 1, "payload": expected},
+        {"id": 2, "payload": expected},
+    ]
+    assert json.loads(row.model_dump_json())["payload"] == expected
+
+
+@pytest.mark.parametrize("vector_type", [Vector, MultiVector])
+@pytest.mark.parametrize("nullable", [False, True])
+def test_integer_vector_validation(vector_type, nullable):
+    annotation = vector_type(2, value_type=pa.int64(), nullable=nullable)
+
+    class Row(LanceModel):
+        payload: annotation
+
+    def shape(values):
+        return [values] if vector_type is MultiVector else values
+
+    assert Row.to_arrow_schema().field("payload").nullable is nullable
+    assert Row(payload=shape([1.0, -2.0])).model_dump()["payload"] == shape([1, -2])
+    with pytest.raises(pydantic.ValidationError, match="valid integer"):
+        Row(payload=shape([1.5, 2]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=shape([1]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=shape([1, 2, 3]))
+    with pytest.raises(pydantic.ValidationError):
+        Row(payload=None)
+
+    class OptionalRow(LanceModel):
+        payload: Optional[annotation] = None
+
+    assert OptionalRow(payload=None).payload is None
+
+
 def test_bare_vector_raises_clear_error():
     namespace = {
         "__name__": "test_model_without_pyarrow",
@@ -733,3 +794,52 @@ def test_enum_types():
     assert schema.field("priority").type == pa.int64()
     assert schema.field("opt_status").type == pa.dictionary(pa.int32(), pa.utf8())
     assert schema.field("opt_status").nullable
+
+
+def test_plain_enum_values_are_written(mem_db):
+    """Members of an Enum without a str/int mixin are written as their values."""
+
+    class Color(Enum):
+        RED = "red"
+        BLUE = "blue"
+
+    class Priority(Enum):
+        LOW = 1
+        HIGH = 2
+
+    class Tag(LanceModel):
+        color: Color
+
+    class TestModel(LanceModel):
+        color: Color
+        priority: Priority
+        opt_color: Optional[Color] = None
+        colors: List[Color]
+        tag: Tag
+        vector: Vector(2)
+
+    rows = [
+        TestModel(
+            color=Color.RED,
+            priority=Priority.HIGH,
+            opt_color=Color.BLUE,
+            colors=[Color.BLUE, Color.RED],
+            tag=Tag(color=Color.BLUE),
+            vector=[1.0, 2.0],
+        ),
+        TestModel(
+            color=Color.BLUE,
+            priority=Priority.LOW,
+            colors=[],
+            tag=Tag(color=Color.RED),
+            vector=[3.0, 4.0],
+        ),
+    ]
+    tbl = mem_db.create_table("test", schema=TestModel)
+    tbl.add(rows)
+
+    assert tbl.to_arrow().select(["color", "priority", "opt_color"]).to_pylist() == [
+        {"color": "red", "priority": 2, "opt_color": "blue"},
+        {"color": "blue", "priority": 1, "opt_color": None},
+    ]
+    assert tbl.search().to_pydantic(TestModel) == rows

python/python/tests/test_pydantic_excluded_fields.py

@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import lancedb
+import pytest
+from pydantic import BaseModel, Field
+from lancedb.pydantic import LanceModel
+
+
+class Details(BaseModel):
+    label: str
+    transient: str = Field(default="nested default", exclude=True)
+
+
+class Row(LanceModel):
+    id: int
+    details: Details
+    transient: str = Field(default="local default", exclude=True)
+    generated: list[str] = Field(default_factory=lambda: ["local"], exclude=True)
+    retained: str = Field(default="persisted", exclude=False)
+
+
+def rows():
+    return [
+        Row(
+            id=i,
+            details=Details(label=str(i), transient="private"),
+            transient="private",
+            generated=["private"],
+        )
+        for i in (1, 2)
+    ]
+
+
+def assert_schema(schema):
+    assert schema.names == ["id", "details", "retained"]
+    assert [field.name for field in schema.field("details").type] == ["label"]
+    assert Row.field_names() == ["id", "details", "transient", "generated", "retained"]
+
+
+def assert_restored(restored):
+    assert sorted(row.id for row in restored) == [1, 2]
+    for row in restored:
+        assert row.transient == "local default"
+        assert row.generated == ["local"]
+        assert row.details.transient == "nested default"
+        assert row.details.label == str(row.id)
+        assert row.retained == "persisted"
+
+
+@pytest.mark.parametrize("explicit_schema", [False, True])
+def test_excluded_fields_round_trip(tmp_path, explicit_schema):
+    data = rows()
+    kwargs = {"schema": Row} if explicit_schema else {}
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("rows", data=data[:1], **kwargs)
+    table.add(data[1:])
+    assert_schema(table.schema)
+    reopened = lancedb.connect(tmp_path).open_table("rows")
+    assert_restored(reopened.search().to_pydantic(Row))
+    assert data[0].transient == "private"
+    assert data[0].generated == ["private"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("explicit_schema", [False, True])
+async def test_excluded_fields_round_trip_async(tmp_path, explicit_schema):
+    data = rows()
+    kwargs = {"schema": Row} if explicit_schema else {}
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("rows", data=data[:1], **kwargs)
+    await table.add(data[1:])
+    assert_schema(await table.schema())
+    reopened_db = await lancedb.connect_async(tmp_path)
+    reopened = await reopened_db.open_table("rows")
+    assert_restored(await reopened.query().to_pydantic(Row))
+    assert data[0].details.transient == "private"
+
+
+@pytest.mark.parametrize("excluded", ["transient", "source", "vector"])
+def test_excluded_fields_keep_embedding_metadata_consistent(
+    tmp_path, monkeypatch, excluded
+):
+    from lancedb.embeddings.base import TextEmbeddingFunction
+    from lancedb.embeddings.registry import get_registry
+    from lancedb.pydantic import Vector
+
+    calls = []
+
+    class LocalEmbedding(TextEmbeddingFunction):
+        def ndims(self):
+            return 2
+
+        def generate_embeddings(self, texts):
+            calls.extend(texts)
+            return [[1.0, 0.0] for _ in texts]
+
+    registry = get_registry()
+    monkeypatch.setattr(registry, "_functions", registry._functions.copy())
+    registry.register("test-excluded-fields")(LocalEmbedding)
+    function = LocalEmbedding.create(max_retries=0)
+
+    class EmbeddedRow(LanceModel):
+        text: str = function.SourceField(default="alpha", exclude=excluded == "source")
+        vector: Vector(2) = function.VectorField(
+            default_factory=lambda: [1.0, 0.0], exclude=excluded == "vector"
+        )
+        transient: str = Field(default="local", exclude=True)
+
+    configs = EmbeddedRow.parse_embedding_functions()
+    assert len(configs) == (1 if excluded == "transient" else 0)
+    db = lancedb.connect(tmp_path)
+    data = [{"text": "alpha"}] if excluded == "transient" else [EmbeddedRow()]
+    table = db.create_table("embeddings", schema=EmbeddedRow, data=data)
+    assert "transient" not in table.schema.names
+    assert ("text" in table.schema.names) == (excluded != "source")
+    assert ("vector" in table.schema.names) == (excluded != "vector")
+    reopened = lancedb.connect(tmp_path).open_table("embeddings")
+    assert len(reopened.embedding_functions) == len(configs)
+    restored = reopened.search().to_pydantic(EmbeddedRow)
+    assert restored[0].text == "alpha"
+    assert restored[0].vector == [1.0, 0.0]
+    assert restored[0].transient == "local"
+    if excluded == "transient":
+        assert calls == ["alpha"]
+        assert reopened.search("alpha").limit(1).to_list()[0]["text"] == "alpha"
+    else:
+        assert calls == []

python/python/tests/test_query.py

@@ -111,6 +111,26 @@ async def _create_blob_v2_query_table_async(db, name):
     return table
 
 
+def _nested_blob_v2_query_schema():
+    return pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("vector", pa.list_(pa.float32(), list_size=2)),
+            pa.field(
+                "info",
+                pa.struct([pa.field("label", pa.string()), lancedb.blob("blob")]),
+            ),
+        ]
+    )
+
+
+def _nested_blob_v2_query_rows():
+    return [
+        {"id": 1, "vector": [1.0, 0.0], "info": {"label": "one", "blob": b"one"}},
+        {"id": 2, "vector": [2.0, 0.0], "info": {"label": "two", "blob": b"two"}},
+    ]
+
+
 def _assert_lazy_blob(value, expected: bytes):
     assert hasattr(value, "readall")
     assert value.readall() == expected
@@ -503,7 +523,7 @@ def test_vector_query_to_pandas_blob_mode_requires_native_path(tmp_db):
         storage_options=LEGACY_BLOB_STORAGE_OPTIONS,
     )
 
-    with pytest.raises(RuntimeError, match="Lance native pandas conversion"):
+    with pytest.raises(RuntimeError, match="no fetchable blob v2 source"):
         table.search([1.0, 0.0]).select(["blob", "vector"]).limit(1).to_pandas(
             blob_mode="lazy"
         )
@@ -535,6 +555,193 @@ def test_vector_query_to_pandas_legacy_blob_descriptions_and_bytes(tmp_db):
     ]
 
 
+@pytest.mark.parametrize("blob_mode", ["default", "lazy", "bytes", "descriptions"])
+@pytest.mark.parametrize("with_row_id", [None, True, False])
+def test_vector_query_to_pandas_blob_v2_modes(tmp_db, blob_mode, with_row_id):
+    pytest.importorskip("lance")
+    table = _create_blob_v2_query_table(tmp_db, f"test_vector_blob_v2_{blob_mode}")
+
+    query = table.search([1.0, 0.0]).select(["id", "blob", "vector"]).limit(1)
+    if with_row_id is not None:
+        query = query.with_row_id(with_row_id)
+    df = (
+        query.to_pandas()
+        if blob_mode == "default"
+        else query.to_pandas(blob_mode=blob_mode)
+    )
+
+    assert df["id"].tolist() == [1]
+    assert ("_rowid" in df.columns) == (with_row_id is True)
+    if blob_mode == "bytes":
+        assert df["blob"].tolist() == [b"one"]
+    else:
+        descriptor = df["blob"].iloc[0]
+        assert descriptor != b"one"
+        assert not hasattr(descriptor, "readall")
+        assert "_lance_row_id" not in descriptor
+
+
+@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"])
+def test_blob_v2_non_scan_query_shapes_to_pandas(tmp_db, blob_mode):
+    pytest.importorskip("lance")
+    table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_query_shapes")
+    table.create_index("tag", config=FTS())
+
+    queries = [
+        table.search("keep", query_type="fts").limit(2),
+        table.search(query_type="hybrid").vector([1.0, 0.0]).text("keep").limit(2),
+        table.take_offsets([1, 2]).select(["id", "blob"]),
+    ]
+    for query in queries:
+        df = query.to_pandas(blob_mode=blob_mode)
+        assert len(df) == 2
+        if blob_mode == "bytes":
+            assert df["blob"].tolist() == [
+                {1: b"one", 2: b"two", 3: b"three", 4: b"four"}[row_id]
+                for row_id in df["id"]
+            ]
+        else:
+            assert all("_lance_row_id" not in blob for blob in df["blob"])
+        assert "_rowid" not in df.columns
+
+
+@pytest.mark.parametrize("rename_parent", [False, True])
+def test_vector_query_to_pandas_nested_blob_bytes(tmp_db, rename_parent):
+    pytest.importorskip("lance")
+    table = tmp_db.create_table(
+        f"test_vector_nested_blob_{rename_parent}",
+        schema=_nested_blob_v2_query_schema(),
+    )
+    table.add(_nested_blob_v2_query_rows())
+
+    projection = {"id": "id", "renamed": "info"} if rename_parent else ["id", "info"]
+    df = (
+        table.search([1.0, 0.0])
+        .select(projection)
+        .limit(2)
+        .to_pandas(blob_mode="bytes")
+    )
+
+    assert df["id"].tolist() == [1, 2]
+    parent = "renamed" if rename_parent else "info"
+    assert df[parent].tolist() == [
+        {"label": "one", "blob": b"one"},
+        {"label": "two", "blob": b"two"},
+    ]
+    for blob_mode in ("lazy", "descriptions"):
+        descriptors = (
+            table.search([1.0, 0.0])
+            .select(projection)
+            .limit(1)
+            .to_pandas(blob_mode=blob_mode)
+        )
+        assert "_lance_row_id" not in descriptors[parent].iloc[0]["blob"]
+
+
+@pytest.mark.asyncio
+async def test_async_vector_query_to_pandas_nested_blob_bytes(tmp_db_async):
+    pytest.importorskip("lance")
+    table = await tmp_db_async.create_table(
+        "test_async_vector_nested_blob",
+        schema=_nested_blob_v2_query_schema(),
+    )
+    await table.add(_nested_blob_v2_query_rows())
+
+    df = await (
+        table.vector_search([1.0, 0.0])
+        .select(["id", "info"])
+        .limit(2)
+        .to_pandas(blob_mode="bytes")
+    )
+
+    assert df["id"].tolist() == [1, 2]
+    assert df["info"].tolist() == [
+        {"label": "one", "blob": b"one"},
+        {"label": "two", "blob": b"two"},
+    ]
+
+
+def test_blob_v2_vector_projection_and_plain_scan_options(tmp_db, monkeypatch):
+    pytest.importorskip("lance")
+    table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_pandas_options")
+
+    df = (
+        table.search([1.0, 0.0])
+        .select({"payload": "blob", "id": "id"})
+        .limit(1)
+        .to_pandas(blob_mode="bytes")
+    )
+    assert df["payload"].tolist() == [b"one"]
+
+    query = table.search().where("id = 1")
+
+    def fail_to_arrow(*args, **kwargs):
+        raise AssertionError("the native scanner should handle bytes with flatten")
+
+    monkeypatch.setattr(query, "to_arrow", fail_to_arrow)
+    assert query.to_pandas(blob_mode="bytes", flatten=True)["blob"].tolist() == [b"one"]
+
+    flattened_lazy = table.search().where("id = 1").to_pandas(flatten=True)
+    assert flattened_lazy["id"].tolist() == [1]
+    assert not any("_lance_row_id" in col for col in flattened_lazy.columns)
+
+    timed = (
+        table.search()
+        .where("id = 1")
+        .to_pandas(blob_mode="bytes", timeout=timedelta(seconds=10))
+    )
+    assert timed["blob"].tolist() == [b"one"]
+
+    timed_lazy = table.search().where("id = 1").to_pandas(timeout=timedelta(seconds=10))
+    assert timed_lazy["id"].tolist() == [1]
+
+
+def test_blob_v2_pandas_preserves_fetch_error(tmp_db, monkeypatch):
+    pytest.importorskip("lance")
+    table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_fetch_error")
+
+    def missing_blob(*args, **kwargs):
+        raise FileNotFoundError("missing obj.bin")
+
+    monkeypatch.setattr(table, "fetch_blobs", missing_blob)
+    with pytest.raises(FileNotFoundError, match="missing obj.bin"):
+        table.search().to_pandas(blob_mode="bytes")
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("blob_mode", ["lazy", "bytes", "descriptions"])
+@pytest.mark.parametrize("with_row_id", [False, True])
+async def test_async_vector_query_to_pandas_blob_v2_modes(
+    tmp_db_async, blob_mode, with_row_id
+):
+    pytest.importorskip("lance")
+    table = await _create_blob_v2_query_table_async(
+        tmp_db_async, f"test_async_vector_blob_v2_{blob_mode}"
+    )
+
+    query = table.vector_search([1.0, 0.0]).limit(1)
+    if with_row_id:
+        query = query.with_row_id()
+    df = await query.to_pandas(blob_mode=blob_mode)
+    assert df["id"].tolist() == [1]
+    assert ("_rowid" in df.columns) == with_row_id
+    if blob_mode == "bytes":
+        assert df["blob"].tolist() == [b"one"]
+    else:
+        assert "_lance_row_id" not in df["blob"].iloc[0]
+
+
+@pytest.mark.asyncio
+async def test_async_take_query_to_pandas_blob_v2_bytes(tmp_db_async):
+    pytest.importorskip("lance")
+    table = await _create_blob_v2_query_table_async(
+        tmp_db_async, "test_async_take_blob_v2"
+    )
+
+    df = await table.take_offsets([1, 2]).to_pandas(blob_mode="bytes")
+    assert set(df["blob"]) == {b"two", b"three"}
+
+
 def test_order_by_plain_query(mem_db):
     table = mem_db.create_table(
         "test_order_by",
@@ -626,6 +833,28 @@ def test_with_row_id(table: lancedb.table.Table):
     assert rs["_rowid"].to_pylist() == [0, 1]
 
 
+def test_filter_preserves_deeply_nested_struct_fields(tmp_db):
+    # Regression test for https://github.com/lancedb/lancedb/issues/2217.
+    data = [
+        {
+            "item": "test",
+            "sub_obj": {
+                "second_sub_obj": [
+                    {
+                        "third_sub_obj": {"int_val": 1},
+                        "str_val": "test",
+                    }
+                ]
+            },
+        }
+    ]
+    table = tmp_db.create_table("deeply_nested_struct", data=data)
+
+    result = table.search().where("item = 'test'").limit(1).to_arrow()
+
+    assert result.to_pylist() == data
+
+
 def test_blob_v2_query_omits_auto_row_id(tmp_db):
     table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_omits_auto_rowid")
 

python/python/tests/test_read_consistency_interval.py

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from datetime import timedelta
+
+import lancedb
+import pytest
+from lance_namespace import connect as connect_namespace_client
+from lancedb import _lancedb
+from lancedb.namespace import LanceNamespaceDBConnection
+
+
+@pytest.mark.parametrize("seconds", [-1.0, -0.000001])
+@pytest.mark.parametrize("kind", ["sync", "namespace", "namespace_async", "client"])
+def test_negative_read_consistency_interval(tmp_path, seconds, kind):
+    interval = timedelta(seconds=seconds)
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        if kind == "sync":
+            lancedb.connect(tmp_path, read_consistency_interval=interval)
+        elif kind == "client":
+            client = connect_namespace_client("dir", {"root": str(tmp_path)})
+            LanceNamespaceDBConnection(client, read_consistency_interval=interval)
+        else:
+            connect = (
+                lancedb.connect_namespace
+                if kind == "namespace"
+                else lancedb.connect_namespace_async
+            )
+            connect("dir", {"root": str(tmp_path)}, read_consistency_interval=interval)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("seconds", [-1.0, -0.000001])
+async def test_negative_async_read_consistency_interval(tmp_path, seconds):
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        await lancedb.connect_async(
+            tmp_path, read_consistency_interval=timedelta(seconds=seconds)
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("seconds", [float("nan"), float("inf"), -float("inf"), 1e30])
+@pytest.mark.parametrize("kind", ["connect", "namespace", "client"])
+async def test_binding_rejects_invalid_read_consistency_interval(
+    tmp_path, seconds, kind
+):
+    with pytest.raises(ValueError, match="read_consistency_interval"):
+        if kind == "connect":
+            await _lancedb.connect(str(tmp_path), read_consistency_interval=seconds)
+        elif kind == "namespace":
+            _lancedb.connect_namespace(
+                "dir", {"root": str(tmp_path)}, read_consistency_interval=seconds
+            )
+        else:
+            client = connect_namespace_client("dir", {"root": str(tmp_path)})
+            _lancedb.connect_namespace_client(client, read_consistency_interval=seconds)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "interval", [None, timedelta(0), timedelta(microseconds=1), timedelta(seconds=5)]
+)
+async def test_valid_read_consistency_intervals(tmp_path, interval):
+    db = lancedb.connect(tmp_path / "sync", read_consistency_interval=interval)
+    assert db.read_consistency_interval == interval
+    db.close()
+
+    db = await lancedb.connect_async(
+        tmp_path / "async", read_consistency_interval=interval
+    )
+    assert await db.get_read_consistency_interval() == interval
+    db.close()
+
+    db = lancedb.connect_namespace(
+        "dir", {"root": str(tmp_path / "namespace")}, read_consistency_interval=interval
+    )
+    assert db.read_consistency_interval == interval
+    db.close()
+
+    client = connect_namespace_client("dir", {"root": str(tmp_path / "client")})
+    db = LanceNamespaceDBConnection(client, read_consistency_interval=interval)
+    assert db.read_consistency_interval == interval
+    db.close()

python/python/tests/test_remote_db.py

@@ -1623,8 +1623,6 @@ def remote_unsupported_operations():
     return [
         ("uses_v2_manifest_paths", (), {}, "uses_v2_manifest_paths"),
         ("migrate_v2_manifest_paths", (), {}, "migrate_manifest_paths_v2"),
-        ("set_unenforced_primary_key", (["id"],), {}, "set_unenforced_primary_key"),
-        ("fetch_blob_ranges", ("image", [(1, 0, 1)]), {}, "fetch_blob_ranges"),
         ("optimize", (), {}, "optimize"),
         ("create_index", ("vector",), {"config": HnswPq()}, "IVF_HNSW_PQ"),
     ]
@@ -1795,6 +1793,79 @@ def test_table_wait_for_index_timeout():
             table.wait_for_index(["id_idx"], timedelta(seconds=1))
 
 
+def _jieba_fts_handler(request):
+    """Serve a table with a ``jieba/default`` FTS index and no other state."""
+    if request.path == "/v1/table/test/describe/":
+        body = {
+            "version": 1,
+            "schema": {
+                "fields": [
+                    {"name": "text", "type": {"type": "string"}, "nullable": True}
+                ]
+            },
+        }
+    elif request.path == "/v1/table/test/index/list/":
+        index_details = {
+            "base_tokenizer": "jieba/default",
+            "language": "English",
+            "with_position": False,
+            "max_token_length": 40,
+            "lower_case": True,
+            "stem": False,
+            "remove_stop_words": False,
+            "ascii_folding": False,
+        }
+        body = {
+            "indexes": [
+                {
+                    "index_name": "text_idx",
+                    "columns": ["text"],
+                    "index_type": "FTS",
+                    "index_details": json.dumps(index_details),
+                }
+            ]
+        }
+    else:
+        request.send_response(404)
+        request.end_headers()
+        return
+    request.send_response(200)
+    request.send_header("Content-Type", "application/json")
+    request.end_headers()
+    request.wfile.write(json.dumps(body).encode())
+
+
+_MODEL_FILES_ERROR = (
+    r"(?s)Invalid directory path.*"
+    r"full text search index 'text_idx'.*"
+    r"tokenize\(\) runs on the client.*'jieba/default'.*"
+    r"LANCE_LANGUAGE_MODEL_HOME.*match the files the server uses"
+)
+
+
+def test_remote_tokenize_model_backed_without_model_files(tmp_path, monkeypatch):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    with mock_lancedb_connection(_jieba_fts_handler) as db:
+        table = db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            table.tokenize("我们是中国人", column="text")
+
+
+@pytest.mark.asyncio
+async def test_remote_tokenize_model_backed_without_model_files_async(
+    tmp_path, monkeypatch
+):
+    monkeypatch.setenv("LANCE_LANGUAGE_MODEL_HOME", str(tmp_path))
+    async with mock_lancedb_connection_async(_jieba_fts_handler) as db:
+        table = await db.open_table("test")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", index_name="text_idx")
+        with pytest.raises(ValueError, match=_MODEL_FILES_ERROR):
+            await table.tokenize("我们是中国人", column="text")
+
+
 def test_stats():
     stats = {
         "total_bytes": 38,
@@ -1846,10 +1917,10 @@ def lsm_test_table(lsm_handler):
     """A remote table whose LSM routes are served by ``lsm_handler``.
 
     ``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
-    where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is
+    where route is one of flush_rte, compact_rte, get_rte_stats, and is
     responsible for writing the response.
     """
-    routes = ("flush_lsm", "compact_lsm", "get_lsm_stats")
+    routes = ("flush_rte", "compact_rte", "get_rte_stats")
 
     def handler(request):
         match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
@@ -1881,9 +1952,9 @@ def send_json(request, payload, status=200):
     request.wfile.write(json.dumps(payload).encode())
 
 
-def test_get_lsm_stats_sync():
+def test_get_rte_stats_sync():
     """The sync wrapper round-trips the server payload into a dict."""
-    bucket = {
+    table_shard = {
         "shard_id": "b0",
         "status": "Active",
         "writer_epoch": 3,
@@ -1891,7 +1962,7 @@ def test_get_lsm_stats_sync():
         "current_generation": 6,
         "replay_after_wal_entry_position": 40,
         "wal_entry_position_last_seen": 42,
-        "generations": [{"generation": 5, "bytes": 1024, "rows": 7}],
+        "sstables": [{"generation": 5, "bytes": 1024, "rows": 7}],
         "compacting": False,
         "memtables": [
             {
@@ -1906,29 +1977,29 @@ def test_get_lsm_stats_sync():
     seen_bodies = []
 
     def lsm_handler(request, route):
-        assert route == "get_lsm_stats"
+        assert route == "get_rte_stats"
         seen_bodies.append(read_json_body(request))
-        send_json(request, {"lsm_stats": {"buckets": [bucket]}})
+        send_json(request, {"rte_stats": {"table_shards": [table_shard]}})
 
     with lsm_test_table(lsm_handler) as table:
-        assert table.get_lsm_stats() == {"buckets": [bucket]}
+        assert table.get_rte_stats() == {"table_shards": [table_shard]}
         # Off by default, and forwarded when asked for.
-        assert seen_bodies == [{"include_generation_rows": False}]
-        table.get_lsm_stats(include_generation_rows=True)
-        assert seen_bodies[-1] == {"include_generation_rows": True}
+        assert seen_bodies == [{"include_sstable_rows": False}]
+        table.get_rte_stats(include_sstable_rows=True)
+        assert seen_bodies[-1] == {"include_sstable_rows": True}
 
 
-def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
-    """A null envelope means the LSM write path is not enabled, not an error."""
+def test_get_rte_stats_sync_returns_none_when_rte_disabled():
+    """A null envelope means the RTE write path is not enabled, not an error."""
 
     def lsm_handler(request, route):
-        send_json(request, {"lsm_stats": None})
+        send_json(request, {"rte_stats": None})
 
     with lsm_test_table(lsm_handler) as table:
-        assert table.get_lsm_stats() is None
+        assert table.get_rte_stats() is None
 
 
-def test_flush_and_compact_lsm_sync():
+def test_flush_and_compact_rte_sync():
     """Both are one-shot POSTs answered 202 with no body."""
     called = []
 
@@ -1938,13 +2009,13 @@ def test_flush_and_compact_lsm_sync():
         request.end_headers()
 
     with lsm_test_table(lsm_handler) as table:
-        assert table.flush_lsm() is None
-        assert table.compact_lsm() is None
-        assert called == ["flush_lsm", "compact_lsm"]
+        assert table.flush_rte() is None
+        assert table.compact_rte() is None
+        assert called == ["flush_rte", "compact_rte"]
 
 
-def test_checkpoint_lsm_sync():
-    """Seal, read the watermark, and return once L0 holds nothing.
+def test_checkpoint_rte_sync():
+    """Freeze, read the watermark, and return once no SSTables remain.
 
     The convergence loop itself is covered in Rust; this pins the sync
     binding to the endpoints it drives.
@@ -1953,17 +2024,17 @@ def test_checkpoint_lsm_sync():
 
     def lsm_handler(request, route):
         called.append(route)
-        if route == "get_lsm_stats":
-            # An empty L0 yields no target watermark, so the loop is done
-            # after the seal without ever polling compaction.
-            send_json(request, {"lsm_stats": {"buckets": []}})
+        if route == "get_rte_stats":
+            # An empty SSTable tier yields no target watermark, so the loop is done
+            # after the freeze without ever polling compaction.
+            send_json(request, {"rte_stats": {"table_shards": []}})
         else:
             request.send_response(202)
             request.end_headers()
 
     with lsm_test_table(lsm_handler) as table:
-        assert table.checkpoint_lsm() is None
-        assert called == ["flush_lsm", "get_lsm_stats"]
+        assert table.checkpoint_rte() is None
+        assert called == ["flush_rte", "get_rte_stats"]
 
 
 @contextlib.contextmanager
@@ -3256,6 +3327,89 @@ BLOB_DESCRIBE_RESPONSE = {
 }
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+@pytest.mark.parametrize("mode", ["merge", "overwrite"])
+@pytest.mark.parametrize("target_with_ranges", [False, True])
+async def test_remote_writes_normalize_blob_layouts(
+    use_async, mode, target_with_ranges
+):
+    description = json.loads(json.dumps(BLOB_DESCRIBE_RESPONSE))
+    if target_with_ranges:
+        description["schema"]["fields"][1]["type"]["fields"].extend(
+            {"name": name, "type": {"type": "uint64"}, "nullable": True}
+            for name in ("position", "size")
+        )
+    received = []
+
+    def handler(request):
+        if request.path == "/v1/table/test/describe/":
+            response = description
+        else:
+            expected = "merge_insert" if mode == "merge" else "insert"
+            assert urlparse(request.path).path == f"/v1/table/test/{expected}/"
+            if request.headers.get("Transfer-Encoding") == "chunked":
+                body = bytearray()
+                while size := int(request.rfile.readline().split(b";")[0], 16):
+                    body.extend(request.rfile.read(size))
+                    assert request.rfile.read(2) == b"\r\n"
+                assert request.rfile.read(2) == b"\r\n"
+            else:
+                body = request.rfile.read(int(request.headers["Content-Length"]))
+            received.append(pa.ipc.open_stream(body).read_all())
+            response = {
+                "version": 2,
+                "num_updated_rows": 0,
+                "num_inserted_rows": 2,
+                "num_deleted_rows": 0,
+            }
+        request.send_response(200)
+        request.send_header("Content-Type", "application/json")
+        request.send_header("phalanx-version", "0.5.0")
+        request.end_headers()
+        request.wfile.write(json.dumps(response).encode())
+
+    field = lancedb.blob("image")
+    storage_type = field.type.storage_type
+    if target_with_ranges:
+        storage_type = pa.struct(list(storage_type)[:2])
+        field = pa.field(
+            "image",
+            storage_type,
+            metadata={"ARROW:extension:name": "lance.blob.v2"},
+        )
+    images = pa.array([{"data": b"payload"}, None], type=storage_type)
+    if isinstance(field.type, pa.ExtensionType):
+        images = pa.ExtensionArray.from_storage(field.type, images)
+    data = pa.Table.from_arrays(
+        [pa.array([1, 2], type=pa.int64()), images],
+        schema=pa.schema([pa.field("id", pa.int64()), field]),
+    )
+
+    with mock_lancedb_connection(handler) as db:
+        remote_table = db.open_table("test")
+        table = remote_table._table if use_async else remote_table
+        if mode == "merge":
+            result = (
+                table.merge_insert("id").when_not_matched_insert_all().execute(data)
+            )
+        else:
+            result = table.add(data, mode="overwrite")
+        if use_async:
+            await result
+
+    assert len(received) == 1
+    images = received[0]["image"].combine_chunks()
+    if isinstance(images.type, pa.ExtensionType):
+        images = images.storage
+    with_ranges = target_with_ranges if mode == "merge" else not target_with_ranges
+    assert images.type.num_fields == (4 if with_ranges else 2)
+    expected = {"data": b"payload", "uri": None}
+    if with_ranges:
+        expected.update(position=None, size=None)
+    assert images.to_pylist() == [expected, None]
+
+
 def blob_query_response_table():
     image_field = pa.field(
         "image",
@@ -3363,13 +3517,20 @@ def blob_remote_table(*, server_version=Version("0.5.0")):
             else:
                 assert body["with_row_id"] is True
                 if columns == ["image"]:
-                    # fetch_blob_files sizes its handles from a descriptor take.
+                    # Blob handles are sized from a descriptor take.
                     assert body["filter"].startswith("_rowid IN")
                     response_table = blob_descriptor_take_table()
+                    row_ids = [
+                        int(value) for value in re.findall(r"\d+", body["filter"])
+                    ]
+                    response_table = response_table.filter(
+                        pa.compute.is_in(response_table["_rowid"], pa.array(row_ids))
+                    )
                 else:
                     response_table = blob_query_response_table()
             request.send_response(200)
             request.send_header("Content-Type", "application/vnd.apache.arrow.file")
+            request.send_header("x-lancedb-version", "1")
             request.end_headers()
             with pa.ipc.new_file(request.wfile, response_table.schema) as writer:
                 writer.write_table(response_table)
@@ -3401,6 +3562,42 @@ def test_remote_blob_columns_and_fetch():
         assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True])
+async def test_remote_fetch_blob_ranges(use_async):
+    with blob_remote_table() as remote_table:
+        table = remote_table._table if use_async else remote_table
+        requests = [
+            (30, 2, 3),
+            (10, 1, 3),
+            (20, 0, 1),
+            (10, 0, 2),
+            (40, 0, 0),
+            (10, 5, 0),
+            (30, 2, 3),
+        ]
+        result = table.fetch_blob_ranges("image", requests)
+        if use_async:
+            result = await result
+        assert isinstance(result, pa.LargeBinaryArray)
+        assert result.to_pylist() == [b"mma", b"lph", None, b"al", b"", b"", b"mma"]
+
+        for invalid, message in [
+            ([(10, 0, 1), (99, 0, 1)], "only 1 exist"),
+            ([(10, 4, 2)], "exceeds blob size"),
+            ([(20, 2**64 - 1, 1)], "overflows u64"),
+        ]:
+            with pytest.raises(ValueError, match=message):
+                result = table.fetch_blob_ranges("image", invalid)
+                if use_async:
+                    await result
+
+        result = table.fetch_blob_ranges("image", [])
+        if use_async:
+            result = await result
+        assert result.to_pylist() == []
+
+
 @pytest.mark.asyncio
 async def test_async_remote_to_lance_rejects_storage_access(monkeypatch):
     with blob_remote_table() as remote_table:
@@ -3539,7 +3736,7 @@ def test_remote_blob_files_are_lazy_seekable_handles():
         assert empty.read_range(0, 0) == b""
         alpha.close()
         assert alpha.closed
-        with pytest.raises(RuntimeError, match="already closed"):
+        with pytest.raises(ValueError, match="read of closed file"):
             alpha.read_range(0, 1)
 
 
@@ -3584,6 +3781,56 @@ def test_remote_blob_query_stashes_row_ids_for_fetch():
     assert blobs.to_pylist() == [b"alpha", None, b"gamma"]
 
 
+def test_remote_blob_query_to_pandas():
+    with blob_remote_table() as table:
+        query = table.search().select(["id", "image"]).limit(3)
+        df = query.to_pandas(blob_mode="descriptions", split_blocks=True)
+
+    assert df["id"].tolist() == [1, 2, 3]
+    assert "_rowid" not in df.columns
+    assert df["image"].iloc[1] is None
+    assert "_lance_row_id" not in df["image"].iloc[0]
+
+
+@pytest.mark.parametrize("blob_mode", ["default", "lazy", "bytes"])
+def test_remote_blob_query_to_pandas_rejects_unpinned_fetch(monkeypatch, blob_mode):
+    with blob_remote_table() as table:
+
+        def fail_fetch(*args, **kwargs):
+            raise AssertionError("blob fetch must not use an unpinned remote table")
+
+        monkeypatch.setattr(table, "fetch_blobs", fail_fetch)
+        query = table.search().select(["id", "image"]).limit(3)
+        pandas_kwargs = {} if blob_mode == "default" else {"blob_mode": blob_mode}
+        with pytest.raises(NotImplementedError, match="stable table snapshot"):
+            query.to_pandas(**pandas_kwargs)
+
+
+@pytest.mark.parametrize("blob_mode", ["lazy", "bytes"])
+def test_remote_nonblob_projection_to_pandas(blob_mode):
+    with blob_remote_table() as table:
+        df = table.search().select(["id"]).to_pandas(blob_mode=blob_mode)
+
+    assert df["id"].tolist() == [1, 2, 3]
+    assert list(df.columns) == ["id"]
+
+
+@pytest.mark.asyncio
+async def test_async_remote_blob_query_to_pandas_flatten():
+    with blob_remote_table() as table:
+        df = (
+            await table._table.query()
+            .select(["id", "image"])
+            .limit(3)
+            .to_pandas(blob_mode="descriptions", flatten=True, split_blocks=True)
+        )
+
+    assert df["id"].tolist() == [1, 2, 3]
+    assert df["image.size"].iloc[0] == 5
+    assert "image._lance_row_id" not in df.columns
+    assert "_rowid" not in df.columns
+
+
 def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
     def handler(request):
         if request.path == "/v1/table/test/describe/":
@@ -3613,6 +3860,8 @@ def test_remote_blob_query_survives_a_server_that_ignores_the_row_id_request():
         assert "_lance_row_id" not in hits.schema.field("image").type.names
         with pytest.raises(ValueError, match="pass a list of row ids"):
             table.fetch_blobs("image", hits)
+        with pytest.raises(NotImplementedError, match="stable table snapshot"):
+            table.search().select(["id", "image"]).to_pandas(blob_mode="bytes")
 
 
 def test_remote_blob_byte_apis_not_supported_on_old_server():

python/python/tests/test_reranker_credentials.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import sys
+import types
+
+import pytest
+
+from lancedb.rerankers import CohereReranker, VoyageAIReranker
+
+RERANKERS = [
+    pytest.param("cohere", "COHERE_API_KEY", CohereReranker, {}, id="cohere"),
+    pytest.param(
+        "voyageai",
+        "VOYAGE_API_KEY",
+        VoyageAIReranker,
+        {"model_name": "rerank-2.5"},
+        id="voyageai",
+    ),
+]
+
+
+def fake_sdk(monkeypatch, module_name):
+    """Replace the SDK module with one whose Client records the API key."""
+    keys = []
+
+    def client(api_key=None):
+        keys.append(api_key)
+        return object()
+
+    module = types.ModuleType(module_name)
+    module.Client = client
+    monkeypatch.setitem(sys.modules, module_name, module)
+    return keys
+
+
+@pytest.mark.parametrize("module_name, env_var, reranker_cls, kwargs", RERANKERS)
+@pytest.mark.parametrize(
+    "explicit_key, environment_key, expected_key",
+    [
+        ("explicit-test-key", "environment-test-key", "explicit-test-key"),
+        ("explicit-test-key", None, "explicit-test-key"),
+        (None, "environment-test-key", "environment-test-key"),
+    ],
+)
+def test_reranker_api_key_precedence(
+    monkeypatch,
+    module_name,
+    env_var,
+    reranker_cls,
+    kwargs,
+    explicit_key,
+    environment_key,
+    expected_key,
+):
+    keys = fake_sdk(monkeypatch, module_name)
+    if environment_key is None:
+        monkeypatch.delenv(env_var, raising=False)
+    else:
+        monkeypatch.setenv(env_var, environment_key)
+
+    reranker_cls(api_key=explicit_key, **kwargs)._client
+    assert keys == [expected_key]
+
+
+@pytest.mark.parametrize("module_name, env_var, reranker_cls, kwargs", RERANKERS)
+def test_reranker_requires_api_key(
+    monkeypatch, module_name, env_var, reranker_cls, kwargs
+):
+    keys = fake_sdk(monkeypatch, module_name)
+    monkeypatch.delenv(env_var, raising=False)
+    with pytest.raises(ValueError, match=f"{env_var} not set"):
+        reranker_cls(**kwargs)._client
+    assert keys == []

python/python/tests/test_rerankers.py

@@ -785,6 +785,154 @@ def test_empty_hybrid_result_reranker():
     assert "_rowid" in result.column_names
 
 
+@pytest.mark.parametrize("norm", ["score", "rank"])
+def test_hybrid_return_all_restores_original_scores(norm):
+    from lancedb.query import LanceHybridQueryBuilder
+
+    vector_results = pa.table(
+        {
+            "text": ["a", "b", "c"],
+            "_rowid": pa.array([0, 1, 2], type=pa.uint64()),
+            "_distance": pa.array([0.5, 2.5, 10.0], type=pa.float32()),
+        }
+    )
+    fts_results = pa.table(
+        {
+            "text": ["b", "a"],
+            "_rowid": pa.array([1, 0], type=pa.uint64()),
+            "_score": pa.array([3.25, 1.75], type=pa.float32()),
+        }
+    )
+
+    result = LanceHybridQueryBuilder._combine_hybrid_results(
+        fts_results=fts_results,
+        vector_results=vector_results,
+        norm=norm,
+        fts_query="query",
+        reranker=RRFReranker(return_score="all"),
+        limit=10,
+        with_row_ids=True,
+    )
+
+    # The raw distances and scores are returned, not the normalized values
+    # or ranks used for reranking.
+    rows = {row["_rowid"]: row for row in result.to_pylist()}
+    assert rows[0]["_distance"] == 0.5
+    assert rows[1]["_distance"] == 2.5
+    assert rows[2]["_distance"] == 10.0
+    assert rows[0]["_score"] == 1.75
+    assert rows[1]["_score"] == 3.25
+    assert rows[2]["_score"] is None
+
+
+@pytest.mark.parametrize("return_score", ["relevance", "all"])
+@pytest.mark.parametrize("first_row_id", [1, 2**63])
+@pytest.mark.parametrize("score_type", [pa.float32(), pa.float64()])
+def test_model_reranker_preserves_arrow_schema(
+    mocker, return_score, first_row_id, score_type
+):
+    common_schema = pa.schema(
+        [
+            pa.field("_rowid", pa.uint64()),
+            pa.field("text", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+        ],
+        metadata={b"source": b"reranker-test"},
+    )
+    vector_results = pa.Table.from_pylist(
+        [
+            {
+                "_rowid": first_row_id,
+                "text": "vector",
+                "vector": [1, 0],
+                "_distance": 0.1,
+            },
+            {
+                "_rowid": first_row_id + 1,
+                "text": "both",
+                "vector": [0, 1],
+                "_distance": 0.2,
+            },
+        ],
+        schema=common_schema.append(pa.field("_distance", score_type)),
+    )
+    fts_results = pa.Table.from_pylist(
+        [
+            {
+                "_rowid": first_row_id + 1,
+                "text": "both",
+                "vector": [0, 1],
+                "_score": 0.9,
+            },
+            {
+                "_rowid": first_row_id + 2,
+                "text": "fts",
+                "vector": [1, 1],
+                "_score": 0.8,
+            },
+        ],
+        schema=common_schema.append(pa.field("_score", score_type)),
+    )
+    reranker = CohereReranker(return_score=return_score)
+    reranker._client = mocker.Mock()
+    reranker._client.rerank.return_value.results = [
+        mocker.Mock(index=index, relevance_score=score)
+        for index, score in [(2, 0.9), (1, 0.8), (0, 0.7)]
+    ]
+
+    result = reranker.rerank_hybrid("query", vector_results, fts_results)
+
+    assert result.select(common_schema.names).schema.equals(
+        common_schema, check_metadata=True
+    )
+    assert result["_rowid"].to_pylist() == [
+        first_row_id + 2,
+        first_row_id + 1,
+        first_row_id,
+    ]
+    assert result["text"].to_pylist() == ["fts", "both", "vector"]
+    if return_score == "all":
+        assert (
+            result["_distance"]
+            .combine_chunks()
+            .equals(pa.array([None, 0.2, 0.1], type=score_type))
+        )
+        assert (
+            result["_score"]
+            .combine_chunks()
+            .equals(pa.array([0.8, 0.9, None], type=score_type))
+        )
+
+
+@pytest.mark.parametrize("return_score", ["relevance", "all"])
+def test_model_reranker_empty_results_preserve_schema(mocker, return_score):
+    common_schema = pa.schema(
+        [
+            ("_rowid", pa.uint64()),
+            ("text", pa.string()),
+            ("vector", pa.list_(pa.float32(), 2)),
+        ],
+        metadata={b"source": b"reranker-test"},
+    )
+    vector_results = pa.Table.from_pylist(
+        [], schema=common_schema.append(pa.field("_distance", pa.float32()))
+    )
+    fts_results = pa.Table.from_pylist(
+        [], schema=common_schema.append(pa.field("_score", pa.float32()))
+    )
+    reranker = CohereReranker(return_score=return_score)
+    reranker._client = mocker.Mock()
+
+    result = reranker.rerank_hybrid("query", vector_results, fts_results)
+
+    assert len(result) == 0
+    assert result.select(common_schema.names).schema.equals(
+        common_schema, check_metadata=True
+    )
+    assert result.schema.field("_relevance_score").type == pa.float32()
+    reranker._client.rerank.assert_not_called()
+
+
 def test_cross_encoder_reranker_return_all(tmp_path):
     pytest.importorskip("sentence_transformers")
     reranker = CrossEncoderReranker(return_score="all")

python/python/tests/test_rte_spec.py

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-"""Tests for installing and clearing an LsmWriteSpec via
-`Table.set_lsm_write_spec` / `Table.unset_lsm_write_spec`.
+"""Tests for installing and clearing an RTEWriteSpec via
+`Table.enable_rte` / `Table.disable_rte`.
 """
 
 from datetime import timedelta
@@ -10,7 +10,7 @@ from datetime import timedelta
 import lancedb
 import pyarrow as pa
 import pytest
-from lancedb._lancedb import LsmWriteSpec
+from lancedb._lancedb import RTEWriteSpec
 from lancedb.index import BTree
 
 SCHEMA = pa.schema(
@@ -21,9 +21,9 @@ SCHEMA = pa.schema(
 )
 
 
-def test_lsm_write_spec_module_metadata():
-    assert lancedb.LsmWriteSpec is LsmWriteSpec
-    assert LsmWriteSpec.__module__ == "lancedb._lancedb"
+def test_rte_spec_module_metadata():
+    assert lancedb.RTEWriteSpec is RTEWriteSpec
+    assert RTEWriteSpec.__module__ == "lancedb._lancedb"
 
 
 def _batch(ids, vs):
@@ -43,35 +43,49 @@ def _make_table(tmp_path):
     return db, table
 
 
-def test_set_lsm_write_spec_validates(tmp_path):
+def test_enable_rte_validates(tmp_path):
     _db, table = _make_table(tmp_path)
+    table.set_unenforced_primary_key("id")
 
     # Out-of-range num_buckets.
     with pytest.raises(Exception, match="num_buckets"):
-        table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 0))
+        table.enable_rte(RTEWriteSpec.bucket("id", 0))
     with pytest.raises(Exception, match="num_buckets"):
-        table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
+        table.enable_rte(RTEWriteSpec.bucket("id", 1025))
 
     # Happy path, then a set over the installed spec is refused.
-    table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
+    table.enable_rte(RTEWriteSpec.bucket("id", 4))
     with pytest.raises(Exception, match="already set"):
-        table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
+        table.enable_rte(RTEWriteSpec.bucket("id", 8))
 
 
-def test_unset_lsm_write_spec(tmp_path):
+def test_disable_rte(tmp_path):
     _db, table = _make_table(tmp_path)
+    table.set_unenforced_primary_key("id")
 
     # unset errors when no spec is set.
-    with pytest.raises(Exception, match="no LSM write spec"):
-        table.unset_lsm_write_spec()
+    with pytest.raises(Exception, match="no RTE write spec"):
+        table.disable_rte()
 
     # Install a spec, then remove it; afterwards a fresh spec can be set.
-    table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
-    table.unset_lsm_write_spec()
+    table.enable_rte(RTEWriteSpec.bucket("id", 4))
+    table.disable_rte()
     # A second unset errors — there is no spec left to remove.
-    with pytest.raises(Exception, match="no LSM write spec"):
-        table.unset_lsm_write_spec()
-    table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
+    with pytest.raises(Exception, match="no RTE write spec"):
+        table.disable_rte()
+    table.enable_rte(RTEWriteSpec.bucket("id", 8))
+
+
+def test_enable_rte_requires_primary_key(tmp_path):
+    _db, table = _make_table(tmp_path)
+    for spec in (
+        RTEWriteSpec.unsharded(),
+        RTEWriteSpec.bucket("id", 4),
+        RTEWriteSpec.identity("id"),
+    ):
+        with pytest.raises(Exception, match="primary key"):
+            table.enable_rte(spec)
+    assert table.get_rte_spec() is None
 
 
 def test_set_unsharded_spec(tmp_path):
@@ -79,12 +93,12 @@ def test_set_unsharded_spec(tmp_path):
     # Lance MemWAL still requires a primary key on the dataset; Unsharded
     # just skips per-row hashing.
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded())
-    table.unset_lsm_write_spec()
+    table.enable_rte(RTEWriteSpec.unsharded())
+    table.disable_rte()
 
 
-def test_lsm_write_spec_repr():
-    s = LsmWriteSpec.bucket("id", 4)
+def test_rte_spec_repr():
+    s = RTEWriteSpec.bucket("id", 4)
     assert s.spec_type == "bucket"
     assert s.column == "id"
     assert s.num_buckets == 4
@@ -95,20 +109,20 @@ def test_lsm_write_spec_repr():
     assert "id" in repr(s)
     assert "4" in repr(s)
 
-    u = LsmWriteSpec.unsharded()
+    u = RTEWriteSpec.unsharded()
     assert u.spec_type == "unsharded"
     assert u.column is None
     assert u.num_buckets is None
     assert "unsharded" in repr(u)
 
 
-def test_lsm_write_spec_with_maintained_indexes():
-    s = LsmWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"])
+def test_rte_spec_with_maintained_indexes():
+    s = RTEWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"])
     assert s.maintained_indexes == ["idx_a", "idx_b"]
 
 
 @pytest.mark.asyncio
-async def test_async_set_unset_lsm_write_spec(tmp_path):
+async def test_async_set_disable_rte(tmp_path):
     db = await lancedb.connect_async(
         tmp_path, read_consistency_interval=timedelta(seconds=0)
     )
@@ -118,11 +132,11 @@ async def test_async_set_unset_lsm_write_spec(tmp_path):
     )
 
     await table.set_unenforced_primary_key("id")
-    await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
-    await table.unset_lsm_write_spec()
+    await table.enable_rte(RTEWriteSpec.bucket("id", 4))
+    await table.disable_rte()
     # A second unset errors.
-    with pytest.raises(Exception, match="no LSM write spec"):
-        await table.unset_lsm_write_spec()
+    with pytest.raises(Exception, match="no RTE write spec"):
+        await table.disable_rte()
 
 
 def test_set_identity_spec(tmp_path):
@@ -130,12 +144,12 @@ def test_set_identity_spec(tmp_path):
     # Identity sharding still requires an unenforced primary key on the
     # table; it shards by the raw value of the given column.
     table.set_unenforced_primary_key("id")
-    table.set_lsm_write_spec(LsmWriteSpec.identity("v"))
-    table.unset_lsm_write_spec()
+    table.enable_rte(RTEWriteSpec.identity("v"))
+    table.disable_rte()
 
 
-def test_lsm_write_spec_identity_and_writer_config_defaults():
-    s = LsmWriteSpec.identity("v")
+def test_rte_spec_identity_and_writer_config_defaults():
+    s = RTEWriteSpec.identity("v")
     assert s.spec_type == "identity"
     assert s.column == "v"
     assert s.num_buckets is None
@@ -146,12 +160,12 @@ def test_lsm_write_spec_identity_and_writer_config_defaults():
     assert "durable_write" in repr(s)
 
 
-def test_get_lsm_write_spec(tmp_path):
+def test_get_rte_spec(tmp_path):
     _db, table = _make_table(tmp_path)
     table.set_unenforced_primary_key("id")
 
     # None when nothing is installed.
-    assert table.get_lsm_write_spec() is None
+    assert table.get_rte_spec() is None
 
     # A real scalar index is needed to name it as a maintained index.
     table.create_index("id", config=BTree())
@@ -159,12 +173,12 @@ def test_get_lsm_write_spec(tmp_path):
 
     # Bucket spec round-trips, including maintained indexes and writer config
     # defaults.
-    table.set_lsm_write_spec(
-        LsmWriteSpec.bucket("id", 4)
+    table.enable_rte(
+        RTEWriteSpec.bucket("id", 4)
         .with_maintained_indexes([idx_name])
         .with_writer_config_defaults({"durable_write": "false"})
     )
-    spec = table.get_lsm_write_spec()
+    spec = table.get_rte_spec()
     assert spec is not None
     assert spec.spec_type == "bucket"
     assert spec.column == "id"
@@ -173,30 +187,30 @@ def test_get_lsm_write_spec(tmp_path):
     assert spec.writer_config_defaults == {"durable_write": "false"}
 
     # After unset, None again.
-    table.unset_lsm_write_spec()
-    assert table.get_lsm_write_spec() is None
+    table.disable_rte()
+    assert table.get_rte_spec() is None
 
     # Identity round-trips (column recovered from the schema). An unnamed
     # maintained set reads back as None: what is stored is the intent to
     # maintain every index, not the set it resolves to now.
-    table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
-    spec = table.get_lsm_write_spec()
+    table.enable_rte(RTEWriteSpec.identity("id"))
+    spec = table.get_rte_spec()
     assert spec.spec_type == "identity"
     assert spec.column == "id"
     assert spec.maintained_indexes is None
-    table.unset_lsm_write_spec()
+    table.disable_rte()
 
     # Unsharded round-trips (no routing column). Opting out is distinct from
     # the unnamed default.
-    table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
-    spec = table.get_lsm_write_spec()
+    table.enable_rte(RTEWriteSpec.unsharded().with_maintained_indexes([]))
+    spec = table.get_rte_spec()
     assert spec.spec_type == "unsharded"
     assert spec.column is None
     assert spec.maintained_indexes == []
 
 
 @pytest.mark.asyncio
-async def test_async_get_lsm_write_spec(tmp_path):
+async def test_async_get_rte_spec(tmp_path):
     db = await lancedb.connect_async(
         tmp_path, read_consistency_interval=timedelta(seconds=0)
     )
@@ -205,20 +219,21 @@ async def test_async_get_lsm_write_spec(tmp_path):
         pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
     )
 
-    assert await table.get_lsm_write_spec() is None
+    assert await table.get_rte_spec() is None
+    await table.set_unenforced_primary_key("id")
 
     # A real scalar index is needed to name it as a maintained index.
     await table.create_index("id", config=BTree())
     idx_name = (await table.list_indices())[0].name
 
-    await table.set_lsm_write_spec(
-        LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
+    await table.enable_rte(
+        RTEWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
     )
-    spec = await table.get_lsm_write_spec()
+    spec = await table.get_rte_spec()
     assert spec is not None
     assert spec.spec_type == "bucket"
     assert spec.column == "id"
     assert spec.num_buckets == 8
     assert spec.maintained_indexes == [idx_name]
-    await table.unset_lsm_write_spec()
-    assert await table.get_lsm_write_spec() is None
+    await table.disable_rte()
+    assert await table.get_rte_spec() is None

python/python/tests/test_session.py

@@ -36,3 +36,13 @@ def test_session_cache_configuration(tmp_path):
     assert final_cache_size > initial_cache_size  # Cache should have grown
     assert final_cache_items >= initial_cache_items  # Items should not decrease
     assert initial_cache_size < index_cache_size + metadata_cache_size
+
+
+def test_connection_session_returns_supplied_session(tmp_path):
+    """Sync connections must expose the session they were opened with."""
+    db = lancedb.connect(tmp_path)
+    assert db.session is None
+
+    session = lancedb.Session()
+    db = lancedb.connect(tmp_path, session=session)
+    assert db.session is session

python/python/tests/test_table.py

@@ -16,7 +16,7 @@ from datetime import date, datetime, timedelta
 from decimal import Decimal, localcontext
 from time import sleep
 from typing import List
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import lancedb
 from lancedb import table as table_module
@@ -281,6 +281,30 @@ def test_create_table_infers_large_int_vectors(mem_db: DBConnection):
     assert vector_column.to_pylist() == [[0.0, 300.0]]
 
 
+def test_create_table_all_null_vector_column(mem_db: DBConnection):
+    # A vector column can legitimately arrive with every value null, e.g. when
+    # the embeddings have not been computed yet. The dimension is unknowable in
+    # that case, so schema inference must not crash and should leave the column
+    # as a variable-length list rather than a fixed-size one.
+    data = pa.table(
+        {
+            "id": pa.array([1, 2]),
+            "vector": pa.array([None, None], type=pa.list_(pa.float32())),
+        }
+    )
+
+    table = mem_db.create_table("all_null_vector", data=data)
+
+    vector_field = table.schema.field("vector")
+    assert pa.types.is_list(vector_field.type)
+    assert vector_field.type.value_type == pa.float32()
+
+    result = table.to_arrow()
+    assert result.num_rows == 2
+    assert result.column("vector").to_pylist() == [None, None]
+    assert table.count_rows() == 2
+
+
 @pytest.mark.asyncio
 async def test_create_table_async_infers_large_int_vectors(
     mem_db_async: AsyncConnection,
@@ -519,6 +543,30 @@ def test_add(mem_db: DBConnection):
     _add(table, schema)
 
 
+def test_add_from_iterator_that_queries_table(mem_db: DBConnection):
+    source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
+    target = mem_db.create_table("target", schema=source.schema)
+
+    def batches():
+        for _ in range(5):
+            yield source.search().limit(10).to_arrow()
+
+    target.add(batches())
+
+    assert target.count_rows() == 50
+
+
+def test_create_table_from_iterator_that_queries_table(mem_db: DBConnection):
+    source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
+
+    def batches():
+        yield source.search().limit(10).to_arrow()
+
+    target = mem_db.create_table("target", data=batches())
+
+    assert target.count_rows() == 10
+
+
 def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection):
     """Regression test for https://github.com/lancedb/lancedb/issues/2512."""
     schema = pa.schema([pa.field("x", pa.int64())])
@@ -1586,6 +1634,25 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
     job.cancel()
 
 
+@pytest.mark.parametrize("method", ["create_index", "create_index_async"])
+@pytest.mark.parametrize("wait_timeout", ["invalid", 1, []])
+def test_create_index_invalid_wait_timeout(mem_db: DBConnection, method, wait_timeout):
+    table = mem_db.create_table("invalid_timeout", [{"id": 1}])
+    with pytest.raises(TypeError):
+        getattr(table, method)("id", config=BTree(), wait_timeout=wait_timeout)
+    assert table.list_indices() == []
+
+
+@pytest.mark.parametrize("method", ["create_index", "create_index_async"])
+@pytest.mark.parametrize("wait_timeout", [None, timedelta(seconds=1)])
+def test_create_index_valid_wait_timeout(mem_db: DBConnection, method, wait_timeout):
+    table = mem_db.create_table("valid_timeout", [{"id": 1}])
+    result = getattr(table, method)("id", config=BTree(), wait_timeout=wait_timeout)
+    if result is not None:
+        result.wait()
+    assert len(table.list_indices()) == 1
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_method(mock_create_index, mem_db: DBConnection):
     table = mem_db.create_table(
@@ -1726,6 +1793,51 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
     )
 
 
+@pytest.mark.parametrize("api", ["legacy", "unified"])
+@pytest.mark.parametrize("cuda_failure", ["unavailable", "init_error"])
+def test_create_index_cuda_failure_suggests_cpu(
+    api, cuda_failure, mem_db: DBConnection
+):
+    table = mem_db.create_table(
+        f"test_cuda_unavailable_{api}",
+        data=[{"vector": [3.1, 4.1]}, {"vector": [5.9, 26.5]}],
+    )
+    torch = MagicMock()
+    torch.cuda.is_available.return_value = cuda_failure == "init_error"
+    if cuda_failure == "init_error":
+        torch.cuda.init.side_effect = RuntimeError("CUDA driver initialization failed")
+
+    with (
+        patch.dict(sys.modules, {"torch": torch}),
+        patch.object(table, "to_lance") as to_lance,
+    ):
+        if api == "legacy":
+            with (
+                pytest.warns(DeprecationWarning, match="create_index"),
+                pytest.raises(RuntimeError, match="accelerator=None.*CPU"),
+            ):
+                table.create_index(
+                    metric="dot",
+                    num_partitions=1,
+                    num_sub_vectors=1,
+                    vector_column_name="vector",
+                    accelerator="cuda",
+                )
+        else:
+            with pytest.raises(RuntimeError, match="accelerator=None.*CPU"):
+                table.create_index(
+                    "vector",
+                    config=IvfPq(
+                        distance_type="dot",
+                        num_partitions=1,
+                        num_sub_vectors=1,
+                        accelerator="cuda",
+                    ),
+                )
+
+    to_lance.return_value.create_index.assert_not_called()
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_name_and_train_parameters(
     mock_create_index, mem_db: DBConnection
@@ -3928,6 +4040,42 @@ def test_multiple_vector_columns(mem_db: DBConnection):
     assert result1["text"][0] != result2["text"][0]
 
 
+@pytest.mark.parametrize(
+    "query",
+    [
+        (0.0, 0.0),
+        pa.array([0.0, 0.0]),
+        pa.chunked_array([[0.0], [0.0]]),
+    ],
+)
+@pytest.mark.parametrize("query_type", ["auto", "vector"])
+def test_sync_search_accepts_supported_vector_containers(
+    mem_db: DBConnection, query, query_type
+):
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("small", pa.list_(pa.float32(), 2)),
+            pa.field("large", pa.list_(pa.float32(), 3)),
+        ]
+    )
+    table = mem_db.create_table(
+        "supported_vector_containers",
+        data=pa.table(
+            {
+                "id": [1, 2],
+                "small": [[0.0, 0.0], [1.0, 1.0]],
+                "large": [[1.0, 1.0, 1.0], [0.0, 0.0, 0.0]],
+            },
+            schema=schema,
+        ),
+    )
+
+    result = table.search(query, query_type=query_type).limit(1).to_arrow()
+
+    assert result["id"].to_pylist() == [1]
+
+
 def test_create_scalar_index(mem_db: DBConnection):
     vec_array = pa.array(
         [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], pa.list_(pa.float32(), 2)
@@ -4271,6 +4419,26 @@ def test_search_nested_vector_no_candidates(mem_db: DBConnection):
         table.search([0.0, 1.0]).limit(1).to_arrow()
 
 
+def test_search_single_vector_column_wrong_query_dim(mem_db: DBConnection):
+    """A query vector whose dimension doesn't match the (only) vector column
+    must still resolve that column, surfacing a dimension-mismatch error from
+    query execution rather than a misleading "no vector column" error."""
+    schema = pa.schema(
+        [
+            pa.field("id", pa.uint32(), False),
+            pa.field("vector_col", pa.list_(pa.float32(), 32), False),
+        ]
+    )
+    table = mem_db.create_table(
+        "wrong_query_dim",
+        data=[{"id": 1, "vector_col": [0.1] * 32}],
+        schema=schema,
+    )
+
+    with pytest.raises(ValueError, match="dim"):
+        table.search([0.01] * 120).limit(1).to_arrow()
+
+
 def test_compact_cleanup(tmp_db: DBConnection):
     pytest.importorskip("lance")
     table = tmp_db.create_table(
@@ -4613,6 +4781,58 @@ async def test_alter_columns_rejects_invalid_async(
     assert await table.schema() == initial_schema
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("list_type", [pa.list_, pa.large_list])
+@pytest.mark.parametrize(
+    "value_type, values, inferred_type",
+    [
+        (pa.float64(), [1.25, 2.5], pa.float32()),
+        (pa.int32(), [0, 255], pa.uint8()),
+        (pa.int32(), [-1, 256], pa.float32()),
+    ],
+)
+async def test_inferred_vector_field_metadata(
+    tmp_db: DBConnection,
+    tmp_db_async: AsyncConnection,
+    asynchronous,
+    list_type,
+    value_type,
+    values,
+    inferred_type,
+):
+    metadata = {
+        "lancedb:description": "Embedding values",
+        "lancedb:tag:model": "test-model",
+    }
+    schema = pa.schema(
+        [
+            pa.field(
+                "vector", list_type(value_type), nullable=False, metadata=metadata
+            ),
+            pa.field("text", pa.string(), metadata={"description": "Source text"}),
+        ],
+        metadata={"description": "Source dataset"},
+    )
+    rows = [{"vector": values, "text": "first"}, {"vector": values, "text": "second"}]
+    data = pa.Table.from_pylist(rows, schema=schema)
+    if asynchronous:
+        await tmp_db_async.create_table("vector_metadata", data=data)
+        table = await tmp_db_async.open_table("vector_metadata")
+        actual_schema = await table.schema()
+        actual_data = await table.to_arrow()
+    else:
+        tmp_db.create_table("vector_metadata", data=data)
+        table = tmp_db.open_table("vector_metadata")
+        actual_schema = table.schema
+        actual_data = table.to_arrow()
+    expected_schema = schema.set(
+        0, schema.field("vector").with_type(pa.list_(inferred_type, 2))
+    )
+    assert actual_schema.equals(expected_schema, check_metadata=True)
+    assert actual_data.to_pylist() == rows
+
+
 def test_update_field_metadata(mem_db: DBConnection):
     data = pa.table({"id": [0, 1], "category": ["a", "b"]})
     table = mem_db.create_table("my_table", data=data)
@@ -4827,6 +5047,57 @@ def test_optimize_warns_on_zero_cleanup_sync(mem_db: DBConnection):
     assert all("concurrent" in str(w.message) for w in seen)
 
 
+def test_optimize_retains_tagged_versions(tmp_db: DBConnection):
+    # A tagged version older than the cleanup cutoff must be retained, not make
+    # optimize() fail part-way (after compaction, before the index step).
+    table = tmp_db.create_table(
+        "test", data=pa.table({"id": list(range(100)), "text": ["a"] * 100})
+    )
+    table.create_index("id", config=BTree())
+    table.tags.create("release-1", table.version)
+    tagged_version = table.version
+    for i in range(3):
+        table.add(pa.table({"id": [1000 + i], "text": ["new"]}))
+    assert table.stats()["fragment_stats"]["num_fragments"] == 4
+    assert table.index_stats("id_idx").num_unindexed_rows == 3
+    sleep(1.1)
+
+    table.optimize(cleanup_older_than=timedelta(seconds=1))
+
+    # The index step ran after the prune step
+    assert table.index_stats("id_idx").num_unindexed_rows == 0
+    # It stays optimizable
+    table.optimize(cleanup_older_than=timedelta(seconds=1))
+    assert table.stats()["fragment_stats"]["num_fragments"] == 1
+
+    # Untagged old versions were pruned, the tagged one was retained
+    versions = [v["version"] for v in table.list_versions()]
+    assert tagged_version in versions
+    for added in range(tagged_version + 1, tagged_version + 4):
+        assert added not in versions
+    assert len(table) == 103
+
+    table.checkout("release-1")
+    assert len(table) == 100
+    assert table.to_arrow().num_rows == 100
+
+
+@pytest.mark.asyncio
+async def test_optimize_retains_tagged_versions_async(tmp_db_async: AsyncConnection):
+    table = await tmp_db_async.create_table("test", data=[{"x": [1]}])
+    await table.tags.create("keep", await table.version())
+    await table.add(data=[{"x": [2]}])
+    await table.add(data=[{"x": [3]}])
+    sleep(1.1)
+
+    stats = await table.optimize(cleanup_older_than=timedelta(seconds=1))
+    assert stats.prune.old_versions_removed >= 1
+
+    await table.checkout("keep")
+    assert await table.count_rows() == 1
+    assert await table.query().to_arrow() == pa.table({"x": [[1]]})
+
+
 def test_replace_field_metadata(tmp_path):
     db = lancedb.connect(tmp_path)
     table = db.create_table("my_table", data=[{"x": 0}])
@@ -4882,6 +5153,81 @@ def test_stats(mem_db: DBConnection):
     assert stats_with_index["fragment_stats"]["num_fragments"] == 1
 
 
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("empty", [False, True])
+@pytest.mark.parametrize("with_schema", [False, True])
+def test_create_table_recordbatch_iterable(
+    mem_db: DBConnection, iterable_type, empty, with_schema
+):
+    schema = pa.schema([pa.field("id", pa.int64())])
+    batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)]
+    data = iterable_type(batches)
+    if empty and not with_schema:
+        with pytest.raises(ValueError, match="empty iterable without a schema"):
+            mem_db.create_table("test", data=data)
+        return
+
+    table = mem_db.create_table(
+        "test", data=data, schema=schema if with_schema else None
+    )
+    assert table.schema == schema
+    assert table.to_arrow().to_pylist() == ([] if empty else [{"id": 1}])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("empty", [False, True])
+@pytest.mark.parametrize("with_schema", [False, True])
+async def test_create_table_recordbatch_iterable_async(
+    mem_db_async: AsyncConnection, iterable_type, empty, with_schema
+):
+    schema = pa.schema([pa.field("id", pa.int64())])
+    batches = [] if empty else [pa.RecordBatch.from_pydict({"id": [1]}, schema)]
+    data = iterable_type(batches)
+    if empty and not with_schema:
+        with pytest.raises(ValueError, match="empty iterable without a schema"):
+            await mem_db_async.create_table("test", data=data)
+        return
+
+    table = await mem_db_async.create_table(
+        "test", data=data, schema=schema if with_schema else None
+    )
+    assert await table.schema() == schema
+    assert (await table.to_arrow()).to_pylist() == ([] if empty else [{"id": 1}])
+
+
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
+def test_add_empty_recordbatch_iterable(
+    mem_db: DBConnection, iterable_type, on_bad_vectors
+):
+    table = mem_db.create_table("test", [{"id": 1}])
+    if on_bad_vectors == "error":
+        # The default scannable path still requires an input schema. This
+        # fix does not change its validation policy for schema-less input.
+        with pytest.raises(ValueError, match="empty iterator"):
+            table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    else:
+        # Sanitization has the existing table schema available.
+        table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    assert table.to_arrow().to_pylist() == [{"id": 1}]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("iterable_type", [iter, tuple])
+@pytest.mark.parametrize("on_bad_vectors", ["error", "drop", "fill", "null"])
+async def test_add_empty_recordbatch_iterable_async(
+    mem_db_async: AsyncConnection, iterable_type, on_bad_vectors
+):
+    table = await mem_db_async.create_table("test", [{"id": 1}])
+    if on_bad_vectors == "error":
+        with pytest.raises(ValueError, match="empty iterator"):
+            await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    else:
+        await table.add(iterable_type([]), on_bad_vectors=on_bad_vectors)
+    assert (await table.to_arrow()).to_pylist() == [{"id": 1}]
+
+
 def test_create_table_empty_list_with_schema(mem_db: DBConnection):
     """Test creating table with empty list data and schema
 
@@ -5104,3 +5450,53 @@ async def test_refresh_column_async_job_async_table(tmp_path):
     assert result.published_version == 4
     assert await job.status() == "finished"
     assert (await table.to_arrow())["tripled"].to_pylist() == [9]
+
+
+@pytest.mark.parametrize("interval", [None, 300])
+def test_computed_auto_refresh_options(tmp_path, interval):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("refresh_options", [{"x": 1}])
+    before = table.version
+    table.add_columns(
+        computed={"doubled": "x * 2", "tripled": "x * 3"},
+        auto_refresh_enabled=False,
+        auto_refresh_interval_seconds=interval,
+    )
+    assert table.version == before + 1
+    for column in ("doubled", "tripled"):
+        metadata = table.schema.field(column).metadata
+        assert metadata[b"computed_column.auto_refresh_enabled"] == b"false"
+        if interval is None:
+            assert b"computed_column.auto_refresh_interval_seconds" not in metadata
+        else:
+            assert metadata[b"computed_column.auto_refresh_interval_seconds"] == b"300"
+    table.refresh_column("doubled")
+    assert table.to_arrow()["doubled"].to_pylist() == [2]
+
+
+@pytest.mark.parametrize("value", [0, -1, True, 1.5, "300", 2**64])
+def test_computed_auto_refresh_rejects_invalid_interval(tmp_path, value):
+    table = lancedb.connect(tmp_path).create_table("invalid_options", [{"x": 1}])
+    with pytest.raises(ValueError, match="auto_refresh_interval_seconds"):
+        table.add_columns(
+            computed={"doubled": "x * 2"}, auto_refresh_interval_seconds=value
+        )
+    assert table.schema.names == ["x"]
+
+
+def test_auto_refresh_options_require_computed_columns(tmp_path):
+    table = lancedb.connect(tmp_path).create_table("invalid_kind", [{"x": 1}])
+    with pytest.raises(ValueError, match="require computed"):
+        table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True)
+
+
+@pytest.mark.asyncio
+async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("server_interval", [{"x": 1}])
+    before = await table.version()
+    await table.add_columns(computed={"doubled": "x * 2"})
+    assert await table.version() == before + 1
+    metadata = (await table.schema()).field("doubled").metadata
+    assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
+    assert b"computed_column.auto_refresh_interval_seconds" not in metadata

python/python/tests/test_util.py

@@ -21,6 +21,7 @@ from lancedb.table import (
     _infer_target_schema,
     _merge_metadata,
     _sanitize_data,
+    has_nan_values,
     sanitize_create_table,
 )
 import pyarrow as pa
@@ -28,7 +29,14 @@ import pandas as pd
 import polars as pl
 import pytest
 import lancedb
-from lancedb.util import flatten_columns, get_uri_scheme, join_uri, value_to_sql
+from lancedb.util import (
+    flatten_columns,
+    get_uri_location,
+    get_uri_scheme,
+    inf_vector_column_query,
+    join_uri,
+    value_to_sql,
+)
 from utils import exception_output
 
 
@@ -80,6 +88,22 @@ def test_normalize_uri():
         assert parsed_scheme == expected_scheme
 
 
+def test_get_uri_scheme_and_location_pathlib():
+    # Test pathlib.Path instances
+    abs_path = pathlib.Path("/tmp/data")
+    rel_path = pathlib.Path("relative/data")
+
+    assert get_uri_scheme(abs_path) == "file"
+    assert get_uri_scheme(rel_path) == "file"
+
+    assert get_uri_location(abs_path) == str(abs_path)
+    assert get_uri_location(rel_path) == str(rel_path)
+
+    # String URIs still behave identically
+    assert get_uri_scheme("s3://bucket/data") == "s3"
+    assert get_uri_location("s3://bucket/data") == "bucket/data"
+
+
 def test_join_uri_remote():
     schemes = ["s3", "az", "gs"]
     for scheme in schemes:
@@ -466,6 +490,20 @@ def test_handle_bad_vectors_nan(on_bad_vectors):
     assert output["vector"].combine_chunks() == expected
 
 
+@pytest.mark.parametrize("dtype", [pa.float16(), pa.float32(), pa.float64()])
+@pytest.mark.parametrize("large", [False, True])
+@pytest.mark.parametrize("chunked", [False, True])
+def test_has_nan_values_skips_hidden_children_of_null_lists(dtype, large, chunked):
+    array_type = pa.LargeListArray if large else pa.ListArray
+    values = pa.array([float("nan"), None, float("nan"), 3.0, 4.0, 5.0], type=dtype)
+    arr = array_type.from_arrays(
+        [0, 2, 4, 6], values, mask=pa.array([True, False, False])
+    )
+    if chunked:
+        arr = pa.chunked_array([arr.slice(0, 2), arr.slice(2, 1)])
+    assert has_nan_values(arr).to_pylist() == [False, True, False]
+
+
 def test_handle_bad_vectors_noop():
     # ChunkedArray should be preserved as-is
     vector = pa.chunked_array(
@@ -1251,3 +1289,41 @@ def test_infer_vector_column_raises_clear_error(tmp_path):
         # Plain vector search on a table with no vector column should raise
         # a clear ValueError, not a cryptic TypeError.
         table.search([1.0, 2.0]).to_list()
+
+
+def test_inf_vector_column_query_dim_mismatch_still_resolves():
+    """Regression: a query dimension that matches no column must not hide an
+    otherwise-unambiguous vector column behind "no vector column in the data".
+
+    `dim` exists to disambiguate between *multiple* vector columns of
+    different sizes. When the schema has exactly one vector column, that
+    column is still the only sensible one to search; a dimension mismatch is
+    a query-execution concern, not a missing-column one. This previously
+    broke remote search (#4409 wired up client-side inference there), which
+    expects the server to reject the mismatched dimension itself.
+    """
+    schema = pa.schema(
+        [
+            pa.field("id", pa.uint32(), False),
+            pa.field("vector_col", pa.list_(pa.float32(), 32), False),
+            pa.field("s", pa.string(), False),
+        ]
+    )
+
+    assert inf_vector_column_query(schema, dim=32) == "vector_col"
+    assert inf_vector_column_query(schema, dim=120) == "vector_col"
+    assert inf_vector_column_query(schema) == "vector_col"
+
+
+def test_inf_vector_column_query_dim_mismatch_with_multiple_columns():
+    """With more than one vector column, a dimension that matches none of
+    them is still ambiguous and must raise, not silently pick one."""
+    schema = pa.schema(
+        [
+            pa.field("vector1", pa.list_(pa.float32(), 10), False),
+            pa.field("vector2", pa.list_(pa.float32(), 20), False),
+        ]
+    )
+
+    with pytest.raises(ValueError, match="more than one vector column"):
+        inf_vector_column_query(schema, dim=99)

python/python/tests/test_util_update_values.py

@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+from datetime import date, datetime
+
+import lancedb
+import numpy as np
+import pyarrow as pa
+import pytest
+
+
+ARRAY_VALUES = [
+    ((1.0, 2.0), [1.0, 2.0]),
+    ([1.0, 2.0], [1.0, 2.0]),
+    (np.array([1.0, 2.0]), [1.0, 2.0]),
+    (None, None),
+]
+
+
+@pytest.mark.parametrize("value,expected", ARRAY_VALUES)
+def test_update_array_values(tmp_path, value, expected):
+    dtype = pa.list_(pa.float64())
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "values", pa.table({"array": pa.array([[0.0, 0.0]], type=dtype)})
+    )
+    table.update(values={"array": value})
+    assert table.to_arrow()["array"].to_pylist() == [expected]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("value,expected", ARRAY_VALUES)
+async def test_update_array_values_async(tmp_path, value, expected):
+    dtype = pa.list_(pa.float64())
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table(
+        "values", pa.table({"array": pa.array([[0.0, 0.0]], type=dtype)})
+    )
+    await table.update({"array": value})
+    assert (await table.to_arrow())["array"].to_pylist() == [expected]
+
+
+TIME_VALUES = [
+    np.datetime64("2025", "Y"),
+    np.datetime64("2025-01", "M"),
+    np.datetime64("2025-01-02", "D"),
+    np.datetime64("2025-01-02T03", "h"),
+    np.datetime64("2025-01-02T03:04", "m"),
+    np.datetime64("2025-01-02T03:04:05", "s"),
+    np.datetime64("2025-01-02T03:04:05.123", "ms"),
+    np.datetime64("2025-01-02T03:04:05.123456", "us"),
+    np.datetime64("2025-01-02T03:04:05.123456789", "ns"),
+    np.datetime64("NaT", "ns"),
+    datetime(2025, 1, 2, 3, 4, 5, 123456),
+    None,
+]
+
+
+def expected_time(value):
+    if value is None or isinstance(value, np.datetime64) and np.isnat(value):
+        return None
+    return int(np.datetime64(value, "ns").astype(np.int64))
+
+
+@pytest.mark.parametrize("value", TIME_VALUES)
+def test_update_timestamp_values(tmp_path, value):
+    db = lancedb.connect(tmp_path)
+    data = pa.table({"time": pa.array([datetime(2025, 1, 1)], type=pa.timestamp("ns"))})
+    table = db.create_table("times", data)
+    table.update(values={"time": value})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == [
+        expected_time(value)
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("value", TIME_VALUES)
+async def test_update_timestamp_values_async(tmp_path, value):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table({"time": pa.array([datetime(2025, 1, 1)], type=pa.timestamp("ns"))})
+    table = await db.create_table("times", data)
+    await table.update({"time": value})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == [
+        expected_time(value)
+    ]
+
+
+@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], []])
+def test_nested_array_tuple_matches_list(tmp_path, values):
+    db = lancedb.connect(tmp_path)
+    data = pa.table(
+        {"array": pa.array([[[0, 0]]], type=pa.list_(pa.list_(pa.int64())))}
+    )
+    control = db.create_table("list_values", data)
+    candidate = db.create_table("tuple_values", data)
+    tuples = tuple(tuple(value) for value in values)
+    try:
+        control.update(values={"array": values})
+    except ValueError as error:
+        reason = (
+            "Expected a literal value in array" if values else "concat requires input"
+        )
+        assert reason in str(error)
+        with pytest.raises(ValueError, match=reason):
+            candidate.update(values={"array": tuples})
+        assert control.to_arrow().equals(data)
+        assert candidate.to_arrow().equals(data)
+    else:
+        candidate.update(values={"array": tuples})
+        assert candidate.to_arrow().equals(control.to_arrow())
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("values", [[[1, 2], [3, 4]], []])
+async def test_nested_array_tuple_matches_list_async(tmp_path, values):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table(
+        {"array": pa.array([[[0, 0]]], type=pa.list_(pa.list_(pa.int64())))}
+    )
+    control = await db.create_table("list_values", data)
+    candidate = await db.create_table("tuple_values", data)
+    tuples = tuple(tuple(value) for value in values)
+    try:
+        await control.update({"array": values})
+    except ValueError as error:
+        reason = (
+            "Expected a literal value in array" if values else "concat requires input"
+        )
+        assert reason in str(error)
+        with pytest.raises(ValueError, match=reason):
+            await candidate.update({"array": tuples})
+        assert (await control.to_arrow()).equals(data)
+        assert (await candidate.to_arrow()).equals(data)
+    else:
+        await candidate.update({"array": tuples})
+        assert (await candidate.to_arrow()).equals(await control.to_arrow())
+
+
+@pytest.mark.parametrize("unit", ["ps", "fs", "as"])
+def test_subnanosecond_update_does_not_round(tmp_path, unit):
+    db = lancedb.connect(tmp_path)
+    data = pa.table({"time": pa.array([datetime(1970, 1, 1)], type=pa.timestamp("ns"))})
+    table = db.create_table("precise", data)
+    with pytest.raises(ValueError, match="cannot preserve sub-nanosecond precision"):
+        table.update(values={"time": np.datetime64(1, unit)})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == [0]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("unit", ["ps", "fs", "as"])
+async def test_subnanosecond_update_does_not_round_async(tmp_path, unit):
+    db = await lancedb.connect_async(tmp_path)
+    data = pa.table({"time": pa.array([datetime(1970, 1, 1)], type=pa.timestamp("ns"))})
+    table = await db.create_table("precise", data)
+    with pytest.raises(ValueError, match="cannot preserve sub-nanosecond precision"):
+        await table.update({"time": np.datetime64(1, unit)})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == [0]
+
+
+@pytest.mark.parametrize("unit", ["Y", "M", "W", "D"])
+def test_update_numpy_date_column(tmp_path, unit):
+    value = np.datetime64("2025-01-02", unit)
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "dates", pa.table({"day": pa.array([date(1970, 1, 1)], type=pa.date32())})
+    )
+    table.update(values={"day": value})
+    assert table.to_arrow()["day"].to_pylist() == [value.astype("datetime64[D]").item()]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("unit", ["Y", "M", "W", "D"])
+async def test_update_numpy_date_column_async(tmp_path, unit):
+    value = np.datetime64("2025-01-02", unit)
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table(
+        "dates", pa.table({"day": pa.array([date(1970, 1, 1)], type=pa.date32())})
+    )
+    await table.update({"day": value})
+    assert (await table.to_arrow())["day"].to_pylist() == [
+        value.astype("datetime64[D]").item()
+    ]
+
+
+@pytest.mark.parametrize("timezone", [None, "UTC", "America/New_York", "Asia/Tokyo"])
+@pytest.mark.parametrize(
+    "value",
+    [np.datetime64("2025-01-02T03:04:05.123456789", "ns"), np.datetime64("NaT", "ns")],
+)
+def test_update_numpy_timestamp_matches_arrow_ingestion(tmp_path, timezone, value):
+    dtype = pa.timestamp("ns", tz=timezone)
+    ingested = pa.array(np.array([value]), type=dtype)
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("timestamps", pa.table({"time": ingested}))
+    expected = table.to_arrow()["time"].cast(pa.int64()).to_pylist()
+    table.update(values={"time": value})
+    assert table.to_arrow()["time"].cast(pa.int64()).to_pylist() == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("timezone", [None, "UTC", "America/New_York", "Asia/Tokyo"])
+@pytest.mark.parametrize(
+    "value",
+    [np.datetime64("2025-01-02T03:04:05.123456789", "ns"), np.datetime64("NaT", "ns")],
+)
+async def test_update_numpy_timestamp_matches_arrow_ingestion_async(
+    tmp_path, timezone, value
+):
+    dtype = pa.timestamp("ns", tz=timezone)
+    ingested = pa.array(np.array([value]), type=dtype)
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("timestamps", pa.table({"time": ingested}))
+    expected = (await table.to_arrow())["time"].cast(pa.int64()).to_pylist()
+    await table.update({"time": value})
+    assert (await table.to_arrow())["time"].cast(pa.int64()).to_pylist() == expected

python/python/tests/test_voyageai_embeddings.py

@@ -6,6 +6,8 @@
 These tests verify model registration and configuration without requiring API calls.
 """
 
+import numpy as np
+import pyarrow as pa
 import pytest
 from unittest.mock import MagicMock, patch
 
@@ -122,3 +124,58 @@ class TestVoyageAIModelRegistration:
         assert "voyage-4" not in func.multimodal_embedding_models
         assert "voyage-4-lite" not in func.multimodal_embedding_models
         assert "voyage-4-large" not in func.multimodal_embedding_models
+
+
+@pytest.mark.parametrize(
+    "inputs",
+    [
+        ["first", "second"],
+        np.array(["first", "second"]),
+        pa.array(["first", "second"]),
+        pa.chunked_array([["first"], ["second"]]),
+    ],
+    ids=["list", "numpy", "arrow", "chunked-arrow"],
+)
+def test_text_source_embeddings_accept_batch_containers(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    client.tokenize.return_value = [[1], [2]]
+    client.embed.return_value.embeddings = [[0.1, 0.2], [0.3, 0.4]]
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    assert func.compute_source_embeddings(inputs) == [[0.1, 0.2], [0.3, 0.4]]
+    client.embed.assert_called_once_with(
+        texts=["first", "second"], model="voyage-3", input_type="document"
+    )
+
+
+@pytest.mark.parametrize("inputs", [[], np.array([], dtype=str)])
+def test_text_source_embeddings_accept_empty_batch(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    assert func.compute_source_embeddings(inputs) == []
+    client.tokenize.assert_not_called()
+    client.embed.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "inputs",
+    [["text", 42], np.array(["text", None]), np.array([["first", "second"]])],
+)
+def test_text_source_embeddings_reject_non_text_batch_entries(inputs):
+    from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+    client = MagicMock()
+    VoyageAIEmbeddingFunction.client = client
+    func = get_registry().get("voyageai").create(name="voyage-3")
+
+    with pytest.raises(ValueError, match="Each input should be str"):
+        func.compute_source_embeddings(inputs)
+    client.tokenize.assert_not_called()
+    client.embed.assert_not_called()

python/python/tests/test_voyageai_image_inputs.py

@@ -0,0 +1,88 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import base64
+from io import BytesIO
+from unittest.mock import MagicMock
+
+import pytest
+
+from lancedb.embeddings import get_registry
+from lancedb.embeddings.voyageai import VoyageAIEmbeddingFunction
+
+
+@pytest.mark.parametrize("model_name", ["voyage-multimodal-3", "voyage-multimodal-3.5"])
+@pytest.mark.parametrize("input_kind", ["pil", "bytes", "path"])
+@pytest.mark.parametrize(
+    "mode,color,expected_rgb",
+    [
+        ("RGBA", (64, 128, 192, 128), (64, 128, 192)),
+        ("LA", (96, 128), (96, 96, 96)),
+        ("P", 0, (64, 128, 192)),
+        ("RGB", (64, 128, 192), (64, 128, 192)),
+    ],
+)
+def test_multimodal_source_embeddings_encode_image_modes(
+    model_name, input_kind, mode, color, expected_rgb, tmp_path, monkeypatch
+):
+    Image = pytest.importorskip("PIL.Image")
+    image = Image.new(mode, (8, 6), color)
+    if mode == "P":
+        image.putpalette([64, 128, 192] + [0, 0, 0] * 255)
+        image.info["transparency"] = 0
+    original_pixels = image.tobytes()
+    image_input = image
+    if input_kind != "pil":
+        png = BytesIO()
+        image.save(png, format="PNG")
+        if input_kind == "bytes":
+            image_input = png.getvalue()
+        else:
+            image_input = tmp_path / "image.png"
+            image_input.write_bytes(png.getvalue())
+
+    client = MagicMock()
+    client.multimodal_embed.return_value.embeddings = [[0.1, 0.2]]
+    monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
+    func = get_registry().get("voyageai").create(name=model_name)
+
+    assert func.compute_source_embeddings(image_input) == [[0.1, 0.2]]
+
+    client.multimodal_embed.assert_called_once()
+    request = client.multimodal_embed.call_args.kwargs
+    assert request["model"] == model_name
+    assert request["input_type"] == "document"
+    assert len(request["inputs"]) == 1
+    (content,) = request["inputs"][0]["content"]
+    assert content["type"] == "image_base64"
+    prefix, encoded = content["image_base64"].split(",", 1)
+    assert prefix == "data:image/jpeg;base64"
+    with Image.open(BytesIO(base64.b64decode(encoded, validate=True))) as decoded:
+        assert decoded.format == "JPEG"
+        assert decoded.mode == "RGB"
+        assert decoded.size == (8, 6)
+        assert decoded.getpixel((0, 0)) == pytest.approx(expected_rgb, abs=3)
+    assert image.mode == mode
+    assert image.tobytes() == original_pixels
+    client.embed.assert_not_called()
+
+
+@pytest.mark.parametrize("input_kind", ["bytes", "path"])
+def test_multimodal_source_embeddings_reject_invalid_image(
+    input_kind, tmp_path, monkeypatch
+):
+    PIL = pytest.importorskip("PIL")
+    image_input = b"not an image"
+    if input_kind == "path":
+        image_input = tmp_path / "invalid.png"
+        image_input.write_bytes(b"not an image")
+
+    client = MagicMock()
+    monkeypatch.setattr(VoyageAIEmbeddingFunction, "client", client)
+    func = get_registry().get("voyageai").create(name="voyage-multimodal-3")
+
+    with pytest.raises(PIL.UnidentifiedImageError):
+        func.compute_source_embeddings(image_input)
+
+    client.multimodal_embed.assert_not_called()
+    client.embed.assert_not_called()

python/python/tests/test_wal_hybrid.py

@@ -14,7 +14,7 @@ from lancedb.rerankers import RRFReranker
 
 
 def _wal_table(schema: pa.Schema) -> mock.Mock:
-    """A MemWAL-backed table a previous query has already been refused
+    """An RTE-backed table a previous query has already been refused
     `_rowid` on, so the fusion goes straight to the primary key."""
     table = mock.Mock()
     table._hybrid_pk_fusion_learned.return_value = True
@@ -51,7 +51,7 @@ def test_wal_hybrid_projects_the_key_instead_of_asking_for_row_ids():
     assert builder._pk_fusion.injected == ["id"]
     for leg in (builder._vector_query, builder._fts_query):
         assert leg._columns == ["text", "id"]
-        # A MemWAL table rejects with_row_id before it plans.
+        # An RTE table rejects with_row_id before it plans.
         assert leg._with_row_id is not True
 
 
@@ -81,15 +81,15 @@ def test_base_table_hybrid_still_joins_on_row_ids():
     assert builder._vector_query._columns is None
 
 
-def test_wal_hybrid_with_use_lsm_false_joins_on_row_ids():
-    builder = _wal_hybrid().use_lsm(False)
+def test_wal_hybrid_with_use_rte_false_joins_on_row_ids():
+    builder = _wal_hybrid().use_rte(False)
     builder._create_query_builders()
     assert builder._pk_fusion is None
 
 
 def test_wal_hybrid_refuses_an_explicit_with_row_id():
     builder = _wal_hybrid().with_row_id(True)
-    with pytest.raises(NotImplementedError, match="use_lsm"):
+    with pytest.raises(NotImplementedError, match="use_rte"):
         builder._create_query_builders()
 
 
@@ -141,7 +141,7 @@ def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool
     else:
         builder = builder.rerank(reranker).with_row_id(True)
 
-    with pytest.raises(NotImplementedError, match="use_lsm"):
+    with pytest.raises(NotImplementedError, match="use_rte"):
         builder._create_query_builders()
 
 
@@ -199,7 +199,7 @@ def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids():
     def run():
         raise _REFUSAL
 
-    with pytest.raises(NotImplementedError, match="use_lsm") as info:
+    with pytest.raises(NotImplementedError, match="use_rte") as info:
         with_pk_fallback(
             run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True
         )

python/python/tests/utils.py

@@ -73,10 +73,10 @@ class MockPermutationServer:
                     },
                 },
             )
-        if path == f"/v1/table/{self.name}/get_lsm_write_spec/":
+        if path == f"/v1/table/{self.name}/get_rte_spec/":
             self._read_body(request)
-            # Null spec: this table has no LSM write path.
-            return self._json(request, {"lsm_write_spec": None})
+            # Null spec: this table has no RTE write path.
+            return self._json(request, {"rte_spec": None})
         if path == f"/v1/table/{self.name}/count_rows/":
             self._read_body(request)
             return self._json(request, self.num_rows)

python/src/connection.rs

@@ -49,6 +49,18 @@ impl Connection {
     }
 }
 
+fn parse_read_consistency_interval(interval: Option<f64>) -> PyResult<Option<Duration>> {
+    interval
+        .map(|seconds| {
+            Duration::try_from_secs_f64(seconds).map_err(|_| {
+                PyValueError::new_err(
+                    "read_consistency_interval must be finite, non-negative, and within the supported range",
+                )
+            })
+        })
+        .transpose()
+}
+
 fn parse_namespace_client_pushdown_operations(
     operations: Option<Vec<String>>,
 ) -> PyResult<HashSet<NamespaceClientPushdownOperation>> {
@@ -412,7 +424,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn create_materialized_view(
         self_: PyRef<'_, Self>,
         name: String,
@@ -421,10 +434,18 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<f64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(enabled) = auto_refresh_enabled {
+                builder = builder.auto_refresh_enabled(enabled);
+            }
+            if let Some(seconds) = auto_refresh_interval_seconds {
+                builder = builder.auto_refresh_interval_seconds(seconds);
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }
@@ -440,7 +461,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn create_materialized_view_async(
         self_: PyRef<'_, Self>,
         name: String,
@@ -449,10 +471,18 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<f64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(enabled) = auto_refresh_enabled {
+                builder = builder.auto_refresh_enabled(enabled);
+            }
+            if let Some(seconds) = auto_refresh_interval_seconds {
+                builder = builder.auto_refresh_interval_seconds(seconds);
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }
@@ -1059,6 +1089,7 @@ pub fn connect(
     namespace_client_properties: Option<HashMap<String, String>>,
     oauth_config: Option<crate::oauth::PyOAuthConfig>,
 ) -> PyResult<Bound<'_, PyAny>> {
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     future_into_py(py, async move {
         let mut builder = lancedb::connect(&uri);
         if let Some(api_key) = api_key {
@@ -1077,7 +1108,6 @@ pub fn connect(
         #[cfg(not(feature = "remote"))]
         let _ = sql_host_override;
         if let Some(read_consistency_interval) = read_consistency_interval {
-            let read_consistency_interval = Duration::from_secs_f64(read_consistency_interval);
             builder = builder.read_consistency_interval(read_consistency_interval);
         }
         if let Some(storage_options) = storage_options {
@@ -1126,7 +1156,7 @@ pub fn connect_namespace_client(
     namespace_client_impl: Option<String>,
     namespace_client_properties: Option<HashMap<String, String>>,
 ) -> PyResult<Connection> {
-    let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     let namespace_client_pushdown_operations =
         parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
     let ns_properties = namespace_client_properties.unwrap_or_default();
@@ -1183,7 +1213,7 @@ pub fn connect_namespace(
     session: Option<crate::session::Session>,
     namespace_client_pushdown_operations: Option<Vec<String>>,
 ) -> PyResult<Connection> {
-    let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
+    let read_consistency_interval = parse_read_consistency_interval(read_consistency_interval)?;
     let namespace_client_pushdown_operations =
         parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
 

python/src/expr.rs

@@ -240,22 +240,60 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
 
 fn parse_decimal(s: &str) -> PyResult<(i128, u8, i8)> {
     let s = s.trim();
-    let dot_pos = s.find('.');
-    let scale = if let Some(pos) = dot_pos {
-        (s.len() - pos - 1) as i8
-    } else {
-        0
+    let (mantissa, exponent) = match s.split_once(['e', 'E']) {
+        Some((mantissa, exponent)) => (
+            mantissa,
+            exponent.parse::<i64>().map_err(|e| {
+                PyValueError::new_err(format!("failed to parse decimal exponent: {}", e))
+            })?,
+        ),
+        None => (s, 0),
     };
+    let fractional_digits = mantissa.find('.').map_or(0, |pos| mantissa.len() - pos - 1);
+    // Keep the coefficient intact instead of expanding an exponent into zeros.
+    // Arrow Decimal128 has at most 38 digits and an i8 scale (at most 38).
+    let mut scale = i64::try_from(fractional_digits)
+        .ok()
+        .and_then(|fractional_digits| fractional_digits.checked_sub(exponent))
+        .and_then(|scale| i8::try_from(scale).ok())
+        .filter(|scale| *scale <= 38)
+        .ok_or_else(|| PyValueError::new_err("decimal scale is outside Decimal128 range"))?;
 
-    let digits = s.replace('.', "");
-    let val = digits
+    let digits = mantissa.replace('.', "");
+    let mut val = digits
         .parse::<i128>()
         .map_err(|e| PyValueError::new_err(format!("failed to parse decimal digits: {}", e)))?;
 
-    // Precision is total number of digits
-    let precision = digits.trim_start_matches('-').len() as u8;
+    // Positive scale cannot exceed precision, including for tiny values such as 1E-38.
+    let mut precision = digits
+        .trim_start_matches('-')
+        .len()
+        .max(usize::try_from(scale).unwrap_or(0));
+    if scale < 0 {
+        // Arrow cannot cast SQL text to a decimal with negative scale. Normalize
+        // positive exponents within Decimal128's bounded coefficient range.
+        if val != 0 {
+            let shift = -i16::from(scale) as u32;
+            precision += shift as usize;
+            if precision > 38 {
+                return Err(PyValueError::new_err(
+                    "decimal precision exceeds Decimal128 maximum of 38",
+                ));
+            }
+            val = 10_i128
+                .checked_pow(shift)
+                .and_then(|factor| val.checked_mul(factor))
+                .ok_or_else(|| PyValueError::new_err("decimal coefficient exceeds i128 range"))?;
+        }
+        scale = 0;
+    }
+    if precision > 38 {
+        return Err(PyValueError::new_err(
+            "decimal precision exceeds Decimal128 maximum of 38",
+        ));
+    }
 
-    Ok((val, precision, scale))
+    Ok((val, precision as u8, scale))
 }
 
 /// Call an arbitrary registered SQL function by name.

python/src/lib.rs

@@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
 use session::Session;
 use table::{
     AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
-    FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, LsmWriteSpec, MergeResult,
-    PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table,
+    FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, MergeResult, PyBlobFile,
+    RTEWriteSpec, RefreshColumnResult, RefreshMaterializedViewResult, Table,
     UpdateFieldMetadataResult, UpdateResult,
 };
 
@@ -104,7 +104,7 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<UpdateFieldMetadataResult>()?;
     m.add_class::<AddResult>()?;
     m.add_class::<MergeResult>()?;
-    m.add_class::<LsmWriteSpec>()?;
+    m.add_class::<RTEWriteSpec>()?;
     m.add_class::<DeleteResult>()?;
     m.add_class::<DropColumnsResult>()?;
     m.add_class::<UpdateResult>()?;

python/src/permutation.rs

@@ -178,6 +178,50 @@ impl PyAsyncPermutationBuilder {
         })
     }
 
+    #[pyo3(signature = (num_samples, *, ratios=None, counts=None, fixed=None, seed=None, split_names=None))]
+    pub fn bootstrap_sample(
+        slf: PyRefMut<'_, Self>,
+        num_samples: u64,
+        ratios: Option<Vec<f64>>,
+        counts: Option<Vec<u64>>,
+        fixed: Option<u64>,
+        seed: Option<u64>,
+        split_names: Option<Vec<String>>,
+    ) -> PyResult<Self> {
+        // Check that exactly one split type is provided
+        let split_args_count = [ratios.is_some(), counts.is_some(), fixed.is_some()]
+            .iter()
+            .filter(|&&x| x)
+            .count();
+
+        if split_args_count != 1 {
+            return Err(pyo3::exceptions::PyValueError::new_err(
+                "Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
+            ));
+        }
+
+        let sizes = if let Some(ratios) = ratios {
+            SplitSizes::Percentages(ratios)
+        } else if let Some(counts) = counts {
+            SplitSizes::Counts(counts)
+        } else if let Some(fixed) = fixed {
+            SplitSizes::Fixed(fixed)
+        } else {
+            unreachable!("One of the split arguments must be provided");
+        };
+
+        slf.modify(|builder| {
+            builder.with_split_strategy(
+                SplitStrategy::Bootstrap {
+                    num_samples,
+                    sizes,
+                    seed,
+                },
+                split_names,
+            )
+        })
+    }
+
     pub fn split_calculated(
         slf: PyRefMut<'_, Self>,
         calculation: String,

python/src/query.rs

@@ -333,7 +333,7 @@ pub struct PyQueryRequest {
     pub select_source_columns: Option<HashMap<String, String>>,
     pub fast_search: Option<bool>,
     pub with_row_id: Option<bool>,
-    pub use_lsm: Option<bool>,
+    pub use_rte: Option<bool>,
     pub column: Option<String>,
     pub query_vector: Option<PyQueryVectors>,
     pub nprobes: Option<usize>,
@@ -366,7 +366,7 @@ impl From<AnyQuery> for PyQueryRequest {
                 select: PySelect(query_request.select),
                 fast_search: Some(query_request.fast_search),
                 with_row_id: Some(query_request.with_row_id),
-                use_lsm: query_request.use_lsm,
+                use_rte: query_request.use_rte,
                 column: None,
                 query_vector: None,
                 nprobes: None,
@@ -395,7 +395,7 @@ impl From<AnyQuery> for PyQueryRequest {
                 select: PySelect(vector_query.base.select),
                 fast_search: Some(vector_query.base.fast_search),
                 with_row_id: Some(vector_query.base.with_row_id),
-                use_lsm: vector_query.base.use_lsm,
+                use_rte: vector_query.base.use_rte,
                 column: vector_query.column,
                 query_vector: Some(PyQueryVectors(vector_query.query_vector)),
                 nprobes: vector_query.nprobes,
@@ -540,8 +540,8 @@ impl Query {
         self.inner = self.inner.clone().fast_search();
     }
 
-    pub fn use_lsm(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_lsm(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
     }
 
     pub fn with_row_id(&mut self) {
@@ -706,8 +706,8 @@ impl TakeQuery {
         self.inner = self.inner.clone().with_row_id();
     }
 
-    pub fn use_lsm(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_lsm(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
     }
 
     #[pyo3(signature = ())]
@@ -819,8 +819,8 @@ impl FTSQuery {
         self.inner = self.inner.clone().fast_search();
     }
 
-    pub fn use_lsm(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_lsm(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
     }
 
     pub fn with_row_id(&mut self) {
@@ -970,8 +970,8 @@ impl VectorQuery {
         self.inner = self.inner.clone().fast_search();
     }
 
-    pub fn use_lsm(&mut self, enable: bool) {
-        self.inner = self.inner.clone().use_lsm(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner = self.inner.clone().use_rte(enable);
     }
 
     pub fn with_row_id(&mut self) {
@@ -1178,9 +1178,9 @@ impl HybridQuery {
         self.inner_fts.postfilter();
     }
 
-    pub fn use_lsm(&mut self, enable: bool) {
-        self.inner_vec.use_lsm(enable);
-        self.inner_fts.use_lsm(enable);
+    pub fn use_rte(&mut self, enable: bool) {
+        self.inner_vec.use_rte(enable);
+        self.inner_fts.use_rte(enable);
     }
 
     pub fn add_query_vector(&mut self, vector: Bound<'_, PyAny>) -> PyResult<()> {

python/src/table.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
+use lancedb::table::computed_columns::ComputedColumnOptions;
 use std::{
     collections::HashMap,
     sync::{
@@ -39,16 +40,16 @@ use pyo3::{
 
 mod scannable;
 
-/// Convert `LsmStats` to a Python dict, preserving the per-bucket list.
+/// Convert `RTEStats` to a Python dict, preserving the per-table-shard list.
 ///
 /// Deliberately not flattened to a table-level summary: a table is N
-/// buckets on one node, and the per-bucket detail is the reason the
-/// endpoint exists — flattening hides the single hot bucket someone opened
+/// table shards on one node, and the per-shard detail is the reason the
+/// endpoint exists — flattening hides the single hot table shard someone opened
 /// it to find.
-fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult<Py<PyDict>> {
+fn rte_stats_to_py(py: Python<'_>, stats: &lancedb::table::RTEStats) -> PyResult<Py<PyDict>> {
     let out = PyDict::new(py);
-    let buckets = PyList::empty(py);
-    for b in &stats.buckets {
+    let table_shards = PyList::empty(py);
+    for b in &stats.table_shards {
         let e = PyDict::new(py);
         e.set_item("shard_id", &b.shard_id)?;
         e.set_item("status", &b.status)?;
@@ -64,15 +65,15 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
             b.wal_entry_position_last_seen,
         )?;
 
-        let generations = PyList::empty(py);
-        for g in &b.generations {
+        let sstables = PyList::empty(py);
+        for g in &b.sstables {
             let ge = PyDict::new(py);
             ge.set_item("generation", g.generation)?;
             ge.set_item("bytes", g.bytes)?;
             ge.set_item("rows", g.rows)?;
-            generations.append(ge)?;
+            sstables.append(ge)?;
         }
-        e.set_item("generations", generations)?;
+        e.set_item("sstables", sstables)?;
         e.set_item("compacting", b.compacting)?;
 
         e.set_item(
@@ -94,9 +95,9 @@ fn lsm_stats_to_py(py: Python<'_>, stats: &lancedb::table::LsmStats) -> PyResult
                 })
                 .transpose()?,
         )?;
-        buckets.append(e)?;
+        table_shards.append(e)?;
     }
-    out.set_item("buckets", buckets)?;
+    out.set_item("table_shards", table_shards)?;
     Ok(out.unbind())
 }
 
@@ -261,8 +262,8 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
     }
 }
 
-/// Specification selecting Lance's MemWAL LSM-style write path for
-/// `merge_insert`.
+/// Specification enabling the realtime engine (RTE), which routes
+/// `merge_insert` through Lance's MemWAL LSM-style write path.
 ///
 /// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
 /// classmethods, then optionally chain `with_maintained_indexes(...)` and
@@ -270,17 +271,17 @@ fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
 /// table has, including ones created later.
 #[pyclass(module = "lancedb._lancedb", from_py_object)]
 #[derive(Clone, Debug)]
-pub struct LsmWriteSpec {
-    inner: lancedb::table::LsmWriteSpec,
+pub struct RTEWriteSpec {
+    inner: lancedb::table::RTEWriteSpec,
 }
 
 #[pymethods]
-impl LsmWriteSpec {
+impl RTEWriteSpec {
     /// Hash-bucket sharding by the unenforced primary key column.
     #[staticmethod]
     pub fn bucket(column: String, num_buckets: u32) -> Self {
         Self {
-            inner: lancedb::table::LsmWriteSpec::bucket(column, num_buckets),
+            inner: lancedb::table::RTEWriteSpec::bucket(column, num_buckets),
         }
     }
 
@@ -294,20 +295,20 @@ impl LsmWriteSpec {
     #[staticmethod]
     pub fn identity(column: String) -> Self {
         Self {
-            inner: lancedb::table::LsmWriteSpec::identity(column),
+            inner: lancedb::table::RTEWriteSpec::identity(column),
         }
     }
 
     /// No sharding — every `merge_insert` call writes to a single
-    /// MemWAL shard.
+    /// RTE shard.
     #[staticmethod]
     pub fn unsharded() -> Self {
         Self {
-            inner: lancedb::table::LsmWriteSpec::unsharded(),
+            inner: lancedb::table::RTEWriteSpec::unsharded(),
         }
     }
 
-    /// Set which indexes the MemWAL maintains. `None` (the default) is
+    /// Set which indexes the RTE maintains. `None` (the default) is
     /// every index the table has, including ones created later; a list is
     /// verbatim, and an empty list maintains nothing.
     #[pyo3(signature = (indexes))]
@@ -327,33 +328,33 @@ impl LsmWriteSpec {
 
     pub fn __repr__(&self) -> String {
         match &self.inner {
-            lancedb::table::LsmWriteSpec::Bucket {
+            lancedb::table::RTEWriteSpec::Bucket {
                 column,
                 num_buckets,
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
                 column,
                 num_buckets,
                 fmt_maintained(maintained_indexes),
                 writer_config_defaults,
             ),
-            lancedb::table::LsmWriteSpec::Identity {
+            lancedb::table::RTEWriteSpec::Identity {
                 column,
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
                 column,
                 fmt_maintained(maintained_indexes),
                 writer_config_defaults,
             ),
-            lancedb::table::LsmWriteSpec::Unsharded {
+            lancedb::table::RTEWriteSpec::Unsharded {
                 maintained_indexes,
                 writer_config_defaults,
             } => format!(
-                "LsmWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
+                "RTEWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
                 fmt_maintained(maintained_indexes),
                 writer_config_defaults,
             ),
@@ -365,9 +366,9 @@ impl LsmWriteSpec {
     #[getter]
     pub fn spec_type(&self) -> &'static str {
         match &self.inner {
-            lancedb::table::LsmWriteSpec::Bucket { .. } => "bucket",
-            lancedb::table::LsmWriteSpec::Identity { .. } => "identity",
-            lancedb::table::LsmWriteSpec::Unsharded { .. } => "unsharded",
+            lancedb::table::RTEWriteSpec::Bucket { .. } => "bucket",
+            lancedb::table::RTEWriteSpec::Identity { .. } => "identity",
+            lancedb::table::RTEWriteSpec::Unsharded { .. } => "unsharded",
         }
     }
 
@@ -375,9 +376,9 @@ impl LsmWriteSpec {
     #[getter]
     pub fn column(&self) -> Option<String> {
         match &self.inner {
-            lancedb::table::LsmWriteSpec::Bucket { column, .. }
-            | lancedb::table::LsmWriteSpec::Identity { column, .. } => Some(column.clone()),
-            lancedb::table::LsmWriteSpec::Unsharded { .. } => None,
+            lancedb::table::RTEWriteSpec::Bucket { column, .. }
+            | lancedb::table::RTEWriteSpec::Identity { column, .. } => Some(column.clone()),
+            lancedb::table::RTEWriteSpec::Unsharded { .. } => None,
         }
     }
 
@@ -385,12 +386,12 @@ impl LsmWriteSpec {
     #[getter]
     pub fn num_buckets(&self) -> Option<u32> {
         match &self.inner {
-            lancedb::table::LsmWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
+            lancedb::table::RTEWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
             _ => None,
         }
     }
 
-    /// Indexes the MemWAL keeps up to date, or `None` for every supported one.
+    /// Indexes the RTE keeps up to date, or `None` for every supported one.
     #[getter]
     pub fn maintained_indexes(&self) -> Option<Vec<String>> {
         self.inner.maintained_indexes().map(<[String]>::to_vec)
@@ -403,14 +404,14 @@ impl LsmWriteSpec {
     }
 }
 
-impl From<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
-    fn from(spec: LsmWriteSpec) -> Self {
+impl From<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
+    fn from(spec: RTEWriteSpec) -> Self {
         spec.inner
     }
 }
 
-impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
-    fn from(inner: lancedb::table::LsmWriteSpec) -> Self {
+impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
+    fn from(inner: lancedb::table::RTEWriteSpec) -> Self {
         Self { inner }
     }
 }
@@ -1133,7 +1134,9 @@ impl Table {
         train: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let index = extract_index_params(&index)?;
-        let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
+        let timeout = wait_timeout
+            .map(|t| t.extract::<std::time::Duration>())
+            .transpose()?;
         let mut op = self_
             .inner_ref()?
             .create_index_with_timeout(&[column], index, timeout);
@@ -1164,7 +1167,9 @@ impl Table {
         train: Option<bool>,
     ) -> PyResult<Bound<'a, PyAny>> {
         let index = extract_index_params(&index)?;
-        let timeout = wait_timeout.map(|t| t.extract::<std::time::Duration>().unwrap());
+        let timeout = wait_timeout
+            .map(|t| t.extract::<std::time::Duration>())
+            .transpose()?;
         let mut op = self_
             .inner_ref()?
             .create_index_with_timeout(&[column], index, timeout);
@@ -1427,8 +1432,8 @@ impl Table {
         })
     }
 
-    pub fn query(&self) -> Query {
-        Query::new(self.inner_ref().unwrap().query())
+    pub fn query(&self) -> PyResult<Query> {
+        Ok(Query::new(self.inner_ref()?.query()))
     }
 
     #[getter]
@@ -1636,8 +1641,8 @@ impl Table {
         if let Some(use_index) = parameters.use_index {
             builder.use_index(use_index);
         }
-        if let Some(use_lsm) = parameters.use_lsm {
-            builder.use_lsm(use_lsm);
+        if let Some(use_rte) = parameters.use_rte {
+            builder.use_rte(use_rte);
         }
         if let Some(validate_single_shard) = parameters.validate_single_shard {
             builder.validate_single_shard(validate_single_shard);
@@ -1662,29 +1667,29 @@ impl Table {
         })
     }
 
-    pub fn set_lsm_write_spec<'a>(
+    pub fn enable_rte<'a>(
         self_: PyRef<'a, Self>,
-        spec: LsmWriteSpec,
+        spec: RTEWriteSpec,
     ) -> PyResult<Bound<'a, PyAny>> {
         let inner = self_.inner_ref()?.clone();
-        let native_spec = lancedb::table::LsmWriteSpec::from(spec);
+        let native_spec = lancedb::table::RTEWriteSpec::from(spec);
         future_into_py(self_.py(), async move {
-            inner.set_lsm_write_spec(native_spec).await.infer_error()
+            inner.enable_rte(native_spec).await.infer_error()
         })
     }
 
-    pub fn unset_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn disable_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
-            inner.unset_lsm_write_spec().await.infer_error()
+            inner.disable_rte().await.infer_error()
         })
     }
 
-    pub fn get_lsm_write_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn get_rte_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
-            let spec = inner.get_lsm_write_spec().await.infer_error()?;
-            Ok(spec.map(LsmWriteSpec::from))
+            let spec = inner.get_rte_spec().await.infer_error()?;
+            Ok(spec.map(RTEWriteSpec::from))
         })
     }
 
@@ -1703,55 +1708,55 @@ impl Table {
         Ok(())
     }
 
-    /// Converge the table's LSM write path into its base table.
+    /// Converge the table's RTE write path into its base table.
     ///
     /// Best-effort: with writes flowing, new rows may land after the last
     /// pass. Errors if the table stops making progress.
-    pub fn checkpoint_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn checkpoint_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
-            inner.checkpoint_lsm().await.infer_error()
+            inner.checkpoint_rte().await.infer_error()
         })
     }
 
-    /// Seal every bucket's active memtable into L0.
-    pub fn flush_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    /// Freeze every table shard's active memtable into an SSTable.
+    pub fn flush_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(
             self_.py(),
-            async move { inner.flush_lsm().await.infer_error() },
+            async move { inner.flush_rte().await.infer_error() },
         )
     }
 
-    /// Trigger a background L0 → base pass per bucket. Returns once the
-    /// passes are dispatched, not once they finish — watch `get_lsm_stats`.
-    pub fn compact_lsm(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    /// Trigger a background SSTable compaction pass per table shard. Returns once the
+    /// passes are dispatched, not once they finish — watch `get_rte_stats`.
+    pub fn compact_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
-            inner.compact_lsm().await.infer_error()
+            inner.compact_rte().await.infer_error()
         })
     }
 
-    /// Live LSM state, or `None` when the LSM write path is not enabled.
-    #[pyo3(signature = (include_generation_rows=false))]
-    pub fn get_lsm_stats(
+    /// Live RTE state, or `None` when the RTE write path is not enabled.
+    #[pyo3(signature = (include_sstable_rows=false))]
+    pub fn get_rte_stats(
         self_: PyRef<'_, Self>,
-        include_generation_rows: bool,
+        include_sstable_rows: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let stats = inner
-                .get_lsm_stats(include_generation_rows)
+                .get_rte_stats(include_sstable_rows)
                 .await
                 .infer_error()?;
-            Python::attach(|py| stats.map(|s| lsm_stats_to_py(py, &s)).transpose())
+            Python::attach(|py| stats.map(|s| rte_stats_to_py(py, &s)).transpose())
         })
     }
 
-    pub fn close_lsm_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
+    pub fn close_rte_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
-            inner.close_lsm_writers().await.infer_error()
+            inner.close_rte_writers().await.infer_error()
         })
     }
 
@@ -1803,35 +1808,59 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (columns, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn add_computed_columns(
         self_: PyRef<'_, Self>,
         columns: Vec<(String, String)>,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<u64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
+        let options = ComputedColumnOptions {
+            auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
+            auto_refresh_interval_seconds,
+        };
+        options.validate().infer_error()?;
         future_into_py(self_.py(), async move {
             let mut builder = inner.add_columns();
             for (name, expression) in columns {
                 builder = builder.computed(name, expression);
             }
-            let result = builder.execute().await.infer_error()?;
+            let result = builder
+                .computed_options(options)
+                .execute()
+                .await
+                .infer_error()?;
             Ok(AddColumnsResult::from(result))
         })
     }
 
+    #[pyo3(signature = (application_json, output_name, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn add_function_columns(
         self_: PyRef<'_, Self>,
         application_json: String,
         output_name: Option<String>,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<u64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let application =
             lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
         let inner = self_.inner_ref()?.clone();
+        let options = ComputedColumnOptions {
+            auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
+            auto_refresh_interval_seconds,
+        };
+        options.validate().infer_error()?;
         future_into_py(self_.py(), async move {
             let builder = match output_name {
                 Some(name) => inner.add_columns().function_as(name, application),
                 None => inner.add_columns().function(application),
             };
-            let result = builder.execute().await.infer_error()?;
+            let result = builder
+                .computed_options(options)
+                .execute()
+                .await
+                .infer_error()?;
             Ok(AddColumnsResult::from(result))
         })
     }
@@ -2083,7 +2112,7 @@ pub struct MergeInsertParams {
     when_not_matched_by_source_condition_expr: Option<PyExpr>,
     timeout: Option<std::time::Duration>,
     use_index: Option<bool>,
-    use_lsm: Option<bool>,
+    use_rte: Option<bool>,
     validate_single_shard: Option<bool>,
 }
 

python/uv.lock

@@ -443,11 +443,11 @@ wheels = [
 
 [[package]]
 name = "certifi"
-version = "2026.6.17"
+version = "2026.10.19"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
 ]
 
 [[package]]

python/vendor/lance-arrow/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "lance-arrow"
+version = "15.0.0-beta.1"
+edition = "2024"
+authors = ["Lance Devs <dev@lance.org>"]
+license = "Apache-2.0"
+repository = "https://github.com/lance-format/lance"
+readme = "README.md"
+description = "Arrow Extension for Lance"
+keywords = ["data-format", "data-science", "machine-learning", "apache-arrow", "data-analytics"]
+categories = ["database-implementations"]
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+arrow-array.workspace = true
+arrow-buffer.workspace = true
+arrow-data.workspace = true
+arrow-ipc = { workspace = true, features = ["zstd"] }
+arrow-ord.workspace = true
+arrow-schema.workspace = true
+arrow-select.workspace = true
+bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] }
+bytes.workspace = true
+futures.workspace = true
+half = { workspace = true, features = ["std", "bytemuck"] }
+lance-jsonb.workspace = true
+num-traits.workspace = true
+rand = { workspace = true, features = ["small_rng"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+getrandom = { version = "0.2", features = ["js"] }

python/vendor/lance-arrow/README.lancedb.md

@@ -0,0 +1,7 @@
+This directory contains `rust/lance-arrow` from Lance tag `v15.0.0-beta.1`
+(commit `46a7cfe1fba5efa1f90deea4c7b14d414b991409`), licensed under Apache-2.0.
+
+LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
+until a Lance release includes the fix. The patch is selected in the root
+`Cargo.toml`; the rest of Lance stays on the pinned release. An explicit unsafe
+block in `src/bfloat16.rs` lets this copy pass LanceDB's Rust 2024 warning gate.

python/vendor/lance-arrow/README.md

@@ -0,0 +1,6 @@
+# lance-arrow
+
+`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs)
+extensions used by [Lance](https://github.com/lance-format/lance).
+
+**Important Note**: This crate is **not intended for external usage**.

python/vendor/lance-arrow/src/bfloat16.rs

@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! bfloat16 support for Apache Arrow.
+
+use std::fmt::Formatter;
+use std::slice;
+
+use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder};
+use arrow_buffer::{Buffer, MutableBuffer};
+use arrow_data::ArrayData;
+use arrow_schema::{ArrowError, DataType, Field as ArrowField};
+use half::bf16;
+
+use crate::{ARROW_EXT_NAME_KEY, FloatArray};
+
+/// The name of the bfloat16 extension in Arrow metadata
+pub const BFLOAT16_EXT_NAME: &str = "lance.bfloat16";
+
+/// Check whether the given field is a bfloat16 field
+///
+/// A field is a bfloat16 field if it has a data type of `FixedSizeBinary(2)` and the metadata
+/// contains the bfloat16 extension name.
+pub fn is_bfloat16_field(field: &ArrowField) -> bool {
+    field.data_type() == &DataType::FixedSizeBinary(2)
+        && field
+            .metadata()
+            .get(ARROW_EXT_NAME_KEY)
+            .map(|name| name == BFLOAT16_EXT_NAME)
+            .unwrap_or_default()
+}
+
+/// The bfloat16 data type
+///
+/// This implements the [`ArrowFloatType`](crate::floats::ArrowFloatType) trait for bfloat16 values.
+#[derive(Debug)]
+pub struct BFloat16Type {}
+
+/// An array of bfloat16 values
+///
+/// Note that bfloat16 is not the same thing as fp16 which is supported natively by arrow-rs.
+#[derive(Clone)]
+pub struct BFloat16Array {
+    inner: FixedSizeBinaryArray,
+}
+
+impl std::fmt::Debug for BFloat16Array {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "BFloat16Array\n[\n")?;
+        from_arrow::print_long_array(&self.inner, f, |array, i, f| {
+            if array.is_null(i) {
+                write!(f, "null")
+            } else {
+                let binary_values = array.value(i);
+                let value =
+                    bf16::from_bits(u16::from_le_bytes([binary_values[0], binary_values[1]]));
+                write!(f, "{:?}", value)
+            }
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl BFloat16Array {
+    pub fn from_iter_values(iter: impl IntoIterator<Item = bf16>) -> Self {
+        let values: Vec<bf16> = iter.into_iter().collect();
+        values.into()
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+
+    pub fn is_null(&self, i: usize) -> bool {
+        self.inner.is_null(i)
+    }
+
+    pub fn null_count(&self) -> usize {
+        self.inner.null_count()
+    }
+
+    pub fn iter(&self) -> BFloat16Iter<'_> {
+        BFloat16Iter {
+            array: self,
+            index: 0,
+        }
+    }
+
+    pub fn value(&self, i: usize) -> bf16 {
+        assert!(
+            i < self.len(),
+            "Trying to access an element at index {} from a BFloat16Array of length {}",
+            i,
+            self.len()
+        );
+        // Safety:
+        // `i < self.len()
+        unsafe { self.value_unchecked(i) }
+    }
+
+    /// # Safety
+    /// Caller must ensure that `i < self.len()`
+    pub unsafe fn value_unchecked(&self, i: usize) -> bf16 {
+        // SAFETY: the caller guarantees that `i` is within this array's bounds.
+        let binary_value = unsafe { self.inner.value_unchecked(i) };
+        bf16::from_bits(u16::from_le_bytes([binary_value[0], binary_value[1]]))
+    }
+
+    pub fn into_inner(self) -> FixedSizeBinaryArray {
+        self.inner
+    }
+}
+
+impl FromIterator<Option<bf16>> for BFloat16Array {
+    fn from_iter<I: IntoIterator<Item = Option<bf16>>>(iter: I) -> Self {
+        let mut buffer = MutableBuffer::new(10);
+        // No null buffer builder :(
+        let mut nulls = BooleanBufferBuilder::new(10);
+        let mut len = 0;
+
+        for maybe_value in iter {
+            if let Some(value) = maybe_value {
+                let bytes = value.to_le_bytes();
+                buffer.extend(bytes);
+            } else {
+                buffer.extend([0u8, 0u8]);
+            }
+            nulls.append(maybe_value.is_some());
+            len += 1;
+        }
+
+        let null_buffer = nulls.finish();
+        let num_valid = null_buffer.count_set_bits();
+        let null_buffer = if num_valid == len {
+            None
+        } else {
+            Some(null_buffer.into_inner())
+        };
+
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
+            .len(len)
+            .add_buffer(buffer.into())
+            .null_bit_buffer(null_buffer);
+        // SAFETY: the value buffer contains exactly `2 * len` bytes (two bytes
+        // pushed per iteration of the loop above, including the zero-fill for
+        // null slots), which matches the `FixedSizeBinary(2)` storage layout.
+        // The null bit buffer, when present, has `len` bits appended above, so
+        // its length covers the array's logical range.
+        let array_data = unsafe { array_data.build_unchecked() };
+        Self {
+            inner: FixedSizeBinaryArray::from(array_data),
+        }
+    }
+}
+
+impl FromIterator<bf16> for BFloat16Array {
+    fn from_iter<I: IntoIterator<Item = bf16>>(iter: I) -> Self {
+        Self::from_iter_values(iter)
+    }
+}
+
+impl From<Vec<bf16>> for BFloat16Array {
+    fn from(data: Vec<bf16>) -> Self {
+        let len = data.len();
+        // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives
+        // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place —
+        // no per-element copy or heap alloc. The crate-root `compile_error!`
+        // pins `target_endian = "little"`, so the resulting bytes match the
+        // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere.
+        let raw: Vec<u16> = bytemuck::cast_vec(data);
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
+            .len(len)
+            .add_buffer(Buffer::from_vec(raw));
+        // SAFETY: the value buffer contains exactly `2 * len` bytes — one
+        // `u16` per element after the layout-compatible cast — matching the
+        // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so
+        // every element is logically valid.
+        let array_data = unsafe { array_data.build_unchecked() };
+        Self {
+            inner: FixedSizeBinaryArray::from(array_data),
+        }
+    }
+}
+
+impl TryFrom<FixedSizeBinaryArray> for BFloat16Array {
+    type Error = ArrowError;
+
+    fn try_from(value: FixedSizeBinaryArray) -> Result<Self, Self::Error> {
+        if value.value_length() == 2 {
+            Ok(Self { inner: value })
+        } else {
+            Err(ArrowError::InvalidArgumentError(
+                "FixedSizeBinaryArray must have a value length of 2".to_string(),
+            ))
+        }
+    }
+}
+
+impl PartialEq<Self> for BFloat16Array {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner.eq(&other.inner)
+    }
+}
+
+pub struct BFloat16Iter<'a> {
+    array: &'a BFloat16Array,
+    index: usize,
+}
+
+impl<'a> Iterator for BFloat16Iter<'a> {
+    type Item = Option<bf16>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index >= self.array.len() {
+            return None;
+        }
+        let i = self.index;
+        self.index += 1;
+        if self.array.is_null(i) {
+            Some(None)
+        } else {
+            Some(Some(self.array.value(i)))
+        }
+    }
+}
+
+/// Methods that are lifted from arrow-rs temporarily until they are made public.
+mod from_arrow {
+    use arrow_array::Array;
+
+    /// Helper function for printing potentially long arrays.
+    pub(super) fn print_long_array<A, F>(
+        array: &A,
+        f: &mut std::fmt::Formatter,
+        print_item: F,
+    ) -> std::fmt::Result
+    where
+        A: Array,
+        F: Fn(&A, usize, &mut std::fmt::Formatter) -> std::fmt::Result,
+    {
+        let head = std::cmp::min(10, array.len());
+
+        for i in 0..head {
+            if array.is_null(i) {
+                writeln!(f, "  null,")?;
+            } else {
+                write!(f, "  ")?;
+                print_item(array, i, f)?;
+                writeln!(f, ",")?;
+            }
+        }
+        if array.len() > 10 {
+            if array.len() > 20 {
+                writeln!(f, "  ...{} elements...,", array.len() - 20)?;
+            }
+
+            let tail = std::cmp::max(head, array.len() - 10);
+
+            for i in tail..array.len() {
+                if array.is_null(i) {
+                    writeln!(f, "  null,")?;
+                } else {
+                    write!(f, "  ")?;
+                    print_item(array, i, f)?;
+                    writeln!(f, ",")?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
+    type FloatType = BFloat16Type;
+
+    /// Returns the underlying `bf16` values as a borrowed slice.
+    ///
+    /// # Preconditions
+    ///
+    /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape
+    ///   used by [`BFloat16Array`]). Asserted at entry.
+    /// - The value buffer must be at least 2-byte aligned. Lance's in-tree
+    ///   constructors always satisfy this: value buffers are built either via
+    ///   `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32
+    ///   bytes) or via `Buffer::from_vec::<u16>` (aligned to `align_of::<u16>()`
+    ///   == 2); both meet `bf16`'s 2-byte requirement. Externally-built
+    ///   `FixedSizeBinaryArray`s arriving via FFI, IPC, or
+    ///   `Buffer::from_custom_allocation` are not required by arrow-rs to be
+    ///   aligned beyond a single byte; passing one to this method violates the
+    ///   precondition. A `debug_assert` below catches such inputs in debug and
+    ///   test builds.
+    ///
+    /// # Endianness
+    ///
+    /// `lance-arrow` is gated on `target_endian = "little"` at the crate root,
+    /// so this method always returns values in the same byte order Lance writes
+    /// (see [`BFloat16Array::value`] and the [`FromIterator`] impls).
+    fn as_slice(&self) -> &[bf16] {
+        assert_eq!(
+            self.value_length(),
+            2,
+            "BFloat16 arrays must use FixedSizeBinary(2) storage"
+        );
+        debug_assert_eq!(
+            (self.value_data().as_ptr() as usize) % std::mem::align_of::<bf16>(),
+            0,
+            "BFloat16 value buffer must be at least 2-byte aligned"
+        );
+        // SAFETY:
+        // - The assert above pins `value_size == 2`, so `value_data().len() / 2`
+        //   equals the array's logical element count.
+        //   `FixedSizeBinaryArray::From<ArrayData>` constructs its value buffer
+        //   as `buffers[0].slice_with_length(offset * 2, len * 2)` (arrow-array
+        //   `fixed_size_binary_array.rs`), so `value_data()` already returns
+        //   the offset-adjusted slice. Do not replace `value_data()` with an
+        //   accessor that returns the un-sliced backing buffer.
+        // - `bf16` is `#[repr(transparent)]` over `u16` (size 2, alignment 2);
+        //   every `u16` bit pattern is a valid `bf16`, so any byte content
+        //   yields a defined value — never UB.
+        // - Alignment is the caller's responsibility per the precondition
+        //   documented above. The `debug_assert_eq!` immediately preceding this
+        //   block catches violations in debug and test builds only — release
+        //   builds rely on callers honoring the precondition. arrow-rs
+        //   declares `FixedSizeBinary(n)`'s
+        //   `BufferSpec::FixedWidth { alignment: align_of::<u8>() == 1 }`
+        //   (arrow-data `data.rs`), so arrow-rs alone does not guarantee
+        //   2-byte alignment. Lance's in-tree construction paths build value
+        //   buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant,
+        //   ≥32 bytes) or `Buffer::from_vec::<u16>` (2-byte aligned), both of
+        //   which satisfy `bf16`'s 2-byte requirement.
+        // - The returned slice borrows from `self`; the underlying ref-counted,
+        //   immutable Arrow buffer cannot be mutated or freed for the slice's
+        //   lifetime.
+        unsafe {
+            slice::from_raw_parts(
+                self.value_data().as_ptr() as *const bf16,
+                self.value_data().len() / 2,
+            )
+        }
+    }
+
+    fn from_values(values: Vec<bf16>) -> Self {
+        BFloat16Array::from(values).into_inner()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basics() {
+        let values: Vec<f32> = vec![1.0, 2.0, 3.0];
+        let values: Vec<bf16> = values.iter().map(|v| bf16::from_f32(*v)).collect();
+
+        let array = BFloat16Array::from_iter_values(values.clone());
+        let array2 = BFloat16Array::from(values.clone());
+        assert_eq!(array, array2);
+        assert_eq!(array.len(), 3);
+
+        // Pin the raw little-endian bytes emitted by `From<Vec<bf16>>` (rewritten to
+        // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order
+        // regression is caught directly rather than only through Debug formatting.
+        // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040.
+        let inner = array2.clone().into_inner();
+        let raw_bytes: Vec<u8> = (0..inner.len())
+            .flat_map(|i| inner.value(i).to_vec())
+            .collect();
+        assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]);
+
+        let expected_fmt = "BFloat16Array\n[\n  1.0,\n  2.0,\n  3.0,\n]";
+        assert_eq!(expected_fmt, format!("{:?}", array));
+
+        for (expected, value) in values.iter().zip(array.iter()) {
+            assert_eq!(Some(*expected), value);
+        }
+
+        for (expected, value) in values.as_slice().iter().zip(array2.iter()) {
+            assert_eq!(Some(*expected), value);
+        }
+
+        let arrow_array = array.into_inner();
+        assert_eq!(arrow_array.as_slice(), values.as_slice());
+    }
+
+    #[test]
+    fn test_nulls() {
+        let values: Vec<Option<bf16>> =
+            vec![Some(bf16::from_f32(1.0)), None, Some(bf16::from_f32(3.0))];
+        let array = BFloat16Array::from_iter(values.clone());
+        assert_eq!(array.len(), 3);
+        assert_eq!(array.null_count(), 1);
+
+        let expected_fmt = "BFloat16Array\n[\n  1.0,\n  null,\n  3.0,\n]";
+        assert_eq!(expected_fmt, format!("{:?}", array));
+
+        for (expected, value) in values.iter().zip(array.iter()) {
+            assert_eq!(*expected, value);
+        }
+    }
+}

python/vendor/lance-arrow/src/deepcopy.rs

@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use arrow_array::{Array, RecordBatch, make_array};
+use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer};
+use arrow_data::{ArrayData, ArrayDataBuilder, transform::MutableArrayData};
+use arrow_schema::DataType;
+
+pub fn deep_copy_buffer(buffer: &Buffer) -> Buffer {
+    Buffer::from(buffer.as_slice())
+}
+
+pub fn deep_copy_nulls(nulls: Option<&NullBuffer>) -> Option<NullBuffer> {
+    let nulls = nulls?;
+    let bit_buffer = deep_copy_buffer(nulls.inner().inner());
+    // SAFETY: `null_count` is taken from the source `NullBuffer`, which already
+    // upheld `NullBuffer::new_unchecked`'s invariant — the unset-bit count over
+    // the logical bit slice `[bit_offset, bit_offset + bit_len)`. `NullBuffer::slice`
+    // adjusts only `BooleanBuffer::bit_offset` / `bit_len` and never byte-advances
+    // the inner `Buffer`, so `deep_copy_buffer` (which copies the source `Buffer`'s
+    // `as_slice()` view from byte 0) reproduces the exact bit pattern at the same
+    // bit offsets; the unset-bit count is therefore preserved. `BooleanBuffer::new`
+    // panics (does not UB) if `bit_offset + bit_len > 8 * buffer.len()`, and the
+    // copy has the same length, so that check still passes.
+    Some(unsafe {
+        NullBuffer::new_unchecked(
+            BooleanBuffer::new(bit_buffer, nulls.offset(), nulls.len()),
+            nulls.null_count(),
+        )
+    })
+}
+
+pub fn deep_copy_array_data(data: &ArrayData) -> ArrayData {
+    let data_type = data.data_type().clone();
+    let len = data.len();
+    let nulls = deep_copy_nulls(data.nulls());
+    let offset = data.offset();
+    let buffers = data
+        .buffers()
+        .iter()
+        .map(deep_copy_buffer)
+        .collect::<Vec<_>>();
+    let child_data = data
+        .child_data()
+        .iter()
+        .map(deep_copy_array_data)
+        .collect::<Vec<_>>();
+    // SAFETY: `build_unchecked` inherits `ArrayData::new_unchecked`'s contract —
+    // `(data_type, len, offset, nulls, buffers, child_data)` must form a valid
+    // Arrow array. This call reproduces `data` structurally: `data_type`, `len`,
+    // and `offset` are forwarded unchanged; each buffer is replaced by a byte-
+    // identical copy of its offset-applied `as_slice()` view (the output buffer
+    // is `MutableBuffer`-allocated, at least as aligned as the source); `nulls`
+    // is deep-copied with the same bit offset/length and unset-bit count (see
+    // `deep_copy_nulls`); `child_data` is recursively cloned with the same
+    // guarantee. Every value-level invariant the source upheld — UTF-8 validity,
+    // monotonic offsets, in-bounds dictionary indices, run-end monotonicity,
+    // struct child-length matching — therefore transfers to the copy. If the
+    // source `ArrayData` was itself constructed via `new_unchecked` with an
+    // invalid payload, this function faithfully reproduces that invalidity.
+    unsafe {
+        ArrayDataBuilder::new(data_type)
+            .len(len)
+            .nulls(nulls)
+            .offset(offset)
+            .buffers(buffers)
+            .child_data(child_data)
+            .build_unchecked()
+    }
+}
+
+pub fn deep_copy_array(array: &dyn Array) -> Arc<dyn Array> {
+    let data = array.to_data();
+    let data = deep_copy_array_data(&data);
+    make_array(data)
+}
+
+pub fn deep_copy_batch(batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    let arrays = batch
+        .columns()
+        .iter()
+        .map(|array| deep_copy_array(array))
+        .collect::<Vec<_>>();
+    RecordBatch::try_new(batch.schema(), arrays)
+}
+
+/// Deep copy array data, extracting only the sliced portion using MutableArrayData
+/// This is the most efficient and correct way to copy just the sliced data
+pub fn deep_copy_array_data_sliced(data: &ArrayData) -> ArrayData {
+    // Use MutableArrayData to efficiently copy just the slice
+    let mut mutable = MutableArrayData::new(vec![data], false, data.len());
+
+    // Which index this takes depends on the layout, because arrow's extenders
+    // disagree about who applies `ArrayData::offset()`. The bit-packed one adds
+    // it to the raw values buffer itself, so an offset-applied index there
+    // reads from twice the offset and runs off the end of the buffer. Every
+    // other layout either reads through an already-offset-applied view or
+    // forwards the index to children carrying their own offsets, and takes the
+    // absolute index it has always been given.
+    let start = match data.data_type() {
+        DataType::Boolean => 0,
+        _ => data.offset(),
+    };
+    mutable.extend(0, start, start + data.len());
+
+    // Freeze into immutable ArrayData
+    mutable.freeze()
+}
+
+/// Deep copy an array, extracting only the sliced portion using MutableArrayData
+pub fn deep_copy_array_sliced(array: &dyn Array) -> Arc<dyn Array> {
+    let data = array.to_data();
+    let data = deep_copy_array_data_sliced(&data);
+    make_array(data)
+}
+
+/// Deep copy a RecordBatch, extracting only the sliced portion using MutableArrayData
+pub fn deep_copy_batch_sliced(batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    let arrays = batch
+        .columns()
+        .iter()
+        .map(|array| deep_copy_array_sliced(array))
+        .collect::<Vec<_>>();
+    RecordBatch::try_new(batch.schema(), arrays)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch, StringArray};
+    use arrow_data::ArrayDataBuilder;
+    use arrow_schema::{DataType, Field, Schema};
+
+    #[test]
+    fn raw_sliced_fixed_size_list_data_keeps_its_child_offset() {
+        // `ArrayData::slice` records the offset on the parent and leaves the
+        // child whole -- a shape `FixedSizeListArray::slice` never produces,
+        // but a legal one this public helper can be handed directly. Arrow's
+        // extender forwards `start * size` to that unsliced child and adds
+        // nothing, so the index it gets has to be the absolute one; this is
+        // here to stop the boolean fix from being generalised over it.
+        let child = Int32Array::from(vec![10, 11, 20, 21]).to_data();
+        let field = Arc::new(Field::new_list_field(DataType::Int32, false));
+        let data = ArrayDataBuilder::new(DataType::FixedSizeList(field, 2))
+            .len(2)
+            .add_child_data(child)
+            .build()
+            .unwrap();
+        let sliced = data.slice(1, 1);
+
+        let copied = super::deep_copy_array_data_sliced(&sliced);
+        let copied_child = Int32Array::from(copied.child_data()[0].clone());
+        assert_eq!(copied_child.values().as_ref(), &[20, 21]);
+    }
+
+    #[test]
+    fn raw_sliced_struct_of_fixed_size_list_reaches_the_right_values() {
+        // Slicing raw struct data pushes the window into the struct's children,
+        // and a fixed-size-list child takes it as its own parent offset with
+        // its values left whole. Two levels of offset, and the absolute index
+        // has to remain right through both of them.
+        let values = Int32Array::from(vec![10, 11, 20, 21, 30, 31]).to_data();
+        let item = Arc::new(Field::new_list_field(DataType::Int32, false));
+        let list = ArrayDataBuilder::new(DataType::FixedSizeList(item, 2))
+            .len(3)
+            .add_child_data(values)
+            .build()
+            .unwrap();
+        let field = Arc::new(Field::new("l", list.data_type().clone(), false));
+        let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
+            .len(3)
+            .add_child_data(list)
+            .build()
+            .unwrap();
+        let sliced = data.slice(2, 1);
+
+        let copied = super::deep_copy_array_data_sliced(&sliced);
+        let copied_values = Int32Array::from(copied.child_data()[0].child_data()[0].clone());
+        assert_eq!(copied_values.values().as_ref(), &[30, 31]);
+    }
+
+    #[test]
+    fn raw_parent_offset_struct_keeps_the_selected_child_row() {
+        // A struct whose children are whole and whose window is the parent
+        // offset: arrow's struct extender forwards the index to those children
+        // untouched, so it has to be the absolute one.
+        let field = Arc::new(Field::new("a", DataType::Int32, false));
+        let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
+            .len(1)
+            .offset(1)
+            .add_child_data(Int32Array::from(vec![10, 20]).to_data())
+            .build()
+            .unwrap();
+
+        let copied = super::deep_copy_array_data_sliced(&data);
+        let copied_child = Int32Array::from(copied.child_data()[0].clone());
+        assert_eq!(copied_child.values().as_ref(), &[20]);
+    }
+
+    #[test]
+    fn sliced_boolean_deep_copy_reads_from_the_slice() {
+        // A boolean slice keeps its `ArrayData::offset()` -- the buffer cannot
+        // advance by a fraction of a byte -- so a copy that adds the offset on
+        // top of what `MutableArrayData` already applies reads past the end of
+        // the values buffer and panics.
+        let array = BooleanArray::from(vec![true, false, true, false, true, false, true, false]);
+        for (offset, len) in [(0usize, 8usize), (1, 7), (3, 5), (7, 1)] {
+            let sliced = array.slice(offset, len);
+            let copied = super::deep_copy_array_sliced(&sliced);
+            let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
+            let expected: Vec<bool> = (0..len).map(|i| sliced.value(i)).collect();
+            let actual: Vec<bool> = (0..len).map(|i| copied.value(i)).collect();
+            assert_eq!(actual, expected, "offset={offset} len={len}");
+        }
+    }
+
+    #[test]
+    fn sliced_boolean_deep_copy_keeps_its_nulls() {
+        let array = BooleanArray::from(vec![
+            Some(true),
+            None,
+            Some(false),
+            Some(true),
+            None,
+            Some(false),
+        ]);
+        let sliced = array.slice(1, 4);
+        let copied = super::deep_copy_array_sliced(&sliced);
+        let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
+        let expected: Vec<Option<bool>> = (0..sliced.len())
+            .map(|i| (!sliced.is_null(i)).then(|| sliced.value(i)))
+            .collect();
+        let actual: Vec<Option<bool>> = (0..copied.len())
+            .map(|i| (!copied.is_null(i)).then(|| copied.value(i)))
+            .collect();
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn test_deep_copy_sliced_array_with_nulls() {
+        let array = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            None,
+            Some(5),
+        ]));
+        let sliced_array = array.slice(1, 3);
+        let copied_array = super::deep_copy_array(&sliced_array);
+        assert_eq!(sliced_array.len(), copied_array.len());
+        assert_eq!(sliced_array.nulls(), copied_array.nulls());
+    }
+
+    #[test]
+    fn test_deep_copy_array_data_sliced() {
+        let array = Int32Array::from((0..1000).collect::<Vec<i32>>());
+        let sliced = array.slice(100, 10);
+
+        let sliced_data = sliced.to_data();
+        let copied_data = super::deep_copy_array_data_sliced(&sliced_data);
+
+        assert_eq!(copied_data.len(), 10);
+        assert_eq!(copied_data.offset(), 0);
+
+        // Verify data correctness
+        let copied_array = Int32Array::from(copied_data);
+        for i in 0..10 {
+            assert_eq!(copied_array.value(i), 100 + i as i32);
+        }
+    }
+
+    #[test]
+    fn test_deep_copy_array_sliced() {
+        let array = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        let sliced = array.slice(1, 3);
+
+        let copied = super::deep_copy_array_sliced(&sliced);
+
+        assert_eq!(copied.len(), 3);
+        let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(copied_int.value(0), 2);
+        assert_eq!(copied_int.value(1), 3);
+        assert_eq!(copied_int.value(2), 4);
+    }
+
+    #[test]
+    fn test_deep_copy_batch_sliced() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+        ]));
+
+        let id_array = Arc::new(Int32Array::from((0..100).collect::<Vec<i32>>()));
+        let name_array = Arc::new(StringArray::from(
+            (0..100)
+                .map(|i| format!("name_{}", i))
+                .collect::<Vec<String>>(),
+        ));
+
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![id_array as Arc<dyn Array>, name_array as Arc<dyn Array>],
+        )
+        .unwrap();
+
+        let sliced = batch.slice(10, 5);
+        let copied = super::deep_copy_batch_sliced(&sliced).unwrap();
+
+        assert_eq!(copied.num_rows(), 5);
+        assert_eq!(copied.num_columns(), 2);
+
+        // Verify data correctness
+        let id_col = copied
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        let name_col = copied
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+
+        for i in 0..5 {
+            assert_eq!(id_col.value(i), 10 + i as i32);
+            assert_eq!(name_col.value(i), format!("name_{}", 10 + i));
+        }
+    }
+
+    #[test]
+    fn test_deep_copy_array_sliced_with_nulls() {
+        let array = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            None,
+            Some(5),
+        ]));
+        let sliced = array.slice(1, 3); // [None, Some(3), None]
+
+        let copied = super::deep_copy_array_sliced(&sliced);
+
+        assert_eq!(copied.len(), 3);
+        assert_eq!(copied.null_count(), 2); // Two nulls in the slice
+
+        let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert!(!copied_int.is_valid(0)); // None
+        assert!(copied_int.is_valid(1)); // Some(3)
+        assert!(!copied_int.is_valid(2)); // None
+        assert_eq!(copied_int.value(1), 3);
+    }
+}

python/vendor/lance-arrow/src/floats.rs

@@ -0,0 +1,292 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Floats Array
+
+use std::fmt::{Debug, Display};
+use std::iter::Sum;
+use std::sync::Arc;
+use std::{
+    fmt::Formatter,
+    ops::{AddAssign, DivAssign},
+};
+
+use arrow_array::{
+    Array, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array,
+    types::{Float16Type, Float32Type, Float64Type},
+};
+use arrow_schema::{DataType, Field};
+use half::{bf16, f16};
+use num_traits::{AsPrimitive, Bounded, Float, FromPrimitive};
+
+use super::bfloat16::{BFloat16Array, BFloat16Type};
+use crate::Result;
+use crate::bfloat16::is_bfloat16_field;
+
+/// Float data type.
+///
+/// This helps differentiate between the different float types,
+/// because bf16 is not officially supported [DataType] in arrow-rs.
+#[derive(Debug)]
+pub enum FloatType {
+    BFloat16,
+    Float16,
+    Float32,
+    Float64,
+}
+
+impl std::fmt::Display for FloatType {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::BFloat16 => write!(f, "bfloat16"),
+            Self::Float16 => write!(f, "float16"),
+            Self::Float32 => write!(f, "float32"),
+            Self::Float64 => write!(f, "float64"),
+        }
+    }
+}
+
+/// Try to convert a [DataType] to a [FloatType]. To support bfloat16, always
+/// prefer using the `TryFrom<&Field>` implementation.
+impl TryFrom<&DataType> for FloatType {
+    type Error = crate::ArrowError;
+
+    fn try_from(value: &DataType) -> Result<Self> {
+        match *value {
+            DataType::Float16 => Ok(Self::Float16),
+            DataType::Float32 => Ok(Self::Float32),
+            DataType::Float64 => Ok(Self::Float64),
+            _ => Err(crate::ArrowError::InvalidArgumentError(format!(
+                "{:?} is not a floating type",
+                value
+            ))),
+        }
+    }
+}
+
+impl TryFrom<&Field> for FloatType {
+    type Error = crate::ArrowError;
+
+    fn try_from(field: &Field) -> Result<Self> {
+        match field.data_type() {
+            DataType::FixedSizeBinary(2) if is_bfloat16_field(field) => Ok(Self::BFloat16),
+            _ => Self::try_from(field.data_type()),
+        }
+    }
+}
+
+/// Trait for float types used in Lance indexes
+///
+/// This mimics the utilities provided by [`arrow_array::ArrowPrimitiveType`]
+/// but applies to all float types (including bfloat16)
+pub trait ArrowFloatType: Debug {
+    type Native: FromPrimitive
+        + FloatToArrayType<ArrowType = Self>
+        + AsPrimitive<f32>
+        + Debug
+        + Display;
+
+    const FLOAT_TYPE: FloatType;
+    const MIN: Self::Native;
+    const MAX: Self::Native;
+
+    /// Arrow Float Array Type.
+    type ArrayType: FloatArray<Self>;
+
+    /// Returns empty array of this type.
+    fn empty_array() -> Self::ArrayType {
+        <Self::ArrayType as FloatArray<Self>>::from_values(Vec::new())
+    }
+}
+
+/// Trait to be implemented by native types that have a corresponding [`ArrowFloatType`]
+/// implementation.
+///
+/// This helps define what operations are supported by native floats and also helps convert
+/// from a native type back to the corresponding Arrow float type.
+pub trait FloatToArrayType:
+    Float
+    + Bounded
+    + Sum
+    + AddAssign<Self>
+    + AsPrimitive<f64>
+    + AsPrimitive<f32>
+    + DivAssign
+    + Send
+    + Sync
+    + Copy
+{
+    /// The corresponding [`ArrowFloatType`] implementation for this native type
+    type ArrowType: ArrowFloatType<Native = Self>;
+}
+
+impl FloatToArrayType for bf16 {
+    type ArrowType = BFloat16Type;
+}
+
+impl FloatToArrayType for f16 {
+    type ArrowType = Float16Type;
+}
+
+impl FloatToArrayType for f32 {
+    type ArrowType = Float32Type;
+}
+
+impl FloatToArrayType for f64 {
+    type ArrowType = Float64Type;
+}
+
+impl ArrowFloatType for BFloat16Type {
+    type Native = bf16;
+
+    const FLOAT_TYPE: FloatType = FloatType::BFloat16;
+    const MIN: Self::Native = bf16::MIN;
+    const MAX: Self::Native = bf16::MAX;
+
+    type ArrayType = FixedSizeBinaryArray;
+}
+
+impl ArrowFloatType for Float16Type {
+    type Native = f16;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float16;
+    const MIN: Self::Native = f16::MIN;
+    const MAX: Self::Native = f16::MAX;
+
+    type ArrayType = Float16Array;
+}
+
+impl ArrowFloatType for Float32Type {
+    type Native = f32;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float32;
+    const MIN: Self::Native = f32::MIN;
+    const MAX: Self::Native = f32::MAX;
+
+    type ArrayType = Float32Array;
+}
+
+impl ArrowFloatType for Float64Type {
+    type Native = f64;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float64;
+    const MIN: Self::Native = f64::MIN;
+    const MAX: Self::Native = f64::MAX;
+
+    type ArrayType = Float64Array;
+}
+
+/// [FloatArray] is a trait that is implemented by all float type arrays
+///
+/// This is similar to [`arrow_array::PrimitiveArray`] but applies to all float types (including bfloat16)
+/// and is implemented as a trait and not a struct
+pub trait FloatArray<T: ArrowFloatType + ?Sized>: Array + Clone + 'static {
+    type FloatType: ArrowFloatType;
+
+    /// Returns a reference to the underlying data as a slice.
+    ///
+    /// # Panics
+    ///
+    /// Implementations may panic if the array's storage shape does not match
+    /// the expected element layout. In particular, the `bf16` impl panics if
+    /// `value_length() != 2` (the `FixedSizeBinary(2)` shape required by
+    /// `BFloat16Array`).
+    ///
+    /// # Preconditions
+    ///
+    /// Implementations may impose additional invariants on the underlying
+    /// buffer. The `bf16` impl requires the value buffer to be at least
+    /// 2-byte aligned — satisfied automatically by every in-tree Lance
+    /// constructor, but external callers passing externally-built arrays
+    /// (FFI, IPC, `Buffer::from_custom_allocation`) must ensure alignment.
+    /// See the impl's docstring for details.
+    fn as_slice(&self) -> &[T::Native];
+
+    /// Construct an array from a vector of values.
+    fn from_values(values: Vec<T::Native>) -> Self;
+
+    /// Construct an array from an iterator of values.
+    fn from_iter_values(values: impl IntoIterator<Item = T::Native>) -> Self
+    where
+        Self: Sized,
+    {
+        Self::from_values(values.into_iter().collect())
+    }
+}
+
+impl FloatArray<Float16Type> for Float16Array {
+    type FloatType = Float16Type;
+
+    fn as_slice(&self) -> &[<Float16Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float16Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+impl FloatArray<Float32Type> for Float32Array {
+    type FloatType = Float32Type;
+
+    fn as_slice(&self) -> &[<Float32Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float32Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+impl FloatArray<Float64Type> for Float64Array {
+    type FloatType = Float64Type;
+
+    fn as_slice(&self) -> &[<Float64Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float64Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+/// Convert a float32 array to another float array
+///
+/// This is used during queries as query vectors are always provided as float32 arrays
+/// and need to be converted to the appropriate float type for the index.
+pub fn coerce_float_vector(input: &Float32Array, float_type: FloatType) -> Result<Arc<dyn Array>> {
+    match float_type {
+        FloatType::BFloat16 => Ok(Arc::new(
+            BFloat16Array::from_iter_values(input.values().iter().map(|v| bf16::from_f32(*v)))
+                .into_inner(),
+        )),
+        FloatType::Float16 => Ok(Arc::new(Float16Array::from_iter_values(
+            input.values().iter().map(|v| f16::from_f32(*v)),
+        ))),
+        FloatType::Float32 => Ok(Arc::new(input.clone())),
+        FloatType::Float64 => Ok(Arc::new(Float64Array::from_iter_values(
+            input.values().iter().map(|v| *v as f64),
+        ))),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_coerce_float_vector_bfloat16() {
+        let input = Float32Array::from(vec![1.0f32, 2.0, 3.0]);
+        let array = coerce_float_vector(&input, FloatType::BFloat16).unwrap();
+
+        assert_eq!(array.data_type(), &DataType::FixedSizeBinary(2));
+
+        let fixed = array
+            .as_any()
+            .downcast_ref::<FixedSizeBinaryArray>()
+            .unwrap();
+        let expected: Vec<bf16> = input.values().iter().map(|v| bf16::from_f32(*v)).collect();
+        assert_eq!(fixed.as_slice(), expected.as_slice());
+    }
+}

python/vendor/lance-arrow/src/ipc.rs

@@ -0,0 +1,621 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Zero-copy Arrow IPC stream read/write utilities.
+//!
+//! Provides helpers for serializing and deserializing [`RecordBatch`]es as
+//! self-delimiting Arrow IPC streams using synchronous [`Read`]/[`Write`] I/O.
+//!
+//! These are designed for embedding IPC streams inside larger binary formats
+//! (e.g. a cache entry that contains multiple IPC sections). Each stream is
+//! self-delimiting (schema + batches + EOS marker) and can be read back
+//! independently.
+//!
+//! # Zero-copy reads
+//!
+//! [`read_ipc_stream`] and [`read_ipc_stream_single`] take `&Bytes` and use
+//! [`Bytes::slice`] to produce each message buffer. Because `Bytes::slice`
+//! increments a reference count rather than copying, the resulting
+//! [`Buffer`]s — and the array data decoded from them by [`FileDecoder`] —
+//! are all backed by the same allocation as the input.
+
+use std::io::{Read, Write};
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_buffer::Buffer;
+use arrow_ipc::convert::fb_to_schema;
+use arrow_ipc::reader::FileDecoder;
+use arrow_ipc::root_as_message;
+use arrow_ipc::writer::StreamWriter;
+use arrow_schema::ArrowError;
+use bytes::Bytes;
+
+// ---------------------------------------------------------------------------
+// Length-prefixed byte utilities
+// ---------------------------------------------------------------------------
+
+/// Write `data` prefixed by its length as a little-endian `u64`.
+///
+/// Paired with [`read_len_prefixed_bytes`].
+pub fn write_len_prefixed_bytes(writer: &mut dyn Write, data: &[u8]) -> Result<(), ArrowError> {
+    writer
+        .write_all(&(data.len() as u64).to_le_bytes())
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    writer
+        .write_all(data)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))
+}
+
+/// Read a byte slice written by [`write_len_prefixed_bytes`].
+///
+/// Reads an 8-byte little-endian length then exactly that many bytes.
+pub fn read_len_prefixed_bytes(reader: &mut dyn Read) -> Result<Vec<u8>, ArrowError> {
+    let mut len_buf = [0u8; 8];
+    reader
+        .read_exact(&mut len_buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    let len = u64::from_le_bytes(len_buf) as usize;
+    let mut buf = vec![0u8; len];
+    reader
+        .read_exact(&mut buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    Ok(buf)
+}
+
+// ---------------------------------------------------------------------------
+// IPC stream utilities
+// ---------------------------------------------------------------------------
+
+// 4-byte continuation marker used by modern Arrow IPC streams.
+const IPC_CONTINUATION: [u8; 4] = [0xff; 4];
+
+/// Write `batch` as a single-batch Arrow IPC stream to `writer`.
+pub fn write_ipc_stream(batch: &RecordBatch, writer: &mut dyn Write) -> Result<(), ArrowError> {
+    let mut sw = StreamWriter::try_new(&mut *writer, batch.schema_ref())?;
+    sw.write(batch)?;
+    sw.finish()
+}
+
+/// Write all batches from `iter` as a single Arrow IPC stream to `writer`.
+///
+/// `iter` must yield at least one batch; the schema is inferred from the first
+/// batch. Returns `ArrowError::InvalidArgumentError` if the iterator is empty.
+/// If you need to write an empty stream (schema only, no rows), construct a
+/// `StreamWriter` directly.
+pub fn write_ipc_stream_batches<I>(iter: I, writer: &mut dyn Write) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    let mut iter = iter.into_iter();
+    let first = iter
+        .next()
+        .ok_or_else(|| ArrowError::InvalidArgumentError("no batches to serialize".into()))?;
+    let mut sw = StreamWriter::try_new(&mut *writer, first.schema_ref())?;
+    sw.write(&first)?;
+    for batch in iter {
+        sw.write(&batch)?;
+    }
+    sw.finish()
+}
+
+/// Read one complete Arrow IPC stream message from `data` as a zero-copy [`Buffer`].
+///
+/// Parses the first message starting at byte 0 of `data`. Returns `None` on
+/// EOS (size field == 0) or empty input. The returned [`Buffer`] is backed by
+/// `data`'s allocation — no bytes are copied.
+///
+/// The caller should advance its position by `buf.len()` after each call.
+fn read_one_ipc_message(data: &Bytes) -> Result<Option<Buffer>, ArrowError> {
+    let bytes = data.as_ref();
+
+    if bytes.is_empty() {
+        return Ok(None);
+    }
+    if bytes.len() < 4 {
+        return Err(ArrowError::IoError(
+            "IPC: truncated header".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC header"),
+        ));
+    }
+
+    let has_continuation = bytes[..4] == IPC_CONTINUATION;
+    let (size_bytes, prefix_len): ([u8; 4], usize) = if has_continuation {
+        if bytes.len() < 8 {
+            return Err(ArrowError::IoError(
+                "IPC: truncated header after continuation".into(),
+                std::io::Error::new(
+                    std::io::ErrorKind::UnexpectedEof,
+                    "truncated after continuation",
+                ),
+            ));
+        }
+        (bytes[4..8].try_into().unwrap(), 8)
+    } else {
+        (bytes[..4].try_into().unwrap(), 4)
+    };
+
+    let meta_size = u32::from_le_bytes(size_bytes) as usize;
+    if meta_size == 0 {
+        return Ok(None); // EOS
+    }
+
+    let meta_end = prefix_len + meta_size;
+    if bytes.len() < meta_end {
+        return Err(ArrowError::IoError(
+            "IPC: truncated metadata".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC metadata"),
+        ));
+    }
+
+    let msg = root_as_message(&bytes[prefix_len..meta_end])
+        .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+    let body_len = msg.bodyLength() as usize;
+
+    let total = meta_end + body_len;
+    if bytes.len() < total {
+        return Err(ArrowError::IoError(
+            "IPC: truncated body".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC body"),
+        ));
+    }
+
+    // Zero-copy: Bytes::slice shares the backing allocation; Buffer::from
+    // wraps it without copying.
+    Ok(Some(Buffer::from(data.slice(0..total))))
+}
+
+/// Read a length-prefixed byte slice at `offset` in `data`, advancing `offset`.
+///
+/// Reads an 8-byte little-endian length, then slices exactly that many bytes
+/// from `data`. The returned [`Bytes`] is zero-copy (shares `data`'s allocation).
+pub fn read_len_prefixed_bytes_at(data: &Bytes, offset: &mut usize) -> Result<Bytes, ArrowError> {
+    let bytes = data.as_ref();
+    let len_end = offset
+        .checked_add(8)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated length field".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated length"),
+            )
+        })?;
+    let len = u64::from_le_bytes(bytes[*offset..len_end].try_into().unwrap()) as usize;
+    *offset = len_end;
+    let data_end = offset
+        .checked_add(len)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated data".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated data"),
+            )
+        })?;
+    let result = data.slice(*offset..data_end);
+    *offset = data_end;
+    Ok(result)
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    let batches = read_ipc_stream(&data.slice(*offset..))?;
+
+    // Recompute how many bytes were consumed by re-parsing message sizes.
+    // We can't get this from read_ipc_stream directly, so we re-walk the
+    // message headers (metadata only, no body re-read) to sum up lengths.
+    let slice = &data.as_ref()[*offset..];
+    let mut consumed = 0usize;
+    loop {
+        let rem = &slice[consumed..];
+        if rem.is_empty() {
+            break;
+        }
+        let has_cont = rem.len() >= 4 && rem[..4] == IPC_CONTINUATION;
+        let (size_bytes, prefix_len): ([u8; 4], usize) = if has_cont {
+            if rem.len() < 8 {
+                break;
+            }
+            (rem[4..8].try_into().unwrap(), 8)
+        } else {
+            if rem.len() < 4 {
+                break;
+            }
+            (rem[..4].try_into().unwrap(), 4)
+        };
+        let meta_size = u32::from_le_bytes(size_bytes) as usize;
+        if meta_size == 0 {
+            // EOS — consume it and stop.
+            consumed += prefix_len;
+            break;
+        }
+        let meta_end = prefix_len + meta_size;
+        if rem.len() < meta_end {
+            break;
+        }
+        let msg = root_as_message(&rem[prefix_len..meta_end])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+        consumed += meta_end + body_len;
+    }
+    *offset += consumed;
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_single_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream_at(data, offset)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+/// Extract the prefix length and metadata size from a raw IPC message buffer.
+///
+/// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`.
+/// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`.
+fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> {
+    let has_continuation = buf.len() >= 4 && buf[..4] == IPC_CONTINUATION;
+    if has_continuation {
+        if buf.len() < 8 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize;
+        Ok((8, meta_size))
+    } else {
+        if buf.len() < 4 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[..4].try_into().unwrap()) as usize;
+        Ok((4, meta_size))
+    }
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream.
+///
+/// Zero-copy: each batch's array data buffers are borrowed from the input
+/// message buffer(s) and not copied during decoding.
+///
+/// Uses [`FileDecoder`] directly (rather than `StreamDecoder`) to avoid a
+/// known edge case where `StreamDecoder` does not produce a batch for messages
+/// with a zero-length body when the message exactly fills the decode buffer.
+pub fn read_ipc_stream(data: &Bytes) -> Result<Vec<RecordBatch>, ArrowError> {
+    let mut offset = 0usize;
+
+    let schema_buf = read_one_ipc_message(&data.slice(offset..))?.ok_or_else(|| {
+        ArrowError::ParseError("IPC stream: expected schema message, got EOS".into())
+    })?;
+    offset += schema_buf.len();
+
+    let (prefix_len, meta_size) = parse_ipc_message_prefix(&schema_buf)?;
+    let schema_msg = root_as_message(&schema_buf[prefix_len..prefix_len + meta_size])
+        .map_err(|e| ArrowError::ParseError(format!("IPC schema parse error: {e}")))?;
+    let schema = Arc::new(fb_to_schema(schema_msg.header_as_schema().ok_or_else(
+        || ArrowError::ParseError("IPC stream: first message is not a schema".into()),
+    )?));
+    let mut decoder = FileDecoder::new(schema, schema_msg.version());
+
+    let mut batches = Vec::new();
+
+    loop {
+        let Some(buf) = read_one_ipc_message(&data.slice(offset..))? else {
+            break;
+        };
+        offset += buf.len();
+
+        let (prefix_len, meta_size) = parse_ipc_message_prefix(&buf)?;
+        let msg = root_as_message(&buf[prefix_len..prefix_len + meta_size])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+
+        // Block offset = 0 since the buffer starts at the message boundary.
+        // metaDataLength = prefix_len + meta_size (prefix + flatbuf + padding).
+        let block = arrow_ipc::Block::new(0, (prefix_len + meta_size) as i32, body_len as i64);
+
+        match msg.header_type() {
+            arrow_ipc::MessageHeader::RecordBatch => {
+                if let Some(batch) = decoder.read_record_batch(&block, &buf)? {
+                    batches.push(batch);
+                }
+            }
+            arrow_ipc::MessageHeader::DictionaryBatch => {
+                decoder.read_dictionary(&block, &buf)?;
+            }
+            _ => break,
+        }
+    }
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream.
+pub fn read_ipc_stream_single(data: &Bytes) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream(data)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Aligned IPC sections
+// ---------------------------------------------------------------------------
+
+/// Byte alignment that each IPC section's stream start is padded to.
+///
+/// When several IPC streams are concatenated into one larger blob (e.g. a
+/// cache entry), a section that starts at an arbitrary offset would leave its
+/// array data misaligned. [`FileDecoder`] with `require_alignment = false`
+/// then silently copies each buffer into a freshly aligned allocation on
+/// every read, defeating zero-copy. Padding each section start to a 64-byte
+/// boundary keeps the decoded buffers borrowed directly from the input.
+pub const IPC_SECTION_ALIGNMENT: usize = 64;
+
+/// Number of zero-padding bytes needed to advance `pos` to the next
+/// [`IPC_SECTION_ALIGNMENT`] boundary.
+fn section_padding(pos: usize) -> usize {
+    (IPC_SECTION_ALIGNMENT - (pos % IPC_SECTION_ALIGNMENT)) % IPC_SECTION_ALIGNMENT
+}
+
+/// A [`Write`] adapter that counts the bytes written through it.
+struct CountingWriter<'a> {
+    inner: &'a mut dyn Write,
+    count: usize,
+}
+
+impl Write for CountingWriter<'_> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        let n = self.inner.write(buf)?;
+        self.count += n;
+        Ok(n)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+}
+
+/// Write zero padding so the next byte lands on an [`IPC_SECTION_ALIGNMENT`]
+/// boundary, advancing `pos` past it.
+fn write_section_padding(writer: &mut dyn Write, pos: &mut usize) -> Result<(), ArrowError> {
+    let pad = section_padding(*pos);
+    if pad > 0 {
+        const ZEROS: [u8; IPC_SECTION_ALIGNMENT] = [0u8; IPC_SECTION_ALIGNMENT];
+        writer
+            .write_all(&ZEROS[..pad])
+            .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+        *pos += pad;
+    }
+    Ok(())
+}
+
+/// Write `batch` as a 64-byte-aligned single-batch Arrow IPC section.
+///
+/// `pos` is the absolute byte offset of `writer` within the enclosing blob.
+/// Zero padding is written first so the IPC stream begins on an
+/// [`IPC_SECTION_ALIGNMENT`] boundary, then the stream itself. `pos` is
+/// advanced past both the padding and the stream so the caller can write
+/// further aligned sections.
+///
+/// Paired with [`read_ipc_section_at`]. For the decoded buffers to be borrowed
+/// zero-copy, the blob must ultimately be read back from a buffer whose base
+/// address is at least 64-byte aligned.
+pub fn write_ipc_section(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    batch: &RecordBatch,
+) -> Result<(), ArrowError> {
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream(batch, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read a single [`RecordBatch`] from an aligned IPC section at `offset`.
+///
+/// Skips the alignment padding written by [`write_ipc_section`], then reads
+/// the stream, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section`]).
+pub fn read_ipc_section_at(data: &Bytes, offset: &mut usize) -> Result<RecordBatch, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_single_at(data, offset)
+}
+
+/// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC section.
+///
+/// Like [`write_ipc_section`] but emits every batch from `iter` into one IPC
+/// stream (schema + N batches + EOS). `iter` must yield at least one batch.
+/// Paired with [`read_ipc_section_batches_at`].
+pub fn write_ipc_section_batches<I>(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    iter: I,
+) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream_batches(iter, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read all [`RecordBatch`]es from an aligned multi-batch IPC section at
+/// `offset`, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section_batches`]).
+pub fn read_ipc_section_batches_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_at(data, offset)
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::{ArrayRef, record_batch};
+
+    use super::*;
+
+    #[test]
+    fn test_ipc_roundtrip() {
+        let batch1 = record_batch!(
+            ("int", Int32, [1, 2, 3]),
+            ("str", Utf8, ["foo", "bar", "baz"])
+        )
+        .unwrap();
+        let batch2 = record_batch!(("int", Int32, [4, 5]), ("str", Utf8, ["qux", "quux"])).unwrap();
+        let batches = vec![batch1.clone(), batch2.clone()];
+
+        let mut buf = Vec::new();
+        write_ipc_stream_batches(batches, &mut buf).unwrap();
+
+        let data = Bytes::from(buf);
+
+        let batches = read_ipc_stream(&data).unwrap();
+        assert_eq!(batches.len(), 2);
+        assert_eq!(batches[0], batch1);
+        assert_eq!(batches[1], batch2);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        let assert_col_zero_copy = |array: &ArrayRef| {
+            for buffer in array.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "buffer at {ptr:#x} is not backed by the input Bytes allocation \
+                     [{data_base:#x}..{data_end:#x})"
+                );
+            }
+        };
+
+        for batch in &batches {
+            assert_eq!(batch.schema(), batch1.schema());
+            assert_col_zero_copy(batch.column(0));
+            assert_col_zero_copy(batch.column(1));
+        }
+    }
+
+    /// Allocate a [`Bytes`] whose base address is 64-byte aligned, modelling a
+    /// backend that reads cache entries into an aligned buffer. A plain
+    /// `Bytes::from(vec)` only guarantees the allocator's alignment for `u8`.
+    fn aligned_bytes(payload: &[u8]) -> Bytes {
+        let mut v = vec![0u8; payload.len() + IPC_SECTION_ALIGNMENT];
+        let pad = section_padding(v.as_ptr() as usize);
+        v[pad..pad + payload.len()].copy_from_slice(payload);
+        Bytes::from(v).slice(pad..pad + payload.len())
+    }
+
+    #[test]
+    fn test_aligned_ipc_sections_are_zero_copy() {
+        // A LargeBinary column exercises the i64-offset buffer whose 8-byte
+        // alignment requirement triggers a realigning memcpy when misaligned.
+        let blocks = arrow_array::LargeBinaryArray::from_vec(vec![&b"hello"[..], b"world"]);
+        let section_a = RecordBatch::try_from_iter([("a", Arc::new(blocks) as ArrayRef)]).unwrap();
+        let section_b = record_batch!(("b", Int64, [10i64, 20, 30, 40, 50])).unwrap();
+
+        let mut buf = Vec::new();
+        // Arbitrary, deliberately non-64-aligned preamble so the first section
+        // must be padded rather than landing at offset 0 by luck.
+        buf.extend_from_slice(&[0xABu8; 7]);
+        let mut pos = buf.len();
+        // The first section's stream begins after padding the 7-byte preamble
+        // up to the next 64-byte boundary.
+        assert_eq!(7 + section_padding(7), IPC_SECTION_ALIGNMENT);
+        write_ipc_section(&mut buf, &mut pos, &section_a).unwrap();
+        write_ipc_section(&mut buf, &mut pos, &section_b).unwrap();
+
+        let data = aligned_bytes(&buf);
+        assert_eq!(
+            section_padding(data.as_ptr() as usize),
+            0,
+            "base not aligned"
+        );
+
+        let mut offset = 7;
+        let read_a = read_ipc_section_at(&data, &mut offset).unwrap();
+        let read_b = read_ipc_section_at(&data, &mut offset).unwrap();
+        assert_eq!(read_a, section_a);
+        assert_eq!(read_b, section_b);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for batch in [&read_a, &read_b] {
+            for buffer in batch.column(0).to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "section buffer at {ptr:#x} was realigned out of the input \
+                     [{data_base:#x}..{data_end:#x}) — misaligned section",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_aligned_multi_batch_section_roundtrip_zero_copy() {
+        // A multi-batch section (e.g. IVF SQ storage chunks) must round-trip
+        // every batch and decode the first batch's buffers zero-copy.
+        let b1 = record_batch!(("v", Int64, [1i64, 2, 3])).unwrap();
+        let b2 = record_batch!(("v", Int64, [4i64, 5])).unwrap();
+        let b3 = record_batch!(("v", Int64, [6i64])).unwrap();
+
+        let mut buf = vec![0xCDu8; 5];
+        let mut pos = buf.len();
+        write_ipc_section_batches(&mut buf, &mut pos, [b1.clone(), b2.clone(), b3.clone()])
+            .unwrap();
+
+        let data = aligned_bytes(&buf);
+        let mut offset = 5;
+        let read = read_ipc_section_batches_at(&data, &mut offset).unwrap();
+        assert_eq!(read, vec![b1, b2, b3]);
+        assert_eq!(offset, buf.len(), "offset should land at section end");
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for buffer in read[0].column(0).to_data().buffers() {
+            let ptr = buffer.as_ptr() as usize;
+            assert!(
+                ptr >= data_base && ptr < data_end,
+                "first batch buffer at {ptr:#x} was realigned out of the input",
+            );
+        }
+    }
+}

python/vendor/lance-arrow/src/list.rs

@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use arrow_array::{Array, BooleanArray, GenericListArray, OffsetSizeTrait};
+use arrow_buffer::{BooleanBufferBuilder, OffsetBuffer, ScalarBuffer};
+use arrow_schema::Field;
+
+pub trait ListArrayExt {
+    /// Filters out masked null items from the list array
+    ///
+    /// It is legal for a list array to have a null entry with a non-zero length.  The
+    /// values inside the entry are "garbage" and should be ignored.  This function
+    /// filters the values array to remove the garbage values.
+    ///
+    /// The output list will always have zero-length nulls.
+    fn filter_garbage_nulls(&self) -> Self;
+    /// Returns a copy of the list's values array that has been sliced to size
+    ///
+    /// It is legal for a list array's offsets to not start with zero.  It's also legal
+    /// for a list array's offsets to not extend to the entire values array.  This function
+    /// behaves similarly to `values()` except it slices the array so that it starts at
+    /// the first list offset and ends at the last list offset.
+    fn trimmed_values(&self) -> Arc<dyn Array>;
+    /// The offset type of the underlying list array.
+    type Offset: OffsetSizeTrait;
+    /// Returns offsets shifted so the first offset is zero, matching
+    /// [`Self::trimmed_values`].
+    ///
+    /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the
+    /// original values buffer, so combining them with trimmed values produces
+    /// offsets that exceed the values length. Use this together with
+    /// `trimmed_values` when constructing a new list array.
+    fn trimmed_offsets(&self) -> OffsetBuffer<Self::Offset>;
+}
+
+impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize> {
+    fn filter_garbage_nulls(&self) -> Self {
+        if self.is_empty() {
+            return self.clone();
+        }
+        let Some(validity) = self.nulls().cloned() else {
+            return self.clone();
+        };
+
+        let mut should_keep = BooleanBufferBuilder::new(self.values().len());
+
+        // Handle case where offsets do not start at 0
+        let preamble_len = self.offsets().first().unwrap().to_usize().unwrap();
+        should_keep.append_n(preamble_len, false);
+
+        let mut new_offsets: Vec<OffsetSize> = Vec::with_capacity(self.len() + 1);
+        new_offsets.push(OffsetSize::zero());
+        let mut cur_len = OffsetSize::zero();
+        for (offset, is_valid) in self.offsets().windows(2).zip(validity.iter()) {
+            let len = offset[1] - offset[0];
+            if is_valid {
+                cur_len += len;
+                should_keep.append_n(len.to_usize().unwrap(), true);
+                new_offsets.push(cur_len);
+            } else {
+                should_keep.append_n(len.to_usize().unwrap(), false);
+                new_offsets.push(cur_len);
+            }
+        }
+
+        // Offsets may not reference entire values buffer
+        let trailer = self.values().len() - should_keep.len();
+        should_keep.append_n(trailer, false);
+
+        let should_keep = should_keep.finish();
+        let should_keep = BooleanArray::new(should_keep, None);
+        let new_values = arrow_select::filter::filter(self.values(), &should_keep).unwrap();
+        let new_offsets = ScalarBuffer::from(new_offsets);
+        let new_offsets = OffsetBuffer::new(new_offsets);
+
+        Self::new(
+            Arc::new(Field::new(
+                "item",
+                self.value_type(),
+                self.values().is_nullable(),
+            )),
+            new_offsets,
+            new_values,
+            Some(validity),
+        )
+    }
+
+    fn trimmed_values(&self) -> Arc<dyn Array> {
+        let first_value = self
+            .offsets()
+            .first()
+            .map(|v| v.to_usize().unwrap())
+            .unwrap_or(0);
+        let last_value = self
+            .offsets()
+            .last()
+            .map(|v| v.to_usize().unwrap())
+            .unwrap_or(0);
+        self.values().slice(first_value, last_value - first_value)
+    }
+
+    type Offset = OffsetSize;
+
+    fn trimmed_offsets(&self) -> OffsetBuffer<OffsetSize> {
+        let offsets = self.offsets();
+        let Some(&first) = offsets.first() else {
+            return offsets.clone();
+        };
+        if first == OffsetSize::zero() {
+            return offsets.clone();
+        }
+        let shifted: Vec<OffsetSize> = offsets.iter().map(|&o| o - first).collect();
+        OffsetBuffer::new(ScalarBuffer::from(shifted))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{ListArray, UInt64Array};
+    use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer};
+    use arrow_schema::{DataType, Field};
+
+    use super::ListArrayExt;
+
+    #[test]
+    fn test_filter_garbage_nulls() {
+        let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 8, 9]);
+        let offsets = OffsetBuffer::new(offsets);
+        let list_validity = NullBuffer::new(BooleanBuffer::from(vec![true, false, true]));
+        let list_arr = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, true)),
+            offsets,
+            Arc::new(items),
+            Some(list_validity.clone()),
+        );
+
+        let filtered = list_arr.filter_garbage_nulls();
+
+        let expected_items = UInt64Array::from(vec![2, 3, 4, 8]);
+        let offsets = ScalarBuffer::<i32>::from(vec![0, 3, 3, 4]);
+        let expected = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, false)),
+            OffsetBuffer::new(offsets),
+            Arc::new(expected_items),
+            Some(list_validity),
+        );
+
+        assert_eq!(filtered, expected);
+    }
+
+    #[test]
+    fn test_trim_values() {
+        let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 6, 8, 9]);
+        let offsets = OffsetBuffer::new(offsets);
+        let list_arr = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, true)),
+            offsets,
+            Arc::new(items),
+            None,
+        );
+        let list_arr = list_arr.slice(1, 2);
+
+        let trimmed = list_arr.trimmed_values();
+
+        let expected_items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let expected_items = expected_items.slice(5, 3);
+
+        assert_eq!(trimmed.as_ref(), &expected_items);
+    }
+}

python/vendor/lance-arrow/src/memory.rs

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashSet;
+
+use arrow_array::{Array, RecordBatch};
+use arrow_data::ArrayData;
+
+/// Counts memory used by buffers of Arrow arrays and RecordBatches.
+///
+/// This is meant to capture how much memory is being used by the Arrow data
+/// structures as they are. It does not represent the memory used if the data
+/// were to be serialized and then deserialized. In particular:
+///
+/// * This does not double count memory used by buffers shared by multiple
+///   arrays or batches. Round-tripped data may use more memory because of this.
+/// * This counts the **total** size of the buffers, even if the array is a slice.
+///   Round-tripped data may use less memory because of this.
+#[derive(Default)]
+pub struct MemoryAccumulator {
+    seen: HashSet<usize>,
+    total: usize,
+}
+
+impl MemoryAccumulator {
+    pub fn record_array(&mut self, array: &dyn Array) {
+        let data = array.to_data();
+        self.record_array_data(&data);
+    }
+
+    fn record_array_data(&mut self, data: &ArrayData) {
+        for buffer in data.buffers() {
+            let ptr = buffer.as_ptr();
+            if self.seen.insert(ptr as usize) {
+                self.total += buffer.capacity();
+            }
+        }
+
+        if let Some(nulls) = data.nulls() {
+            let null_buf = nulls.inner().inner();
+            let ptr = null_buf.as_ptr();
+            if self.seen.insert(ptr as usize) {
+                self.total += null_buf.capacity();
+            }
+        }
+
+        for child in data.child_data() {
+            self.record_array_data(child);
+        }
+    }
+
+    pub fn record_batch(&mut self, batch: &RecordBatch) {
+        for array in batch.columns() {
+            self.record_array(array);
+        }
+    }
+
+    pub fn total(&self) -> usize {
+        self.total
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::Int32Array;
+    use arrow_schema::{DataType, Field, Schema};
+
+    use super::*;
+
+    #[test]
+    fn test_memory_accumulator() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let slice = batch.slice(1, 2);
+
+        let mut acc = MemoryAccumulator::default();
+
+        // Should record whole buffer, not just slice
+        acc.record_batch(&slice);
+        assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
+
+        // Should not double count
+        acc.record_batch(&slice);
+        assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
+    }
+}

python/vendor/lance-arrow/src/scalar.rs

@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use arrow_array::{ArrayRef, UInt64Array, make_array};
+use arrow_buffer::Buffer;
+use arrow_data::ArrayDataBuilder;
+use arrow_schema::{ArrowError, DataType};
+use arrow_select::take::take;
+
+use crate::DataTypeExt;
+
+type Result<T> = std::result::Result<T, ArrowError>;
+
+pub const INLINE_VALUE_MAX_BYTES: usize = 32;
+
+pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result<ArrayRef> {
+    if idx >= array.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar index out of bounds".to_string(),
+        ));
+    }
+
+    take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None)
+}
+
+fn read_u32(buf: &[u8], offset: &mut usize) -> Result<u32> {
+    if *offset + 4 > buf.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: unexpected EOF".to_string(),
+        ));
+    }
+    let bytes = [
+        buf[*offset],
+        buf[*offset + 1],
+        buf[*offset + 2],
+        buf[*offset + 3],
+    ];
+    *offset += 4;
+    Ok(u32::from_le_bytes(bytes))
+}
+
+fn read_bytes<'a>(buf: &'a [u8], offset: &mut usize, len: usize) -> Result<&'a [u8]> {
+    if *offset + len > buf.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: unexpected EOF".to_string(),
+        ));
+    }
+    let slice = &buf[*offset..*offset + len];
+    *offset += len;
+    Ok(slice)
+}
+
+fn write_u32(out: &mut Vec<u8>, v: u32) {
+    out.extend_from_slice(&v.to_le_bytes());
+}
+
+fn write_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
+    out.extend_from_slice(bytes);
+}
+
+pub fn encode_scalar_value_buffer(scalar: &ArrayRef) -> Result<Vec<u8>> {
+    if scalar.len() != 1 || scalar.null_count() != 0 {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer must be a single non-null value".to_string(),
+        ));
+    }
+    let data = scalar.to_data();
+    if data.offset() != 0 {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer must have offset=0".to_string(),
+        ));
+    }
+    if !data.child_data().is_empty() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer does not support nested types".to_string(),
+        ));
+    }
+
+    // Minimal format (RFC): store the Arrow value buffers for a length-1 array.
+    // Null bitmap and child data are intentionally not supported here.
+    //
+    // | u32 num_buffers |
+    // | u32 buffer_0_len | ... | u32 buffer_{n-1}_len |
+    // | buffer_0 bytes | ... | buffer_{n-1} bytes |
+    let mut out = Vec::with_capacity(128);
+    let buffers = data.buffers();
+    write_u32(&mut out, buffers.len() as u32);
+    for b in buffers {
+        write_u32(&mut out, b.len() as u32);
+    }
+    for b in buffers {
+        write_bytes(&mut out, b.as_slice());
+    }
+    Ok(out)
+}
+
+pub fn decode_scalar_from_value_buffer(
+    data_type: &DataType,
+    value_buffer: &[u8],
+) -> Result<ArrayRef> {
+    if matches!(
+        data_type,
+        DataType::Struct(_) | DataType::FixedSizeList(_, _)
+    ) {
+        return Err(ArrowError::InvalidArgumentError(format!(
+            "Scalar value buffer does not support nested data type {:?}",
+            data_type
+        )));
+    }
+
+    let mut offset = 0;
+    let num_buffers = read_u32(value_buffer, &mut offset)? as usize;
+    let buffer_lens = (0..num_buffers)
+        .map(|_| read_u32(value_buffer, &mut offset).map(|l| l as usize))
+        .collect::<Result<Vec<_>>>()?;
+
+    let mut buffers = Vec::with_capacity(num_buffers);
+    for len in buffer_lens {
+        let bytes = read_bytes(value_buffer, &mut offset, len)?;
+        buffers.push(Buffer::from_vec(bytes.to_vec()));
+    }
+
+    if offset != value_buffer.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: trailing bytes".to_string(),
+        ));
+    }
+
+    let mut builder = ArrayDataBuilder::new(data_type.clone())
+        .len(1)
+        .null_count(0);
+    for b in buffers {
+        builder = builder.add_buffer(b);
+    }
+    Ok(make_array(builder.build()?))
+}
+
+pub fn decode_scalar_from_inline_value(
+    data_type: &DataType,
+    inline_value: &[u8],
+) -> Result<ArrayRef> {
+    // I expect our input to be safe here, but I added some debug_assert_eq statements just in case.
+    // If they are triggered, we may need to change them to return actual errors.
+    //
+    // Boolean values are bit-packed in Arrow and therefore are not "fixed-stride" in bytes.
+    // As a result, `byte_width_opt()` returns `None` for `DataType::Boolean`, even though a
+    // length-1 scalar can be represented inline using a single byte (matching `try_inline_value`).
+    if matches!(data_type, DataType::Boolean) {
+        debug_assert_eq!(
+            inline_value.len(),
+            1,
+            "Invalid boolean inline scalar length (expected 1 byte, got {})",
+            inline_value.len()
+        );
+    } else if let Some(byte_width) = data_type.byte_width_opt() {
+        debug_assert_eq!(
+            inline_value.len(),
+            byte_width,
+            "Inline constant length mismatch for {:?}: expected {} bytes but got {}",
+            data_type,
+            byte_width,
+            inline_value.len()
+        );
+    }
+
+    let data = ArrayDataBuilder::new(data_type.clone())
+        .len(1)
+        .null_count(0)
+        .add_buffer(Buffer::from_vec(inline_value.to_vec()))
+        .build()?;
+    Ok(make_array(data))
+}
+
+pub fn try_inline_value(scalar: &ArrayRef) -> Option<Vec<u8>> {
+    if scalar.null_count() != 0 || scalar.len() != 1 {
+        return None;
+    }
+    let data = scalar.to_data();
+    if !data.child_data().is_empty() {
+        return None;
+    }
+    if data.buffers().len() != 1 {
+        return None;
+    }
+    let bytes = data.buffers()[0].as_slice();
+    if bytes.len() > INLINE_VALUE_MAX_BYTES {
+        return None;
+    }
+    Some(bytes.to_vec())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{
+        BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray,
+        cast::AsArray, types::Int8Type,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_extract_scalar_value() {
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+        let scalar = extract_scalar_value(&array, 2).unwrap();
+        assert_eq!(scalar.len(), 1);
+        assert_eq!(
+            scalar
+                .as_primitive::<arrow_array::types::Int32Type>()
+                .value(0),
+            3
+        );
+    }
+
+    #[test]
+    fn test_extract_scalar_value_from_full_dictionary() {
+        let values = Arc::new(StringArray::from(
+            (0..=i8::MAX)
+                .map(|value| format!("value-{value}"))
+                .collect::<Vec<_>>(),
+        ));
+        let keys = Int8Array::from((0..=i8::MAX).collect::<Vec<_>>());
+        let array: ArrayRef = Arc::new(DictionaryArray::<Int8Type>::new(keys, values));
+
+        let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap();
+
+        let scalar = scalar.as_dictionary::<Int8Type>();
+        assert_eq!(scalar.len(), 1);
+        assert_eq!(scalar.key(0), Some(i8::MAX as usize));
+        assert_eq!(scalar.values().len(), i8::MAX as usize + 1);
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_utf8_round_trip() {
+        let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"]));
+        let buf = encode_scalar_value_buffer(&scalar).unwrap();
+        let decoded = decode_scalar_from_value_buffer(&DataType::Utf8, &buf).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.null_count(), 0);
+        assert_eq!(decoded.as_string::<i32>().value(0), "hello");
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_fixed_size_binary_round_trip() {
+        let val = vec![0xABu8; 33];
+        let scalar: ArrayRef = Arc::new(
+            FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                std::iter::once(Some(val.as_slice())),
+                33,
+            )
+            .unwrap(),
+        );
+        let buf = encode_scalar_value_buffer(&scalar).unwrap();
+        let decoded =
+            decode_scalar_from_value_buffer(&DataType::FixedSizeBinary(33), &buf).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.as_fixed_size_binary().value(0), val.as_slice());
+    }
+
+    #[test]
+    fn test_inline_value_boolean_round_trip() {
+        let scalar: ArrayRef = Arc::new(BooleanArray::from_iter([Some(true)]));
+        let inline = try_inline_value(&scalar).unwrap();
+        let decoded = decode_scalar_from_inline_value(&DataType::Boolean, &inline).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.null_count(), 0);
+        assert!(decoded.as_boolean().value(0));
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_rejects_nested_type() {
+        let field = Arc::new(arrow_schema::Field::new("item", DataType::Int32, false));
+        let list: ArrayRef = Arc::new(arrow_array::FixedSizeListArray::new(
+            field,
+            2,
+            Arc::new(Int32Array::from(vec![1, 2])),
+            None,
+        ));
+        let scalar = list.slice(0, 1);
+        assert!(encode_scalar_value_buffer(&scalar).is_err());
+    }
+
+    #[test]
+    fn test_decode_scalar_from_value_buffer_rejects_nested_type() {
+        let buf = Vec::<u8>::new();
+        let res =
+            decode_scalar_from_value_buffer(&DataType::Struct(arrow_schema::Fields::empty()), &buf);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_decode_scalar_from_value_buffer_trailing_bytes() {
+        // num_buffers = 0, plus an extra byte
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&0u32.to_le_bytes());
+        bytes.push(1);
+        let res = decode_scalar_from_value_buffer(&DataType::Int32, &bytes);
+        assert!(res.is_err());
+    }
+}

python/vendor/lance-arrow/src/schema.rs

@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Extension to arrow schema
+
+use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema};
+
+use crate::{ARROW_EXT_NAME_KEY, BLOB_META_KEY, BLOB_V2_EXT_NAME};
+
+pub enum Indentation {
+    OneLine,
+    MultiLine(u8),
+}
+
+impl Indentation {
+    fn value(&self) -> String {
+        match self {
+            Self::OneLine => "".to_string(),
+            Self::MultiLine(spaces) => " ".repeat(*spaces as usize),
+        }
+    }
+
+    fn deepen(&self) -> Self {
+        match self {
+            Self::OneLine => Self::OneLine,
+            Self::MultiLine(spaces) => Self::MultiLine(spaces + 2),
+        }
+    }
+}
+
+/// Extends the functionality of [arrow_schema::Field].
+pub trait FieldExt {
+    /// Create a compact string representation of the field
+    ///
+    /// This is intended for display purposes and not for serialization
+    fn to_compact_string(&self, indent: Indentation) -> String;
+
+    /// Check if the field is marked as a packed struct
+    fn is_packed_struct(&self) -> bool;
+
+    /// Check if the field is marked as a blob
+    fn is_blob(&self) -> bool;
+
+    /// Check if the field is marked as a blob
+    fn is_blob_v2(&self) -> bool;
+}
+
+impl FieldExt for Field {
+    fn to_compact_string(&self, indent: Indentation) -> String {
+        let mut result = format!("{}: ", self.name().clone());
+        match self.data_type() {
+            DataType::Struct(fields) => {
+                result += "{";
+                result += &indent.value();
+                for (field_idx, field) in fields.iter().enumerate() {
+                    result += field.to_compact_string(indent.deepen()).as_str();
+                    if field_idx < fields.len() - 1 {
+                        result += ",";
+                    }
+                    result += indent.value().as_str();
+                }
+                result += "}";
+            }
+            DataType::List(field)
+            | DataType::LargeList(field)
+            | DataType::ListView(field)
+            | DataType::LargeListView(field) => {
+                result += "[";
+                result += field.to_compact_string(indent.deepen()).as_str();
+                result += "]";
+            }
+            DataType::FixedSizeList(child, dimension) => {
+                result += &format!(
+                    "[{}; {}]",
+                    child.to_compact_string(indent.deepen()),
+                    dimension
+                );
+            }
+            DataType::Dictionary(key_type, value_type) => {
+                result += &value_type.to_string();
+                result += "@";
+                result += &key_type.to_string();
+            }
+            _ => {
+                result += &self.data_type().to_string();
+            }
+        }
+        if self.is_nullable() {
+            result += "?";
+        }
+        result
+    }
+
+    // Check if field has metadata `packed` set to true, this check is case insensitive.
+    fn is_packed_struct(&self) -> bool {
+        let field_metadata = self.metadata();
+        const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
+        PACKED_KEYS.iter().any(|key| {
+            field_metadata
+                .get(*key)
+                .map(|value| value.eq_ignore_ascii_case("true"))
+                .unwrap_or(false)
+        })
+    }
+
+    fn is_blob(&self) -> bool {
+        let field_metadata = self.metadata();
+        field_metadata.get(BLOB_META_KEY).is_some()
+            || field_metadata
+                .get(ARROW_EXT_NAME_KEY)
+                .map(|value| value == BLOB_V2_EXT_NAME)
+                .unwrap_or(false)
+    }
+
+    fn is_blob_v2(&self) -> bool {
+        let field_metadata = self.metadata();
+        field_metadata
+            .get(ARROW_EXT_NAME_KEY)
+            .map(|value| value == BLOB_V2_EXT_NAME)
+            .unwrap_or(false)
+    }
+}
+
+/// Extends the functionality of [arrow_schema::Schema].
+pub trait SchemaExt {
+    /// Create a new [`Schema`] with one extra field.
+    fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError>;
+
+    fn try_with_column_at(
+        &self,
+        index: usize,
+        field: Field,
+    ) -> std::result::Result<Schema, ArrowError>;
+
+    fn field_names(&self) -> Vec<&String>;
+
+    fn without_column(&self, column_name: &str) -> Schema;
+
+    /// Create a compact string representation of the schema
+    ///
+    /// This is intended for display purposes and not for serialization
+    fn to_compact_string(&self, indent: Indentation) -> String;
+}
+
+impl SchemaExt for Schema {
+    fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError> {
+        if self.column_with_name(field.name()).is_some() {
+            return Err(ArrowError::SchemaError(format!(
+                "Can not append column {} on schema: {:?}",
+                field.name(),
+                self
+            )));
+        };
+        let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
+        fields.push(FieldRef::new(field));
+        Ok(Self::new_with_metadata(fields, self.metadata.clone()))
+    }
+
+    fn try_with_column_at(
+        &self,
+        index: usize,
+        field: Field,
+    ) -> std::result::Result<Schema, ArrowError> {
+        if self.column_with_name(field.name()).is_some() {
+            return Err(ArrowError::SchemaError(format!(
+                "Failed to modify schema: Inserting column {} would create a duplicate column in schema: {:?}",
+                field.name(),
+                self
+            )));
+        };
+        let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
+        fields.insert(index, FieldRef::new(field));
+        Ok(Self::new_with_metadata(fields, self.metadata.clone()))
+    }
+
+    /// Project the schema to remove the given column.
+    ///
+    /// This only works on top-level fields right now. If a field does not exist,
+    /// the schema will be returned as is.
+    fn without_column(&self, column_name: &str) -> Schema {
+        let fields: Vec<FieldRef> = self
+            .fields()
+            .iter()
+            .filter(|f| f.name() != column_name)
+            .cloned()
+            .collect();
+        Self::new_with_metadata(fields, self.metadata.clone())
+    }
+
+    fn field_names(&self) -> Vec<&String> {
+        self.fields().iter().map(|f| f.name()).collect()
+    }
+
+    fn to_compact_string(&self, indent: Indentation) -> String {
+        let mut result = "{".to_string();
+        result += &indent.value();
+        for (field_idx, field) in self.fields.iter().enumerate() {
+            result += field.to_compact_string(indent.deepen()).as_str();
+            if field_idx < self.fields.len() - 1 {
+                result += ",";
+            }
+            result += indent.value().as_str();
+        }
+        result += "}";
+        result
+    }
+}

python/vendor/lance-arrow/src/stream.rs

@@ -0,0 +1,554 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Utilities for working with streams of [`RecordBatch`].
+
+use arrow_array::RecordBatch;
+use arrow_schema::{ArrowError, SchemaRef};
+use futures::stream::{self, Stream, StreamExt};
+use std::pin::Pin;
+
+use crate::deepcopy::deep_copy_batch_sliced;
+
+/// Rechunks a stream of [`RecordBatch`] so that each output batch has
+/// approximately `target_bytes` of array data.
+///
+/// Small input batches are accumulated (by concatenation) until at least
+/// `min_bytes` of data has been collected. If the resulting batch exceeds
+/// `max_bytes`, it is sliced into roughly equal pieces of ~`max_bytes`
+/// (assuming uniform row sizes).
+pub fn rechunk_stream_by_size<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, false)
+}
+
+/// Like [`rechunk_stream_by_size`] but deep-copies slices so that
+/// `get_array_memory_size` reflects the true size of each output batch.
+///
+/// After a normal `RecordBatch::slice`, the backing buffers are shared with
+/// the original batch, so `get_array_memory_size` still reports the full
+/// parent size.  This variant deep-copies every slice produced during the
+/// splitting phase, which allows the stream to detect and re-split slices
+/// that still exceed `max_bytes` (e.g. because a single row is much larger
+/// than average).
+///
+/// The deep copy is a last resort and potentially expensive for large
+/// batches.  However, it is only performed when a batch actually needs to be
+/// sliced — batches that are already within the target range pass through at
+/// zero cost.  Use this only when the hard cap on `max_bytes` is a
+/// correctness requirement, not merely a performance hint.
+pub fn rechunk_stream_by_size_deep_copy<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, true)
+}
+
+fn rechunk_stream_by_size_inner<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+    deep_copy: bool,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    stream::try_unfold(
+        RechunkState {
+            input: Box::pin(input),
+            accumulated: Vec::new(),
+            acc_bytes: 0,
+            done: false,
+            input_schema,
+            min_bytes,
+            max_bytes,
+            deep_copy,
+        },
+        |mut state| async move {
+            if state.done && state.accumulated.is_empty() {
+                return Ok(None);
+            }
+
+            // Pull batches until we reach the byte target or exhaust input.
+            // Always pull at least one batch so that min_bytes=0 works.
+            while !state.done && (state.accumulated.is_empty() || state.acc_bytes < state.min_bytes)
+            {
+                match state.input.next().await {
+                    Some(Ok(batch)) => {
+                        state.acc_bytes += batch.get_array_memory_size();
+                        state.accumulated.push(batch);
+                    }
+                    Some(Err(e)) => return Err(e),
+                    None => {
+                        state.done = true;
+                    }
+                }
+            }
+
+            if state.accumulated.is_empty() {
+                return Ok(None);
+            }
+
+            // Fast path: if the first accumulated batch already meets the
+            // byte threshold, deliver it directly instead of concatenating
+            // everything together (which would just get sliced back apart).
+            if state.accumulated.len() > 1
+                && state.accumulated[0].get_array_memory_size() >= state.min_bytes
+            {
+                let b = state.accumulated.remove(0);
+                state.acc_bytes -= b.get_array_memory_size();
+                return Ok(Some((b, state)));
+            }
+
+            let batch = if state.accumulated.len() == 1 {
+                state.accumulated.pop().unwrap()
+            } else {
+                let b =
+                    arrow_select::concat::concat_batches(&state.input_schema, &state.accumulated)
+                        .map_err(E::from)?;
+                state.accumulated.clear();
+                b
+            };
+            state.acc_bytes = 0;
+
+            // Slice the batch into ~max_bytes pieces assuming uniform row sizes.
+            let mut slices =
+                slice_batch(batch, state.max_bytes, state.deep_copy).map_err(E::from)?;
+
+            if slices.len() == 1 {
+                Ok(Some((slices.pop().unwrap(), state)))
+            } else {
+                let first = slices.remove(0);
+
+                // Stash leftover slices for subsequent iterations.
+                for a in &slices {
+                    state.acc_bytes += a.get_array_memory_size();
+                }
+                state.accumulated = slices;
+
+                Ok(Some((first, state)))
+            }
+        },
+    )
+}
+
+/// Slice a batch into pieces of at most `max_bytes`.
+///
+/// When `deep_copy` is false, slices share buffers with the original batch
+/// and `get_array_memory_size` will still report the parent buffer size.
+/// This is fine when the caller only needs approximate sizing.
+///
+/// When `deep_copy` is true, each slice is deep-copied so that
+/// `get_array_memory_size` reflects the true size.  If a deep-copied slice
+/// still exceeds `max_bytes` (due to non-uniform row sizes), it is
+/// recursively split until every piece is within budget or contains only a
+/// single row.
+fn slice_batch(
+    batch: RecordBatch,
+    max_bytes: usize,
+    deep_copy: bool,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    let batch_bytes = batch.get_array_memory_size();
+    let num_rows = batch.num_rows();
+
+    if batch_bytes <= max_bytes {
+        return Ok(vec![batch]);
+    }
+
+    if num_rows <= 1 {
+        // A single row cannot be split further, but the size just measured may
+        // belong to its source buffer rather than to the row:
+        // `get_array_memory_size` reports whole buffers, and a slice shares
+        // them. Returning it uncopied hands the caller that inflated figure --
+        // a 100 KB row inside a 200 MB buffer measures as 200 MB, and a caller
+        // budgeting on it concludes the row cannot fit anywhere.
+        if deep_copy {
+            return Ok(vec![deep_copy_batch_sliced(&batch)?]);
+        }
+        return Ok(vec![batch]);
+    }
+
+    let rows_per_chunk = (max_bytes as u64 * num_rows as u64 / batch_bytes as u64).max(1) as usize;
+
+    let mut result = Vec::new();
+    let mut offset = 0;
+    while offset < num_rows {
+        let len = rows_per_chunk.min(num_rows - offset);
+        let slice = batch.slice(offset, len);
+        if deep_copy {
+            let copied = deep_copy_batch_sliced(&slice)?;
+            // Recurse: the deep-copied slice has accurate sizes, so if it
+            // still exceeds max_bytes we can split further.
+            result.extend(slice_batch(copied, max_bytes, true)?);
+        } else {
+            result.push(slice);
+        }
+        offset += len;
+    }
+
+    Ok(result)
+}
+
+/// Internal state for [`rechunk_stream`].
+///
+/// Kept as a named struct so the `try_unfold` closure stays readable.
+struct RechunkState<S> {
+    input: Pin<Box<S>>,
+    accumulated: Vec<RecordBatch>,
+    acc_bytes: usize,
+    done: bool,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+    deep_copy: bool,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::Arc;
+
+    use arrow_array::Int32Array;
+    use arrow_schema::{DataType, Field, Schema};
+    use futures::executor::block_on;
+
+    /// A single row that shares a large source buffer must report its own
+    /// size, not the buffer's. Callers budget on this figure, and an inflated
+    /// one makes a small row look impossible to place.
+    #[test]
+    fn a_single_row_slice_reports_its_own_size() {
+        use arrow_array::LargeStringArray;
+
+        let row = "x".repeat(100 * 1024);
+        let values: Vec<&str> = (0..2000).map(|_| row.as_str()).collect();
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "text",
+            DataType::LargeUtf8,
+            false,
+        )]));
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(LargeStringArray::from(values))]).unwrap();
+        let whole = batch.get_array_memory_size();
+
+        let one_row = batch.slice(7, 1);
+        assert_eq!(
+            one_row.get_array_memory_size(),
+            whole,
+            "a slice shares its source buffer, which is the reason this test exists"
+        );
+
+        // Budget far below the row so the split recurses to the single-row floor.
+        let chunks = slice_batch(one_row, 1024, true).unwrap();
+        assert_eq!(chunks.len(), 1);
+        let measured = chunks[0].get_array_memory_size();
+        assert_eq!(chunks[0].num_rows(), 1);
+        assert!(
+            measured < whole / 100,
+            "single row still measured as {measured} bytes against a {whole} byte source"
+        );
+    }
+
+    fn make_batch(num_rows: usize) -> RecordBatch {
+        let schema = test_schema();
+        let values: Vec<i32> = (0..num_rows as i32).collect();
+        RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap()
+    }
+
+    fn test_schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]))
+    }
+
+    fn collect_rechunked(
+        batches: Vec<RecordBatch>,
+        min_bytes: usize,
+        max_bytes: usize,
+    ) -> Vec<RecordBatch> {
+        let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
+        let rechunked = rechunk_stream_by_size(input, test_schema(), min_bytes, max_bytes);
+        block_on(rechunked.collect::<Vec<_>>())
+            .into_iter()
+            .map(|r| r.unwrap())
+            .collect()
+    }
+
+    fn total_rows(batches: &[RecordBatch]) -> usize {
+        batches.iter().map(|b| b.num_rows()).sum()
+    }
+
+    #[test]
+    fn test_empty_stream() {
+        let result = collect_rechunked(vec![], 100, 200);
+        assert!(result.is_empty());
+    }
+
+    #[test]
+    fn test_single_batch_passthrough() {
+        let batch = make_batch(100);
+        let bytes = batch.get_array_memory_size();
+        // Batch is between min and max — should pass through as-is.
+        let result = collect_rechunked(vec![batch], bytes / 2, bytes * 2);
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].num_rows(), 100);
+    }
+
+    #[test]
+    fn test_small_batches_concatenated() {
+        let one_batch_bytes = make_batch(10).get_array_memory_size();
+        let batches: Vec<_> = (0..8).map(|_| make_batch(10)).collect();
+        // min = 5 batches worth, max = 10 batches worth.
+        let result = collect_rechunked(batches, one_batch_bytes * 5, one_batch_bytes * 10);
+        assert_eq!(total_rows(&result), 80);
+        // Should have been concatenated into fewer batches than the 8 inputs.
+        assert!(
+            result.len() < 8,
+            "expected fewer output batches, got {}",
+            result.len()
+        );
+    }
+
+    #[test]
+    fn test_large_batch_sliced() {
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
+        assert_eq!(total_rows(&result), 1000);
+        assert!(
+            result.len() >= 4,
+            "expected at least 4 slices, got {}",
+            result.len()
+        );
+    }
+
+    #[test]
+    fn test_sliced_leftovers_are_not_recombined() {
+        // Key test for the fast-path optimisation. When a large batch is
+        // sliced, leftover slices should be delivered one-at-a-time without
+        // being concatenated back together.  We verify this by checking that
+        // every output buffer pointer falls inside the original batch's
+        // allocation (i.e. they are all zero-copy slices, not fresh copies).
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let orig_data = batch.column(0).to_data();
+        let orig_buf = &orig_data.buffers()[0];
+        let orig_start = orig_buf.as_ptr() as usize;
+        let orig_end = orig_start + orig_buf.len();
+
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
+
+        assert_eq!(total_rows(&result), 1000);
+        assert!(result.len() >= 4);
+
+        for (i, b) in result.iter().enumerate() {
+            let ptr = b.column(0).to_data().buffers()[0].as_ptr() as usize;
+            assert!(
+                ptr >= orig_start && ptr < orig_end,
+                "slice {i} buffer at {ptr:#x} is outside the original allocation \
+                 [{orig_start:#x}, {orig_end:#x}) — it was re-concatenated"
+            );
+        }
+    }
+
+    #[test]
+    fn test_flush_remainder_on_stream_end() {
+        // Data below min_bytes should still be flushed when the stream ends.
+        let batch = make_batch(10);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes * 100, bytes * 200);
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].num_rows(), 10);
+    }
+
+    #[test]
+    fn test_large_then_small_batches() {
+        // After a large batch is fully drained, subsequent small batches
+        // should be accumulated normally.
+        let large = make_batch(1000);
+        let small_bytes = make_batch(10).get_array_memory_size();
+        let batches = vec![
+            large,
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+        ];
+        let result = collect_rechunked(batches, small_bytes * 3, small_bytes * 100);
+        assert_eq!(total_rows(&result), 1050);
+        // The large batch should appear (possibly sliced) followed by
+        // concatenated small batches, so we should have fewer output batches
+        // than the 6 inputs.
+        assert!(result.len() < 6);
+    }
+
+    #[test]
+    fn test_row_preservation_across_slicing() {
+        // Verify that every input row appears exactly once in the output
+        // and in the correct order after slicing.
+        let batch = make_batch(237); // odd count to exercise remainder slice
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 5);
+
+        assert_eq!(total_rows(&result), 237);
+
+        let values: Vec<i32> = result
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .iter()
+                    .copied()
+            })
+            .collect();
+        let expected: Vec<i32> = (0..237).collect();
+        assert_eq!(values, expected);
+    }
+
+    #[test]
+    fn test_min_bytes_zero_still_yields_all_rows() {
+        // When min_bytes=0, the stream should still yield every batch.
+        // This is the "chop only, don't coalesce" use case.
+        let batches: Vec<_> = (0..5).map(|_| make_batch(100)).collect();
+        let batch_bytes = batches[0].get_array_memory_size();
+        let result = collect_rechunked(batches, 0, batch_bytes * 2);
+        assert_eq!(total_rows(&result), 500);
+    }
+
+    #[test]
+    fn test_min_bytes_zero_slices_oversized() {
+        // min_bytes=0 with a small max_bytes should still slice large batches.
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], 0, bytes / 4);
+        assert_eq!(total_rows(&result), 1000);
+        assert!(
+            result.len() >= 4,
+            "expected at least 4 slices, got {}",
+            result.len()
+        );
+    }
+
+    /// Build a batch with one variable-length string column.
+    /// Every row is `small_size` bytes except the row at index `big_row_idx`
+    /// which is `big_size` bytes.
+    fn make_variable_batch(
+        num_rows: usize,
+        small_size: usize,
+        big_row_idx: usize,
+        big_size: usize,
+    ) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
+        let values: Vec<String> = (0..num_rows)
+            .map(|i| {
+                if i == big_row_idx {
+                    "X".repeat(big_size)
+                } else {
+                    "x".repeat(small_size)
+                }
+            })
+            .collect();
+        let array = arrow_array::StringArray::from(values);
+        RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap()
+    }
+
+    fn variable_schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]))
+    }
+
+    fn collect_rechunked_variable(
+        batches: Vec<RecordBatch>,
+        min_bytes: usize,
+        max_bytes: usize,
+    ) -> Vec<RecordBatch> {
+        let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
+        let rechunked =
+            rechunk_stream_by_size_deep_copy(input, variable_schema(), min_bytes, max_bytes);
+        block_on(rechunked.collect::<Vec<_>>())
+            .into_iter()
+            .map(|r| r.unwrap())
+            .collect()
+    }
+
+    #[test]
+    fn test_oversized_row_at_end() {
+        // 100 rows: 99 small (64 bytes each) + 1 large (100KiB) at the end.
+        let batch = make_variable_batch(100, 64, 99, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_oversized_row_at_start() {
+        // 100 rows: 1 large (100KiB) at the start + 99 small (64 bytes each).
+        let batch = make_variable_batch(100, 64, 0, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_oversized_row_in_middle() {
+        // 100 rows: 1 large (100KiB) in the middle + 99 small (64 bytes each).
+        let batch = make_variable_batch(100, 64, 50, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_error_propagation() {
+        let input = stream::iter(vec![
+            Ok(make_batch(10)),
+            Err(ArrowError::ComputeError("boom".into())),
+            Ok(make_batch(10)),
+        ]);
+        let rechunked = rechunk_stream_by_size(input, test_schema(), 1, usize::MAX);
+        let results: Vec<Result<RecordBatch, ArrowError>> = block_on(rechunked.collect());
+        assert!(results.iter().any(|r| r.is_err()));
+    }
+}

python/vendor/lance-arrow/src/struct.rs

@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Extension to arrow struct arrays
+
+use arrow_array::{Array, StructArray, cast::AsArray, make_array};
+use arrow_buffer::NullBuffer;
+use arrow_data::{ArrayData, ArrayDataBuilder};
+use arrow_schema::{ArrowError, DataType};
+
+pub trait StructArrayExt {
+    /// Removes the offset / length of the struct array by pushing it into the children
+    ///
+    /// In arrow-rs when slice is called it recursively slices the children.
+    /// In arrow-cpp when slice is called it just sets the offset/length of
+    ///   the struct array and leaves the children as-is
+    ///
+    /// Both are legal approaches (╥﹏╥)
+    ///
+    /// This method helps reduce complexity by folding into the arrow-rs approach
+    fn normalize_slicing(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized;
+
+    /// Structs are allowed to mask valid items.  For example, a struct array might be:
+    ///
+    /// [ {"items": [1, 2, 3]}, NULL, {"items": NULL}]
+    ///
+    /// However, the underlying items array might be: [[1, 2, 3], [4, 5], NULL]
+    ///
+    /// The [4, 5] list is masked out because the struct array is null.
+    ///
+    /// The struct validity would be [true, false, true] and the list validity would be [true, true, false]
+    ///
+    /// This method pushes nulls down into all children.  In the above example the list validity would become
+    /// [true, false, false].
+    ///
+    /// This method is not recursive.  If a child is a struct array it will not push that child's nulls down.
+    ///
+    /// This method does not remove garbage lists.  It only updates the validity so a future call to
+    /// [crate::list::ListArrayExt::filter_garbage_nulls] will remove the garbage lists (without
+    /// this pushdown it would not)
+    fn pushdown_nulls(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized;
+}
+
+fn normalized_struct_array_data(data: ArrayData) -> Result<ArrayData, ArrowError> {
+    let parent_offset = data.offset();
+    let parent_len = data.len();
+    let modified_children = data
+        .child_data()
+        .iter()
+        .map(|d| {
+            let d = normalized_struct_array_data(d.clone())?;
+            let offset = d.offset();
+            let len = d.len();
+            if len < parent_len + parent_offset {
+                return Err(ArrowError::InvalidArgumentError(format!(
+                    "Child array {} has length {} which is less than the parent length {} plus the parent offset {}",
+                    d.data_type(),
+                    len,
+                    parent_len,
+                    parent_offset
+                )));
+            }
+            let new_offset = offset + parent_offset;
+            d.into_builder().offset(new_offset)
+                .len(parent_len)
+                .build()
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+    ArrayDataBuilder::new(data.data_type().clone())
+        .len(parent_len)
+        .offset(0)
+        .buffers(data.buffers().to_vec())
+        .child_data(modified_children)
+        .build()
+}
+
+impl StructArrayExt for StructArray {
+    fn normalize_slicing(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized,
+    {
+        if self.offset() == 0 && self.columns().iter().all(|c| c.len() == self.len()) {
+            return Ok(self.clone());
+        }
+
+        let data = normalized_struct_array_data(self.to_data())?;
+        Ok(Self::from(data))
+    }
+
+    fn pushdown_nulls(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized,
+    {
+        let Some(validity) = self.nulls() else {
+            return Ok(self.clone());
+        };
+        let data = self.to_data();
+        let children = data
+            .child_data()
+            .iter()
+            .map(|c| {
+                if c.data_type() == &DataType::Null {
+                    // A Null array is already null in every slot, and arrow-rs rejects
+                    // a validity buffer on it.
+                    Ok(c.clone())
+                } else if let Some(child_validity) = c.nulls() {
+                    let new_validity = child_validity.inner() & validity.inner();
+                    c.clone()
+                        .into_builder()
+                        .nulls(Some(NullBuffer::from(new_validity)))
+                        .build()
+                } else {
+                    Ok(c.clone()
+                        .into_builder()
+                        .nulls(Some(validity.clone()))
+                        .build()?)
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let arr = make_array(data.into_builder().child_data(children).build()?);
+        Ok(arr.as_struct().clone())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::{
+        Array, Int32Array, NullArray, StructArray, cast::AsArray, make_array, types::Int32Type,
+    };
+    use arrow_buffer::NullBuffer;
+    use arrow_schema::{DataType, Field, Fields};
+    use std::sync::Arc;
+
+    use crate::r#struct::StructArrayExt;
+
+    #[test]
+    fn test_normalize_slicing_no_offset() {
+        let x = Int32Array::from(vec![1, 2, 3]);
+        let y = Int32Array::from(vec![4, 5, 6]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        let normalized = struct_array.normalize_slicing().unwrap();
+        assert_eq!(normalized, struct_array);
+    }
+
+    #[test]
+    fn test_arrow_rs_slicing() {
+        let x = Int32Array::from(vec![1, 2, 3, 4]);
+        let y = Int32Array::from(vec![5, 6, 7, 8]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        // Slicing with arrow-rs propagates the slicing to the children so there should
+        // be no change needed to the struct array
+        let sliced = struct_array.slice(1, 2);
+        let normalized = sliced.normalize_slicing().unwrap();
+
+        assert_eq!(normalized, sliced);
+    }
+
+    #[test]
+    fn test_arrow_cpp_slicing() {
+        let x = Int32Array::from(vec![1, 2, 3, 4]);
+        let y = Int32Array::from(vec![5, 6, 7, 8]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        let data = struct_array.to_data();
+        let sliced = data.into_builder().offset(1).len(2).build().unwrap();
+        let sliced = make_array(sliced);
+        let normalized = sliced.as_struct().clone().normalize_slicing().unwrap();
+
+        assert_eq!(normalized, struct_array.slice(1, 2));
+    }
+
+    #[test]
+    fn test_pushdown_nulls_with_null_typed_child() {
+        // `pa.Table.from_pylist([{"s": {"a": 1, "b": None}}, {"s": None}])` infers a
+        // struct with a Null-typed child. Arrow forbids a validity buffer on a Null
+        // array, so the struct's nulls must not be pushed into it.
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Null, true),
+            ]),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(NullArray::new(3)),
+            ],
+            Some(NullBuffer::from(vec![true, false, true])),
+        );
+
+        let pushed = struct_array.pushdown_nulls().unwrap();
+
+        assert_eq!(pushed.nulls(), struct_array.nulls());
+        assert_eq!(
+            pushed.column(0).as_primitive::<Int32Type>(),
+            &Int32Array::from(vec![Some(1), None, Some(3)])
+        );
+        assert_eq!(pushed.column(1).data_type(), &DataType::Null);
+        assert_eq!(pushed.column(1).logical_null_count(), 3);
+    }
+}