Merge main into gatekeeper/fix-2325-1

Preserve the compaction option tests and upstream CUDA tests by retaining both Any and MagicMock imports.

Validation: rebuilt the Python extension against Lance v15; 29 optimize/CUDA tests and 7 storage documentation tests passed. Rust formatting and Python source checks passed. Full repository Ruff still reports 49 inherited documentation example errors.

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.41.0-beta.10"
+version = "0.41.0-beta.13"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -22,6 +22,8 @@ bytes.workspace = true
 lancedb.workspace = true
 datafusion-common.workspace = true
 lance-core.workspace = true
+# The workspace path dependency includes the patched crate in Maturin sources.
+lance-arrow.workspace = true
 lance-namespace.workspace = true
 lance-namespace-impls.workspace = true
 lance-io.workspace = true

python/python/lancedb/_lancedb.pyi

@@ -270,6 +270,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Table: ...
     async def create_materialized_view_async(
         self,
@@ -279,6 +281,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job: ...
     def list_materialized_views(
         self, *, page_token: Optional[str] = None, page_limit: Optional[int] = None
@@ -501,10 +505,17 @@ class Table:
     async def delete(self, filter: Union[str, PyExpr]) -> DeleteResult: ...
     async def add_columns(self, columns: list[tuple[str, str]]) -> AddColumnsResult: ...
     async def add_computed_columns(
-        self, columns: list[tuple[str, str]]
+        self,
+        columns: list[tuple[str, str]],
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[int] = None,
     ) -> AddColumnsResult: ...
     async def add_function_columns(
-        self, application_json: str, output_name: Optional[str]
+        self,
+        application_json: str,
+        output_name: Optional[str],
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[int] = None,
     ) -> AddColumnsResult: ...
     async def refresh_column(self, column: str) -> RefreshColumnResult: ...
     async def refresh_column_async(self, column: str) -> Job: ...
@@ -851,9 +862,9 @@ class RTEWriteSpec:
     @staticmethod
     def unsharded() -> "RTEWriteSpec": ...
     def with_maintained_indexes(self, indexes: Optional[List[str]]) -> "RTEWriteSpec":
-        """Set which indexes the RTE keeps up to date. None resolves every
-        index on the table at install, failing if one cannot be maintained;
-        a list is verbatim, empty means none."""
+        """Set which indexes the RTE keeps up to date. None is every index on
+        the table, now and later, skipping any it cannot maintain; a list is
+        verbatim, empty means none."""
         ...
     def with_writer_config_defaults(self, defaults: Dict[str, str]) -> "RTEWriteSpec":
         """Return a copy of this spec recording the given default
@@ -941,6 +952,15 @@ class AsyncPermutationBuilder:
         counts: Optional[List[int]] = None,
         fixed: Optional[int] = None,
     ) -> "AsyncPermutationBuilder": ...
+    def bootstrap_sample(
+        self,
+        num_samples: int,
+        *,
+        ratios: Optional[List[float]] = None,
+        counts: Optional[List[int]] = None,
+        fixed: Optional[int] = None,
+        seed: Optional[int] = None,
+    ) -> "AsyncPermutationBuilder": ...
     def split_calculated(self, calculation: str) -> "AsyncPermutationBuilder": ...
     def shuffle(
         self, seed: Optional[int], clump_size: Optional[int]

python/python/lancedb/db.py

@@ -5,6 +5,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
+import asyncio
 from datetime import timedelta
 from pathlib import Path
 import sys
@@ -607,6 +608,8 @@ class DBConnection(EnforceOverrides):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
 
@@ -631,6 +634,13 @@ class DBConnection(EnforceOverrides):
             Cap the view at this many rows, in materialization order.
         with_no_data: bool, default False
             Skip the initial refresh and leave the backing table empty.
+        auto_refresh_enabled: bool, optional
+            Enable or disable automatic refresh on Cloud and Enterprise.
+            When omitted, the server determines whether automatic refresh is enabled.
+            Embedded databases do not support automatic refresh.
+        auto_refresh_interval_seconds: float, optional
+            Positive, finite refresh interval in seconds on Cloud and Enterprise.
+            When omitted, the server determines when to refresh.
 
         Returns
         -------
@@ -649,6 +659,8 @@ class DBConnection(EnforceOverrides):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         """Submit materialized-view creation and return its job.
 
@@ -1706,6 +1718,8 @@ class LanceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
         See
@@ -1734,6 +1748,8 @@ class LanceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return MaterializedView(self.open_table(name))
@@ -1748,6 +1764,8 @@ class LanceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._conn.create_materialized_view_async(
@@ -1757,6 +1775,8 @@ class LanceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)
@@ -2633,8 +2653,16 @@ class AsyncConnection(object):
         if fill_value is None:
             fill_value = 0.0
 
-        data, schema = sanitize_create_table(
-            data, schema, metadata, on_bad_vectors, fill_value
+        # Input preparation may advance a user-provided iterator.  Keep that work
+        # off the background event loop so an iterator can use the synchronous
+        # LanceDB API without blocking the loop that API needs to make progress.
+        data, schema = await asyncio.to_thread(
+            sanitize_create_table,
+            data,
+            schema,
+            metadata,
+            on_bad_vectors,
+            fill_value,
         )
         validate_schema(schema)
 
@@ -2767,6 +2795,8 @@ class AsyncConnection(object):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncMaterializedView:
         """Define a materialized view named ``name`` over the table ``source``.
         See
@@ -2782,6 +2812,8 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             **native_options,
         )
         return AsyncMaterializedView(AsyncTable(inner))
@@ -2795,6 +2827,8 @@ class AsyncConnection(object):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncJob[None]:
         """Submit materialized-view creation and return its job.
 
@@ -2810,6 +2844,8 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             **native_options,
         )
         return AsyncJob(inner)

python/python/lancedb/embeddings/openai.py

@@ -101,24 +101,35 @@ class OpenAIEmbeddings(TextEmbeddingFunction):
                 valid_indices.append(idx)
 
         # TODO retry, rate limit, token limit
-        try:
+        def embed_batch(texts_batch, indices_batch):
             kwargs = {
-                "input": valid_texts,
+                "input": texts_batch,
                 "model": self.name,
             }
             if self.name != "text-embedding-ada-002":
                 kwargs["dimensions"] = self.dim
 
             rs = self._openai_client.embeddings.create(**kwargs)
-            valid_embeddings = {
-                idx: v.embedding for v, idx in zip(rs.data, valid_indices)
-            }
+            return {idx: v.embedding for v, idx in zip(rs.data, indices_batch)}
+
+        def embed_one_or_log(idx, text):
+            # Retry a single rejected input
+            try:
+                return embed_batch([text], [idx])
+            except openai.BadRequestError:
+                logging.exception("Bad request for input at index %d: %r", idx, text)
+                return {}
+
+        try:
+            valid_embeddings = embed_batch(valid_texts, valid_indices)
         except openai.AuthenticationError:
             logging.error("Authentication failed: Invalid API key provided")
             raise
         except openai.BadRequestError:
-            logging.exception("Bad request: %s", texts)
-            return [None] * len(texts)
+            # Retry each input individually
+            valid_embeddings = {}
+            for idx, text in zip(valid_indices, valid_texts):
+                valid_embeddings.update(embed_one_or_log(idx, text))
         except Exception:
             logging.exception("OpenAI embeddings error")
             raise

python/python/lancedb/namespace.py

@@ -665,6 +665,8 @@ class LanceNamespaceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> "MaterializedView":
         """Define a materialized view over a table in the root namespace.
         See
@@ -680,6 +682,8 @@ class LanceNamespaceDBConnection(DBConnection):
                         where=where,
                         limit=limit,
                         with_no_data=with_no_data,
+                        auto_refresh_enabled=auto_refresh_enabled,
+                        auto_refresh_interval_seconds=auto_refresh_interval_seconds,
                     )
                 ).name
             )
@@ -695,6 +699,8 @@ class LanceNamespaceDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._inner.create_materialized_view_async(
@@ -704,6 +710,8 @@ class LanceNamespaceDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)
@@ -1288,6 +1296,8 @@ class AsyncLanceNamespaceDBConnection:
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> "AsyncMaterializedView":
         """Define a materialized view over a table in the root namespace."""
         view = await self._inner.create_materialized_view(
@@ -1297,6 +1307,8 @@ class AsyncLanceNamespaceDBConnection:
             where=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
         )
         # Reopen through the namespace so the view's table carries the
         # namespace client and pushdown configuration a bare inner table lacks.
@@ -1311,6 +1323,8 @@ class AsyncLanceNamespaceDBConnection:
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> AsyncJob[None]:
         """Submit materialized-view creation and return its job."""
         return await self._inner.create_materialized_view_async(
@@ -1320,6 +1334,8 @@ class AsyncLanceNamespaceDBConnection:
             where=where,
             limit=limit,
             with_no_data=with_no_data,
+            auto_refresh_enabled=auto_refresh_enabled,
+            auto_refresh_interval_seconds=auto_refresh_interval_seconds,
         )
 
     async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":

python/python/lancedb/permutation.py

@@ -169,6 +169,50 @@ class PermutationBuilder:
         )
         return self
 
+    def bootstrap_sample(
+        self,
+        num_samples: int,
+        *,
+        ratios: Optional[list[float]] = None,
+        counts: Optional[list[int]] = None,
+        fixed: Optional[int] = None,
+        seed: Optional[int] = None,
+        split_names: Optional[list[str]] = None,
+    ) -> "PermutationBuilder":
+        """
+        Sample rows uniformly **with replacement**, independently per split.
+
+        Unlike the other split strategies, the number of output rows is
+        ``num_samples``, not the number of (optionally filtered) input rows: the
+        same row can appear zero, one, or many times, and ``num_samples`` may be
+        larger or smaller than the input row count. Useful for bootstrap
+        resampling, oversampling rare classes, or any training workload that
+        wants a fixed-size stream of random draws rather than a single
+        deterministic pass over every row. Combine with :meth:`filter` to
+        restrict the eligible population.
+
+        One of ratios, counts, or fixed must be provided, with the same meaning
+        as in :meth:`split_random`, except they divide ``num_samples`` across
+        splits rather than the input row count: if fixed is 4, each split gets
+        ``num_samples // 4`` samples.
+
+        Each split draws its own sample using a seed derived independently from
+        ``(seed, split_index)``, so a split's content does not change if the samples are
+        later redistributed across a different number of splits.
+
+        The optional split_names can be provided to name the splits.  If not provided,
+        the splits can only be referenced by their index.
+        """
+        self._async.bootstrap_sample(
+            num_samples,
+            ratios=ratios,
+            counts=counts,
+            fixed=fixed,
+            seed=seed,
+            split_names=split_names,
+        )
+        return self
+
     def split_calculated(
         self, calculation: str, split_names: Optional[list[str]] = None
     ) -> "PermutationBuilder":

python/python/lancedb/query.py

@@ -82,6 +82,7 @@ if TYPE_CHECKING:
 
 T = TypeVar("T", bound="LanceModel")
 AnalyzePlanDistributedMetrics = Literal["aggregate", "per_worker", "full"]
+_SYNC_VECTOR_QUERY_TYPES = (list, tuple, np.ndarray, pa.Array, pa.ChunkedArray)
 
 # Number of rows a hybrid query returns when no limit was set on it. This
 # mirrors the default the Rust query builder applies to its sub-queries.
@@ -1024,10 +1025,12 @@ class LanceQueryBuilder(ABC):
                 fast_search=fast_search,
             )
 
-        if isinstance(query, list):
+        if isinstance(query, (list, tuple)):
             query = np.array(query, dtype=np.float32)
         elif isinstance(query, np.ndarray):
             query = query.astype(np.float32)
+        elif isinstance(query, (pa.Array, pa.ChunkedArray)):
+            query = np.array(query.to_pylist(), dtype=np.float32)
         else:
             raise TypeError(f"Unsupported query type: {type(query)}")
 
@@ -1049,7 +1052,7 @@ class LanceQueryBuilder(ABC):
             query = cls._query_to_vector(table, query, vector_column_name)
             return query, query_type
         elif query_type == "auto":
-            if isinstance(query, (list, np.ndarray)):
+            if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
                 return query, "vector"
             else:
                 conf = table.embedding_functions.get(vector_column_name)
@@ -1065,7 +1068,7 @@ class LanceQueryBuilder(ABC):
 
     @classmethod
     def _query_to_vector(cls, table, query, vector_column_name):
-        if isinstance(query, (list, np.ndarray)):
+        if isinstance(query, _SYNC_VECTOR_QUERY_TYPES):
             return query
         conf = table.embedding_functions.get(vector_column_name)
         if conf is not None:
@@ -2438,7 +2441,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         if query is not None and (vector is not None or text is not None):
             raise ValueError(
                 "You can either provide a string query in search() method "
-                "or set `vector()` and `text()` explicitly for hybrid search."
+                "or set `vector()` and `text()` explicitly for hybrid search. "
                 "But not both."
             )
 

python/python/lancedb/remote/db.py

@@ -723,6 +723,8 @@ class RemoteDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> MaterializedView:
         from .table import RemoteTable
 
@@ -734,6 +736,8 @@ class RemoteDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return MaterializedView(
@@ -755,6 +759,8 @@ class RemoteDBConnection(DBConnection):
         where: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        auto_refresh_enabled: Optional[bool] = None,
+        auto_refresh_interval_seconds: Optional[float] = None,
     ) -> Job[None]:
         job = LOOP.run(
             self._conn.create_materialized_view_async(
@@ -764,6 +770,8 @@ class RemoteDBConnection(DBConnection):
                 where=where,
                 limit=limit,
                 with_no_data=with_no_data,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
             )
         )
         return Job(job)

python/python/lancedb/remote/header.py

@@ -141,7 +141,7 @@ class OAuthProvider(HeaderProvider):
 
                 # Set expiration if provided
                 expires_in = token_data.get("expires_in")
-                if expires_in:
+                if expires_in is not None:
                     self._token_expires_at = time.time() + expires_in
                 else:
                     # Token doesn't expire or expiration unknown

python/python/lancedb/remote/table.py

@@ -1003,8 +1003,17 @@ class RemoteTable(Table):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+        return LOOP.run(
+            self._table.add_columns(
+                transforms,
+                computed=computed,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
+            )
+        )
 
     def refresh_column(self, column: str):
         return LOOP.run(self._table.refresh_column(column))

python/python/lancedb/rerankers/base.py

@@ -241,6 +241,7 @@ class Reranker(ABC):
         This op is slower than just keeping relevance score but can be useful
         for debugging.
         """
+        combined_schema = pa.unify_schemas([vector_results.schema, fts_results.schema])
         # add nulls to fts results for _distance
         if "_distance" not in fts_results.column_names:
             fts_results = fts_results.append_column(
@@ -265,7 +266,9 @@ class Reranker(ABC):
             else:
                 vector_results_dict[key] = value
 
-        combined = pa.Table.from_pylist(list(vector_results_dict.values()))
+        combined = pa.Table.from_pylist(
+            list(vector_results_dict.values()), schema=combined_schema
+        )
         return combined
 
     def _keep_relevance_score(self, combined_results: pa.Table):

python/python/lancedb/rerankers/voyageai.py

@@ -32,7 +32,7 @@ class VoyageAIReranker(Reranker):
     return_score : str, default "relevance"
         options are "relevance" or "all". Only "relevance" is supported for now.
     api_key : str, default None
-        The API key to use. If None, will use the OPENAI_API_KEY environment variable.
+        The API key to use. If None, will use the VOYAGE_API_KEY environment variable.
     truncation : Optional[bool], default None
     """
 

python/python/lancedb/streaming.py

@@ -19,6 +19,7 @@ above.
 """
 
 import ctypes
+import hashlib
 import heapq
 import logging
 import os
@@ -39,6 +40,7 @@ from torch.utils.data import DataLoader, IterableDataset, get_worker_info
 
 from .permutation import (
     Permutation,
+    PermutationBuilder,
     Transforms,
     permutation_builder,
     _drop_base_version,
@@ -244,6 +246,20 @@ class StreamingDataset(IterableDataset):
         default) rows are shuffled using ``shuffle_seed`` and ``epoch``.
         When ``False`` rows are divided into splits sequentially in storage
         order, which can be useful for deterministic debugging or evaluation.
+        Ignored when ``split_strategy`` is provided.
+    split_strategy:
+        Escape hatch for any ``PermutationBuilder`` split strategy this class
+        doesn't otherwise expose a dedicated parameter for -- ``split_hash``,
+        ``split_calculated``, ``bootstrap_sample`` (uniform sampling with
+        replacement), or any strategy added later.  A callable that receives a
+        fresh ``PermutationBuilder`` for ``table`` and returns a configured one;
+        it is called and ``.execute()``'d directly, taking over permutation
+        construction entirely.  When provided, ``shuffle``, ``shuffle_seed``,
+        ``epoch``, and ``shuffle_clump_size`` are ignored -- the callable is
+        solely responsible for how rows are split and ordered.  For example,
+        ``split_strategy=lambda b: b.bootstrap_sample(10_000, fixed=num_splits,
+        seed=42)`` draws 10,000 samples with replacement, divided evenly across
+        ``num_splits`` splits.
     shuffle_seed:
         Base seed for the random permutation.  Combined with ``epoch`` so
         each epoch produces a different ordering.  Pass ``None`` to generate
@@ -393,6 +409,9 @@ class StreamingDataset(IterableDataset):
         *,
         num_splits: Optional[int] = None,
         shuffle: bool = True,
+        split_strategy: Optional[
+            Callable[[PermutationBuilder], PermutationBuilder]
+        ] = None,
         shuffle_seed: Optional[int] = 0,
         epoch: int = 0,
         rank: int = 0,
@@ -583,13 +602,70 @@ class StreamingDataset(IterableDataset):
         builder = permutation_builder(table)
         if filter is not None:
             builder = builder.filter(filter)
-        if shuffle:
+        if split_strategy is not None:
+            # Escape hatch: lets any PermutationBuilder split strategy drive this
+            # dataset (split_hash, split_calculated, bootstrap_sample, or any future
+            # strategy) without StreamingDataset needing a bespoke parameter for each
+            # one. shuffle/shuffle_seed/epoch/shuffle_clump_size are ignored in this
+            # path -- the caller's split_strategy is solely responsible for ordering.
+            self._perm_table = split_strategy(builder).execute()
+            # _rank_splits below assumes split ids are exactly 0..num_splits-1 with no
+            # gaps (every other path guarantees this by always building with
+            # fixed=num_splits) -- a split_strategy callable has no such guarantee, and
+            # a mismatch would otherwise silently drop rows (ids >= num_splits) or leave
+            # some ranks reading an empty split (missing ids < num_splits).
+            split_value_counts = (
+                self._perm_table.to_arrow().column("split_id").value_counts()
+            )
+            counts_by_split = dict(
+                zip(
+                    split_value_counts.field("values").to_pylist(),
+                    split_value_counts.field("counts").to_pylist(),
+                )
+            )
+            expected_splits = set(range(num_splits))
+            if set(counts_by_split) != expected_splits:
+                raise ValueError(
+                    f"split_strategy produced split ids {sorted(counts_by_split)}, "
+                    f"but num_splits={num_splits} requires exactly "
+                    f"{{0, ..., {num_splits - 1}}}. Pass num_splits matching the "
+                    "strategy's own split count (e.g. `fixed=num_splits` on "
+                    "bootstrap_sample/split_random/etc.)."
+                )
+            # The round-robin iteration below (and in _iter_owned) assumes every
+            # split has the same row count -- it stops as soon as any one owned
+            # split is exhausted (see its own comment: "all splits exhaust
+            # simultaneously"). Every other path guarantees this by always building
+            # with fixed=num_splits; a split_strategy using ratios/counts/split_hash
+            # weights has no such guarantee, and unequal sizes would otherwise
+            # silently drop rows from the larger splits -- or, with world_size > 1
+            # where each rank owns different splits, end ranks after different row
+            # counts and desync or hang a distributed run's end-of-epoch collectives.
+            if len(set(counts_by_split.values())) > 1:
+                raise ValueError(
+                    f"split_strategy produced unequal split sizes "
+                    f"{counts_by_split} (split id -> row count). Every split must "
+                    "have the same row count. Use `fixed=num_splits` rather than "
+                    "`ratios`/`counts`/weighted `split_hash`, which can produce "
+                    "unequal sizes, or split unevenly-sized groups into separate "
+                    "StreamingDatasets instead."
+                )
+        elif shuffle:
             perm_seed = shuffle_seed + epoch * _EPOCH_PRIME
             self._perm_table = builder.split_random(
                 fixed=num_splits, seed=perm_seed, clump_size=shuffle_clump_size
             ).execute()
         else:
             self._perm_table = builder.split_sequential(fixed=num_splits).execute()
+        # Only the split_strategy path needs a permutation fingerprint in the
+        # checkpoint (see `_checkpoint_snapshot`): split_random/split_sequential are
+        # already fully pinned by shuffle_seed/num_splits, and a fingerprint over raw
+        # row ids would spuriously reject an otherwise-valid resume after those row
+        # ids get renumbered by compaction (row ids are not stable by default).
+        self._uses_split_strategy = split_strategy is not None
+        # Lazily computed and cached by `_perm_fingerprint` -- `_perm_table` never
+        # changes after construction, so this is correct to compute once.
+        self._perm_fingerprint_cache: Optional[str] = None
 
         if self._blocks_per_epoch == "auto":
             self._blocks_per_epoch = self._estimate_blocks_per_epoch()
@@ -1406,11 +1482,51 @@ class StreamingDataset(IterableDataset):
                 )
         return state
 
+    @property
+    def _perm_fingerprint(self) -> str:
+        """Cheap identity check for the built permutation, computed once and cached
+        (``_perm_table`` never changes after construction).
+
+        Only meaningful -- and only recorded in a checkpoint -- for the
+        ``split_strategy`` path (see ``_checkpoint_snapshot``): ``num_splits``/
+        ``shuffle_seed`` alone cannot detect a `load_state_dict` resuming saved
+        per-split progress against a *different* permutation than the one that
+        produced it, since a ``split_strategy`` callable ignores ``shuffle_seed``
+        entirely -- e.g. a ``bootstrap_sample`` with no seed (or any changed
+        callable) can rebuild a completely different permutation on the next run
+        while ``shuffle_seed`` keeps its same default value and the existing check
+        passes anyway. ``split_random``/``split_sequential`` are already fully
+        pinned by ``shuffle_seed``/``num_splits`` and must *not* additionally be
+        fingerprinted: this hashes physical row ids, which are not stable across
+        compaction by default, so checking it on those paths would reject an
+        otherwise-valid resume after the table gets compacted between save and
+        load even though the seeded permutation itself is unchanged.
+        """
+        if self._perm_fingerprint_cache is None:
+            arrow_tbl = self._perm_table.to_arrow()
+            hasher = hashlib.sha256()
+            hasher.update(str(arrow_tbl.num_rows).encode())
+            for name in ("row_id", "split_id"):
+                column = arrow_tbl.column(name).combine_chunks()
+                hasher.update(column.to_numpy(zero_copy_only=False).tobytes())
+            self._perm_fingerprint_cache = hasher.hexdigest()
+        return self._perm_fingerprint_cache
+
     def _checkpoint_snapshot(self) -> dict:
+        # Only recorded for split_strategy: split_random/split_sequential are
+        # already fully pinned by shuffle_seed/num_splits below, and row ids are not
+        # stable across compaction by default, so fingerprinting them unconditionally
+        # would reject an otherwise-valid resume after the table got compacted.
+        perm_fingerprint = (
+            {"perm_fingerprint": self._perm_fingerprint}
+            if (self._uses_split_strategy)
+            else {}
+        )
         if self._pack_sequences is not None:
             return {
                 "shuffle_seed": self._shuffle_seed,
                 "num_splits": self._num_splits,
+                **perm_fingerprint,
                 "epoch": self._epoch,
                 "pack_sequences": self._pack_sequences,
                 "eos_id": self._eos_id,
@@ -1431,6 +1547,7 @@ class StreamingDataset(IterableDataset):
         return {
             "shuffle_seed": self._shuffle_seed,
             "num_splits": self._num_splits,
+            **perm_fingerprint,
             "epoch": self._epoch,
             "samples_consumed_per_split": samples,
             "positions_consumed_per_split": positions,
@@ -1560,8 +1677,16 @@ class StreamingDataset(IterableDataset):
 
         Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ
         from the checkpoint, since a different split structure or shuffle order
-        makes mid-epoch resumption meaningless.  Packed checkpoints
-        pin ``pack_sequences``, ``eos_id``, ``pad_id``,
+        makes mid-epoch resumption meaningless. Also raises if the checkpoint
+        was built with ``split_strategy`` and records a permutation fingerprint
+        that does not match the current dataset's permutation -- this catches a
+        changed or non-deterministically-reseeded ``split_strategy`` that
+        ``num_splits``/``shuffle_seed`` alone cannot detect, since ``shuffle_seed``
+        is unused on that path. Checkpoints from ``split_random``/
+        ``split_sequential`` never carry this fingerprint (row ids are not stable
+        across compaction, so it would reject an otherwise-valid resume), and
+        continue to rely on ``shuffle_seed``/``num_splits`` alone as before.
+        Packed checkpoints pin ``pack_sequences``, ``eos_id``, ``pad_id``,
         ``blocks_per_epoch``, and ``epoch``.
         """
         if state["num_splits"] != self._num_splits:
@@ -1574,6 +1699,19 @@ class StreamingDataset(IterableDataset):
                 f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, "
                 f"current dataset has {self._shuffle_seed}"
             )
+        checkpoint_fingerprint = state.get("perm_fingerprint")
+        if (
+            checkpoint_fingerprint is not None
+            and checkpoint_fingerprint != self._perm_fingerprint
+        ):
+            raise ValueError(
+                "permutation mismatch: this checkpoint was saved against a "
+                "different permutation (e.g. a split_strategy that resamples "
+                "differently between runs, such as bootstrap_sample with no "
+                "seed, or a changed callable) than the current dataset built. "
+                "Resuming would apply saved per-split positions to the wrong "
+                "rows."
+            )
         self._consumer_checkpoint_requires_uniform = False
 
         if "pack_buffers" in state or self._pack_sequences is not None:
@@ -1675,6 +1813,10 @@ class StreamingDataset(IterableDataset):
         first = states[0]
         packed = "pack_buffers" in first
         config_keys = ["shuffle_seed", "num_splits", "epoch"]
+        if "perm_fingerprint" in first:
+            # Absent only in checkpoints saved before this key existed; skip rather
+            # than KeyError so old checkpoints keep merging.
+            config_keys.append("perm_fingerprint")
         if packed:
             config_keys.extend(
                 ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"]
@@ -1684,6 +1826,8 @@ class StreamingDataset(IterableDataset):
             if ("pack_buffers" in state) != packed:
                 raise ValueError("cannot merge packed and unpacked state dicts")
             for key in config_keys:
+                if key not in state:
+                    continue
                 if state[key] != first[key]:
                     raise ValueError(
                         f"{key} mismatch across state dicts: "

python/python/lancedb/table.py

@@ -120,6 +120,33 @@ def _should_push_down_query_table(
     return namespace_client is not None and "QueryTable" in pushdown_operations
 
 
+_CUDA_UNAVAILABLE_MESSAGE = (
+    "CUDA acceleration is unavailable in this Python environment. Install a "
+    "CUDA-enabled PyTorch build and verify that the CUDA device is accessible, or "
+    "omit `accelerator` (or set `accelerator=None`) to create the index on CPU."
+)
+
+
+def _validate_index_accelerator(accelerator: Any) -> None:
+    if not isinstance(accelerator, str) or accelerator.split(":", 1)[0] != "cuda":
+        return
+
+    try:
+        import torch
+
+        cuda_available = torch.cuda.is_available()
+    except (ImportError, RuntimeError, AssertionError) as err:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
+
+    if not cuda_available:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE)
+
+    try:
+        torch.cuda.init()
+    except (RuntimeError, AssertionError) as err:
+        raise RuntimeError(_CUDA_UNAVAILABLE_MESSAGE) from err
+
+
 def _polars_predicate_pushdown_barrier(frame: Any) -> Any:
     """Return a Polars frame unchanged while blocking predicate pushdown."""
     return frame
@@ -2653,6 +2680,8 @@ class Table(ABC):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ):
         """
         Add new columns with defined values.
@@ -2697,6 +2726,15 @@ class Table(ABC):
             [`refresh_column_async`][lancedb.table.Table.refresh_column_async].
             Cannot be combined with ``transforms``.
 
+        auto_refresh_enabled: bool, optional
+            Enable automatic refresh for each declared computed output. Defaults
+            to True. Local tables store this option; scheduling requires a server.
+        auto_refresh_interval_seconds: int, optional
+            Positive minimum interval after a successful refresh, in seconds.
+            When omitted, the server determines when to refresh. Applies to every
+            computed output in this call.
+            Scheduling options require computed or Function columns.
+
         Returns
         -------
         AddColumnsResult
@@ -3681,7 +3719,9 @@ class LanceTable(Table):
 
             # Handle accelerator through pylance
             if accelerator is not None:
-                self.to_lance().create_index(
+                dataset = self.to_lance()
+                _validate_index_accelerator(accelerator)
+                dataset.create_index(
                     column=column,
                     index_type=index_type,
                     metric=metric,
@@ -3706,6 +3746,8 @@ class LanceTable(Table):
                 acc = getattr(config, "accelerator", None)
                 if acc is not None:
                     # Dispatch to pylance for GPU acceleration
+                    dataset = self.to_lance()
+                    _validate_index_accelerator(acc)
                     index_type_map = {
                         "IvfFlat": "IVF_FLAT",
                         "IvfSq": "IVF_SQ",
@@ -3717,7 +3759,7 @@ class LanceTable(Table):
                     cfg_type = type(config).__name__
                     lance_index_type = index_type_map.get(cfg_type, "IVF_PQ")
 
-                    self.to_lance().create_index(
+                    dataset.create_index(
                         column=column,
                         index_type=lance_index_type,
                         metric=getattr(config, "distance_type", "l2"),
@@ -4902,8 +4944,17 @@ class LanceTable(Table):
         | None = None,
         *,
         computed: Dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
-        return LOOP.run(self._table.add_columns(transforms, computed=computed))
+        return LOOP.run(
+            self._table.add_columns(
+                transforms,
+                computed=computed,
+                auto_refresh_enabled=auto_refresh_enabled,
+                auto_refresh_interval_seconds=auto_refresh_interval_seconds,
+            )
+        )
 
     def refresh_column(self, column: str) -> "RefreshColumnResult":
         """Fill a computed column's unfilled rows and recompute those whose
@@ -5414,6 +5465,12 @@ def _infer_target_schema(
             # Use the most common length of the list as the dimensions
             dim = _modal_list_size(peeked.column(i))
 
+            if dim is None:
+                # Every list entry is null (e.g. embeddings not computed yet),
+                # so the dimension is unknowable. Leave the column as a
+                # variable-length list instead of guessing a fixed size.
+                continue
+
             # Determine target type based on value type
             if pa.types.is_floating(field.type.value_type):
                 target_type = pa.list_(pa.float32(), dim)
@@ -5453,9 +5510,14 @@ def _infer_target_schema(
     return schema, reader
 
 
-def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> int:
-    # Use the most common length of the list as the dimensions
-    return pc.mode(pc.list_value_length(arr))[0].as_py()["mode"]
+def _modal_list_size(arr: Union[pa.ListArray, pa.ChunkedArray]) -> Optional[int]:
+    # Use the most common length of the list as the dimensions. pyarrow's
+    # `mode` skips nulls, so an all-null column yields an empty result; return
+    # None in that case so the caller can keep a variable-length list.
+    modes = pc.mode(pc.list_value_length(arr))
+    if len(modes) == 0:
+        return None
+    return modes[0].as_py()["mode"]
 
 
 def _infer_vector_dim(arr: Union[pa.Array, pa.ChunkedArray]) -> Optional[int]:
@@ -5663,12 +5725,12 @@ class AsyncTable:
         via [`set_unenforced_primary_key`]; bucket sharding additionally
         requires it to be the single column being bucketed.
 
-        By default the RTE maintains every index on the table, resolved
-        here — a snapshot, so an index created afterwards needs the spec unset
-        and set again. This fails if one cannot be maintained; name the set
-        with ``with_maintained_indexes`` to install anyway. That pins an exact
-        set (a still-building index is rejected, not omitted); ``[]`` maintains
-        none.
+        By default the RTE maintains every index on the table, re-read as the
+        table changes, so an index created later is maintained too; one of a
+        kind the RTE cannot mirror is skipped. Name the set with
+        ``with_maintained_indexes`` to pin it instead: each name must already
+        exist and be maintainable (a still-building index is rejected, not
+        omitted); ``[]`` maintains none.
 
         Parameters
         ----------
@@ -6292,8 +6354,12 @@ class AsyncTable:
         if mode == "overwrite":
             # For overwrite, apply the same preprocessing as create_table
             # so vector columns are inferred as FixedSizeList.
-            data, _ = sanitize_create_table(
-                data, None, on_bad_vectors=on_bad_vectors, fill_value=fill_value
+            data, _ = await asyncio.to_thread(
+                sanitize_create_table,
+                data,
+                None,
+                on_bad_vectors=on_bad_vectors,
+                fill_value=fill_value,
             )
             if blob_v2_column_paths(schema):
                 # Plain bytes cannot be inferred as blob v2. Keep matching blob
@@ -6303,7 +6369,8 @@ class AsyncTable:
         elif on_bad_vectors != "error" or (
             schema.metadata is not None and b"embedding_functions" in schema.metadata
         ):
-            data = _sanitize_data(
+            data = await asyncio.to_thread(
+                _sanitize_data,
                 data,
                 schema,
                 metadata=schema.metadata,
@@ -6312,7 +6379,10 @@ class AsyncTable:
                 allow_subschema=True,
             )
         _register_optional_converters()
-        data = to_scannable(data)
+        # Converting an iterator peeks at its first item.  A synchronous query in
+        # that iterator schedules work on LOOP, so peeking on LOOP's own thread
+        # would deadlock waiting for itself.
+        data = await asyncio.to_thread(to_scannable, data)
         progress, owns = _normalize_progress(progress)
         try:
             return await self._inner.add(
@@ -6951,6 +7021,8 @@ class AsyncTable:
         | None = None,
         *,
         computed: dict[str, str] | None = None,
+        auto_refresh_enabled: bool | None = None,
+        auto_refresh_interval_seconds: int | None = None,
     ) -> AddColumnsResult:
         """
         Add new columns with defined values.
@@ -6990,12 +7062,46 @@ class AsyncTable:
             On LanceDB Cloud and Enterprise the expression is planned by
             the server. Cannot be combined with ``transforms``.
 
+        auto_refresh_enabled: bool, optional
+            Enable automatic refresh for each computed output declared by this
+            call. Defaults to True. Local tables store the setting; automatic
+            scheduling requires Cloud or Enterprise.
+        auto_refresh_interval_seconds: int, optional
+            Positive minimum seconds after a successful refresh before another
+            automatic refresh is eligible. When omitted, the server determines when
+            to refresh. Unchanged inputs do
+            not force recomputation. Applies to every declared computed output.
+            These options cannot be used with one-time transforms or ordinary
+            all-null columns.
+
+        Examples
+        --------
+        >>> await table.add_columns(  # doctest: +SKIP
+        ...     computed={"total": "price * quantity"},
+        ...     auto_refresh_enabled=True,
+        ...     auto_refresh_interval_seconds=300,
+        ... )
+
         Returns
         -------
         AddColumnsResult
             version: the new version number of the table after adding columns.
 
         """
+        if auto_refresh_enabled is not None and type(auto_refresh_enabled) is not bool:
+            raise ValueError("auto_refresh_enabled must be a boolean")
+        if auto_refresh_interval_seconds is not None and (
+            type(auto_refresh_interval_seconds) is not int
+            or not 0 < auto_refresh_interval_seconds <= (2**64 - 1) // 1000
+        ):
+            raise ValueError(
+                "auto_refresh_interval_seconds must be a positive integer "
+                "no greater than u64::MAX / 1000"
+            )
+        options_supplied = (
+            auto_refresh_enabled is not None
+            or auto_refresh_interval_seconds is not None
+        )
         function_application = None
         function_output_name = None
         if isinstance(transforms, FunctionApplication):
@@ -7019,7 +7125,10 @@ class AsyncTable:
                 )
             function_application._ensure_declarable()
             return await self._inner.add_function_columns(
-                function_application.to_canonical_json(), function_output_name
+                function_application.to_canonical_json(),
+                function_output_name,
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
             )
 
         if isinstance(transforms, pa.Field):
@@ -7033,7 +7142,15 @@ class AsyncTable:
                 raise ValueError(
                     "add_columns cannot take both transforms and computed columns"
                 )
-            return await self._inner.add_computed_columns(list(computed.items()))
+            return await self._inner.add_computed_columns(
+                list(computed.items()),
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
+            )
+        if options_supplied:
+            raise ValueError(
+                "automatic refresh options require computed or Function columns"
+            )
         if transforms is None:
             raise ValueError("add_columns requires transforms or computed columns")
         if isinstance(transforms, pa.Schema):

python/python/lancedb/util.py

@@ -34,20 +34,20 @@ def safe_import_adlfs():
 adlfs = safe_import_adlfs()
 
 
-def get_uri_scheme(uri: str) -> str:
+def get_uri_scheme(uri: Union[str, pathlib.Path]) -> str:
     """
     Get the scheme of a URI. If the URI does not have a scheme, assume it is a file URI.
 
     Parameters
     ----------
-    uri : str
+    uri : str or pathlib.Path
         The URI to parse.
 
     Returns
     -------
     str: The scheme of the URI.
     """
-    parsed = urlparse(uri)
+    parsed = urlparse(str(uri))
     scheme = parsed.scheme
     if not scheme:
         scheme = "file"
@@ -61,25 +61,26 @@ def get_uri_scheme(uri: str) -> str:
     return scheme
 
 
-def get_uri_location(uri: str) -> str:
+def get_uri_location(uri: Union[str, pathlib.Path]) -> str:
     """
     Get the location of a URI. If the parameter is not a url, assumes it is just a path
 
     Parameters
     ----------
-    uri : str
+    uri : str or pathlib.Path
         The URI to parse.
 
     Returns
     -------
     str: Location part of the URL, without scheme
     """
-    parsed = urlparse(uri)
+    uri_str = str(uri)
+    parsed = urlparse(uri_str)
     if len(parsed.scheme) == 1:
         # Windows drive names are parsed as the scheme
         # e.g. "c:\path" -> ParseResult(scheme="c", netloc="", path="/path", ...)
         # So we add special handling here for schemes that are a single character
-        return uri
+        return uri_str
 
     if not parsed.netloc:
         return parsed.path
@@ -309,7 +310,16 @@ def _query_vector_dim(query: Optional[Any]) -> Optional[int]:
         if query.ndim == 0:
             return None
         return query.shape[-1]
-    if isinstance(query, list) and query:
+    if isinstance(query, (pa.Array, pa.ChunkedArray)):
+        if len(query) == 0:
+            return None
+        if pa.types.is_fixed_size_list(query.type):
+            return query.type.list_size
+        if pa.types.is_list(query.type) or pa.types.is_large_list(query.type):
+            first = query[0].as_py()
+            return len(first) if first is not None else None
+        return len(query)
+    if isinstance(query, (list, tuple)) and query:
         first = query[0]
         if isinstance(first, (list, tuple, np.ndarray)):
             return len(first)

python/python/tests/test_elastic_dataloader.py

@@ -3297,3 +3297,230 @@ def test_streaming_dataset_over_remote_table():
     assert len(server.scans) == 1, "the permutation is built with one row-id scan"
     assert server.takes, "rows must be fetched with row-id takes"
     assert_server_safe_row_id_requests(server)
+
+
+# ---------------------------------------------------------------------------
+# split_strategy escape hatch
+# ---------------------------------------------------------------------------
+
+
+def test_split_strategy_bootstrap_sample(lance_table):
+    """split_strategy lets StreamingDataset use bootstrap_sample directly, producing
+    more rows than the base table has (sampling with replacement)."""
+    num_samples = 4 * NUM_ROWS
+    ds = StreamingDataset(
+        lance_table,
+        split_strategy=lambda b: b.bootstrap_sample(
+            num_samples, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+        ),
+        num_splits=NUM_SPLITS,
+    )
+    ids = [row["id"] for row in ds]
+
+    assert len(ids) == num_samples
+    # 4x oversampling a small table must produce at least one repeat.
+    assert len(set(ids)) < len(ids)
+    assert set(ids).issubset(set(range(NUM_ROWS)))
+
+
+def test_split_strategy_matches_direct_permutation_builder_use(lance_table):
+    """StreamingDataset(split_strategy=...), read single-process (no DataLoader
+    workers, so this one process owns every split and round-robins across all of
+    them), must yield exactly what calling the same strategy directly on a
+    PermutationBuilder would read, interleaved one row per split per cycle.
+
+    Reads each split via __getitems__ (explicit indices), not plain iteration:
+    Permutation's sequential __iter__ returns empty for a permutation with repeated
+    row_ids (confirmed directly -- len() is correct, __getitems__ returns the right
+    rows, but `for row in perm` yields nothing). That's a pre-existing limitation of
+    Permutation's iterator, unrelated to Bootstrap sampling's correctness: it doesn't
+    affect StreamingDataset or this benchmark, since both only ever read via
+    explicit-index batched fetches (take_offsets/__getitems__), never the plain
+    iterator -- flagged separately, not fixed here (out of scope for this change).
+    """
+    from lancedb.permutation import Permutation, permutation_builder
+
+    def strategy(builder):
+        return builder.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=1)
+
+    direct_perm_tbl = strategy(permutation_builder(lance_table)).execute()
+    expected_by_split = []
+    for split in range(NUM_SPLITS):
+        perm = Permutation.from_tables(lance_table, direct_perm_tbl, split=split)
+        rows = perm.__getitems__(list(range(len(perm))))
+        expected_by_split.append([row["id"] for row in rows])
+    samples_per_split = len(expected_by_split[0])
+    expected = [
+        expected_by_split[split][cycle]
+        for cycle in range(samples_per_split)
+        for split in range(NUM_SPLITS)
+    ]
+
+    ds = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+    actual = [row["id"] for row in ds]
+    assert actual == expected
+
+
+def test_split_strategy_is_reproducible_across_instances(lance_table):
+    def strategy(builder):
+        return builder.bootstrap_sample(3 * NUM_ROWS, fixed=NUM_SPLITS, seed=99)
+
+    ds1 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+    ds2 = StreamingDataset(lance_table, split_strategy=strategy, num_splits=NUM_SPLITS)
+
+    assert [row["id"] for row in ds1] == [row["id"] for row in ds2]
+
+
+def test_split_strategy_num_splits_mismatch_raises(lance_table):
+    """StreamingDataset(split_strategy=...) must reject a strategy whose split
+    count doesn't match num_splits. `_rank_splits` assumes split ids are exactly
+    0..num_splits-1 with no gaps; a mismatch would otherwise silently drop rows
+    (split ids >= num_splits) or leave some ranks reading an empty split (missing
+    ids < num_splits)."""
+    with pytest.raises(ValueError, match="num_splits"):
+        StreamingDataset(
+            lance_table,
+            split_strategy=lambda b: b.bootstrap_sample(
+                2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+            ),
+            num_splits=NUM_SPLITS + 1,  # the strategy above always builds NUM_SPLITS
+        )
+
+
+def test_split_strategy_unequal_split_sizes_raises(lance_table):
+    """StreamingDataset(split_strategy=...) must reject a strategy whose splits
+    all exist (ids 0..num_splits-1) but aren't the same size. The round-robin
+    iteration (see _iter_owned) assumes every owned split has the same row count
+    and stops as soon as any one is exhausted -- `ratios`/`counts` (unlike
+    `fixed`) can produce unequal split sizes, which would otherwise silently
+    drop rows from the larger splits."""
+    with pytest.raises(ValueError, match="unequal split sizes"):
+        StreamingDataset(
+            lance_table,
+            split_strategy=lambda b: b.bootstrap_sample(
+                100, ratios=[0.7, 0.3], seed=SHUFFLE_SEED
+            ),
+            num_splits=2,
+        )
+
+
+def test_split_strategy_permutation_mismatch_on_resume_raises(lance_table):
+    """load_state_dict must reject resuming against a different permutation than
+    the one that produced the checkpoint. num_splits/shuffle_seed alone can't catch
+    this for split_strategy, since that path ignores shuffle_seed entirely: a
+    bootstrap_sample reseeded differently resamples completely differently while
+    shuffle_seed keeps whatever value the caller passed either way, so the
+    pre-existing checks would pass despite the mismatch."""
+
+    def strategy(seed):
+        return lambda b: b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=seed)
+
+    ds1 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy(1),
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    it = iter(ds1)
+    for _ in range(NUM_SPLITS):  # one full cycle: every split advances uniformly
+        next(it)
+    checkpoint = ds1.state_dict()
+
+    ds2 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy(2),  # different seed -> a different permutation
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,  # unchanged -- irrelevant on this path
+    )
+    with pytest.raises(ValueError, match="permutation mismatch"):
+        ds2.load_state_dict(checkpoint)
+
+
+def test_split_strategy_resume_with_matching_permutation_succeeds(lance_table):
+    """The common case: resuming against the same split_strategy (same seed) must
+    work exactly like the non-split_strategy checkpoint/resume path, continuing
+    from exactly the same position with no gaps or repeats."""
+
+    def strategy(b):
+        return b.bootstrap_sample(2 * NUM_ROWS, fixed=NUM_SPLITS, seed=7)
+
+    ds1 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    it = iter(ds1)
+    for _ in range(NUM_SPLITS):
+        next(it)
+    checkpoint = ds1.state_dict()
+    remaining_original = [row["id"] for row in it]
+
+    ds2 = StreamingDataset(
+        lance_table,
+        split_strategy=strategy,
+        num_splits=NUM_SPLITS,
+        shuffle_seed=SHUFFLE_SEED,
+    )
+    ds2.load_state_dict(checkpoint)  # must not raise
+    remaining_resumed = [row["id"] for row in ds2]
+
+    assert remaining_original == remaining_resumed
+
+
+def test_default_checkpoint_omits_perm_fingerprint(lance_table):
+    """split_random/split_sequential checkpoints must not carry a
+    perm_fingerprint -- see test_default_checkpoint_resumes_after_compaction for
+    why checking one on this path would be wrong, not just unnecessary."""
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    assert "perm_fingerprint" not in ds.state_dict()
+
+
+def test_split_strategy_checkpoint_includes_perm_fingerprint(lance_table):
+    ds = StreamingDataset(
+        lance_table,
+        split_strategy=lambda b: b.bootstrap_sample(
+            2 * NUM_ROWS, fixed=NUM_SPLITS, seed=SHUFFLE_SEED
+        ),
+        num_splits=NUM_SPLITS,
+    )
+    assert "perm_fingerprint" in ds.state_dict()
+
+
+def test_default_checkpoint_resumes_after_compaction(lance_table):
+    """split_random/split_sequential checkpoints must resume normally even after
+    the table is compacted between save and load. Lance row ids are not stable
+    across compaction by default (confirmed empirically: a 2-fragment table's
+    row ids differ before and after `optimize()`), so fingerprinting raw row ids
+    on this path -- instead of relying on shuffle_seed/num_splits alone, as it
+    does -- would reject this resume even though the seeded permutation itself
+    is unchanged."""
+    lance_table.add(pa.table({"id": [NUM_ROWS]}))  # a second fragment to compact
+
+    ds = StreamingDataset(lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED)
+    it = iter(ds)
+    for _ in range(NUM_SPLITS):
+        next(it)
+    checkpoint = ds.state_dict()
+    assert "perm_fingerprint" not in checkpoint
+
+    before_compaction = set(
+        lance_table.to_lance()
+        .to_table(columns=[], with_row_id=True)["_rowid"]
+        .to_pylist()
+    )
+    lance_table.optimize()
+    after_compaction = set(
+        lance_table.to_lance()
+        .to_table(columns=[], with_row_id=True)["_rowid"]
+        .to_pylist()
+    )
+    assert before_compaction != after_compaction, (
+        "test setup didn't actually renumber row ids -- fix the fixture, not "
+        "the assertion below"
+    )
+
+    ds_resumed = StreamingDataset(
+        lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
+    )
+    ds_resumed.load_state_dict(checkpoint)  # must not raise

python/python/tests/test_embeddings.py

@@ -596,6 +596,51 @@ def test_openai_no_retry_on_401(mock_sleep):
     assert mock_sleep.call_count == 0
 
 
+def test_openai_bad_request_partial_batch_recovery():
+    # Regression test for issue #1677
+
+    # Fake exception types, avoids requiring the optional openai/httpx deps
+    class FakeBadRequestError(Exception):
+        pass
+
+    class FakeAuthenticationError(Exception):
+        pass
+
+    fake_openai_module = MagicMock()
+    fake_openai_module.BadRequestError = FakeBadRequestError
+    fake_openai_module.AuthenticationError = FakeAuthenticationError
+
+    registry = get_registry()
+    model = registry.get("openai").create(name="text-embedding-ada-002")
+
+    def fake_create(input, model=None, **kwargs):
+        # Full batch always rejected
+        if len(input) > 1:
+            raise FakeBadRequestError("bad request")
+        if input[0] == "bad text":
+            raise FakeBadRequestError("bad request")
+        embedding = MagicMock()
+        embedding.embedding = [0.1, 0.2, 0.3]
+        result = MagicMock()
+        result.data = [embedding]
+        return result
+
+    mock_client = MagicMock()
+    mock_client.embeddings.create.side_effect = fake_create
+
+    with (
+        patch(
+            "lancedb.embeddings.openai.attempt_import_or_raise",
+            return_value=fake_openai_module,
+        ),
+        patch.object(type(model), "_openai_client", new_callable=lambda: mock_client),
+    ):
+        embeddings = model.generate_embeddings(["good text", "bad text"])
+
+    assert embeddings[0] == [0.1, 0.2, 0.3]
+    assert embeddings[1] is None
+
+
 def test_ollama_embeddings_pickle():
     """OllamaEmbeddings must pickle even after the cached client is created."""
     registry = get_registry()

python/python/tests/test_first_class_function_slice1.py

@@ -398,8 +398,21 @@ class _FunctionDeclarationInner:
     def __init__(self):
         self.calls = []
 
-    async def add_function_columns(self, application_json, output_name):
-        self.calls.append((json.loads(application_json), output_name))
+    async def add_function_columns(
+        self,
+        application_json,
+        output_name,
+        auto_refresh_enabled,
+        auto_refresh_interval_seconds,
+    ):
+        self.calls.append(
+            (
+                json.loads(application_json),
+                output_name,
+                auto_refresh_enabled,
+                auto_refresh_interval_seconds,
+            )
+        )
         return "declared"
 
 
@@ -420,14 +433,18 @@ async def test_add_columns_routes_struct_as_one_and_multi_output_binding_atomica
     )
     assert result == "declared"
     assert inner.calls[-1][1] == "features"
+    assert inner.calls[-1][2:] == (None, None)
 
     bare = application._copy(update={"columns": {}}).rename(
         columns={"normalized_text": "search_text"}
     )
-    result = await table.add_columns(bare)
+    result = await table.add_columns(
+        bare, auto_refresh_enabled=False, auto_refresh_interval_seconds=300
+    )
     assert result == "declared"
     assert inner.calls[-1][1] is None
     assert inner.calls[-1][0]["columns"] == {"normalized_text": "search_text"}
+    assert inner.calls[-1][2:] == (False, 300)
 
 
 @pytest.mark.asyncio

python/python/tests/test_header_provider.py

@@ -114,6 +114,24 @@ class TestOAuthProvider:
             "x-lancedb-credential-type": "oidc",
         }
 
+    def test_zero_lifetime_token_is_refreshed(self):
+        """A token that expires immediately must not be cached indefinitely."""
+        call_count = 0
+
+        def fetcher():
+            nonlocal call_count
+            call_count += 1
+            return {
+                "access_token": f"token{call_count}",
+                "expires_in": 0,
+            }
+
+        provider = OAuthProvider(fetcher, refresh_buffer_seconds=0)
+
+        assert provider.get_headers() == {"Authorization": "Bearer token1"}
+        assert provider.get_headers() == {"Authorization": "Bearer token2"}
+        assert call_count == 2
+
     def test_missing_access_token(self):
         """Test error handling when access_token is missing."""
 

python/python/tests/test_hybrid_query.py

@@ -536,3 +536,15 @@ def test_normalize_scores():
             assert pc.equal(result, expected), (
                 f"Expected {expected} but got {result} for invert={invert}"
             )
+
+
+def test_hybrid_query_rejects_query_with_vector(sync_table: Table):
+    # Pins the full message: the space between "hybrid search." and "But" is
+    # load-bearing (see the _validate_query literal concatenation).
+    with pytest.raises(ValueError) as excinfo:
+        sync_table.search("dog", query_type="hybrid").vector([0.5, -0.5]).to_arrow()
+    assert str(excinfo.value) == (
+        "You can either provide a string query in search() method "
+        "or set `vector()` and `text()` explicitly for hybrid search. "
+        "But not both."
+    )

python/python/tests/test_index.py

@@ -430,6 +430,28 @@ async def test_create_ivfrq_index(some_table: AsyncTable):
     assert indices[0].name == "vector_idx"
 
 
+@pytest.mark.asyncio
+async def test_read_ivfrq_index_created_with_lance(db_async, some_table: AsyncTable):
+    pytest.importorskip("lance")
+    dataset = await some_table.to_lance()
+    dataset.create_index(
+        "vector",
+        index_type="IVF_RQ",
+        metric="cosine",
+        num_partitions=1,
+    )
+
+    # Open a fresh LanceDB handle so the index is discovered from disk.
+    table = await db_async.open_table("some_table")
+    indices = await table.list_indices()
+    assert len(indices) == 1
+    assert indices[0].index_type == "IvfRq"
+    assert indices[0].columns == ["vector"]
+
+    results = await table.query().nearest_to(list(range(DIM))).limit(8).to_arrow()
+    assert len(results) == 8
+
+
 @pytest.mark.asyncio
 async def test_create_hnswpq_index(some_table: AsyncTable):
     await some_table.create_index("vector", config=HnswPq(num_partitions=10))

python/python/tests/test_materialized_views.py

@@ -129,8 +129,21 @@ def test_sync_remote_create_uses_public_async_connection():
             where=None,
             limit=None,
             with_no_data=False,
+            auto_refresh_enabled=None,
+            auto_refresh_interval_seconds=None,
         ):
-            calls.append((name, source, select, where, limit, with_no_data))
+            calls.append(
+                (
+                    name,
+                    source,
+                    select,
+                    where,
+                    limit,
+                    with_no_data,
+                    auto_refresh_enabled,
+                    auto_refresh_interval_seconds,
+                )
+            )
             return StubAsyncMaterializedView()
 
         async def drop_materialized_view(self, name, *, namespace_path=None):
@@ -148,9 +161,11 @@ def test_sync_remote_create_uses_public_async_connection():
         where="age >= 18",
         limit=10,
         with_no_data=True,
+        auto_refresh_enabled=True,
+        auto_refresh_interval_seconds=0.5,
     )
     assert view.name == "adults"
-    assert calls == [("adults", "people", ["name"], "age >= 18", 10, True)]
+    assert calls == [("adults", "people", ["name"], "age >= 18", 10, True, True, 0.5)]
 
     db.drop_materialized_view("adults", namespace_path=["analytics"])
     assert calls[-1] == ("drop", "adults", ["analytics"])
@@ -550,3 +565,89 @@ def test_local_materialized_view_listing_pages_and_resume(tmp_path):
     assert sorted([first, *resumed]) == ["a_view", "z_view"]
     assert resumed.page_token() is None
     assert list(resumed) == []
+
+
+@pytest.mark.parametrize(
+    "options",
+    [
+        {},
+        {"auto_refresh_enabled": True},
+        {"auto_refresh_enabled": False, "auto_refresh_interval_seconds": 0.5},
+    ],
+)
+def test_remote_create_forwards_refresh_options_without_interval_default(options):
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = lancedb.connect("db://dev", api_key="fake", host_override=host)
+        job = db.create_materialized_view_async("adults", "people", **options)
+        assert job.id == "mv-create-123"
+    assert requests == [
+        (
+            "/v1/materialized_view/adults/create",
+            {
+                "query": 'SELECT * FROM "people"',
+                **options,
+            },
+        )
+    ]
+
+
+@pytest.mark.asyncio
+async def test_async_remote_create_keeps_interval_unspecified():
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = await lancedb.connect_async("db://dev", api_key="fake", host_override=host)
+        job = await db.create_materialized_view_async(
+            "adults",
+            "people",
+            auto_refresh_enabled=True,
+        )
+        assert job.id == "mv-create-123"
+    assert requests[0][1] == {
+        "query": 'SELECT * FROM "people"',
+        "auto_refresh_enabled": True,
+    }
+
+
+@pytest.mark.parametrize(
+    "options",
+    [
+        {"auto_refresh_enabled": True},
+        {"auto_refresh_interval_seconds": 300},
+    ],
+)
+def test_embedded_create_refuses_automatic_refresh_before_creating_view(
+    tmp_path, options
+):
+    db = make_db(tmp_path)
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        db.create_materialized_view("adults", "people", **options)
+    assert "adults" not in db.table_names()
+
+
+@pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
+def test_remote_create_rejects_invalid_interval_without_request(seconds):
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = lancedb.connect("db://dev", api_key="fake", host_override=host)
+        with pytest.raises(ValueError, match="positive, finite"):
+            db.create_materialized_view_async(
+                "adults", "people", auto_refresh_interval_seconds=seconds
+            )
+    assert requests == []
+
+
+def test_namespace_forwards_automatic_refresh_request(tmp_path):
+    db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
+
+
+@pytest.mark.asyncio
+async def test_async_namespace_forwards_automatic_refresh_request(tmp_path):
+    db = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
+    with pytest.raises(NotImplementedError, match="Cloud or Enterprise"):
+        await db.create_materialized_view("mv", "source", auto_refresh_enabled=True)
+
+
+def test_embedded_explicitly_disabled_refresh_is_supported(tmp_path):
+    db = make_db(tmp_path)
+    view = db.create_materialized_view("mv", "people", auto_refresh_enabled=False)
+    assert view.table.count_rows() == 3

python/python/tests/test_permutation.py

@@ -161,6 +161,76 @@ def test_split_random_with_seed(mem_db):
     assert data1["split_id"] == data2["split_id"]
 
 
+def test_bootstrap_sample_fixed(mem_db):
+    """Output row count is num_samples, not the input row count, and splits divide
+    it evenly."""
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
+    permutation_tbl = (
+        permutation_builder(tbl).bootstrap_sample(1000, fixed=4, seed=42).execute()
+    )
+
+    assert permutation_tbl.count_rows() == 1000
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    split_ids = data["split_id"]
+    assert set(split_ids) == {0, 1, 2, 3}
+    for split_id in range(4):
+        assert split_ids.count(split_id) == 250
+
+    # Every sampled row_id must still be one of the 20 real rows.
+    assert set(data["row_id"]).issubset(set(range(20)))
+
+
+def test_bootstrap_sample_draws_with_replacement(mem_db):
+    """Sampling more rows than exist must produce repeats."""
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(5)}))
+    permutation_tbl = (
+        permutation_builder(tbl).bootstrap_sample(500, fixed=1, seed=1).execute()
+    )
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    assert len(data["row_id"]) == 500
+    # Only 5 distinct rows exist, so with 500 draws every one of them must repeat.
+    for row_id in range(5):
+        assert data["row_id"].count(row_id) > 1
+
+
+def test_bootstrap_sample_with_seed_is_reproducible(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(50)}))
+
+    perm1 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
+    perm2 = permutation_builder(tbl).bootstrap_sample(200, fixed=2, seed=42).execute()
+
+    data1 = perm1.search(None).to_arrow().to_pydict()
+    data2 = perm2.search(None).to_arrow().to_pydict()
+    assert data1["row_id"] == data2["row_id"]
+    assert data1["split_id"] == data2["split_id"]
+
+
+def test_bootstrap_sample_respects_filter(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(20)}))
+    permutation_tbl = (
+        permutation_builder(tbl)
+        .filter("x >= 15")
+        .bootstrap_sample(100, fixed=1, seed=7)
+        .execute()
+    )
+
+    data = permutation_tbl.search(None).to_arrow().to_pydict()
+    assert len(data["row_id"]) == 100
+    assert set(data["row_id"]).issubset(set(range(15, 20)))
+
+
+def test_bootstrap_sample_requires_exactly_one_size_arg(mem_db):
+    tbl = mem_db.create_table("test_table", pa.table({"x": range(10)}))
+    with pytest.raises(ValueError):
+        permutation_builder(tbl).bootstrap_sample(10).execute()
+    with pytest.raises(ValueError):
+        permutation_builder(tbl).bootstrap_sample(
+            10, ratios=[0.5, 0.5], fixed=2
+        ).execute()
+
+
 def test_split_hash(mem_db):
     """Test hash-based splitting."""
     tbl = mem_db.create_table(

python/python/tests/test_query.py

@@ -833,6 +833,28 @@ def test_with_row_id(table: lancedb.table.Table):
     assert rs["_rowid"].to_pylist() == [0, 1]
 
 
+def test_filter_preserves_deeply_nested_struct_fields(tmp_db):
+    # Regression test for https://github.com/lancedb/lancedb/issues/2217.
+    data = [
+        {
+            "item": "test",
+            "sub_obj": {
+                "second_sub_obj": [
+                    {
+                        "third_sub_obj": {"int_val": 1},
+                        "str_val": "test",
+                    }
+                ]
+            },
+        }
+    ]
+    table = tmp_db.create_table("deeply_nested_struct", data=data)
+
+    result = table.search().where("item = 'test'").limit(1).to_arrow()
+
+    assert result.to_pylist() == data
+
+
 def test_blob_v2_query_omits_auto_row_id(tmp_db):
     table = _create_blob_v2_query_table(tmp_db, "test_blob_v2_omits_auto_rowid")
 

python/python/tests/test_rerankers.py

@@ -785,6 +785,114 @@ def test_empty_hybrid_result_reranker():
     assert "_rowid" in result.column_names
 
 
+@pytest.mark.parametrize("return_score", ["relevance", "all"])
+@pytest.mark.parametrize("first_row_id", [1, 2**63])
+@pytest.mark.parametrize("score_type", [pa.float32(), pa.float64()])
+def test_model_reranker_preserves_arrow_schema(
+    mocker, return_score, first_row_id, score_type
+):
+    common_schema = pa.schema(
+        [
+            pa.field("_rowid", pa.uint64()),
+            pa.field("text", pa.string()),
+            pa.field("vector", pa.list_(pa.float32(), 2)),
+        ],
+        metadata={b"source": b"reranker-test"},
+    )
+    vector_results = pa.Table.from_pylist(
+        [
+            {
+                "_rowid": first_row_id,
+                "text": "vector",
+                "vector": [1, 0],
+                "_distance": 0.1,
+            },
+            {
+                "_rowid": first_row_id + 1,
+                "text": "both",
+                "vector": [0, 1],
+                "_distance": 0.2,
+            },
+        ],
+        schema=common_schema.append(pa.field("_distance", score_type)),
+    )
+    fts_results = pa.Table.from_pylist(
+        [
+            {
+                "_rowid": first_row_id + 1,
+                "text": "both",
+                "vector": [0, 1],
+                "_score": 0.9,
+            },
+            {
+                "_rowid": first_row_id + 2,
+                "text": "fts",
+                "vector": [1, 1],
+                "_score": 0.8,
+            },
+        ],
+        schema=common_schema.append(pa.field("_score", score_type)),
+    )
+    reranker = CohereReranker(return_score=return_score)
+    reranker._client = mocker.Mock()
+    reranker._client.rerank.return_value.results = [
+        mocker.Mock(index=index, relevance_score=score)
+        for index, score in [(2, 0.9), (1, 0.8), (0, 0.7)]
+    ]
+
+    result = reranker.rerank_hybrid("query", vector_results, fts_results)
+
+    assert result.select(common_schema.names).schema.equals(
+        common_schema, check_metadata=True
+    )
+    assert result["_rowid"].to_pylist() == [
+        first_row_id + 2,
+        first_row_id + 1,
+        first_row_id,
+    ]
+    assert result["text"].to_pylist() == ["fts", "both", "vector"]
+    if return_score == "all":
+        assert (
+            result["_distance"]
+            .combine_chunks()
+            .equals(pa.array([None, 0.2, 0.1], type=score_type))
+        )
+        assert (
+            result["_score"]
+            .combine_chunks()
+            .equals(pa.array([0.8, 0.9, None], type=score_type))
+        )
+
+
+@pytest.mark.parametrize("return_score", ["relevance", "all"])
+def test_model_reranker_empty_results_preserve_schema(mocker, return_score):
+    common_schema = pa.schema(
+        [
+            ("_rowid", pa.uint64()),
+            ("text", pa.string()),
+            ("vector", pa.list_(pa.float32(), 2)),
+        ],
+        metadata={b"source": b"reranker-test"},
+    )
+    vector_results = pa.Table.from_pylist(
+        [], schema=common_schema.append(pa.field("_distance", pa.float32()))
+    )
+    fts_results = pa.Table.from_pylist(
+        [], schema=common_schema.append(pa.field("_score", pa.float32()))
+    )
+    reranker = CohereReranker(return_score=return_score)
+    reranker._client = mocker.Mock()
+
+    result = reranker.rerank_hybrid("query", vector_results, fts_results)
+
+    assert len(result) == 0
+    assert result.select(common_schema.names).schema.equals(
+        common_schema, check_metadata=True
+    )
+    assert result.schema.field("_relevance_score").type == pa.float32()
+    reranker._client.rerank.assert_not_called()
+
+
 def test_cross_encoder_reranker_return_all(tmp_path):
     pytest.importorskip("sentence_transformers")
     reranker = CrossEncoderReranker(return_score="all")

python/python/tests/test_table.py

@@ -15,7 +15,7 @@ from datetime import date, datetime, timedelta
 from decimal import Decimal, localcontext
 from time import sleep
 from typing import Any, List
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 import lancedb
 from lancedb import table as table_module
@@ -280,6 +280,30 @@ def test_create_table_infers_large_int_vectors(mem_db: DBConnection):
     assert vector_column.to_pylist() == [[0.0, 300.0]]
 
 
+def test_create_table_all_null_vector_column(mem_db: DBConnection):
+    # A vector column can legitimately arrive with every value null, e.g. when
+    # the embeddings have not been computed yet. The dimension is unknowable in
+    # that case, so schema inference must not crash and should leave the column
+    # as a variable-length list rather than a fixed-size one.
+    data = pa.table(
+        {
+            "id": pa.array([1, 2]),
+            "vector": pa.array([None, None], type=pa.list_(pa.float32())),
+        }
+    )
+
+    table = mem_db.create_table("all_null_vector", data=data)
+
+    vector_field = table.schema.field("vector")
+    assert pa.types.is_list(vector_field.type)
+    assert vector_field.type.value_type == pa.float32()
+
+    result = table.to_arrow()
+    assert result.num_rows == 2
+    assert result.column("vector").to_pylist() == [None, None]
+    assert table.count_rows() == 2
+
+
 @pytest.mark.asyncio
 async def test_create_table_async_infers_large_int_vectors(
     mem_db_async: AsyncConnection,
@@ -518,6 +542,30 @@ def test_add(mem_db: DBConnection):
     _add(table, schema)
 
 
+def test_add_from_iterator_that_queries_table(mem_db: DBConnection):
+    source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
+    target = mem_db.create_table("target", schema=source.schema)
+
+    def batches():
+        for _ in range(5):
+            yield source.search().limit(10).to_arrow()
+
+    target.add(batches())
+
+    assert target.count_rows() == 50
+
+
+def test_create_table_from_iterator_that_queries_table(mem_db: DBConnection):
+    source = mem_db.create_table("source", data=pa.table({"id": range(16)}))
+
+    def batches():
+        yield source.search().limit(10).to_arrow()
+
+    target = mem_db.create_table("target", data=batches())
+
+    assert target.count_rows() == 10
+
+
 def test_add_releases_arrow_buffers_without_gc(mem_db: DBConnection):
     """Regression test for https://github.com/lancedb/lancedb/issues/2512."""
     schema = pa.schema([pa.field("x", pa.int64())])
@@ -1725,6 +1773,51 @@ def test_create_index_method(mock_create_index, mem_db: DBConnection):
     )
 
 
+@pytest.mark.parametrize("api", ["legacy", "unified"])
+@pytest.mark.parametrize("cuda_failure", ["unavailable", "init_error"])
+def test_create_index_cuda_failure_suggests_cpu(
+    api, cuda_failure, mem_db: DBConnection
+):
+    table = mem_db.create_table(
+        f"test_cuda_unavailable_{api}",
+        data=[{"vector": [3.1, 4.1]}, {"vector": [5.9, 26.5]}],
+    )
+    torch = MagicMock()
+    torch.cuda.is_available.return_value = cuda_failure == "init_error"
+    if cuda_failure == "init_error":
+        torch.cuda.init.side_effect = RuntimeError("CUDA driver initialization failed")
+
+    with (
+        patch.dict(sys.modules, {"torch": torch}),
+        patch.object(table, "to_lance") as to_lance,
+    ):
+        if api == "legacy":
+            with (
+                pytest.warns(DeprecationWarning, match="create_index"),
+                pytest.raises(RuntimeError, match="accelerator=None.*CPU"),
+            ):
+                table.create_index(
+                    metric="dot",
+                    num_partitions=1,
+                    num_sub_vectors=1,
+                    vector_column_name="vector",
+                    accelerator="cuda",
+                )
+        else:
+            with pytest.raises(RuntimeError, match="accelerator=None.*CPU"):
+                table.create_index(
+                    "vector",
+                    config=IvfPq(
+                        distance_type="dot",
+                        num_partitions=1,
+                        num_sub_vectors=1,
+                        accelerator="cuda",
+                    ),
+                )
+
+    to_lance.return_value.create_index.assert_not_called()
+
+
 @patch("lancedb.table.AsyncTable.create_index")
 def test_create_index_name_and_train_parameters(
     mock_create_index, mem_db: DBConnection
@@ -3673,6 +3766,42 @@ def test_multiple_vector_columns(mem_db: DBConnection):
     assert result1["text"][0] != result2["text"][0]
 
 
+@pytest.mark.parametrize(
+    "query",
+    [
+        (0.0, 0.0),
+        pa.array([0.0, 0.0]),
+        pa.chunked_array([[0.0], [0.0]]),
+    ],
+)
+@pytest.mark.parametrize("query_type", ["auto", "vector"])
+def test_sync_search_accepts_supported_vector_containers(
+    mem_db: DBConnection, query, query_type
+):
+    schema = pa.schema(
+        [
+            pa.field("id", pa.int64()),
+            pa.field("small", pa.list_(pa.float32(), 2)),
+            pa.field("large", pa.list_(pa.float32(), 3)),
+        ]
+    )
+    table = mem_db.create_table(
+        "supported_vector_containers",
+        data=pa.table(
+            {
+                "id": [1, 2],
+                "small": [[0.0, 0.0], [1.0, 1.0]],
+                "large": [[1.0, 1.0, 1.0], [0.0, 0.0, 0.0]],
+            },
+            schema=schema,
+        ),
+    )
+
+    result = table.search(query, query_type=query_type).limit(1).to_arrow()
+
+    assert result["id"].to_pylist() == [1]
+
+
 def test_create_scalar_index(mem_db: DBConnection):
     vec_array = pa.array(
         [[1, 1], [2, 2], [3, 3], [4, 4], [5, 5]], pa.list_(pa.float32(), 2)
@@ -4997,3 +5126,53 @@ async def test_refresh_column_async_job_async_table(tmp_path):
     assert result.published_version == 4
     assert await job.status() == "finished"
     assert (await table.to_arrow())["tripled"].to_pylist() == [9]
+
+
+@pytest.mark.parametrize("interval", [None, 300])
+def test_computed_auto_refresh_options(tmp_path, interval):
+    db = lancedb.connect(tmp_path)
+    table = db.create_table("refresh_options", [{"x": 1}])
+    before = table.version
+    table.add_columns(
+        computed={"doubled": "x * 2", "tripled": "x * 3"},
+        auto_refresh_enabled=False,
+        auto_refresh_interval_seconds=interval,
+    )
+    assert table.version == before + 1
+    for column in ("doubled", "tripled"):
+        metadata = table.schema.field(column).metadata
+        assert metadata[b"computed_column.auto_refresh_enabled"] == b"false"
+        if interval is None:
+            assert b"computed_column.auto_refresh_interval_seconds" not in metadata
+        else:
+            assert metadata[b"computed_column.auto_refresh_interval_seconds"] == b"300"
+    table.refresh_column("doubled")
+    assert table.to_arrow()["doubled"].to_pylist() == [2]
+
+
+@pytest.mark.parametrize("value", [0, -1, True, 1.5, "300", 2**64])
+def test_computed_auto_refresh_rejects_invalid_interval(tmp_path, value):
+    table = lancedb.connect(tmp_path).create_table("invalid_options", [{"x": 1}])
+    with pytest.raises(ValueError, match="auto_refresh_interval_seconds"):
+        table.add_columns(
+            computed={"doubled": "x * 2"}, auto_refresh_interval_seconds=value
+        )
+    assert table.schema.names == ["x"]
+
+
+def test_auto_refresh_options_require_computed_columns(tmp_path):
+    table = lancedb.connect(tmp_path).create_table("invalid_kind", [{"x": 1}])
+    with pytest.raises(ValueError, match="require computed"):
+        table.add_columns({"doubled": "x * 2"}, auto_refresh_enabled=True)
+
+
+@pytest.mark.asyncio
+async def test_async_computed_auto_refresh_uses_server_interval(tmp_path):
+    db = await lancedb.connect_async(tmp_path)
+    table = await db.create_table("server_interval", [{"x": 1}])
+    before = await table.version()
+    await table.add_columns(computed={"doubled": "x * 2"})
+    assert await table.version() == before + 1
+    metadata = (await table.schema()).field("doubled").metadata
+    assert metadata[b"computed_column.auto_refresh_enabled"] == b"true"
+    assert b"computed_column.auto_refresh_interval_seconds" not in metadata

python/python/tests/test_util.py

@@ -30,6 +30,7 @@ import pytest
 import lancedb
 from lancedb.util import (
     flatten_columns,
+    get_uri_location,
     get_uri_scheme,
     inf_vector_column_query,
     join_uri,
@@ -86,6 +87,22 @@ def test_normalize_uri():
         assert parsed_scheme == expected_scheme
 
 
+def test_get_uri_scheme_and_location_pathlib():
+    # Test pathlib.Path instances
+    abs_path = pathlib.Path("/tmp/data")
+    rel_path = pathlib.Path("relative/data")
+
+    assert get_uri_scheme(abs_path) == "file"
+    assert get_uri_scheme(rel_path) == "file"
+
+    assert get_uri_location(abs_path) == str(abs_path)
+    assert get_uri_location(rel_path) == str(rel_path)
+
+    # String URIs still behave identically
+    assert get_uri_scheme("s3://bucket/data") == "s3"
+    assert get_uri_location("s3://bucket/data") == "bucket/data"
+
+
 def test_join_uri_remote():
     schemes = ["s3", "az", "gs"]
     for scheme in schemes:

python/src/connection.rs

@@ -412,7 +412,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn create_materialized_view(
         self_: PyRef<'_, Self>,
         name: String,
@@ -421,10 +422,18 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<f64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(enabled) = auto_refresh_enabled {
+                builder = builder.auto_refresh_enabled(enabled);
+            }
+            if let Some(seconds) = auto_refresh_interval_seconds {
+                builder = builder.auto_refresh_interval_seconds(seconds);
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }
@@ -440,7 +449,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn create_materialized_view_async(
         self_: PyRef<'_, Self>,
         name: String,
@@ -449,10 +459,18 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<f64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(enabled) = auto_refresh_enabled {
+                builder = builder.auto_refresh_enabled(enabled);
+            }
+            if let Some(seconds) = auto_refresh_interval_seconds {
+                builder = builder.auto_refresh_interval_seconds(seconds);
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }

python/src/permutation.rs

@@ -178,6 +178,50 @@ impl PyAsyncPermutationBuilder {
         })
     }
 
+    #[pyo3(signature = (num_samples, *, ratios=None, counts=None, fixed=None, seed=None, split_names=None))]
+    pub fn bootstrap_sample(
+        slf: PyRefMut<'_, Self>,
+        num_samples: u64,
+        ratios: Option<Vec<f64>>,
+        counts: Option<Vec<u64>>,
+        fixed: Option<u64>,
+        seed: Option<u64>,
+        split_names: Option<Vec<String>>,
+    ) -> PyResult<Self> {
+        // Check that exactly one split type is provided
+        let split_args_count = [ratios.is_some(), counts.is_some(), fixed.is_some()]
+            .iter()
+            .filter(|&&x| x)
+            .count();
+
+        if split_args_count != 1 {
+            return Err(pyo3::exceptions::PyValueError::new_err(
+                "Exactly one of 'ratios', 'counts', or 'fixed' must be provided",
+            ));
+        }
+
+        let sizes = if let Some(ratios) = ratios {
+            SplitSizes::Percentages(ratios)
+        } else if let Some(counts) = counts {
+            SplitSizes::Counts(counts)
+        } else if let Some(fixed) = fixed {
+            SplitSizes::Fixed(fixed)
+        } else {
+            unreachable!("One of the split arguments must be provided");
+        };
+
+        slf.modify(|builder| {
+            builder.with_split_strategy(
+                SplitStrategy::Bootstrap {
+                    num_samples,
+                    sizes,
+                    seed,
+                },
+                split_names,
+            )
+        })
+    }
+
     pub fn split_calculated(
         slf: PyRefMut<'_, Self>,
         calculation: String,

python/src/table.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
+use lancedb::table::computed_columns::ComputedColumnOptions;
 use std::{
     collections::HashMap,
     sync::{
@@ -1937,35 +1938,59 @@ impl Table {
         })
     }
 
+    #[pyo3(signature = (columns, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn add_computed_columns(
         self_: PyRef<'_, Self>,
         columns: Vec<(String, String)>,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<u64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
+        let options = ComputedColumnOptions {
+            auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
+            auto_refresh_interval_seconds,
+        };
+        options.validate().infer_error()?;
         future_into_py(self_.py(), async move {
             let mut builder = inner.add_columns();
             for (name, expression) in columns {
                 builder = builder.computed(name, expression);
             }
-            let result = builder.execute().await.infer_error()?;
+            let result = builder
+                .computed_options(options)
+                .execute()
+                .await
+                .infer_error()?;
             Ok(AddColumnsResult::from(result))
         })
     }
 
+    #[pyo3(signature = (application_json, output_name, auto_refresh_enabled=None, auto_refresh_interval_seconds=None))]
     pub fn add_function_columns(
         self_: PyRef<'_, Self>,
         application_json: String,
         output_name: Option<String>,
+        auto_refresh_enabled: Option<bool>,
+        auto_refresh_interval_seconds: Option<u64>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let application =
             lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
         let inner = self_.inner_ref()?.clone();
+        let options = ComputedColumnOptions {
+            auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
+            auto_refresh_interval_seconds,
+        };
+        options.validate().infer_error()?;
         future_into_py(self_.py(), async move {
             let builder = match output_name {
                 Some(name) => inner.add_columns().function_as(name, application),
                 None => inner.add_columns().function(application),
             };
-            let result = builder.execute().await.infer_error()?;
+            let result = builder
+                .computed_options(options)
+                .execute()
+                .await
+                .infer_error()?;
             Ok(AddColumnsResult::from(result))
         })
     }

python/uv.lock

@@ -443,11 +443,11 @@ wheels = [
 
 [[package]]
 name = "certifi"
-version = "2026.6.17"
+version = "2026.10.19"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/c9/c7/424b75da314c1045981bd9777432fad05a9e0c69daa4ed7e308bbaffe405/certifi-2026.6.17.tar.gz", hash = "sha256:024c88eeec92ca068db80f02b8b07c9cef7b9fe261d1d535abfd5abd6f6af432", size = 134594, upload-time = "2026-06-17T10:31:07.894Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ef/2f/c5464532e965badff2f4c4c1a3a83f5697f0d7c407ed0cda44aaa99bb451/certifi-2026.6.17-py3-none-any.whl", hash = "sha256:2227dcbaafe0d2f59279d1762ddddc37783ed4354594f194ffc31d20f41fc3db", size = 133289, upload-time = "2026-06-17T10:31:06.348Z" },
+    { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" },
 ]
 
 [[package]]

python/vendor/lance-arrow/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "lance-arrow"
+version = "15.0.0-beta.1"
+edition = "2024"
+authors = ["Lance Devs <dev@lance.org>"]
+license = "Apache-2.0"
+repository = "https://github.com/lance-format/lance"
+readme = "README.md"
+description = "Arrow Extension for Lance"
+keywords = ["data-format", "data-science", "machine-learning", "apache-arrow", "data-analytics"]
+categories = ["database-implementations"]
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+arrow-array.workspace = true
+arrow-buffer.workspace = true
+arrow-data.workspace = true
+arrow-ipc = { workspace = true, features = ["zstd"] }
+arrow-ord.workspace = true
+arrow-schema.workspace = true
+arrow-select.workspace = true
+bytemuck = { version = "1", default-features = false, features = ["extern_crate_alloc"] }
+bytes.workspace = true
+futures.workspace = true
+half = { workspace = true, features = ["std", "bytemuck"] }
+lance-jsonb.workspace = true
+num-traits.workspace = true
+rand = { workspace = true, features = ["small_rng"] }
+
+[target.'cfg(target_arch = "wasm32")'.dependencies]
+getrandom = { version = "0.2", features = ["js"] }

python/vendor/lance-arrow/README.lancedb.md

@@ -0,0 +1,7 @@
+This directory contains `rust/lance-arrow` from Lance tag `v15.0.0-beta.1`
+(commit `46a7cfe1fba5efa1f90deea4c7b14d414b991409`), licensed under Apache-2.0.
+
+LanceDB carries a local fix in `src/lib.rs` for sliced child validity bitmaps
+until a Lance release includes the fix. The patch is selected in the root
+`Cargo.toml`; the rest of Lance stays on the pinned release. An explicit unsafe
+block in `src/bfloat16.rs` lets this copy pass LanceDB's Rust 2024 warning gate.

python/vendor/lance-arrow/README.md

@@ -0,0 +1,6 @@
+# lance-arrow
+
+`lance-arrow` is an internal sub-crate, containing [Apache-Arrow](https://github.com/apache/arrow-rs)
+extensions used by [Lance](https://github.com/lance-format/lance).
+
+**Important Note**: This crate is **not intended for external usage**.

python/vendor/lance-arrow/src/bfloat16.rs

@@ -0,0 +1,406 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! bfloat16 support for Apache Arrow.
+
+use std::fmt::Formatter;
+use std::slice;
+
+use arrow_array::{Array, FixedSizeBinaryArray, builder::BooleanBufferBuilder};
+use arrow_buffer::{Buffer, MutableBuffer};
+use arrow_data::ArrayData;
+use arrow_schema::{ArrowError, DataType, Field as ArrowField};
+use half::bf16;
+
+use crate::{ARROW_EXT_NAME_KEY, FloatArray};
+
+/// The name of the bfloat16 extension in Arrow metadata
+pub const BFLOAT16_EXT_NAME: &str = "lance.bfloat16";
+
+/// Check whether the given field is a bfloat16 field
+///
+/// A field is a bfloat16 field if it has a data type of `FixedSizeBinary(2)` and the metadata
+/// contains the bfloat16 extension name.
+pub fn is_bfloat16_field(field: &ArrowField) -> bool {
+    field.data_type() == &DataType::FixedSizeBinary(2)
+        && field
+            .metadata()
+            .get(ARROW_EXT_NAME_KEY)
+            .map(|name| name == BFLOAT16_EXT_NAME)
+            .unwrap_or_default()
+}
+
+/// The bfloat16 data type
+///
+/// This implements the [`ArrowFloatType`](crate::floats::ArrowFloatType) trait for bfloat16 values.
+#[derive(Debug)]
+pub struct BFloat16Type {}
+
+/// An array of bfloat16 values
+///
+/// Note that bfloat16 is not the same thing as fp16 which is supported natively by arrow-rs.
+#[derive(Clone)]
+pub struct BFloat16Array {
+    inner: FixedSizeBinaryArray,
+}
+
+impl std::fmt::Debug for BFloat16Array {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "BFloat16Array\n[\n")?;
+        from_arrow::print_long_array(&self.inner, f, |array, i, f| {
+            if array.is_null(i) {
+                write!(f, "null")
+            } else {
+                let binary_values = array.value(i);
+                let value =
+                    bf16::from_bits(u16::from_le_bytes([binary_values[0], binary_values[1]]));
+                write!(f, "{:?}", value)
+            }
+        })?;
+        write!(f, "]")
+    }
+}
+
+impl BFloat16Array {
+    pub fn from_iter_values(iter: impl IntoIterator<Item = bf16>) -> Self {
+        let values: Vec<bf16> = iter.into_iter().collect();
+        values.into()
+    }
+
+    pub fn len(&self) -> usize {
+        self.inner.len()
+    }
+
+    pub fn is_empty(&self) -> bool {
+        self.inner.is_empty()
+    }
+
+    pub fn is_null(&self, i: usize) -> bool {
+        self.inner.is_null(i)
+    }
+
+    pub fn null_count(&self) -> usize {
+        self.inner.null_count()
+    }
+
+    pub fn iter(&self) -> BFloat16Iter<'_> {
+        BFloat16Iter {
+            array: self,
+            index: 0,
+        }
+    }
+
+    pub fn value(&self, i: usize) -> bf16 {
+        assert!(
+            i < self.len(),
+            "Trying to access an element at index {} from a BFloat16Array of length {}",
+            i,
+            self.len()
+        );
+        // Safety:
+        // `i < self.len()
+        unsafe { self.value_unchecked(i) }
+    }
+
+    /// # Safety
+    /// Caller must ensure that `i < self.len()`
+    pub unsafe fn value_unchecked(&self, i: usize) -> bf16 {
+        // SAFETY: the caller guarantees that `i` is within this array's bounds.
+        let binary_value = unsafe { self.inner.value_unchecked(i) };
+        bf16::from_bits(u16::from_le_bytes([binary_value[0], binary_value[1]]))
+    }
+
+    pub fn into_inner(self) -> FixedSizeBinaryArray {
+        self.inner
+    }
+}
+
+impl FromIterator<Option<bf16>> for BFloat16Array {
+    fn from_iter<I: IntoIterator<Item = Option<bf16>>>(iter: I) -> Self {
+        let mut buffer = MutableBuffer::new(10);
+        // No null buffer builder :(
+        let mut nulls = BooleanBufferBuilder::new(10);
+        let mut len = 0;
+
+        for maybe_value in iter {
+            if let Some(value) = maybe_value {
+                let bytes = value.to_le_bytes();
+                buffer.extend(bytes);
+            } else {
+                buffer.extend([0u8, 0u8]);
+            }
+            nulls.append(maybe_value.is_some());
+            len += 1;
+        }
+
+        let null_buffer = nulls.finish();
+        let num_valid = null_buffer.count_set_bits();
+        let null_buffer = if num_valid == len {
+            None
+        } else {
+            Some(null_buffer.into_inner())
+        };
+
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
+            .len(len)
+            .add_buffer(buffer.into())
+            .null_bit_buffer(null_buffer);
+        // SAFETY: the value buffer contains exactly `2 * len` bytes (two bytes
+        // pushed per iteration of the loop above, including the zero-fill for
+        // null slots), which matches the `FixedSizeBinary(2)` storage layout.
+        // The null bit buffer, when present, has `len` bits appended above, so
+        // its length covers the array's logical range.
+        let array_data = unsafe { array_data.build_unchecked() };
+        Self {
+            inner: FixedSizeBinaryArray::from(array_data),
+        }
+    }
+}
+
+impl FromIterator<bf16> for BFloat16Array {
+    fn from_iter<I: IntoIterator<Item = bf16>>(iter: I) -> Self {
+        Self::from_iter_values(iter)
+    }
+}
+
+impl From<Vec<bf16>> for BFloat16Array {
+    fn from(data: Vec<bf16>) -> Self {
+        let len = data.len();
+        // Zero-copy: `bf16` is `#[repr(transparent)]` over `u16` and derives
+        // `bytemuck::Pod`, so `cast_vec` reinterprets the allocation in place —
+        // no per-element copy or heap alloc. The crate-root `compile_error!`
+        // pins `target_endian = "little"`, so the resulting bytes match the
+        // `FixedSizeBinary(2)` on-disk order Lance writes elsewhere.
+        let raw: Vec<u16> = bytemuck::cast_vec(data);
+        let array_data = ArrayData::builder(DataType::FixedSizeBinary(2))
+            .len(len)
+            .add_buffer(Buffer::from_vec(raw));
+        // SAFETY: the value buffer contains exactly `2 * len` bytes — one
+        // `u16` per element after the layout-compatible cast — matching the
+        // `FixedSizeBinary(2)` storage layout. No null buffer is attached, so
+        // every element is logically valid.
+        let array_data = unsafe { array_data.build_unchecked() };
+        Self {
+            inner: FixedSizeBinaryArray::from(array_data),
+        }
+    }
+}
+
+impl TryFrom<FixedSizeBinaryArray> for BFloat16Array {
+    type Error = ArrowError;
+
+    fn try_from(value: FixedSizeBinaryArray) -> Result<Self, Self::Error> {
+        if value.value_length() == 2 {
+            Ok(Self { inner: value })
+        } else {
+            Err(ArrowError::InvalidArgumentError(
+                "FixedSizeBinaryArray must have a value length of 2".to_string(),
+            ))
+        }
+    }
+}
+
+impl PartialEq<Self> for BFloat16Array {
+    fn eq(&self, other: &Self) -> bool {
+        self.inner.eq(&other.inner)
+    }
+}
+
+pub struct BFloat16Iter<'a> {
+    array: &'a BFloat16Array,
+    index: usize,
+}
+
+impl<'a> Iterator for BFloat16Iter<'a> {
+    type Item = Option<bf16>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.index >= self.array.len() {
+            return None;
+        }
+        let i = self.index;
+        self.index += 1;
+        if self.array.is_null(i) {
+            Some(None)
+        } else {
+            Some(Some(self.array.value(i)))
+        }
+    }
+}
+
+/// Methods that are lifted from arrow-rs temporarily until they are made public.
+mod from_arrow {
+    use arrow_array::Array;
+
+    /// Helper function for printing potentially long arrays.
+    pub(super) fn print_long_array<A, F>(
+        array: &A,
+        f: &mut std::fmt::Formatter,
+        print_item: F,
+    ) -> std::fmt::Result
+    where
+        A: Array,
+        F: Fn(&A, usize, &mut std::fmt::Formatter) -> std::fmt::Result,
+    {
+        let head = std::cmp::min(10, array.len());
+
+        for i in 0..head {
+            if array.is_null(i) {
+                writeln!(f, "  null,")?;
+            } else {
+                write!(f, "  ")?;
+                print_item(array, i, f)?;
+                writeln!(f, ",")?;
+            }
+        }
+        if array.len() > 10 {
+            if array.len() > 20 {
+                writeln!(f, "  ...{} elements...,", array.len() - 20)?;
+            }
+
+            let tail = std::cmp::max(head, array.len() - 10);
+
+            for i in tail..array.len() {
+                if array.is_null(i) {
+                    writeln!(f, "  null,")?;
+                } else {
+                    write!(f, "  ")?;
+                    print_item(array, i, f)?;
+                    writeln!(f, ",")?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+impl FloatArray<BFloat16Type> for FixedSizeBinaryArray {
+    type FloatType = BFloat16Type;
+
+    /// Returns the underlying `bf16` values as a borrowed slice.
+    ///
+    /// # Preconditions
+    ///
+    /// - `value_length()` must be 2 (the `FixedSizeBinary(2)` storage shape
+    ///   used by [`BFloat16Array`]). Asserted at entry.
+    /// - The value buffer must be at least 2-byte aligned. Lance's in-tree
+    ///   constructors always satisfy this: value buffers are built either via
+    ///   `MutableBuffer` (aligned to arrow-buffer's `ALIGNMENT` constant, ≥32
+    ///   bytes) or via `Buffer::from_vec::<u16>` (aligned to `align_of::<u16>()`
+    ///   == 2); both meet `bf16`'s 2-byte requirement. Externally-built
+    ///   `FixedSizeBinaryArray`s arriving via FFI, IPC, or
+    ///   `Buffer::from_custom_allocation` are not required by arrow-rs to be
+    ///   aligned beyond a single byte; passing one to this method violates the
+    ///   precondition. A `debug_assert` below catches such inputs in debug and
+    ///   test builds.
+    ///
+    /// # Endianness
+    ///
+    /// `lance-arrow` is gated on `target_endian = "little"` at the crate root,
+    /// so this method always returns values in the same byte order Lance writes
+    /// (see [`BFloat16Array::value`] and the [`FromIterator`] impls).
+    fn as_slice(&self) -> &[bf16] {
+        assert_eq!(
+            self.value_length(),
+            2,
+            "BFloat16 arrays must use FixedSizeBinary(2) storage"
+        );
+        debug_assert_eq!(
+            (self.value_data().as_ptr() as usize) % std::mem::align_of::<bf16>(),
+            0,
+            "BFloat16 value buffer must be at least 2-byte aligned"
+        );
+        // SAFETY:
+        // - The assert above pins `value_size == 2`, so `value_data().len() / 2`
+        //   equals the array's logical element count.
+        //   `FixedSizeBinaryArray::From<ArrayData>` constructs its value buffer
+        //   as `buffers[0].slice_with_length(offset * 2, len * 2)` (arrow-array
+        //   `fixed_size_binary_array.rs`), so `value_data()` already returns
+        //   the offset-adjusted slice. Do not replace `value_data()` with an
+        //   accessor that returns the un-sliced backing buffer.
+        // - `bf16` is `#[repr(transparent)]` over `u16` (size 2, alignment 2);
+        //   every `u16` bit pattern is a valid `bf16`, so any byte content
+        //   yields a defined value — never UB.
+        // - Alignment is the caller's responsibility per the precondition
+        //   documented above. The `debug_assert_eq!` immediately preceding this
+        //   block catches violations in debug and test builds only — release
+        //   builds rely on callers honoring the precondition. arrow-rs
+        //   declares `FixedSizeBinary(n)`'s
+        //   `BufferSpec::FixedWidth { alignment: align_of::<u8>() == 1 }`
+        //   (arrow-data `data.rs`), so arrow-rs alone does not guarantee
+        //   2-byte alignment. Lance's in-tree construction paths build value
+        //   buffers via `MutableBuffer` (arrow-buffer `ALIGNMENT` constant,
+        //   ≥32 bytes) or `Buffer::from_vec::<u16>` (2-byte aligned), both of
+        //   which satisfy `bf16`'s 2-byte requirement.
+        // - The returned slice borrows from `self`; the underlying ref-counted,
+        //   immutable Arrow buffer cannot be mutated or freed for the slice's
+        //   lifetime.
+        unsafe {
+            slice::from_raw_parts(
+                self.value_data().as_ptr() as *const bf16,
+                self.value_data().len() / 2,
+            )
+        }
+    }
+
+    fn from_values(values: Vec<bf16>) -> Self {
+        BFloat16Array::from(values).into_inner()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_basics() {
+        let values: Vec<f32> = vec![1.0, 2.0, 3.0];
+        let values: Vec<bf16> = values.iter().map(|v| bf16::from_f32(*v)).collect();
+
+        let array = BFloat16Array::from_iter_values(values.clone());
+        let array2 = BFloat16Array::from(values.clone());
+        assert_eq!(array, array2);
+        assert_eq!(array.len(), 3);
+
+        // Pin the raw little-endian bytes emitted by `From<Vec<bf16>>` (rewritten to
+        // reinterpret the Vec via `bytemuck::cast_vec`), so a layout/byte-order
+        // regression is caught directly rather than only through Debug formatting.
+        // bf16 is the high 16 bits of the f32: 1.0->0x3F80, 2.0->0x4000, 3.0->0x4040.
+        let inner = array2.clone().into_inner();
+        let raw_bytes: Vec<u8> = (0..inner.len())
+            .flat_map(|i| inner.value(i).to_vec())
+            .collect();
+        assert_eq!(raw_bytes, vec![0x80, 0x3F, 0x00, 0x40, 0x40, 0x40]);
+
+        let expected_fmt = "BFloat16Array\n[\n  1.0,\n  2.0,\n  3.0,\n]";
+        assert_eq!(expected_fmt, format!("{:?}", array));
+
+        for (expected, value) in values.iter().zip(array.iter()) {
+            assert_eq!(Some(*expected), value);
+        }
+
+        for (expected, value) in values.as_slice().iter().zip(array2.iter()) {
+            assert_eq!(Some(*expected), value);
+        }
+
+        let arrow_array = array.into_inner();
+        assert_eq!(arrow_array.as_slice(), values.as_slice());
+    }
+
+    #[test]
+    fn test_nulls() {
+        let values: Vec<Option<bf16>> =
+            vec![Some(bf16::from_f32(1.0)), None, Some(bf16::from_f32(3.0))];
+        let array = BFloat16Array::from_iter(values.clone());
+        assert_eq!(array.len(), 3);
+        assert_eq!(array.null_count(), 1);
+
+        let expected_fmt = "BFloat16Array\n[\n  1.0,\n  null,\n  3.0,\n]";
+        assert_eq!(expected_fmt, format!("{:?}", array));
+
+        for (expected, value) in values.iter().zip(array.iter()) {
+            assert_eq!(*expected, value);
+        }
+    }
+}

python/vendor/lance-arrow/src/deepcopy.rs

@@ -0,0 +1,356 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use arrow_array::{Array, RecordBatch, make_array};
+use arrow_buffer::{BooleanBuffer, Buffer, NullBuffer};
+use arrow_data::{ArrayData, ArrayDataBuilder, transform::MutableArrayData};
+use arrow_schema::DataType;
+
+pub fn deep_copy_buffer(buffer: &Buffer) -> Buffer {
+    Buffer::from(buffer.as_slice())
+}
+
+pub fn deep_copy_nulls(nulls: Option<&NullBuffer>) -> Option<NullBuffer> {
+    let nulls = nulls?;
+    let bit_buffer = deep_copy_buffer(nulls.inner().inner());
+    // SAFETY: `null_count` is taken from the source `NullBuffer`, which already
+    // upheld `NullBuffer::new_unchecked`'s invariant — the unset-bit count over
+    // the logical bit slice `[bit_offset, bit_offset + bit_len)`. `NullBuffer::slice`
+    // adjusts only `BooleanBuffer::bit_offset` / `bit_len` and never byte-advances
+    // the inner `Buffer`, so `deep_copy_buffer` (which copies the source `Buffer`'s
+    // `as_slice()` view from byte 0) reproduces the exact bit pattern at the same
+    // bit offsets; the unset-bit count is therefore preserved. `BooleanBuffer::new`
+    // panics (does not UB) if `bit_offset + bit_len > 8 * buffer.len()`, and the
+    // copy has the same length, so that check still passes.
+    Some(unsafe {
+        NullBuffer::new_unchecked(
+            BooleanBuffer::new(bit_buffer, nulls.offset(), nulls.len()),
+            nulls.null_count(),
+        )
+    })
+}
+
+pub fn deep_copy_array_data(data: &ArrayData) -> ArrayData {
+    let data_type = data.data_type().clone();
+    let len = data.len();
+    let nulls = deep_copy_nulls(data.nulls());
+    let offset = data.offset();
+    let buffers = data
+        .buffers()
+        .iter()
+        .map(deep_copy_buffer)
+        .collect::<Vec<_>>();
+    let child_data = data
+        .child_data()
+        .iter()
+        .map(deep_copy_array_data)
+        .collect::<Vec<_>>();
+    // SAFETY: `build_unchecked` inherits `ArrayData::new_unchecked`'s contract —
+    // `(data_type, len, offset, nulls, buffers, child_data)` must form a valid
+    // Arrow array. This call reproduces `data` structurally: `data_type`, `len`,
+    // and `offset` are forwarded unchanged; each buffer is replaced by a byte-
+    // identical copy of its offset-applied `as_slice()` view (the output buffer
+    // is `MutableBuffer`-allocated, at least as aligned as the source); `nulls`
+    // is deep-copied with the same bit offset/length and unset-bit count (see
+    // `deep_copy_nulls`); `child_data` is recursively cloned with the same
+    // guarantee. Every value-level invariant the source upheld — UTF-8 validity,
+    // monotonic offsets, in-bounds dictionary indices, run-end monotonicity,
+    // struct child-length matching — therefore transfers to the copy. If the
+    // source `ArrayData` was itself constructed via `new_unchecked` with an
+    // invalid payload, this function faithfully reproduces that invalidity.
+    unsafe {
+        ArrayDataBuilder::new(data_type)
+            .len(len)
+            .nulls(nulls)
+            .offset(offset)
+            .buffers(buffers)
+            .child_data(child_data)
+            .build_unchecked()
+    }
+}
+
+pub fn deep_copy_array(array: &dyn Array) -> Arc<dyn Array> {
+    let data = array.to_data();
+    let data = deep_copy_array_data(&data);
+    make_array(data)
+}
+
+pub fn deep_copy_batch(batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    let arrays = batch
+        .columns()
+        .iter()
+        .map(|array| deep_copy_array(array))
+        .collect::<Vec<_>>();
+    RecordBatch::try_new(batch.schema(), arrays)
+}
+
+/// Deep copy array data, extracting only the sliced portion using MutableArrayData
+/// This is the most efficient and correct way to copy just the sliced data
+pub fn deep_copy_array_data_sliced(data: &ArrayData) -> ArrayData {
+    // Use MutableArrayData to efficiently copy just the slice
+    let mut mutable = MutableArrayData::new(vec![data], false, data.len());
+
+    // Which index this takes depends on the layout, because arrow's extenders
+    // disagree about who applies `ArrayData::offset()`. The bit-packed one adds
+    // it to the raw values buffer itself, so an offset-applied index there
+    // reads from twice the offset and runs off the end of the buffer. Every
+    // other layout either reads through an already-offset-applied view or
+    // forwards the index to children carrying their own offsets, and takes the
+    // absolute index it has always been given.
+    let start = match data.data_type() {
+        DataType::Boolean => 0,
+        _ => data.offset(),
+    };
+    mutable.extend(0, start, start + data.len());
+
+    // Freeze into immutable ArrayData
+    mutable.freeze()
+}
+
+/// Deep copy an array, extracting only the sliced portion using MutableArrayData
+pub fn deep_copy_array_sliced(array: &dyn Array) -> Arc<dyn Array> {
+    let data = array.to_data();
+    let data = deep_copy_array_data_sliced(&data);
+    make_array(data)
+}
+
+/// Deep copy a RecordBatch, extracting only the sliced portion using MutableArrayData
+pub fn deep_copy_batch_sliced(batch: &RecordBatch) -> crate::Result<RecordBatch> {
+    let arrays = batch
+        .columns()
+        .iter()
+        .map(|array| deep_copy_array_sliced(array))
+        .collect::<Vec<_>>();
+    RecordBatch::try_new(batch.schema(), arrays)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{Array, BooleanArray, Int32Array, RecordBatch, StringArray};
+    use arrow_data::ArrayDataBuilder;
+    use arrow_schema::{DataType, Field, Schema};
+
+    #[test]
+    fn raw_sliced_fixed_size_list_data_keeps_its_child_offset() {
+        // `ArrayData::slice` records the offset on the parent and leaves the
+        // child whole -- a shape `FixedSizeListArray::slice` never produces,
+        // but a legal one this public helper can be handed directly. Arrow's
+        // extender forwards `start * size` to that unsliced child and adds
+        // nothing, so the index it gets has to be the absolute one; this is
+        // here to stop the boolean fix from being generalised over it.
+        let child = Int32Array::from(vec![10, 11, 20, 21]).to_data();
+        let field = Arc::new(Field::new_list_field(DataType::Int32, false));
+        let data = ArrayDataBuilder::new(DataType::FixedSizeList(field, 2))
+            .len(2)
+            .add_child_data(child)
+            .build()
+            .unwrap();
+        let sliced = data.slice(1, 1);
+
+        let copied = super::deep_copy_array_data_sliced(&sliced);
+        let copied_child = Int32Array::from(copied.child_data()[0].clone());
+        assert_eq!(copied_child.values().as_ref(), &[20, 21]);
+    }
+
+    #[test]
+    fn raw_sliced_struct_of_fixed_size_list_reaches_the_right_values() {
+        // Slicing raw struct data pushes the window into the struct's children,
+        // and a fixed-size-list child takes it as its own parent offset with
+        // its values left whole. Two levels of offset, and the absolute index
+        // has to remain right through both of them.
+        let values = Int32Array::from(vec![10, 11, 20, 21, 30, 31]).to_data();
+        let item = Arc::new(Field::new_list_field(DataType::Int32, false));
+        let list = ArrayDataBuilder::new(DataType::FixedSizeList(item, 2))
+            .len(3)
+            .add_child_data(values)
+            .build()
+            .unwrap();
+        let field = Arc::new(Field::new("l", list.data_type().clone(), false));
+        let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
+            .len(3)
+            .add_child_data(list)
+            .build()
+            .unwrap();
+        let sliced = data.slice(2, 1);
+
+        let copied = super::deep_copy_array_data_sliced(&sliced);
+        let copied_values = Int32Array::from(copied.child_data()[0].child_data()[0].clone());
+        assert_eq!(copied_values.values().as_ref(), &[30, 31]);
+    }
+
+    #[test]
+    fn raw_parent_offset_struct_keeps_the_selected_child_row() {
+        // A struct whose children are whole and whose window is the parent
+        // offset: arrow's struct extender forwards the index to those children
+        // untouched, so it has to be the absolute one.
+        let field = Arc::new(Field::new("a", DataType::Int32, false));
+        let data = ArrayDataBuilder::new(DataType::Struct(vec![field].into()))
+            .len(1)
+            .offset(1)
+            .add_child_data(Int32Array::from(vec![10, 20]).to_data())
+            .build()
+            .unwrap();
+
+        let copied = super::deep_copy_array_data_sliced(&data);
+        let copied_child = Int32Array::from(copied.child_data()[0].clone());
+        assert_eq!(copied_child.values().as_ref(), &[20]);
+    }
+
+    #[test]
+    fn sliced_boolean_deep_copy_reads_from_the_slice() {
+        // A boolean slice keeps its `ArrayData::offset()` -- the buffer cannot
+        // advance by a fraction of a byte -- so a copy that adds the offset on
+        // top of what `MutableArrayData` already applies reads past the end of
+        // the values buffer and panics.
+        let array = BooleanArray::from(vec![true, false, true, false, true, false, true, false]);
+        for (offset, len) in [(0usize, 8usize), (1, 7), (3, 5), (7, 1)] {
+            let sliced = array.slice(offset, len);
+            let copied = super::deep_copy_array_sliced(&sliced);
+            let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
+            let expected: Vec<bool> = (0..len).map(|i| sliced.value(i)).collect();
+            let actual: Vec<bool> = (0..len).map(|i| copied.value(i)).collect();
+            assert_eq!(actual, expected, "offset={offset} len={len}");
+        }
+    }
+
+    #[test]
+    fn sliced_boolean_deep_copy_keeps_its_nulls() {
+        let array = BooleanArray::from(vec![
+            Some(true),
+            None,
+            Some(false),
+            Some(true),
+            None,
+            Some(false),
+        ]);
+        let sliced = array.slice(1, 4);
+        let copied = super::deep_copy_array_sliced(&sliced);
+        let copied = copied.as_any().downcast_ref::<BooleanArray>().unwrap();
+        let expected: Vec<Option<bool>> = (0..sliced.len())
+            .map(|i| (!sliced.is_null(i)).then(|| sliced.value(i)))
+            .collect();
+        let actual: Vec<Option<bool>> = (0..copied.len())
+            .map(|i| (!copied.is_null(i)).then(|| copied.value(i)))
+            .collect();
+        assert_eq!(actual, expected);
+    }
+
+    #[test]
+    fn test_deep_copy_sliced_array_with_nulls() {
+        let array = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            None,
+            Some(5),
+        ]));
+        let sliced_array = array.slice(1, 3);
+        let copied_array = super::deep_copy_array(&sliced_array);
+        assert_eq!(sliced_array.len(), copied_array.len());
+        assert_eq!(sliced_array.nulls(), copied_array.nulls());
+    }
+
+    #[test]
+    fn test_deep_copy_array_data_sliced() {
+        let array = Int32Array::from((0..1000).collect::<Vec<i32>>());
+        let sliced = array.slice(100, 10);
+
+        let sliced_data = sliced.to_data();
+        let copied_data = super::deep_copy_array_data_sliced(&sliced_data);
+
+        assert_eq!(copied_data.len(), 10);
+        assert_eq!(copied_data.offset(), 0);
+
+        // Verify data correctness
+        let copied_array = Int32Array::from(copied_data);
+        for i in 0..10 {
+            assert_eq!(copied_array.value(i), 100 + i as i32);
+        }
+    }
+
+    #[test]
+    fn test_deep_copy_array_sliced() {
+        let array = Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]));
+        let sliced = array.slice(1, 3);
+
+        let copied = super::deep_copy_array_sliced(&sliced);
+
+        assert_eq!(copied.len(), 3);
+        let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert_eq!(copied_int.value(0), 2);
+        assert_eq!(copied_int.value(1), 3);
+        assert_eq!(copied_int.value(2), 4);
+    }
+
+    #[test]
+    fn test_deep_copy_batch_sliced() {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("name", DataType::Utf8, false),
+        ]));
+
+        let id_array = Arc::new(Int32Array::from((0..100).collect::<Vec<i32>>()));
+        let name_array = Arc::new(StringArray::from(
+            (0..100)
+                .map(|i| format!("name_{}", i))
+                .collect::<Vec<String>>(),
+        ));
+
+        let batch = RecordBatch::try_new(
+            schema,
+            vec![id_array as Arc<dyn Array>, name_array as Arc<dyn Array>],
+        )
+        .unwrap();
+
+        let sliced = batch.slice(10, 5);
+        let copied = super::deep_copy_batch_sliced(&sliced).unwrap();
+
+        assert_eq!(copied.num_rows(), 5);
+        assert_eq!(copied.num_columns(), 2);
+
+        // Verify data correctness
+        let id_col = copied
+            .column(0)
+            .as_any()
+            .downcast_ref::<Int32Array>()
+            .unwrap();
+        let name_col = copied
+            .column(1)
+            .as_any()
+            .downcast_ref::<StringArray>()
+            .unwrap();
+
+        for i in 0..5 {
+            assert_eq!(id_col.value(i), 10 + i as i32);
+            assert_eq!(name_col.value(i), format!("name_{}", 10 + i));
+        }
+    }
+
+    #[test]
+    fn test_deep_copy_array_sliced_with_nulls() {
+        let array = Arc::new(Int32Array::from(vec![
+            Some(1),
+            None,
+            Some(3),
+            None,
+            Some(5),
+        ]));
+        let sliced = array.slice(1, 3); // [None, Some(3), None]
+
+        let copied = super::deep_copy_array_sliced(&sliced);
+
+        assert_eq!(copied.len(), 3);
+        assert_eq!(copied.null_count(), 2); // Two nulls in the slice
+
+        let copied_int = copied.as_any().downcast_ref::<Int32Array>().unwrap();
+        assert!(!copied_int.is_valid(0)); // None
+        assert!(copied_int.is_valid(1)); // Some(3)
+        assert!(!copied_int.is_valid(2)); // None
+        assert_eq!(copied_int.value(1), 3);
+    }
+}

python/vendor/lance-arrow/src/floats.rs

@@ -0,0 +1,292 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Floats Array
+
+use std::fmt::{Debug, Display};
+use std::iter::Sum;
+use std::sync::Arc;
+use std::{
+    fmt::Formatter,
+    ops::{AddAssign, DivAssign},
+};
+
+use arrow_array::{
+    Array, FixedSizeBinaryArray, Float16Array, Float32Array, Float64Array,
+    types::{Float16Type, Float32Type, Float64Type},
+};
+use arrow_schema::{DataType, Field};
+use half::{bf16, f16};
+use num_traits::{AsPrimitive, Bounded, Float, FromPrimitive};
+
+use super::bfloat16::{BFloat16Array, BFloat16Type};
+use crate::Result;
+use crate::bfloat16::is_bfloat16_field;
+
+/// Float data type.
+///
+/// This helps differentiate between the different float types,
+/// because bf16 is not officially supported [DataType] in arrow-rs.
+#[derive(Debug)]
+pub enum FloatType {
+    BFloat16,
+    Float16,
+    Float32,
+    Float64,
+}
+
+impl std::fmt::Display for FloatType {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::BFloat16 => write!(f, "bfloat16"),
+            Self::Float16 => write!(f, "float16"),
+            Self::Float32 => write!(f, "float32"),
+            Self::Float64 => write!(f, "float64"),
+        }
+    }
+}
+
+/// Try to convert a [DataType] to a [FloatType]. To support bfloat16, always
+/// prefer using the `TryFrom<&Field>` implementation.
+impl TryFrom<&DataType> for FloatType {
+    type Error = crate::ArrowError;
+
+    fn try_from(value: &DataType) -> Result<Self> {
+        match *value {
+            DataType::Float16 => Ok(Self::Float16),
+            DataType::Float32 => Ok(Self::Float32),
+            DataType::Float64 => Ok(Self::Float64),
+            _ => Err(crate::ArrowError::InvalidArgumentError(format!(
+                "{:?} is not a floating type",
+                value
+            ))),
+        }
+    }
+}
+
+impl TryFrom<&Field> for FloatType {
+    type Error = crate::ArrowError;
+
+    fn try_from(field: &Field) -> Result<Self> {
+        match field.data_type() {
+            DataType::FixedSizeBinary(2) if is_bfloat16_field(field) => Ok(Self::BFloat16),
+            _ => Self::try_from(field.data_type()),
+        }
+    }
+}
+
+/// Trait for float types used in Lance indexes
+///
+/// This mimics the utilities provided by [`arrow_array::ArrowPrimitiveType`]
+/// but applies to all float types (including bfloat16)
+pub trait ArrowFloatType: Debug {
+    type Native: FromPrimitive
+        + FloatToArrayType<ArrowType = Self>
+        + AsPrimitive<f32>
+        + Debug
+        + Display;
+
+    const FLOAT_TYPE: FloatType;
+    const MIN: Self::Native;
+    const MAX: Self::Native;
+
+    /// Arrow Float Array Type.
+    type ArrayType: FloatArray<Self>;
+
+    /// Returns empty array of this type.
+    fn empty_array() -> Self::ArrayType {
+        <Self::ArrayType as FloatArray<Self>>::from_values(Vec::new())
+    }
+}
+
+/// Trait to be implemented by native types that have a corresponding [`ArrowFloatType`]
+/// implementation.
+///
+/// This helps define what operations are supported by native floats and also helps convert
+/// from a native type back to the corresponding Arrow float type.
+pub trait FloatToArrayType:
+    Float
+    + Bounded
+    + Sum
+    + AddAssign<Self>
+    + AsPrimitive<f64>
+    + AsPrimitive<f32>
+    + DivAssign
+    + Send
+    + Sync
+    + Copy
+{
+    /// The corresponding [`ArrowFloatType`] implementation for this native type
+    type ArrowType: ArrowFloatType<Native = Self>;
+}
+
+impl FloatToArrayType for bf16 {
+    type ArrowType = BFloat16Type;
+}
+
+impl FloatToArrayType for f16 {
+    type ArrowType = Float16Type;
+}
+
+impl FloatToArrayType for f32 {
+    type ArrowType = Float32Type;
+}
+
+impl FloatToArrayType for f64 {
+    type ArrowType = Float64Type;
+}
+
+impl ArrowFloatType for BFloat16Type {
+    type Native = bf16;
+
+    const FLOAT_TYPE: FloatType = FloatType::BFloat16;
+    const MIN: Self::Native = bf16::MIN;
+    const MAX: Self::Native = bf16::MAX;
+
+    type ArrayType = FixedSizeBinaryArray;
+}
+
+impl ArrowFloatType for Float16Type {
+    type Native = f16;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float16;
+    const MIN: Self::Native = f16::MIN;
+    const MAX: Self::Native = f16::MAX;
+
+    type ArrayType = Float16Array;
+}
+
+impl ArrowFloatType for Float32Type {
+    type Native = f32;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float32;
+    const MIN: Self::Native = f32::MIN;
+    const MAX: Self::Native = f32::MAX;
+
+    type ArrayType = Float32Array;
+}
+
+impl ArrowFloatType for Float64Type {
+    type Native = f64;
+
+    const FLOAT_TYPE: FloatType = FloatType::Float64;
+    const MIN: Self::Native = f64::MIN;
+    const MAX: Self::Native = f64::MAX;
+
+    type ArrayType = Float64Array;
+}
+
+/// [FloatArray] is a trait that is implemented by all float type arrays
+///
+/// This is similar to [`arrow_array::PrimitiveArray`] but applies to all float types (including bfloat16)
+/// and is implemented as a trait and not a struct
+pub trait FloatArray<T: ArrowFloatType + ?Sized>: Array + Clone + 'static {
+    type FloatType: ArrowFloatType;
+
+    /// Returns a reference to the underlying data as a slice.
+    ///
+    /// # Panics
+    ///
+    /// Implementations may panic if the array's storage shape does not match
+    /// the expected element layout. In particular, the `bf16` impl panics if
+    /// `value_length() != 2` (the `FixedSizeBinary(2)` shape required by
+    /// `BFloat16Array`).
+    ///
+    /// # Preconditions
+    ///
+    /// Implementations may impose additional invariants on the underlying
+    /// buffer. The `bf16` impl requires the value buffer to be at least
+    /// 2-byte aligned — satisfied automatically by every in-tree Lance
+    /// constructor, but external callers passing externally-built arrays
+    /// (FFI, IPC, `Buffer::from_custom_allocation`) must ensure alignment.
+    /// See the impl's docstring for details.
+    fn as_slice(&self) -> &[T::Native];
+
+    /// Construct an array from a vector of values.
+    fn from_values(values: Vec<T::Native>) -> Self;
+
+    /// Construct an array from an iterator of values.
+    fn from_iter_values(values: impl IntoIterator<Item = T::Native>) -> Self
+    where
+        Self: Sized,
+    {
+        Self::from_values(values.into_iter().collect())
+    }
+}
+
+impl FloatArray<Float16Type> for Float16Array {
+    type FloatType = Float16Type;
+
+    fn as_slice(&self) -> &[<Float16Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float16Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+impl FloatArray<Float32Type> for Float32Array {
+    type FloatType = Float32Type;
+
+    fn as_slice(&self) -> &[<Float32Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float32Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+impl FloatArray<Float64Type> for Float64Array {
+    type FloatType = Float64Type;
+
+    fn as_slice(&self) -> &[<Float64Type as ArrowFloatType>::Native] {
+        self.values()
+    }
+
+    fn from_values(values: Vec<<Float64Type as ArrowFloatType>::Native>) -> Self {
+        Self::from(values)
+    }
+}
+
+/// Convert a float32 array to another float array
+///
+/// This is used during queries as query vectors are always provided as float32 arrays
+/// and need to be converted to the appropriate float type for the index.
+pub fn coerce_float_vector(input: &Float32Array, float_type: FloatType) -> Result<Arc<dyn Array>> {
+    match float_type {
+        FloatType::BFloat16 => Ok(Arc::new(
+            BFloat16Array::from_iter_values(input.values().iter().map(|v| bf16::from_f32(*v)))
+                .into_inner(),
+        )),
+        FloatType::Float16 => Ok(Arc::new(Float16Array::from_iter_values(
+            input.values().iter().map(|v| f16::from_f32(*v)),
+        ))),
+        FloatType::Float32 => Ok(Arc::new(input.clone())),
+        FloatType::Float64 => Ok(Arc::new(Float64Array::from_iter_values(
+            input.values().iter().map(|v| *v as f64),
+        ))),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_coerce_float_vector_bfloat16() {
+        let input = Float32Array::from(vec![1.0f32, 2.0, 3.0]);
+        let array = coerce_float_vector(&input, FloatType::BFloat16).unwrap();
+
+        assert_eq!(array.data_type(), &DataType::FixedSizeBinary(2));
+
+        let fixed = array
+            .as_any()
+            .downcast_ref::<FixedSizeBinaryArray>()
+            .unwrap();
+        let expected: Vec<bf16> = input.values().iter().map(|v| bf16::from_f32(*v)).collect();
+        assert_eq!(fixed.as_slice(), expected.as_slice());
+    }
+}

python/vendor/lance-arrow/src/ipc.rs

@@ -0,0 +1,621 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Zero-copy Arrow IPC stream read/write utilities.
+//!
+//! Provides helpers for serializing and deserializing [`RecordBatch`]es as
+//! self-delimiting Arrow IPC streams using synchronous [`Read`]/[`Write`] I/O.
+//!
+//! These are designed for embedding IPC streams inside larger binary formats
+//! (e.g. a cache entry that contains multiple IPC sections). Each stream is
+//! self-delimiting (schema + batches + EOS marker) and can be read back
+//! independently.
+//!
+//! # Zero-copy reads
+//!
+//! [`read_ipc_stream`] and [`read_ipc_stream_single`] take `&Bytes` and use
+//! [`Bytes::slice`] to produce each message buffer. Because `Bytes::slice`
+//! increments a reference count rather than copying, the resulting
+//! [`Buffer`]s — and the array data decoded from them by [`FileDecoder`] —
+//! are all backed by the same allocation as the input.
+
+use std::io::{Read, Write};
+use std::sync::Arc;
+
+use arrow_array::RecordBatch;
+use arrow_buffer::Buffer;
+use arrow_ipc::convert::fb_to_schema;
+use arrow_ipc::reader::FileDecoder;
+use arrow_ipc::root_as_message;
+use arrow_ipc::writer::StreamWriter;
+use arrow_schema::ArrowError;
+use bytes::Bytes;
+
+// ---------------------------------------------------------------------------
+// Length-prefixed byte utilities
+// ---------------------------------------------------------------------------
+
+/// Write `data` prefixed by its length as a little-endian `u64`.
+///
+/// Paired with [`read_len_prefixed_bytes`].
+pub fn write_len_prefixed_bytes(writer: &mut dyn Write, data: &[u8]) -> Result<(), ArrowError> {
+    writer
+        .write_all(&(data.len() as u64).to_le_bytes())
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    writer
+        .write_all(data)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))
+}
+
+/// Read a byte slice written by [`write_len_prefixed_bytes`].
+///
+/// Reads an 8-byte little-endian length then exactly that many bytes.
+pub fn read_len_prefixed_bytes(reader: &mut dyn Read) -> Result<Vec<u8>, ArrowError> {
+    let mut len_buf = [0u8; 8];
+    reader
+        .read_exact(&mut len_buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    let len = u64::from_le_bytes(len_buf) as usize;
+    let mut buf = vec![0u8; len];
+    reader
+        .read_exact(&mut buf)
+        .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+    Ok(buf)
+}
+
+// ---------------------------------------------------------------------------
+// IPC stream utilities
+// ---------------------------------------------------------------------------
+
+// 4-byte continuation marker used by modern Arrow IPC streams.
+const IPC_CONTINUATION: [u8; 4] = [0xff; 4];
+
+/// Write `batch` as a single-batch Arrow IPC stream to `writer`.
+pub fn write_ipc_stream(batch: &RecordBatch, writer: &mut dyn Write) -> Result<(), ArrowError> {
+    let mut sw = StreamWriter::try_new(&mut *writer, batch.schema_ref())?;
+    sw.write(batch)?;
+    sw.finish()
+}
+
+/// Write all batches from `iter` as a single Arrow IPC stream to `writer`.
+///
+/// `iter` must yield at least one batch; the schema is inferred from the first
+/// batch. Returns `ArrowError::InvalidArgumentError` if the iterator is empty.
+/// If you need to write an empty stream (schema only, no rows), construct a
+/// `StreamWriter` directly.
+pub fn write_ipc_stream_batches<I>(iter: I, writer: &mut dyn Write) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    let mut iter = iter.into_iter();
+    let first = iter
+        .next()
+        .ok_or_else(|| ArrowError::InvalidArgumentError("no batches to serialize".into()))?;
+    let mut sw = StreamWriter::try_new(&mut *writer, first.schema_ref())?;
+    sw.write(&first)?;
+    for batch in iter {
+        sw.write(&batch)?;
+    }
+    sw.finish()
+}
+
+/// Read one complete Arrow IPC stream message from `data` as a zero-copy [`Buffer`].
+///
+/// Parses the first message starting at byte 0 of `data`. Returns `None` on
+/// EOS (size field == 0) or empty input. The returned [`Buffer`] is backed by
+/// `data`'s allocation — no bytes are copied.
+///
+/// The caller should advance its position by `buf.len()` after each call.
+fn read_one_ipc_message(data: &Bytes) -> Result<Option<Buffer>, ArrowError> {
+    let bytes = data.as_ref();
+
+    if bytes.is_empty() {
+        return Ok(None);
+    }
+    if bytes.len() < 4 {
+        return Err(ArrowError::IoError(
+            "IPC: truncated header".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC header"),
+        ));
+    }
+
+    let has_continuation = bytes[..4] == IPC_CONTINUATION;
+    let (size_bytes, prefix_len): ([u8; 4], usize) = if has_continuation {
+        if bytes.len() < 8 {
+            return Err(ArrowError::IoError(
+                "IPC: truncated header after continuation".into(),
+                std::io::Error::new(
+                    std::io::ErrorKind::UnexpectedEof,
+                    "truncated after continuation",
+                ),
+            ));
+        }
+        (bytes[4..8].try_into().unwrap(), 8)
+    } else {
+        (bytes[..4].try_into().unwrap(), 4)
+    };
+
+    let meta_size = u32::from_le_bytes(size_bytes) as usize;
+    if meta_size == 0 {
+        return Ok(None); // EOS
+    }
+
+    let meta_end = prefix_len + meta_size;
+    if bytes.len() < meta_end {
+        return Err(ArrowError::IoError(
+            "IPC: truncated metadata".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC metadata"),
+        ));
+    }
+
+    let msg = root_as_message(&bytes[prefix_len..meta_end])
+        .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+    let body_len = msg.bodyLength() as usize;
+
+    let total = meta_end + body_len;
+    if bytes.len() < total {
+        return Err(ArrowError::IoError(
+            "IPC: truncated body".into(),
+            std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated IPC body"),
+        ));
+    }
+
+    // Zero-copy: Bytes::slice shares the backing allocation; Buffer::from
+    // wraps it without copying.
+    Ok(Some(Buffer::from(data.slice(0..total))))
+}
+
+/// Read a length-prefixed byte slice at `offset` in `data`, advancing `offset`.
+///
+/// Reads an 8-byte little-endian length, then slices exactly that many bytes
+/// from `data`. The returned [`Bytes`] is zero-copy (shares `data`'s allocation).
+pub fn read_len_prefixed_bytes_at(data: &Bytes, offset: &mut usize) -> Result<Bytes, ArrowError> {
+    let bytes = data.as_ref();
+    let len_end = offset
+        .checked_add(8)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated length field".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated length"),
+            )
+        })?;
+    let len = u64::from_le_bytes(bytes[*offset..len_end].try_into().unwrap()) as usize;
+    *offset = len_end;
+    let data_end = offset
+        .checked_add(len)
+        .filter(|&e| e <= bytes.len())
+        .ok_or_else(|| {
+            ArrowError::IoError(
+                "length-prefixed bytes: truncated data".into(),
+                std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "truncated data"),
+            )
+        })?;
+    let result = data.slice(*offset..data_end);
+    *offset = data_end;
+    Ok(result)
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    let batches = read_ipc_stream(&data.slice(*offset..))?;
+
+    // Recompute how many bytes were consumed by re-parsing message sizes.
+    // We can't get this from read_ipc_stream directly, so we re-walk the
+    // message headers (metadata only, no body re-read) to sum up lengths.
+    let slice = &data.as_ref()[*offset..];
+    let mut consumed = 0usize;
+    loop {
+        let rem = &slice[consumed..];
+        if rem.is_empty() {
+            break;
+        }
+        let has_cont = rem.len() >= 4 && rem[..4] == IPC_CONTINUATION;
+        let (size_bytes, prefix_len): ([u8; 4], usize) = if has_cont {
+            if rem.len() < 8 {
+                break;
+            }
+            (rem[4..8].try_into().unwrap(), 8)
+        } else {
+            if rem.len() < 4 {
+                break;
+            }
+            (rem[..4].try_into().unwrap(), 4)
+        };
+        let meta_size = u32::from_le_bytes(size_bytes) as usize;
+        if meta_size == 0 {
+            // EOS — consume it and stop.
+            consumed += prefix_len;
+            break;
+        }
+        let meta_end = prefix_len + meta_size;
+        if rem.len() < meta_end {
+            break;
+        }
+        let msg = root_as_message(&rem[prefix_len..meta_end])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+        consumed += meta_end + body_len;
+    }
+    *offset += consumed;
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream starting at `offset`,
+/// advancing `offset` past the stream (including the EOS marker).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation.
+pub fn read_ipc_stream_single_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream_at(data, offset)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+/// Extract the prefix length and metadata size from a raw IPC message buffer.
+///
+/// Modern IPC streams have an 8-byte prefix `[continuation: 4][size: 4]`.
+/// Legacy streams have a 4-byte prefix `[size: 4]`. Returns `(prefix_len, meta_size)`.
+fn parse_ipc_message_prefix(buf: &Buffer) -> Result<(usize, usize), ArrowError> {
+    let has_continuation = buf.len() >= 4 && buf[..4] == IPC_CONTINUATION;
+    if has_continuation {
+        if buf.len() < 8 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize;
+        Ok((8, meta_size))
+    } else {
+        if buf.len() < 4 {
+            return Err(ArrowError::ParseError(
+                "IPC message buffer too short".into(),
+            ));
+        }
+        let meta_size = u32::from_le_bytes(buf[..4].try_into().unwrap()) as usize;
+        Ok((4, meta_size))
+    }
+}
+
+/// Read all [`RecordBatch`]es from one Arrow IPC stream.
+///
+/// Zero-copy: each batch's array data buffers are borrowed from the input
+/// message buffer(s) and not copied during decoding.
+///
+/// Uses [`FileDecoder`] directly (rather than `StreamDecoder`) to avoid a
+/// known edge case where `StreamDecoder` does not produce a batch for messages
+/// with a zero-length body when the message exactly fills the decode buffer.
+pub fn read_ipc_stream(data: &Bytes) -> Result<Vec<RecordBatch>, ArrowError> {
+    let mut offset = 0usize;
+
+    let schema_buf = read_one_ipc_message(&data.slice(offset..))?.ok_or_else(|| {
+        ArrowError::ParseError("IPC stream: expected schema message, got EOS".into())
+    })?;
+    offset += schema_buf.len();
+
+    let (prefix_len, meta_size) = parse_ipc_message_prefix(&schema_buf)?;
+    let schema_msg = root_as_message(&schema_buf[prefix_len..prefix_len + meta_size])
+        .map_err(|e| ArrowError::ParseError(format!("IPC schema parse error: {e}")))?;
+    let schema = Arc::new(fb_to_schema(schema_msg.header_as_schema().ok_or_else(
+        || ArrowError::ParseError("IPC stream: first message is not a schema".into()),
+    )?));
+    let mut decoder = FileDecoder::new(schema, schema_msg.version());
+
+    let mut batches = Vec::new();
+
+    loop {
+        let Some(buf) = read_one_ipc_message(&data.slice(offset..))? else {
+            break;
+        };
+        offset += buf.len();
+
+        let (prefix_len, meta_size) = parse_ipc_message_prefix(&buf)?;
+        let msg = root_as_message(&buf[prefix_len..prefix_len + meta_size])
+            .map_err(|e| ArrowError::ParseError(format!("IPC message parse error: {e}")))?;
+        let body_len = msg.bodyLength() as usize;
+
+        // Block offset = 0 since the buffer starts at the message boundary.
+        // metaDataLength = prefix_len + meta_size (prefix + flatbuf + padding).
+        let block = arrow_ipc::Block::new(0, (prefix_len + meta_size) as i32, body_len as i64);
+
+        match msg.header_type() {
+            arrow_ipc::MessageHeader::RecordBatch => {
+                if let Some(batch) = decoder.read_record_batch(&block, &buf)? {
+                    batches.push(batch);
+                }
+            }
+            arrow_ipc::MessageHeader::DictionaryBatch => {
+                decoder.read_dictionary(&block, &buf)?;
+            }
+            _ => break,
+        }
+    }
+
+    Ok(batches)
+}
+
+/// Read exactly one [`RecordBatch`] from one Arrow IPC stream.
+pub fn read_ipc_stream_single(data: &Bytes) -> Result<RecordBatch, ArrowError> {
+    let mut batches = read_ipc_stream(data)?;
+    match batches.len() {
+        1 => Ok(batches.remove(0)),
+        n => Err(ArrowError::ParseError(format!(
+            "expected exactly 1 IPC record batch, got {n}"
+        ))),
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Aligned IPC sections
+// ---------------------------------------------------------------------------
+
+/// Byte alignment that each IPC section's stream start is padded to.
+///
+/// When several IPC streams are concatenated into one larger blob (e.g. a
+/// cache entry), a section that starts at an arbitrary offset would leave its
+/// array data misaligned. [`FileDecoder`] with `require_alignment = false`
+/// then silently copies each buffer into a freshly aligned allocation on
+/// every read, defeating zero-copy. Padding each section start to a 64-byte
+/// boundary keeps the decoded buffers borrowed directly from the input.
+pub const IPC_SECTION_ALIGNMENT: usize = 64;
+
+/// Number of zero-padding bytes needed to advance `pos` to the next
+/// [`IPC_SECTION_ALIGNMENT`] boundary.
+fn section_padding(pos: usize) -> usize {
+    (IPC_SECTION_ALIGNMENT - (pos % IPC_SECTION_ALIGNMENT)) % IPC_SECTION_ALIGNMENT
+}
+
+/// A [`Write`] adapter that counts the bytes written through it.
+struct CountingWriter<'a> {
+    inner: &'a mut dyn Write,
+    count: usize,
+}
+
+impl Write for CountingWriter<'_> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        let n = self.inner.write(buf)?;
+        self.count += n;
+        Ok(n)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.inner.flush()
+    }
+}
+
+/// Write zero padding so the next byte lands on an [`IPC_SECTION_ALIGNMENT`]
+/// boundary, advancing `pos` past it.
+fn write_section_padding(writer: &mut dyn Write, pos: &mut usize) -> Result<(), ArrowError> {
+    let pad = section_padding(*pos);
+    if pad > 0 {
+        const ZEROS: [u8; IPC_SECTION_ALIGNMENT] = [0u8; IPC_SECTION_ALIGNMENT];
+        writer
+            .write_all(&ZEROS[..pad])
+            .map_err(|e| ArrowError::IoError(e.to_string(), e))?;
+        *pos += pad;
+    }
+    Ok(())
+}
+
+/// Write `batch` as a 64-byte-aligned single-batch Arrow IPC section.
+///
+/// `pos` is the absolute byte offset of `writer` within the enclosing blob.
+/// Zero padding is written first so the IPC stream begins on an
+/// [`IPC_SECTION_ALIGNMENT`] boundary, then the stream itself. `pos` is
+/// advanced past both the padding and the stream so the caller can write
+/// further aligned sections.
+///
+/// Paired with [`read_ipc_section_at`]. For the decoded buffers to be borrowed
+/// zero-copy, the blob must ultimately be read back from a buffer whose base
+/// address is at least 64-byte aligned.
+pub fn write_ipc_section(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    batch: &RecordBatch,
+) -> Result<(), ArrowError> {
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream(batch, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read a single [`RecordBatch`] from an aligned IPC section at `offset`.
+///
+/// Skips the alignment padding written by [`write_ipc_section`], then reads
+/// the stream, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section`]).
+pub fn read_ipc_section_at(data: &Bytes, offset: &mut usize) -> Result<RecordBatch, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_single_at(data, offset)
+}
+
+/// Write `batches` as a single 64-byte-aligned multi-batch Arrow IPC section.
+///
+/// Like [`write_ipc_section`] but emits every batch from `iter` into one IPC
+/// stream (schema + N batches + EOS). `iter` must yield at least one batch.
+/// Paired with [`read_ipc_section_batches_at`].
+pub fn write_ipc_section_batches<I>(
+    writer: &mut dyn Write,
+    pos: &mut usize,
+    iter: I,
+) -> Result<(), ArrowError>
+where
+    I: IntoIterator<Item = RecordBatch>,
+{
+    write_section_padding(writer, pos)?;
+
+    let mut counting = CountingWriter {
+        inner: writer,
+        count: 0,
+    };
+    write_ipc_stream_batches(iter, &mut counting)?;
+    *pos += counting.count;
+    Ok(())
+}
+
+/// Read all [`RecordBatch`]es from an aligned multi-batch IPC section at
+/// `offset`, advancing `offset` past the section (padding + stream + EOS).
+///
+/// Zero-copy: array buffers borrow from `data`'s allocation when `data`'s base
+/// address is at least 64-byte aligned (see [`write_ipc_section_batches`]).
+pub fn read_ipc_section_batches_at(
+    data: &Bytes,
+    offset: &mut usize,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    *offset += section_padding(*offset);
+    read_ipc_stream_at(data, offset)
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::{ArrayRef, record_batch};
+
+    use super::*;
+
+    #[test]
+    fn test_ipc_roundtrip() {
+        let batch1 = record_batch!(
+            ("int", Int32, [1, 2, 3]),
+            ("str", Utf8, ["foo", "bar", "baz"])
+        )
+        .unwrap();
+        let batch2 = record_batch!(("int", Int32, [4, 5]), ("str", Utf8, ["qux", "quux"])).unwrap();
+        let batches = vec![batch1.clone(), batch2.clone()];
+
+        let mut buf = Vec::new();
+        write_ipc_stream_batches(batches, &mut buf).unwrap();
+
+        let data = Bytes::from(buf);
+
+        let batches = read_ipc_stream(&data).unwrap();
+        assert_eq!(batches.len(), 2);
+        assert_eq!(batches[0], batch1);
+        assert_eq!(batches[1], batch2);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        let assert_col_zero_copy = |array: &ArrayRef| {
+            for buffer in array.to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "buffer at {ptr:#x} is not backed by the input Bytes allocation \
+                     [{data_base:#x}..{data_end:#x})"
+                );
+            }
+        };
+
+        for batch in &batches {
+            assert_eq!(batch.schema(), batch1.schema());
+            assert_col_zero_copy(batch.column(0));
+            assert_col_zero_copy(batch.column(1));
+        }
+    }
+
+    /// Allocate a [`Bytes`] whose base address is 64-byte aligned, modelling a
+    /// backend that reads cache entries into an aligned buffer. A plain
+    /// `Bytes::from(vec)` only guarantees the allocator's alignment for `u8`.
+    fn aligned_bytes(payload: &[u8]) -> Bytes {
+        let mut v = vec![0u8; payload.len() + IPC_SECTION_ALIGNMENT];
+        let pad = section_padding(v.as_ptr() as usize);
+        v[pad..pad + payload.len()].copy_from_slice(payload);
+        Bytes::from(v).slice(pad..pad + payload.len())
+    }
+
+    #[test]
+    fn test_aligned_ipc_sections_are_zero_copy() {
+        // A LargeBinary column exercises the i64-offset buffer whose 8-byte
+        // alignment requirement triggers a realigning memcpy when misaligned.
+        let blocks = arrow_array::LargeBinaryArray::from_vec(vec![&b"hello"[..], b"world"]);
+        let section_a = RecordBatch::try_from_iter([("a", Arc::new(blocks) as ArrayRef)]).unwrap();
+        let section_b = record_batch!(("b", Int64, [10i64, 20, 30, 40, 50])).unwrap();
+
+        let mut buf = Vec::new();
+        // Arbitrary, deliberately non-64-aligned preamble so the first section
+        // must be padded rather than landing at offset 0 by luck.
+        buf.extend_from_slice(&[0xABu8; 7]);
+        let mut pos = buf.len();
+        // The first section's stream begins after padding the 7-byte preamble
+        // up to the next 64-byte boundary.
+        assert_eq!(7 + section_padding(7), IPC_SECTION_ALIGNMENT);
+        write_ipc_section(&mut buf, &mut pos, &section_a).unwrap();
+        write_ipc_section(&mut buf, &mut pos, &section_b).unwrap();
+
+        let data = aligned_bytes(&buf);
+        assert_eq!(
+            section_padding(data.as_ptr() as usize),
+            0,
+            "base not aligned"
+        );
+
+        let mut offset = 7;
+        let read_a = read_ipc_section_at(&data, &mut offset).unwrap();
+        let read_b = read_ipc_section_at(&data, &mut offset).unwrap();
+        assert_eq!(read_a, section_a);
+        assert_eq!(read_b, section_b);
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for batch in [&read_a, &read_b] {
+            for buffer in batch.column(0).to_data().buffers() {
+                let ptr = buffer.as_ptr() as usize;
+                assert!(
+                    ptr >= data_base && ptr < data_end,
+                    "section buffer at {ptr:#x} was realigned out of the input \
+                     [{data_base:#x}..{data_end:#x}) — misaligned section",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_aligned_multi_batch_section_roundtrip_zero_copy() {
+        // A multi-batch section (e.g. IVF SQ storage chunks) must round-trip
+        // every batch and decode the first batch's buffers zero-copy.
+        let b1 = record_batch!(("v", Int64, [1i64, 2, 3])).unwrap();
+        let b2 = record_batch!(("v", Int64, [4i64, 5])).unwrap();
+        let b3 = record_batch!(("v", Int64, [6i64])).unwrap();
+
+        let mut buf = vec![0xCDu8; 5];
+        let mut pos = buf.len();
+        write_ipc_section_batches(&mut buf, &mut pos, [b1.clone(), b2.clone(), b3.clone()])
+            .unwrap();
+
+        let data = aligned_bytes(&buf);
+        let mut offset = 5;
+        let read = read_ipc_section_batches_at(&data, &mut offset).unwrap();
+        assert_eq!(read, vec![b1, b2, b3]);
+        assert_eq!(offset, buf.len(), "offset should land at section end");
+
+        let data_base = data.as_ptr() as usize;
+        let data_end = data_base + data.len();
+        for buffer in read[0].column(0).to_data().buffers() {
+            let ptr = buffer.as_ptr() as usize;
+            assert!(
+                ptr >= data_base && ptr < data_end,
+                "first batch buffer at {ptr:#x} was realigned out of the input",
+            );
+        }
+    }
+}

python/vendor/lance-arrow/src/list.rs

@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::sync::Arc;
+
+use arrow_array::{Array, BooleanArray, GenericListArray, OffsetSizeTrait};
+use arrow_buffer::{BooleanBufferBuilder, OffsetBuffer, ScalarBuffer};
+use arrow_schema::Field;
+
+pub trait ListArrayExt {
+    /// Filters out masked null items from the list array
+    ///
+    /// It is legal for a list array to have a null entry with a non-zero length.  The
+    /// values inside the entry are "garbage" and should be ignored.  This function
+    /// filters the values array to remove the garbage values.
+    ///
+    /// The output list will always have zero-length nulls.
+    fn filter_garbage_nulls(&self) -> Self;
+    /// Returns a copy of the list's values array that has been sliced to size
+    ///
+    /// It is legal for a list array's offsets to not start with zero.  It's also legal
+    /// for a list array's offsets to not extend to the entire values array.  This function
+    /// behaves similarly to `values()` except it slices the array so that it starts at
+    /// the first list offset and ends at the last list offset.
+    fn trimmed_values(&self) -> Arc<dyn Array>;
+    /// The offset type of the underlying list array.
+    type Offset: OffsetSizeTrait;
+    /// Returns offsets shifted so the first offset is zero, matching
+    /// [`Self::trimmed_values`].
+    ///
+    /// Sliced list arrays (e.g. a filtered batch) keep offsets that reference the
+    /// original values buffer, so combining them with trimmed values produces
+    /// offsets that exceed the values length. Use this together with
+    /// `trimmed_values` when constructing a new list array.
+    fn trimmed_offsets(&self) -> OffsetBuffer<Self::Offset>;
+}
+
+impl<OffsetSize: OffsetSizeTrait> ListArrayExt for GenericListArray<OffsetSize> {
+    fn filter_garbage_nulls(&self) -> Self {
+        if self.is_empty() {
+            return self.clone();
+        }
+        let Some(validity) = self.nulls().cloned() else {
+            return self.clone();
+        };
+
+        let mut should_keep = BooleanBufferBuilder::new(self.values().len());
+
+        // Handle case where offsets do not start at 0
+        let preamble_len = self.offsets().first().unwrap().to_usize().unwrap();
+        should_keep.append_n(preamble_len, false);
+
+        let mut new_offsets: Vec<OffsetSize> = Vec::with_capacity(self.len() + 1);
+        new_offsets.push(OffsetSize::zero());
+        let mut cur_len = OffsetSize::zero();
+        for (offset, is_valid) in self.offsets().windows(2).zip(validity.iter()) {
+            let len = offset[1] - offset[0];
+            if is_valid {
+                cur_len += len;
+                should_keep.append_n(len.to_usize().unwrap(), true);
+                new_offsets.push(cur_len);
+            } else {
+                should_keep.append_n(len.to_usize().unwrap(), false);
+                new_offsets.push(cur_len);
+            }
+        }
+
+        // Offsets may not reference entire values buffer
+        let trailer = self.values().len() - should_keep.len();
+        should_keep.append_n(trailer, false);
+
+        let should_keep = should_keep.finish();
+        let should_keep = BooleanArray::new(should_keep, None);
+        let new_values = arrow_select::filter::filter(self.values(), &should_keep).unwrap();
+        let new_offsets = ScalarBuffer::from(new_offsets);
+        let new_offsets = OffsetBuffer::new(new_offsets);
+
+        Self::new(
+            Arc::new(Field::new(
+                "item",
+                self.value_type(),
+                self.values().is_nullable(),
+            )),
+            new_offsets,
+            new_values,
+            Some(validity),
+        )
+    }
+
+    fn trimmed_values(&self) -> Arc<dyn Array> {
+        let first_value = self
+            .offsets()
+            .first()
+            .map(|v| v.to_usize().unwrap())
+            .unwrap_or(0);
+        let last_value = self
+            .offsets()
+            .last()
+            .map(|v| v.to_usize().unwrap())
+            .unwrap_or(0);
+        self.values().slice(first_value, last_value - first_value)
+    }
+
+    type Offset = OffsetSize;
+
+    fn trimmed_offsets(&self) -> OffsetBuffer<OffsetSize> {
+        let offsets = self.offsets();
+        let Some(&first) = offsets.first() else {
+            return offsets.clone();
+        };
+        if first == OffsetSize::zero() {
+            return offsets.clone();
+        }
+        let shifted: Vec<OffsetSize> = offsets.iter().map(|&o| o - first).collect();
+        OffsetBuffer::new(ScalarBuffer::from(shifted))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{ListArray, UInt64Array};
+    use arrow_buffer::{BooleanBuffer, NullBuffer, OffsetBuffer, ScalarBuffer};
+    use arrow_schema::{DataType, Field};
+
+    use super::ListArrayExt;
+
+    #[test]
+    fn test_filter_garbage_nulls() {
+        let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 8, 9]);
+        let offsets = OffsetBuffer::new(offsets);
+        let list_validity = NullBuffer::new(BooleanBuffer::from(vec![true, false, true]));
+        let list_arr = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, true)),
+            offsets,
+            Arc::new(items),
+            Some(list_validity.clone()),
+        );
+
+        let filtered = list_arr.filter_garbage_nulls();
+
+        let expected_items = UInt64Array::from(vec![2, 3, 4, 8]);
+        let offsets = ScalarBuffer::<i32>::from(vec![0, 3, 3, 4]);
+        let expected = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, false)),
+            OffsetBuffer::new(offsets),
+            Arc::new(expected_items),
+            Some(list_validity),
+        );
+
+        assert_eq!(filtered, expected);
+    }
+
+    #[test]
+    fn test_trim_values() {
+        let items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let offsets = ScalarBuffer::<i32>::from(vec![2, 5, 6, 8, 9]);
+        let offsets = OffsetBuffer::new(offsets);
+        let list_arr = ListArray::new(
+            Arc::new(Field::new("item", DataType::UInt64, true)),
+            offsets,
+            Arc::new(items),
+            None,
+        );
+        let list_arr = list_arr.slice(1, 2);
+
+        let trimmed = list_arr.trimmed_values();
+
+        let expected_items = UInt64Array::from(vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]);
+        let expected_items = expected_items.slice(5, 3);
+
+        assert_eq!(trimmed.as_ref(), &expected_items);
+    }
+}

python/vendor/lance-arrow/src/memory.rs

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashSet;
+
+use arrow_array::{Array, RecordBatch};
+use arrow_data::ArrayData;
+
+/// Counts memory used by buffers of Arrow arrays and RecordBatches.
+///
+/// This is meant to capture how much memory is being used by the Arrow data
+/// structures as they are. It does not represent the memory used if the data
+/// were to be serialized and then deserialized. In particular:
+///
+/// * This does not double count memory used by buffers shared by multiple
+///   arrays or batches. Round-tripped data may use more memory because of this.
+/// * This counts the **total** size of the buffers, even if the array is a slice.
+///   Round-tripped data may use less memory because of this.
+#[derive(Default)]
+pub struct MemoryAccumulator {
+    seen: HashSet<usize>,
+    total: usize,
+}
+
+impl MemoryAccumulator {
+    pub fn record_array(&mut self, array: &dyn Array) {
+        let data = array.to_data();
+        self.record_array_data(&data);
+    }
+
+    fn record_array_data(&mut self, data: &ArrayData) {
+        for buffer in data.buffers() {
+            let ptr = buffer.as_ptr();
+            if self.seen.insert(ptr as usize) {
+                self.total += buffer.capacity();
+            }
+        }
+
+        if let Some(nulls) = data.nulls() {
+            let null_buf = nulls.inner().inner();
+            let ptr = null_buf.as_ptr();
+            if self.seen.insert(ptr as usize) {
+                self.total += null_buf.capacity();
+            }
+        }
+
+        for child in data.child_data() {
+            self.record_array_data(child);
+        }
+    }
+
+    pub fn record_batch(&mut self, batch: &RecordBatch) {
+        for array in batch.columns() {
+            self.record_array(array);
+        }
+    }
+
+    pub fn total(&self) -> usize {
+        self.total
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::Int32Array;
+    use arrow_schema::{DataType, Field, Schema};
+
+    use super::*;
+
+    #[test]
+    fn test_memory_accumulator() {
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])),
+            vec![Arc::new(Int32Array::from(vec![1, 2, 3]))],
+        )
+        .unwrap();
+        let slice = batch.slice(1, 2);
+
+        let mut acc = MemoryAccumulator::default();
+
+        // Should record whole buffer, not just slice
+        acc.record_batch(&slice);
+        assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
+
+        // Should not double count
+        acc.record_batch(&slice);
+        assert_eq!(acc.total(), 3 * std::mem::size_of::<i32>());
+    }
+}

python/vendor/lance-arrow/src/scalar.rs

@@ -0,0 +1,302 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use arrow_array::{ArrayRef, UInt64Array, make_array};
+use arrow_buffer::Buffer;
+use arrow_data::ArrayDataBuilder;
+use arrow_schema::{ArrowError, DataType};
+use arrow_select::take::take;
+
+use crate::DataTypeExt;
+
+type Result<T> = std::result::Result<T, ArrowError>;
+
+pub const INLINE_VALUE_MAX_BYTES: usize = 32;
+
+pub fn extract_scalar_value(array: &ArrayRef, idx: usize) -> Result<ArrayRef> {
+    if idx >= array.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar index out of bounds".to_string(),
+        ));
+    }
+
+    take(array.as_ref(), &UInt64Array::from(vec![idx as u64]), None)
+}
+
+fn read_u32(buf: &[u8], offset: &mut usize) -> Result<u32> {
+    if *offset + 4 > buf.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: unexpected EOF".to_string(),
+        ));
+    }
+    let bytes = [
+        buf[*offset],
+        buf[*offset + 1],
+        buf[*offset + 2],
+        buf[*offset + 3],
+    ];
+    *offset += 4;
+    Ok(u32::from_le_bytes(bytes))
+}
+
+fn read_bytes<'a>(buf: &'a [u8], offset: &mut usize, len: usize) -> Result<&'a [u8]> {
+    if *offset + len > buf.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: unexpected EOF".to_string(),
+        ));
+    }
+    let slice = &buf[*offset..*offset + len];
+    *offset += len;
+    Ok(slice)
+}
+
+fn write_u32(out: &mut Vec<u8>, v: u32) {
+    out.extend_from_slice(&v.to_le_bytes());
+}
+
+fn write_bytes(out: &mut Vec<u8>, bytes: &[u8]) {
+    out.extend_from_slice(bytes);
+}
+
+pub fn encode_scalar_value_buffer(scalar: &ArrayRef) -> Result<Vec<u8>> {
+    if scalar.len() != 1 || scalar.null_count() != 0 {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer must be a single non-null value".to_string(),
+        ));
+    }
+    let data = scalar.to_data();
+    if data.offset() != 0 {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer must have offset=0".to_string(),
+        ));
+    }
+    if !data.child_data().is_empty() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Scalar value buffer does not support nested types".to_string(),
+        ));
+    }
+
+    // Minimal format (RFC): store the Arrow value buffers for a length-1 array.
+    // Null bitmap and child data are intentionally not supported here.
+    //
+    // | u32 num_buffers |
+    // | u32 buffer_0_len | ... | u32 buffer_{n-1}_len |
+    // | buffer_0 bytes | ... | buffer_{n-1} bytes |
+    let mut out = Vec::with_capacity(128);
+    let buffers = data.buffers();
+    write_u32(&mut out, buffers.len() as u32);
+    for b in buffers {
+        write_u32(&mut out, b.len() as u32);
+    }
+    for b in buffers {
+        write_bytes(&mut out, b.as_slice());
+    }
+    Ok(out)
+}
+
+pub fn decode_scalar_from_value_buffer(
+    data_type: &DataType,
+    value_buffer: &[u8],
+) -> Result<ArrayRef> {
+    if matches!(
+        data_type,
+        DataType::Struct(_) | DataType::FixedSizeList(_, _)
+    ) {
+        return Err(ArrowError::InvalidArgumentError(format!(
+            "Scalar value buffer does not support nested data type {:?}",
+            data_type
+        )));
+    }
+
+    let mut offset = 0;
+    let num_buffers = read_u32(value_buffer, &mut offset)? as usize;
+    let buffer_lens = (0..num_buffers)
+        .map(|_| read_u32(value_buffer, &mut offset).map(|l| l as usize))
+        .collect::<Result<Vec<_>>>()?;
+
+    let mut buffers = Vec::with_capacity(num_buffers);
+    for len in buffer_lens {
+        let bytes = read_bytes(value_buffer, &mut offset, len)?;
+        buffers.push(Buffer::from_vec(bytes.to_vec()));
+    }
+
+    if offset != value_buffer.len() {
+        return Err(ArrowError::InvalidArgumentError(
+            "Invalid scalar value buffer: trailing bytes".to_string(),
+        ));
+    }
+
+    let mut builder = ArrayDataBuilder::new(data_type.clone())
+        .len(1)
+        .null_count(0);
+    for b in buffers {
+        builder = builder.add_buffer(b);
+    }
+    Ok(make_array(builder.build()?))
+}
+
+pub fn decode_scalar_from_inline_value(
+    data_type: &DataType,
+    inline_value: &[u8],
+) -> Result<ArrayRef> {
+    // I expect our input to be safe here, but I added some debug_assert_eq statements just in case.
+    // If they are triggered, we may need to change them to return actual errors.
+    //
+    // Boolean values are bit-packed in Arrow and therefore are not "fixed-stride" in bytes.
+    // As a result, `byte_width_opt()` returns `None` for `DataType::Boolean`, even though a
+    // length-1 scalar can be represented inline using a single byte (matching `try_inline_value`).
+    if matches!(data_type, DataType::Boolean) {
+        debug_assert_eq!(
+            inline_value.len(),
+            1,
+            "Invalid boolean inline scalar length (expected 1 byte, got {})",
+            inline_value.len()
+        );
+    } else if let Some(byte_width) = data_type.byte_width_opt() {
+        debug_assert_eq!(
+            inline_value.len(),
+            byte_width,
+            "Inline constant length mismatch for {:?}: expected {} bytes but got {}",
+            data_type,
+            byte_width,
+            inline_value.len()
+        );
+    }
+
+    let data = ArrayDataBuilder::new(data_type.clone())
+        .len(1)
+        .null_count(0)
+        .add_buffer(Buffer::from_vec(inline_value.to_vec()))
+        .build()?;
+    Ok(make_array(data))
+}
+
+pub fn try_inline_value(scalar: &ArrayRef) -> Option<Vec<u8>> {
+    if scalar.null_count() != 0 || scalar.len() != 1 {
+        return None;
+    }
+    let data = scalar.to_data();
+    if !data.child_data().is_empty() {
+        return None;
+    }
+    if data.buffers().len() != 1 {
+        return None;
+    }
+    let bytes = data.buffers()[0].as_slice();
+    if bytes.len() > INLINE_VALUE_MAX_BYTES {
+        return None;
+    }
+    Some(bytes.to_vec())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{
+        BooleanArray, DictionaryArray, FixedSizeBinaryArray, Int8Array, Int32Array, StringArray,
+        cast::AsArray, types::Int8Type,
+    };
+
+    use super::*;
+
+    #[test]
+    fn test_extract_scalar_value() {
+        let array: ArrayRef = Arc::new(Int32Array::from(vec![Some(1), None, Some(3)]));
+        let scalar = extract_scalar_value(&array, 2).unwrap();
+        assert_eq!(scalar.len(), 1);
+        assert_eq!(
+            scalar
+                .as_primitive::<arrow_array::types::Int32Type>()
+                .value(0),
+            3
+        );
+    }
+
+    #[test]
+    fn test_extract_scalar_value_from_full_dictionary() {
+        let values = Arc::new(StringArray::from(
+            (0..=i8::MAX)
+                .map(|value| format!("value-{value}"))
+                .collect::<Vec<_>>(),
+        ));
+        let keys = Int8Array::from((0..=i8::MAX).collect::<Vec<_>>());
+        let array: ArrayRef = Arc::new(DictionaryArray::<Int8Type>::new(keys, values));
+
+        let scalar = extract_scalar_value(&array, i8::MAX as usize).unwrap();
+
+        let scalar = scalar.as_dictionary::<Int8Type>();
+        assert_eq!(scalar.len(), 1);
+        assert_eq!(scalar.key(0), Some(i8::MAX as usize));
+        assert_eq!(scalar.values().len(), i8::MAX as usize + 1);
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_utf8_round_trip() {
+        let scalar: ArrayRef = Arc::new(StringArray::from(vec!["hello"]));
+        let buf = encode_scalar_value_buffer(&scalar).unwrap();
+        let decoded = decode_scalar_from_value_buffer(&DataType::Utf8, &buf).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.null_count(), 0);
+        assert_eq!(decoded.as_string::<i32>().value(0), "hello");
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_fixed_size_binary_round_trip() {
+        let val = vec![0xABu8; 33];
+        let scalar: ArrayRef = Arc::new(
+            FixedSizeBinaryArray::try_from_sparse_iter_with_size(
+                std::iter::once(Some(val.as_slice())),
+                33,
+            )
+            .unwrap(),
+        );
+        let buf = encode_scalar_value_buffer(&scalar).unwrap();
+        let decoded =
+            decode_scalar_from_value_buffer(&DataType::FixedSizeBinary(33), &buf).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.as_fixed_size_binary().value(0), val.as_slice());
+    }
+
+    #[test]
+    fn test_inline_value_boolean_round_trip() {
+        let scalar: ArrayRef = Arc::new(BooleanArray::from_iter([Some(true)]));
+        let inline = try_inline_value(&scalar).unwrap();
+        let decoded = decode_scalar_from_inline_value(&DataType::Boolean, &inline).unwrap();
+        assert_eq!(decoded.len(), 1);
+        assert_eq!(decoded.null_count(), 0);
+        assert!(decoded.as_boolean().value(0));
+    }
+
+    #[test]
+    fn test_scalar_value_buffer_rejects_nested_type() {
+        let field = Arc::new(arrow_schema::Field::new("item", DataType::Int32, false));
+        let list: ArrayRef = Arc::new(arrow_array::FixedSizeListArray::new(
+            field,
+            2,
+            Arc::new(Int32Array::from(vec![1, 2])),
+            None,
+        ));
+        let scalar = list.slice(0, 1);
+        assert!(encode_scalar_value_buffer(&scalar).is_err());
+    }
+
+    #[test]
+    fn test_decode_scalar_from_value_buffer_rejects_nested_type() {
+        let buf = Vec::<u8>::new();
+        let res =
+            decode_scalar_from_value_buffer(&DataType::Struct(arrow_schema::Fields::empty()), &buf);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_decode_scalar_from_value_buffer_trailing_bytes() {
+        // num_buffers = 0, plus an extra byte
+        let mut bytes = Vec::new();
+        bytes.extend_from_slice(&0u32.to_le_bytes());
+        bytes.push(1);
+        let res = decode_scalar_from_value_buffer(&DataType::Int32, &bytes);
+        assert!(res.is_err());
+    }
+}

python/vendor/lance-arrow/src/schema.rs

@@ -0,0 +1,208 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Extension to arrow schema
+
+use arrow_schema::{ArrowError, DataType, Field, FieldRef, Schema};
+
+use crate::{ARROW_EXT_NAME_KEY, BLOB_META_KEY, BLOB_V2_EXT_NAME};
+
+pub enum Indentation {
+    OneLine,
+    MultiLine(u8),
+}
+
+impl Indentation {
+    fn value(&self) -> String {
+        match self {
+            Self::OneLine => "".to_string(),
+            Self::MultiLine(spaces) => " ".repeat(*spaces as usize),
+        }
+    }
+
+    fn deepen(&self) -> Self {
+        match self {
+            Self::OneLine => Self::OneLine,
+            Self::MultiLine(spaces) => Self::MultiLine(spaces + 2),
+        }
+    }
+}
+
+/// Extends the functionality of [arrow_schema::Field].
+pub trait FieldExt {
+    /// Create a compact string representation of the field
+    ///
+    /// This is intended for display purposes and not for serialization
+    fn to_compact_string(&self, indent: Indentation) -> String;
+
+    /// Check if the field is marked as a packed struct
+    fn is_packed_struct(&self) -> bool;
+
+    /// Check if the field is marked as a blob
+    fn is_blob(&self) -> bool;
+
+    /// Check if the field is marked as a blob
+    fn is_blob_v2(&self) -> bool;
+}
+
+impl FieldExt for Field {
+    fn to_compact_string(&self, indent: Indentation) -> String {
+        let mut result = format!("{}: ", self.name().clone());
+        match self.data_type() {
+            DataType::Struct(fields) => {
+                result += "{";
+                result += &indent.value();
+                for (field_idx, field) in fields.iter().enumerate() {
+                    result += field.to_compact_string(indent.deepen()).as_str();
+                    if field_idx < fields.len() - 1 {
+                        result += ",";
+                    }
+                    result += indent.value().as_str();
+                }
+                result += "}";
+            }
+            DataType::List(field)
+            | DataType::LargeList(field)
+            | DataType::ListView(field)
+            | DataType::LargeListView(field) => {
+                result += "[";
+                result += field.to_compact_string(indent.deepen()).as_str();
+                result += "]";
+            }
+            DataType::FixedSizeList(child, dimension) => {
+                result += &format!(
+                    "[{}; {}]",
+                    child.to_compact_string(indent.deepen()),
+                    dimension
+                );
+            }
+            DataType::Dictionary(key_type, value_type) => {
+                result += &value_type.to_string();
+                result += "@";
+                result += &key_type.to_string();
+            }
+            _ => {
+                result += &self.data_type().to_string();
+            }
+        }
+        if self.is_nullable() {
+            result += "?";
+        }
+        result
+    }
+
+    // Check if field has metadata `packed` set to true, this check is case insensitive.
+    fn is_packed_struct(&self) -> bool {
+        let field_metadata = self.metadata();
+        const PACKED_KEYS: [&str; 2] = ["packed", "lance-encoding:packed"];
+        PACKED_KEYS.iter().any(|key| {
+            field_metadata
+                .get(*key)
+                .map(|value| value.eq_ignore_ascii_case("true"))
+                .unwrap_or(false)
+        })
+    }
+
+    fn is_blob(&self) -> bool {
+        let field_metadata = self.metadata();
+        field_metadata.get(BLOB_META_KEY).is_some()
+            || field_metadata
+                .get(ARROW_EXT_NAME_KEY)
+                .map(|value| value == BLOB_V2_EXT_NAME)
+                .unwrap_or(false)
+    }
+
+    fn is_blob_v2(&self) -> bool {
+        let field_metadata = self.metadata();
+        field_metadata
+            .get(ARROW_EXT_NAME_KEY)
+            .map(|value| value == BLOB_V2_EXT_NAME)
+            .unwrap_or(false)
+    }
+}
+
+/// Extends the functionality of [arrow_schema::Schema].
+pub trait SchemaExt {
+    /// Create a new [`Schema`] with one extra field.
+    fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError>;
+
+    fn try_with_column_at(
+        &self,
+        index: usize,
+        field: Field,
+    ) -> std::result::Result<Schema, ArrowError>;
+
+    fn field_names(&self) -> Vec<&String>;
+
+    fn without_column(&self, column_name: &str) -> Schema;
+
+    /// Create a compact string representation of the schema
+    ///
+    /// This is intended for display purposes and not for serialization
+    fn to_compact_string(&self, indent: Indentation) -> String;
+}
+
+impl SchemaExt for Schema {
+    fn try_with_column(&self, field: Field) -> std::result::Result<Schema, ArrowError> {
+        if self.column_with_name(field.name()).is_some() {
+            return Err(ArrowError::SchemaError(format!(
+                "Can not append column {} on schema: {:?}",
+                field.name(),
+                self
+            )));
+        };
+        let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
+        fields.push(FieldRef::new(field));
+        Ok(Self::new_with_metadata(fields, self.metadata.clone()))
+    }
+
+    fn try_with_column_at(
+        &self,
+        index: usize,
+        field: Field,
+    ) -> std::result::Result<Schema, ArrowError> {
+        if self.column_with_name(field.name()).is_some() {
+            return Err(ArrowError::SchemaError(format!(
+                "Failed to modify schema: Inserting column {} would create a duplicate column in schema: {:?}",
+                field.name(),
+                self
+            )));
+        };
+        let mut fields: Vec<FieldRef> = self.fields().iter().cloned().collect();
+        fields.insert(index, FieldRef::new(field));
+        Ok(Self::new_with_metadata(fields, self.metadata.clone()))
+    }
+
+    /// Project the schema to remove the given column.
+    ///
+    /// This only works on top-level fields right now. If a field does not exist,
+    /// the schema will be returned as is.
+    fn without_column(&self, column_name: &str) -> Schema {
+        let fields: Vec<FieldRef> = self
+            .fields()
+            .iter()
+            .filter(|f| f.name() != column_name)
+            .cloned()
+            .collect();
+        Self::new_with_metadata(fields, self.metadata.clone())
+    }
+
+    fn field_names(&self) -> Vec<&String> {
+        self.fields().iter().map(|f| f.name()).collect()
+    }
+
+    fn to_compact_string(&self, indent: Indentation) -> String {
+        let mut result = "{".to_string();
+        result += &indent.value();
+        for (field_idx, field) in self.fields.iter().enumerate() {
+            result += field.to_compact_string(indent.deepen()).as_str();
+            if field_idx < self.fields.len() - 1 {
+                result += ",";
+            }
+            result += indent.value().as_str();
+        }
+        result += "}";
+        result
+    }
+}

python/vendor/lance-arrow/src/stream.rs

@@ -0,0 +1,554 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Utilities for working with streams of [`RecordBatch`].
+
+use arrow_array::RecordBatch;
+use arrow_schema::{ArrowError, SchemaRef};
+use futures::stream::{self, Stream, StreamExt};
+use std::pin::Pin;
+
+use crate::deepcopy::deep_copy_batch_sliced;
+
+/// Rechunks a stream of [`RecordBatch`] so that each output batch has
+/// approximately `target_bytes` of array data.
+///
+/// Small input batches are accumulated (by concatenation) until at least
+/// `min_bytes` of data has been collected. If the resulting batch exceeds
+/// `max_bytes`, it is sliced into roughly equal pieces of ~`max_bytes`
+/// (assuming uniform row sizes).
+pub fn rechunk_stream_by_size<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, false)
+}
+
+/// Like [`rechunk_stream_by_size`] but deep-copies slices so that
+/// `get_array_memory_size` reflects the true size of each output batch.
+///
+/// After a normal `RecordBatch::slice`, the backing buffers are shared with
+/// the original batch, so `get_array_memory_size` still reports the full
+/// parent size.  This variant deep-copies every slice produced during the
+/// splitting phase, which allows the stream to detect and re-split slices
+/// that still exceed `max_bytes` (e.g. because a single row is much larger
+/// than average).
+///
+/// The deep copy is a last resort and potentially expensive for large
+/// batches.  However, it is only performed when a batch actually needs to be
+/// sliced — batches that are already within the target range pass through at
+/// zero cost.  Use this only when the hard cap on `max_bytes` is a
+/// correctness requirement, not merely a performance hint.
+pub fn rechunk_stream_by_size_deep_copy<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    rechunk_stream_by_size_inner(input, input_schema, min_bytes, max_bytes, true)
+}
+
+fn rechunk_stream_by_size_inner<S, E>(
+    input: S,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+    deep_copy: bool,
+) -> impl Stream<Item = Result<RecordBatch, E>>
+where
+    S: Stream<Item = Result<RecordBatch, E>>,
+    E: From<ArrowError>,
+{
+    stream::try_unfold(
+        RechunkState {
+            input: Box::pin(input),
+            accumulated: Vec::new(),
+            acc_bytes: 0,
+            done: false,
+            input_schema,
+            min_bytes,
+            max_bytes,
+            deep_copy,
+        },
+        |mut state| async move {
+            if state.done && state.accumulated.is_empty() {
+                return Ok(None);
+            }
+
+            // Pull batches until we reach the byte target or exhaust input.
+            // Always pull at least one batch so that min_bytes=0 works.
+            while !state.done && (state.accumulated.is_empty() || state.acc_bytes < state.min_bytes)
+            {
+                match state.input.next().await {
+                    Some(Ok(batch)) => {
+                        state.acc_bytes += batch.get_array_memory_size();
+                        state.accumulated.push(batch);
+                    }
+                    Some(Err(e)) => return Err(e),
+                    None => {
+                        state.done = true;
+                    }
+                }
+            }
+
+            if state.accumulated.is_empty() {
+                return Ok(None);
+            }
+
+            // Fast path: if the first accumulated batch already meets the
+            // byte threshold, deliver it directly instead of concatenating
+            // everything together (which would just get sliced back apart).
+            if state.accumulated.len() > 1
+                && state.accumulated[0].get_array_memory_size() >= state.min_bytes
+            {
+                let b = state.accumulated.remove(0);
+                state.acc_bytes -= b.get_array_memory_size();
+                return Ok(Some((b, state)));
+            }
+
+            let batch = if state.accumulated.len() == 1 {
+                state.accumulated.pop().unwrap()
+            } else {
+                let b =
+                    arrow_select::concat::concat_batches(&state.input_schema, &state.accumulated)
+                        .map_err(E::from)?;
+                state.accumulated.clear();
+                b
+            };
+            state.acc_bytes = 0;
+
+            // Slice the batch into ~max_bytes pieces assuming uniform row sizes.
+            let mut slices =
+                slice_batch(batch, state.max_bytes, state.deep_copy).map_err(E::from)?;
+
+            if slices.len() == 1 {
+                Ok(Some((slices.pop().unwrap(), state)))
+            } else {
+                let first = slices.remove(0);
+
+                // Stash leftover slices for subsequent iterations.
+                for a in &slices {
+                    state.acc_bytes += a.get_array_memory_size();
+                }
+                state.accumulated = slices;
+
+                Ok(Some((first, state)))
+            }
+        },
+    )
+}
+
+/// Slice a batch into pieces of at most `max_bytes`.
+///
+/// When `deep_copy` is false, slices share buffers with the original batch
+/// and `get_array_memory_size` will still report the parent buffer size.
+/// This is fine when the caller only needs approximate sizing.
+///
+/// When `deep_copy` is true, each slice is deep-copied so that
+/// `get_array_memory_size` reflects the true size.  If a deep-copied slice
+/// still exceeds `max_bytes` (due to non-uniform row sizes), it is
+/// recursively split until every piece is within budget or contains only a
+/// single row.
+fn slice_batch(
+    batch: RecordBatch,
+    max_bytes: usize,
+    deep_copy: bool,
+) -> Result<Vec<RecordBatch>, ArrowError> {
+    let batch_bytes = batch.get_array_memory_size();
+    let num_rows = batch.num_rows();
+
+    if batch_bytes <= max_bytes {
+        return Ok(vec![batch]);
+    }
+
+    if num_rows <= 1 {
+        // A single row cannot be split further, but the size just measured may
+        // belong to its source buffer rather than to the row:
+        // `get_array_memory_size` reports whole buffers, and a slice shares
+        // them. Returning it uncopied hands the caller that inflated figure --
+        // a 100 KB row inside a 200 MB buffer measures as 200 MB, and a caller
+        // budgeting on it concludes the row cannot fit anywhere.
+        if deep_copy {
+            return Ok(vec![deep_copy_batch_sliced(&batch)?]);
+        }
+        return Ok(vec![batch]);
+    }
+
+    let rows_per_chunk = (max_bytes as u64 * num_rows as u64 / batch_bytes as u64).max(1) as usize;
+
+    let mut result = Vec::new();
+    let mut offset = 0;
+    while offset < num_rows {
+        let len = rows_per_chunk.min(num_rows - offset);
+        let slice = batch.slice(offset, len);
+        if deep_copy {
+            let copied = deep_copy_batch_sliced(&slice)?;
+            // Recurse: the deep-copied slice has accurate sizes, so if it
+            // still exceeds max_bytes we can split further.
+            result.extend(slice_batch(copied, max_bytes, true)?);
+        } else {
+            result.push(slice);
+        }
+        offset += len;
+    }
+
+    Ok(result)
+}
+
+/// Internal state for [`rechunk_stream`].
+///
+/// Kept as a named struct so the `try_unfold` closure stays readable.
+struct RechunkState<S> {
+    input: Pin<Box<S>>,
+    accumulated: Vec<RecordBatch>,
+    acc_bytes: usize,
+    done: bool,
+    input_schema: SchemaRef,
+    min_bytes: usize,
+    max_bytes: usize,
+    deep_copy: bool,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use std::sync::Arc;
+
+    use arrow_array::Int32Array;
+    use arrow_schema::{DataType, Field, Schema};
+    use futures::executor::block_on;
+
+    /// A single row that shares a large source buffer must report its own
+    /// size, not the buffer's. Callers budget on this figure, and an inflated
+    /// one makes a small row look impossible to place.
+    #[test]
+    fn a_single_row_slice_reports_its_own_size() {
+        use arrow_array::LargeStringArray;
+
+        let row = "x".repeat(100 * 1024);
+        let values: Vec<&str> = (0..2000).map(|_| row.as_str()).collect();
+        let schema = Arc::new(Schema::new(vec![Field::new(
+            "text",
+            DataType::LargeUtf8,
+            false,
+        )]));
+        let batch =
+            RecordBatch::try_new(schema, vec![Arc::new(LargeStringArray::from(values))]).unwrap();
+        let whole = batch.get_array_memory_size();
+
+        let one_row = batch.slice(7, 1);
+        assert_eq!(
+            one_row.get_array_memory_size(),
+            whole,
+            "a slice shares its source buffer, which is the reason this test exists"
+        );
+
+        // Budget far below the row so the split recurses to the single-row floor.
+        let chunks = slice_batch(one_row, 1024, true).unwrap();
+        assert_eq!(chunks.len(), 1);
+        let measured = chunks[0].get_array_memory_size();
+        assert_eq!(chunks[0].num_rows(), 1);
+        assert!(
+            measured < whole / 100,
+            "single row still measured as {measured} bytes against a {whole} byte source"
+        );
+    }
+
+    fn make_batch(num_rows: usize) -> RecordBatch {
+        let schema = test_schema();
+        let values: Vec<i32> = (0..num_rows as i32).collect();
+        RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(values))]).unwrap()
+    }
+
+    fn test_schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)]))
+    }
+
+    fn collect_rechunked(
+        batches: Vec<RecordBatch>,
+        min_bytes: usize,
+        max_bytes: usize,
+    ) -> Vec<RecordBatch> {
+        let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
+        let rechunked = rechunk_stream_by_size(input, test_schema(), min_bytes, max_bytes);
+        block_on(rechunked.collect::<Vec<_>>())
+            .into_iter()
+            .map(|r| r.unwrap())
+            .collect()
+    }
+
+    fn total_rows(batches: &[RecordBatch]) -> usize {
+        batches.iter().map(|b| b.num_rows()).sum()
+    }
+
+    #[test]
+    fn test_empty_stream() {
+        let result = collect_rechunked(vec![], 100, 200);
+        assert!(result.is_empty());
+    }
+
+    #[test]
+    fn test_single_batch_passthrough() {
+        let batch = make_batch(100);
+        let bytes = batch.get_array_memory_size();
+        // Batch is between min and max — should pass through as-is.
+        let result = collect_rechunked(vec![batch], bytes / 2, bytes * 2);
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].num_rows(), 100);
+    }
+
+    #[test]
+    fn test_small_batches_concatenated() {
+        let one_batch_bytes = make_batch(10).get_array_memory_size();
+        let batches: Vec<_> = (0..8).map(|_| make_batch(10)).collect();
+        // min = 5 batches worth, max = 10 batches worth.
+        let result = collect_rechunked(batches, one_batch_bytes * 5, one_batch_bytes * 10);
+        assert_eq!(total_rows(&result), 80);
+        // Should have been concatenated into fewer batches than the 8 inputs.
+        assert!(
+            result.len() < 8,
+            "expected fewer output batches, got {}",
+            result.len()
+        );
+    }
+
+    #[test]
+    fn test_large_batch_sliced() {
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
+        assert_eq!(total_rows(&result), 1000);
+        assert!(
+            result.len() >= 4,
+            "expected at least 4 slices, got {}",
+            result.len()
+        );
+    }
+
+    #[test]
+    fn test_sliced_leftovers_are_not_recombined() {
+        // Key test for the fast-path optimisation. When a large batch is
+        // sliced, leftover slices should be delivered one-at-a-time without
+        // being concatenated back together.  We verify this by checking that
+        // every output buffer pointer falls inside the original batch's
+        // allocation (i.e. they are all zero-copy slices, not fresh copies).
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let orig_data = batch.column(0).to_data();
+        let orig_buf = &orig_data.buffers()[0];
+        let orig_start = orig_buf.as_ptr() as usize;
+        let orig_end = orig_start + orig_buf.len();
+
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 4);
+
+        assert_eq!(total_rows(&result), 1000);
+        assert!(result.len() >= 4);
+
+        for (i, b) in result.iter().enumerate() {
+            let ptr = b.column(0).to_data().buffers()[0].as_ptr() as usize;
+            assert!(
+                ptr >= orig_start && ptr < orig_end,
+                "slice {i} buffer at {ptr:#x} is outside the original allocation \
+                 [{orig_start:#x}, {orig_end:#x}) — it was re-concatenated"
+            );
+        }
+    }
+
+    #[test]
+    fn test_flush_remainder_on_stream_end() {
+        // Data below min_bytes should still be flushed when the stream ends.
+        let batch = make_batch(10);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes * 100, bytes * 200);
+        assert_eq!(result.len(), 1);
+        assert_eq!(result[0].num_rows(), 10);
+    }
+
+    #[test]
+    fn test_large_then_small_batches() {
+        // After a large batch is fully drained, subsequent small batches
+        // should be accumulated normally.
+        let large = make_batch(1000);
+        let small_bytes = make_batch(10).get_array_memory_size();
+        let batches = vec![
+            large,
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+            make_batch(10),
+        ];
+        let result = collect_rechunked(batches, small_bytes * 3, small_bytes * 100);
+        assert_eq!(total_rows(&result), 1050);
+        // The large batch should appear (possibly sliced) followed by
+        // concatenated small batches, so we should have fewer output batches
+        // than the 6 inputs.
+        assert!(result.len() < 6);
+    }
+
+    #[test]
+    fn test_row_preservation_across_slicing() {
+        // Verify that every input row appears exactly once in the output
+        // and in the correct order after slicing.
+        let batch = make_batch(237); // odd count to exercise remainder slice
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], bytes / 8, bytes / 5);
+
+        assert_eq!(total_rows(&result), 237);
+
+        let values: Vec<i32> = result
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<Int32Array>()
+                    .unwrap()
+                    .values()
+                    .iter()
+                    .copied()
+            })
+            .collect();
+        let expected: Vec<i32> = (0..237).collect();
+        assert_eq!(values, expected);
+    }
+
+    #[test]
+    fn test_min_bytes_zero_still_yields_all_rows() {
+        // When min_bytes=0, the stream should still yield every batch.
+        // This is the "chop only, don't coalesce" use case.
+        let batches: Vec<_> = (0..5).map(|_| make_batch(100)).collect();
+        let batch_bytes = batches[0].get_array_memory_size();
+        let result = collect_rechunked(batches, 0, batch_bytes * 2);
+        assert_eq!(total_rows(&result), 500);
+    }
+
+    #[test]
+    fn test_min_bytes_zero_slices_oversized() {
+        // min_bytes=0 with a small max_bytes should still slice large batches.
+        let batch = make_batch(1000);
+        let bytes = batch.get_array_memory_size();
+        let result = collect_rechunked(vec![batch], 0, bytes / 4);
+        assert_eq!(total_rows(&result), 1000);
+        assert!(
+            result.len() >= 4,
+            "expected at least 4 slices, got {}",
+            result.len()
+        );
+    }
+
+    /// Build a batch with one variable-length string column.
+    /// Every row is `small_size` bytes except the row at index `big_row_idx`
+    /// which is `big_size` bytes.
+    fn make_variable_batch(
+        num_rows: usize,
+        small_size: usize,
+        big_row_idx: usize,
+        big_size: usize,
+    ) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]));
+        let values: Vec<String> = (0..num_rows)
+            .map(|i| {
+                if i == big_row_idx {
+                    "X".repeat(big_size)
+                } else {
+                    "x".repeat(small_size)
+                }
+            })
+            .collect();
+        let array = arrow_array::StringArray::from(values);
+        RecordBatch::try_new(schema, vec![Arc::new(array)]).unwrap()
+    }
+
+    fn variable_schema() -> SchemaRef {
+        Arc::new(Schema::new(vec![Field::new("s", DataType::Utf8, false)]))
+    }
+
+    fn collect_rechunked_variable(
+        batches: Vec<RecordBatch>,
+        min_bytes: usize,
+        max_bytes: usize,
+    ) -> Vec<RecordBatch> {
+        let input = stream::iter(batches.into_iter().map(Ok::<_, ArrowError>));
+        let rechunked =
+            rechunk_stream_by_size_deep_copy(input, variable_schema(), min_bytes, max_bytes);
+        block_on(rechunked.collect::<Vec<_>>())
+            .into_iter()
+            .map(|r| r.unwrap())
+            .collect()
+    }
+
+    #[test]
+    fn test_oversized_row_at_end() {
+        // 100 rows: 99 small (64 bytes each) + 1 large (100KiB) at the end.
+        let batch = make_variable_batch(100, 64, 99, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_oversized_row_at_start() {
+        // 100 rows: 1 large (100KiB) at the start + 99 small (64 bytes each).
+        let batch = make_variable_batch(100, 64, 0, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_oversized_row_in_middle() {
+        // 100 rows: 1 large (100KiB) in the middle + 99 small (64 bytes each).
+        let batch = make_variable_batch(100, 64, 50, 100 * 1024);
+        let max_bytes = 64 * 1024;
+        let result = collect_rechunked_variable(vec![batch], 0, max_bytes);
+        assert_eq!(total_rows(&result), 100);
+        for (i, b) in result.iter().enumerate() {
+            let size = b.get_array_memory_size();
+            assert!(
+                size <= max_bytes || b.num_rows() == 1,
+                "batch {i} has {size} bytes (max {max_bytes}) and {} rows",
+                b.num_rows()
+            );
+        }
+    }
+
+    #[test]
+    fn test_error_propagation() {
+        let input = stream::iter(vec![
+            Ok(make_batch(10)),
+            Err(ArrowError::ComputeError("boom".into())),
+            Ok(make_batch(10)),
+        ]);
+        let rechunked = rechunk_stream_by_size(input, test_schema(), 1, usize::MAX);
+        let results: Vec<Result<RecordBatch, ArrowError>> = block_on(rechunked.collect());
+        assert!(results.iter().any(|r| r.is_err()));
+    }
+}

python/vendor/lance-arrow/src/struct.rs

@@ -0,0 +1,227 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Extension to arrow struct arrays
+
+use arrow_array::{Array, StructArray, cast::AsArray, make_array};
+use arrow_buffer::NullBuffer;
+use arrow_data::{ArrayData, ArrayDataBuilder};
+use arrow_schema::{ArrowError, DataType};
+
+pub trait StructArrayExt {
+    /// Removes the offset / length of the struct array by pushing it into the children
+    ///
+    /// In arrow-rs when slice is called it recursively slices the children.
+    /// In arrow-cpp when slice is called it just sets the offset/length of
+    ///   the struct array and leaves the children as-is
+    ///
+    /// Both are legal approaches (╥﹏╥)
+    ///
+    /// This method helps reduce complexity by folding into the arrow-rs approach
+    fn normalize_slicing(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized;
+
+    /// Structs are allowed to mask valid items.  For example, a struct array might be:
+    ///
+    /// [ {"items": [1, 2, 3]}, NULL, {"items": NULL}]
+    ///
+    /// However, the underlying items array might be: [[1, 2, 3], [4, 5], NULL]
+    ///
+    /// The [4, 5] list is masked out because the struct array is null.
+    ///
+    /// The struct validity would be [true, false, true] and the list validity would be [true, true, false]
+    ///
+    /// This method pushes nulls down into all children.  In the above example the list validity would become
+    /// [true, false, false].
+    ///
+    /// This method is not recursive.  If a child is a struct array it will not push that child's nulls down.
+    ///
+    /// This method does not remove garbage lists.  It only updates the validity so a future call to
+    /// [crate::list::ListArrayExt::filter_garbage_nulls] will remove the garbage lists (without
+    /// this pushdown it would not)
+    fn pushdown_nulls(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized;
+}
+
+fn normalized_struct_array_data(data: ArrayData) -> Result<ArrayData, ArrowError> {
+    let parent_offset = data.offset();
+    let parent_len = data.len();
+    let modified_children = data
+        .child_data()
+        .iter()
+        .map(|d| {
+            let d = normalized_struct_array_data(d.clone())?;
+            let offset = d.offset();
+            let len = d.len();
+            if len < parent_len + parent_offset {
+                return Err(ArrowError::InvalidArgumentError(format!(
+                    "Child array {} has length {} which is less than the parent length {} plus the parent offset {}",
+                    d.data_type(),
+                    len,
+                    parent_len,
+                    parent_offset
+                )));
+            }
+            let new_offset = offset + parent_offset;
+            d.into_builder().offset(new_offset)
+                .len(parent_len)
+                .build()
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+    ArrayDataBuilder::new(data.data_type().clone())
+        .len(parent_len)
+        .offset(0)
+        .buffers(data.buffers().to_vec())
+        .child_data(modified_children)
+        .build()
+}
+
+impl StructArrayExt for StructArray {
+    fn normalize_slicing(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized,
+    {
+        if self.offset() == 0 && self.columns().iter().all(|c| c.len() == self.len()) {
+            return Ok(self.clone());
+        }
+
+        let data = normalized_struct_array_data(self.to_data())?;
+        Ok(Self::from(data))
+    }
+
+    fn pushdown_nulls(&self) -> Result<Self, ArrowError>
+    where
+        Self: Sized,
+    {
+        let Some(validity) = self.nulls() else {
+            return Ok(self.clone());
+        };
+        let data = self.to_data();
+        let children = data
+            .child_data()
+            .iter()
+            .map(|c| {
+                if c.data_type() == &DataType::Null {
+                    // A Null array is already null in every slot, and arrow-rs rejects
+                    // a validity buffer on it.
+                    Ok(c.clone())
+                } else if let Some(child_validity) = c.nulls() {
+                    let new_validity = child_validity.inner() & validity.inner();
+                    c.clone()
+                        .into_builder()
+                        .nulls(Some(NullBuffer::from(new_validity)))
+                        .build()
+                } else {
+                    Ok(c.clone()
+                        .into_builder()
+                        .nulls(Some(validity.clone()))
+                        .build()?)
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        let arr = make_array(data.into_builder().child_data(children).build()?);
+        Ok(arr.as_struct().clone())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use arrow_array::{
+        Array, Int32Array, NullArray, StructArray, cast::AsArray, make_array, types::Int32Type,
+    };
+    use arrow_buffer::NullBuffer;
+    use arrow_schema::{DataType, Field, Fields};
+    use std::sync::Arc;
+
+    use crate::r#struct::StructArrayExt;
+
+    #[test]
+    fn test_normalize_slicing_no_offset() {
+        let x = Int32Array::from(vec![1, 2, 3]);
+        let y = Int32Array::from(vec![4, 5, 6]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        let normalized = struct_array.normalize_slicing().unwrap();
+        assert_eq!(normalized, struct_array);
+    }
+
+    #[test]
+    fn test_arrow_rs_slicing() {
+        let x = Int32Array::from(vec![1, 2, 3, 4]);
+        let y = Int32Array::from(vec![5, 6, 7, 8]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        // Slicing with arrow-rs propagates the slicing to the children so there should
+        // be no change needed to the struct array
+        let sliced = struct_array.slice(1, 2);
+        let normalized = sliced.normalize_slicing().unwrap();
+
+        assert_eq!(normalized, sliced);
+    }
+
+    #[test]
+    fn test_arrow_cpp_slicing() {
+        let x = Int32Array::from(vec![1, 2, 3, 4]);
+        let y = Int32Array::from(vec![5, 6, 7, 8]);
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("x", DataType::Int32, true),
+                Field::new("y", DataType::Int32, true),
+            ]),
+            vec![Arc::new(x), Arc::new(y)],
+            None,
+        );
+
+        let data = struct_array.to_data();
+        let sliced = data.into_builder().offset(1).len(2).build().unwrap();
+        let sliced = make_array(sliced);
+        let normalized = sliced.as_struct().clone().normalize_slicing().unwrap();
+
+        assert_eq!(normalized, struct_array.slice(1, 2));
+    }
+
+    #[test]
+    fn test_pushdown_nulls_with_null_typed_child() {
+        // `pa.Table.from_pylist([{"s": {"a": 1, "b": None}}, {"s": None}])` infers a
+        // struct with a Null-typed child. Arrow forbids a validity buffer on a Null
+        // array, so the struct's nulls must not be pushed into it.
+        let struct_array = StructArray::new(
+            Fields::from(vec![
+                Field::new("a", DataType::Int32, true),
+                Field::new("b", DataType::Null, true),
+            ]),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2, 3])),
+                Arc::new(NullArray::new(3)),
+            ],
+            Some(NullBuffer::from(vec![true, false, true])),
+        );
+
+        let pushed = struct_array.pushdown_nulls().unwrap();
+
+        assert_eq!(pushed.nulls(), struct_array.nulls());
+        assert_eq!(
+            pushed.column(0).as_primitive::<Int32Type>(),
+            &Int32Array::from(vec![Some(1), None, Some(3)])
+        );
+        assert_eq!(pushed.column(1).data_type(), &DataType::Null);
+        assert_eq!(pushed.column(1).logical_null_count(), 3);
+    }
+}