From 22165860231231d738c246ccd4d0ef46f5baedd8 Mon Sep 17 00:00:00 2001
From: Dan Rammer <dan@rammer.me>
Date: Thu, 24 Sep 2026 22:02:04 -0500
Subject: [PATCH 01/11] feat(query): dedup hybrid search by primary key on
 MemWAL tables (#4268)

Implementing a PK-based dedup to support WAL in hybrid search.
Basically, the existing code just calls a vector query + FTS query and
dedups locally using the row ID. Since row ID is unsupported in WAL
(currently), we update the algorithm to use the tables PK (which is
required in WAL, and uniquely linked to a PK on updates). To enable
PK-dedup branch, we've chosen to fallback on error rather than fire an
exploratory query. Specifically, we issue the regular hybrid query,
using row IDs, and when that fails with "WAL does not support row IDs"
we fallback to the PK-based variant, and then cache the fallback for
future calls from the same client. The alternative is that we chec if
WAL is enabled before issuing the query. This would require a
`get_lsm_write_spec` call, slowing down all the existing non-WAL paths
unnecessarily.

The ultimate goal is to get row ID supported in WAL, but that is pending
stable row ID GA and subsequent integration into the enterprise stack.
Once that lands, we will rip out the PK-based dedup that is only for
WAL-enabled tables and use the existing row ID code. This is the main
reason for building this adaptively. I had spec-ed out a refactor so
that rather than duplicating code between python and rust we build this
in rust and have python call that through PyO3. However, that made the
reverse path (ie. ) more difficult. So I think the tradeoff of
maintaining duplicate code in the short term for the ease of reverting
this in the medium term (hopefully 2-3 months).

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
---
 python/python/lancedb/_lancedb.pyi     |   3 +
 python/python/lancedb/_wal_hybrid.py   | 282 +++++++++++++
 python/python/lancedb/query.py         | 130 +++++-
 python/python/lancedb/remote/table.py  |   8 +
 python/python/lancedb/table.py         |  21 +
 python/python/tests/test_wal_hybrid.py | 219 +++++++++++
 python/src/table.rs                    |  15 +
 rust/lancedb/src/query.rs              |  44 ++-
 rust/lancedb/src/query/wal_fusion.rs   | 523 +++++++++++++++++++++++++
 rust/lancedb/src/remote/table.rs       | 292 ++++++++++++++
 rust/lancedb/src/table.rs              |  25 ++
 11 files changed, 1538 insertions(+), 24 deletions(-)
 create mode 100644 python/python/lancedb/_wal_hybrid.py
 create mode 100644 python/python/tests/test_wal_hybrid.py
 create mode 100644 rust/lancedb/src/query/wal_fusion.rs

diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index 337b736ce..1f26ad6f4 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -514,6 +514,9 @@ class Table:
     async def set_lsm_write_spec(self, spec: LsmWriteSpec) -> None: ...
     async def unset_lsm_write_spec(self) -> None: ...
     async def get_lsm_write_spec(self) -> Optional[LsmWriteSpec]: ...
+    # WAL-PK-FUSION: delete both.
+    def hybrid_pk_fusion_learned(self) -> bool: ...
+    def note_hybrid_pk_fusion(self) -> None: ...
     async def checkpoint_lsm(self) -> None: ...
     async def flush_lsm(self) -> None: ...
     async def compact_lsm(self) -> None: ...
diff --git a/python/python/lancedb/_wal_hybrid.py b/python/python/lancedb/_wal_hybrid.py
new file mode 100644
index 000000000..67597950d
--- /dev/null
+++ b/python/python/lancedb/_wal_hybrid.py
@@ -0,0 +1,282 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables.
+
+Hybrid search joins its vector and full-text legs on `_rowid`, which a MemWAL
+table cannot supply: the fresh tier has no stable row id, so the server rejects
+`with_row_id` before it plans. When that happens the legs are joined on a
+surrogate `_rowid` built from the table's primary key instead.
+
+This is a stopgap until MemWAL supports `_rowid`, and everything it needs lives
+here so it can be removed in one pass:
+
+1. Delete this file and `tests/test_wal_hybrid.py`.
+2. Run `grep -rn WAL-PK-FUSION python/ rust/` and follow the note at each
+   marker; the Rust side has its own copy in
+   `rust/lancedb/src/query/wal_fusion.rs`.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Awaitable, Callable, List, Optional, Tuple, TypeVar
+
+import pyarrow as pa
+
+from ._blob import blob_v2_projection_sources
+
+if TYPE_CHECKING:
+    from .types import QueryProjection
+
+T = TypeVar("T")
+
+
+# Lance records the unenforced primary key as field metadata, which survives the
+# `describe` round trip — so a remote table can resolve the key without opening
+# the dataset. Keys are bytes on a pyarrow field.
+_PK_MARKER = b"lance-schema:unenforced-primary-key"
+_PK_POSITION_MARKER = b"lance-schema:unenforced-primary-key:position"
+_TRUTHY = {"1", "true", "on", "yes", "y"}
+
+
+def _pk_columns(schema: pa.Schema) -> List[str]:
+    """The unenforced primary key columns of `schema`, in key order.
+
+    Mirrors Lance's ordering: fields with an explicit 1-based position first, in
+    position order, then fields carrying only the legacy boolean marker, in
+    declaration order.
+    """
+    keyed = []
+    for index, field in enumerate(schema):
+        metadata = field.metadata or {}
+        position = metadata.get(_PK_POSITION_MARKER)
+        if position is not None:
+            try:
+                position = int(position)
+            except ValueError:
+                position = None
+        if position is None:
+            legacy = metadata.get(_PK_MARKER)
+            if legacy is None or legacy.decode().lower() not in _TRUTHY:
+                continue
+            position = 0
+        keyed.append((position == 0, position, index, field.name))
+    return [name for *_, name in sorted(keyed)]
+
+
+# Both layers raise this same sentence verbatim -- sophon's
+# `LSM_WITH_ROW_ID_UNSUPPORTED` says so explicitly, and the OSS LSM scanner
+# builds it from the same words -- so this substring survives either path and
+# the wrapping each adds. It is a coupling to a message rather than a code,
+# because the refusal arrives as a generic 400; if the server ever rewords it
+# the fallback stops firing and hybrid on a MemWAL table errors visibly again,
+# which is loud rather than silently wrong.
+_WAL_ROW_ID_REFUSAL = "does not support with_row_id"
+
+WAL_ROW_ID_UNSUPPORTED = (
+    "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has "
+    "no stable row id, and the ids the fusion joins on are synthesized from "
+    "the primary key. Set use_lsm(False) to read the base table only (results "
+    "will exclude un-compacted MemWAL data)"
+)
+
+
+def _is_wal_row_id_refusal(error: BaseException) -> bool:
+    """Whether `error` is the server declining `_rowid` because the table is
+    MemWAL-backed, as opposed to any other failure the query might hit."""
+    return _WAL_ROW_ID_REFUSAL in str(error)
+
+
+def _with_pk_columns(columns, pk_columns: List[str]):
+    """Add the key columns a projection is missing.
+
+    Returns the projection to use and the columns added, which the caller drops
+    from the output again so an injected key never reaches a caller who did not
+    select it.
+    """
+    if not columns:
+        # Every non-system column, the key among them.
+        return columns, []
+    # `PyQueryRequest.select` reports a dynamic projection as (name, sql) pairs.
+    if isinstance(columns, (list, tuple)) and isinstance(columns[0], (list, tuple)):
+        columns = dict(columns)
+    missing = [c for c in pk_columns if c not in columns]
+    if isinstance(columns, dict):
+        return {**columns, **{c: c for c in missing}}, missing
+    return list(columns) + missing, missing
+
+
+def _stamp_surrogate_row_ids(
+    tables: List[pa.Table], pk_columns: List[str]
+) -> List[pa.Table]:
+    """Give each table a `_rowid` column derived from its primary key.
+
+    The fusion only ever compares that column for equality — to deduplicate, to
+    group by in a reranker, and to restore the original scores — so on a MemWAL
+    table, where the server has no stable row id to hand out, any injective
+    mapping of the primary key does the same job. Ids are assigned in first-seen
+    order across `tables`, so passing them in fusion order (vector, then FTS)
+    preserves "first occurrence wins" and its tie break.
+
+    Writing it under the `_rowid` name is what lets every reranker, third-party
+    implementations included, run unmodified. The column never reaches the
+    caller.
+    """
+    if not pk_columns:
+        raise ValueError(
+            "hybrid search on a MemWAL table needs an unenforced primary key to "
+            "deduplicate on, and this table declares none"
+        )
+
+    ids = {}
+    stamped = []
+    for table in tables:
+        if table.num_rows == 0:
+            # Both legs empty: the combined schema carries no key column, and no
+            # row that would need one.
+            row_ids = pa.array([], type=pa.uint64())
+        else:
+            missing = [c for c in pk_columns if c not in table.column_names]
+            if missing:
+                raise ValueError(
+                    "hybrid search could not deduplicate: primary key "
+                    f"column(s) {missing} missing from a result set with "
+                    f"{table.num_rows} rows"
+                )
+            keys = zip(*(table.column(c).to_pylist() for c in pk_columns))
+            row_ids = pa.array(
+                [ids.setdefault(key, len(ids)) for key in keys], type=pa.uint64()
+            )
+        stamped.append(table.append_column("_rowid", row_ids))
+    return stamped
+
+
+def pk_fusion_learned(table, use_lsm: Optional[bool]) -> bool:
+    """Whether a previous hybrid query on `table` was refused `_rowid`, so this
+    one should go straight to the primary key.
+
+    Learned from a refusal, never probed, so asking is free.
+    """
+    if table is None or use_lsm is False:
+        return False
+    # Only an unambiguous True forks to the key: a table type that answers the
+    # question loosely keeps the `_rowid` path.
+    return table._hybrid_pk_fusion_learned() is True
+
+
+class PkFusion:
+    """How the legs of a hybrid query on a MemWAL table are joined: on a
+    surrogate `_rowid` built from the primary key."""
+
+    def __init__(self, pk_columns: List[str]):
+        self.pk_columns = pk_columns
+        # Key columns the caller did not select, added to the projection so the
+        # surrogate can be built and dropped again before returning.
+        self.injected: List[str] = []
+
+    @classmethod
+    def for_query(
+        cls,
+        schema: pa.Schema,
+        projection: QueryProjection,
+        *,
+        caller_requested_row_id: bool,
+    ) -> PkFusion:
+        """Resolve the key for a hybrid query, refusing the shapes that need a
+        real `_rowid`."""
+        if caller_requested_row_id:
+            raise NotImplementedError(WAL_ROW_ID_UNSUPPORTED)
+        # A projected blob needs a real row id to fetch through, whatever else
+        # has turned `_with_row_id` on — including `return_score="all"`.
+        if blob_v2_projection_sources(schema, projection):
+            raise NotImplementedError(
+                "hybrid search cannot project a blob column on a MemWAL table: "
+                "fetching blobs needs a real _rowid, and the fresh tier has no "
+                "stable one. Set use_lsm(False) to read the base table only "
+                "(results will exclude un-compacted MemWAL data)"
+            )
+        return cls(_pk_columns(schema))
+
+    def inject(self, columns):
+        """`columns` with the key columns it is missing, to project in place of
+        the `_rowid` a MemWAL table would refuse."""
+        columns, self.injected = _with_pk_columns(columns, self.pk_columns)
+        return columns
+
+    def stamp(
+        self, vector_results: pa.Table, fts_results: pa.Table
+    ) -> Tuple[pa.Table, pa.Table]:
+        """Give both legs a surrogate `_rowid` built from the key."""
+        # Vector first: `merge_results` concatenates in that order and keeps
+        # the first occurrence, so first-seen ids preserve its tie break.
+        vector_results, fts_results = _stamp_surrogate_row_ids(
+            [vector_results, fts_results], self.pk_columns
+        )
+        return vector_results, fts_results
+
+    def strip(self, results: pa.Table) -> pa.Table:
+        """Drop the surrogate and any injected key columns from the reranked
+        results.
+
+        The surrogate is an internal join key, never an answer, and a caller
+        who asked for `_rowid` was refused in `for_query`.
+        """
+        drop = [c for c in ["_rowid", *self.injected] if c in results.column_names]
+        return results.drop(drop) if drop else results
+
+
+def _check_refusal(error: Exception, *, fused: bool, caller_requested_row_id: bool):
+    """Re-raise `error` unless it is a MemWAL `_rowid` refusal worth retrying on
+    the primary key."""
+    if fused or not _is_wal_row_id_refusal(error):
+        raise error
+    # `return_score="all"` turns row ids on for the reranker, not the caller, so
+    # it still falls back; only a caller who asked for them is refused, and with
+    # the reason rather than a bare 400.
+    if caller_requested_row_id:
+        raise NotImplementedError(WAL_ROW_ID_UNSUPPORTED) from error
+
+
+def with_pk_fallback(
+    run: Callable[[], T],
+    table,
+    *,
+    fused: Callable[[], bool],
+    caller_requested_row_id: bool,
+) -> T:
+    """Run a hybrid query, retrying on the primary key if the table refuses
+    `_rowid`.
+
+    `_rowid` is asked for first rather than paying a round trip up front to
+    find out: only a MemWAL table refuses, and the refusal has to be handled
+    anyway -- a write spec installed elsewhere can arrive between any two
+    queries. It costs nothing server-side: it is raised before the query is
+    planned. `fused` reports whether the attempt already used the key.
+    """
+    try:
+        return run()
+    except Exception as e:
+        _check_refusal(
+            e, fused=fused(), caller_requested_row_id=caller_requested_row_id
+        )
+        # Learned, so this retry and later queries go straight to the key.
+        table._note_hybrid_pk_fusion()
+        return run()
+
+
+async def with_pk_fallback_async(
+    run: Callable[[], Awaitable[T]],
+    table,
+    *,
+    fused: Callable[[], bool],
+    caller_requested_row_id: bool,
+) -> T:
+    """The async form of `with_pk_fallback`."""
+    try:
+        return await run()
+    except Exception as e:
+        _check_refusal(
+            e, fused=fused(), caller_requested_row_id=caller_requested_row_id
+        )
+        table._note_hybrid_pk_fusion()
+        return await run()
diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py
index cef89e63c..c42344ad3 100644
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -41,6 +41,7 @@ from .rerankers.rrf import RRFReranker
 from .rerankers.util import check_reranker_result
 from .schema import is_blob_like_field, schema_has_blob_field
 from .util import flatten_columns
+from . import _wal_hybrid  # WAL-PK-FUSION: delete.
 from ._blob import (
     BLOB_MODE_TO_HANDLING,
     FetchBlobsAsync,
@@ -2207,6 +2208,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         self._phrase_query = None
         self._lower_bound = None
         self._upper_bound = None
+        # WAL-PK-FUSION: delete both, and the `with_row_id` override below.
+        # Set by `_create_query_builders` when the legs are fused on the key.
+        self._pk_fusion: Optional[_wal_hybrid.PkFusion] = None
+        # `_with_row_id` is also turned on by `rerank(return_score="all")` for
+        # the reranker's own use, so it cannot answer "did the caller ask?".
+        self._caller_requested_row_id = False
 
     def _validate_query(self, query, vector=None, text=None):
         if query is not None and (vector is not None or text is not None):
@@ -2228,6 +2235,16 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
 
         return vector_query, text_query
 
+    def with_row_id(self, with_row_id: bool) -> Self:  # WAL-PK-FUSION: delete.
+        """Set whether to return row ids.
+
+        Recorded separately from `_with_row_id`, which `rerank(return_score=
+        "all")` also sets for its own use — on a MemWAL table only a caller who
+        asked is refused, and the reranker still falls back to the primary key.
+        """
+        self._caller_requested_row_id = with_row_id
+        return super().with_row_id(with_row_id)
+
     def phrase_query(self, phrase_query: bool = True) -> LanceHybridQueryBuilder:
         """Set whether to use phrase query.
 
@@ -2248,16 +2265,30 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         raise NotImplementedError("to_query_object not yet supported on a hybrid query")
 
     def to_arrow(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
+        # WAL-PK-FUSION: without the fallback, this body is `_run_hybrid`'s.
+        return _wal_hybrid.with_pk_fallback(
+            lambda: self._run_hybrid(timeout=timeout),
+            self._table,
+            fused=lambda: self._pk_fusion is not None,
+            caller_requested_row_id=self._caller_requested_row_id,
+        )
+
+    def _run_hybrid(self, *, timeout: Optional[timedelta] = None) -> pa.Table:
         self._create_query_builders()
+        fts_query, vector_query = self._fts_query, self._vector_query
+        # WAL-PK-FUSION: without the fallback, both legs always ask for row ids.
+        if self._pk_fusion is None:
+            fts_query = fts_query.with_row_id(True)
+            vector_query = vector_query.with_row_id(True)
         with ThreadPoolExecutor() as executor:
-            fts_future = executor.submit(
-                self._fts_query.with_row_id(True).to_arrow, timeout=timeout
-            )
-            vector_future = executor.submit(
-                self._vector_query.with_row_id(True).to_arrow, timeout=timeout
-            )
+            fts_future = executor.submit(fts_query.to_arrow, timeout=timeout)
+            vector_future = executor.submit(vector_query.to_arrow, timeout=timeout)
             fts_results = fts_future.result()
             vector_results = vector_future.result()
+        if self._pk_fusion is not None:  # WAL-PK-FUSION: delete.
+            vector_results, fts_results = self._pk_fusion.stamp(
+                vector_results, fts_results
+            )
 
         results = self._combine_hybrid_results(
             fts_results=fts_results,
@@ -2272,6 +2303,8 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         return self._finish_hybrid_results(results)
 
     def _finish_hybrid_results(self, results: pa.Table) -> pa.Table:
+        if self._pk_fusion is not None:  # WAL-PK-FUSION: delete.
+            return self._pk_fusion.strip(results)
         if self._user_requested_row_id():
             return results
         if self._blob_auto_row_id_enabled():
@@ -2445,7 +2478,10 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         self._norm = normalize
         self._reranker = reranker
         if reranker.score == "all":
-            self.with_row_id(True)
+            # WAL-PK-FUSION: without the fallback, `self.with_row_id(True)`.
+            # Not through `with_row_id`, which records caller intent: the
+            # reranker's need for row ids must not refuse a MemWAL table.
+            self._with_row_id = True
 
         return self
 
@@ -2719,13 +2755,24 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             sub_query_limit = self._limit + (self._offset or 0)
             self._vector_query.limit(sub_query_limit)
             self._fts_query.limit(sub_query_limit)
-        if self._columns:
-            self._vector_query.select(self._columns)
-            self._fts_query.select(self._columns)
+        # WAL-PK-FUSION: without the fallback, select `self._columns` as is.
+        self._pk_fusion = None
+        columns = self._columns
+        if _wal_hybrid.pk_fusion_learned(self._table, self._use_lsm):
+            self._pk_fusion = _wal_hybrid.PkFusion.for_query(
+                self._table.schema,
+                self._columns,
+                caller_requested_row_id=self._caller_requested_row_id,
+            )
+            columns = self._pk_fusion.inject(columns)
+        if columns:
+            self._vector_query.select(columns)
+            self._fts_query.select(columns)
         if self._where:
             self._vector_query.where(self._where, not self._postfilter)
             self._fts_query.where(self._where, not self._postfilter)
-        if self._with_row_id:
+        # WAL-PK-FUSION: without the fallback, `if self._with_row_id:`.
+        if self._with_row_id and self._pk_fusion is None:
             self._vector_query.with_row_id(True)
             self._fts_query.with_row_id(True)
         if self._use_lsm is not None:
@@ -3866,6 +3913,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         self._inner = inner
         self._norm = "score"
         self._reranker = RRFReranker()
+        # WAL-PK-FUSION: delete.
+        # Set by `_create_child_queries` when the legs are fused on the key.
+        self._pk_fusion: Optional[_wal_hybrid.PkFusion] = None
 
     def rerank(
         self, reranker: Reranker = RRFReranker(), normalize: str = "score"
@@ -3897,7 +3947,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
 
         return self
 
-    def _create_child_queries(
+    async def _create_child_queries(
         self,
     ) -> Tuple["AsyncFTSQuery", "AsyncVectorQuery", int, int]:
         """Build the sub-queries that make up this hybrid query.
@@ -3906,7 +3956,8 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         the plans that are reported are the plans that actually run.
 
         Returns the two sub-queries along with the effective limit and offset of
-        the hybrid query itself.
+        the hybrid query itself. The key the legs are fused on, if any, is left
+        on `self._pk_fusion`.
         """
         fts_query = AsyncFTSQuery(self._inner.to_fts_query(), self._table)
         vec_query = AsyncVectorQuery(self._inner.to_vector_query(), self._table)
@@ -3924,8 +3975,27 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
             limit = DEFAULT_HYBRID_LIMIT
         offset = fts_req.offset or vec_req.offset or 0
 
-        fts_query.with_row_id()
-        vec_query.with_row_id()
+        # WAL-PK-FUSION: without the fallback, keep only the two `with_row_id`
+        # calls.
+        self._pk_fusion = None
+        if _wal_hybrid.pk_fusion_learned(self._table, fts_req.use_lsm):
+            self._pk_fusion = _wal_hybrid.PkFusion.for_query(
+                await self._table.schema(),
+                _query_request_projection(fts_req),
+                caller_requested_row_id=self._user_requested_row_id(),
+            )
+        if self._pk_fusion is None:
+            fts_query.with_row_id()
+            vec_query.with_row_id()
+        else:
+            # Ask for the key columns instead, and never for `_rowid`.
+            # `select` carries the whole projection; `select_source_columns`
+            # keeps only plain column references, so rebuilding from it would
+            # drop computed ones.
+            columns = self._pk_fusion.inject(fts_req.select)
+            if self._pk_fusion.injected:
+                fts_query.select(columns)
+                vec_query.select(columns)
 
         # offset() pushes the offset down into both sub-queries, which would make
         # each of them skip its own first `offset` rows. The window has to be
@@ -3944,7 +4014,23 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         max_batch_length: Optional[int] = None,
         timeout: Optional[timedelta] = None,
     ) -> AsyncRecordBatchReader:
-        fts_query, vec_query, limit, offset = self._create_child_queries()
+        # WAL-PK-FUSION: without the fallback, this body is `_run_hybrid`'s.
+        return await _wal_hybrid.with_pk_fallback_async(
+            lambda: self._run_hybrid(
+                max_batch_length=max_batch_length, timeout=timeout
+            ),
+            self._table,
+            fused=lambda: self._pk_fusion is not None,
+            caller_requested_row_id=self._user_requested_row_id(),
+        )
+
+    async def _run_hybrid(
+        self,
+        *,
+        max_batch_length: Optional[int] = None,
+        timeout: Optional[timedelta] = None,
+    ) -> AsyncRecordBatchReader:
+        fts_query, vec_query, limit, offset = await self._create_child_queries()
 
         req = fts_query._inner.to_query_request()
         blob_auto_row_id = False
@@ -3968,6 +4054,10 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
             fts_query.to_arrow(timeout=timeout),
             vec_query.to_arrow(timeout=timeout),
         )
+        if self._pk_fusion is not None:  # WAL-PK-FUSION: delete.
+            vector_results, fts_results = self._pk_fusion.stamp(
+                vector_results, fts_results
+            )
 
         result = LanceHybridQueryBuilder._combine_hybrid_results(
             fts_results=fts_results,
@@ -3979,7 +4069,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
             with_row_ids=True,
             offset=offset,
         )
-        if (
+        if self._pk_fusion is not None:  # WAL-PK-FUSION: delete this branch.
+            result = self._pk_fusion.strip(result)
+        elif (
             not self._user_requested_row_id()
             and not blob_auto_row_id
             and "_rowid" in result.column_names
@@ -4028,7 +4120,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         plan : str
         """  # noqa: E501
 
-        fts_query, vec_query, _, _ = self._create_child_queries()
+        fts_query, vec_query, _, _ = await self._create_child_queries()
         vector_plan = await vec_query.explain_plan(verbose)
         fts_plan = await fts_query.explain_plan(verbose)
         # Indent sub-plans under the reranker
@@ -4057,7 +4149,7 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
         -------
         plan : str
         """
-        fts_query, vec_query, _, _ = self._create_child_queries()
+        fts_query, vec_query, _, _ = await self._create_child_queries()
 
         results = ["Vector Search Query:"]
         results.append(await vec_query.analyze_plan(distributed_metrics))
diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py
index 2a6512fc5..44d16211c 100644
--- a/python/python/lancedb/remote/table.py
+++ b/python/python/lancedb/remote/table.py
@@ -1036,6 +1036,14 @@ class RemoteTable(Table):
         """Read the installed LsmWriteSpec, or ``None``."""
         return LOOP.run(self._table.get_lsm_write_spec())
 
+    # WAL-PK-FUSION: delete both hooks.
+    def _hybrid_pk_fusion_learned(self) -> bool:
+        # `self._table` is the async table, which owns the Rust handle.
+        return self._table._hybrid_pk_fusion_learned()
+
+    def _note_hybrid_pk_fusion(self) -> None:
+        self._table._note_hybrid_pk_fusion()
+
     def checkpoint_lsm(self) -> None:
         """Synchronous version of
         [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index d87012bcf..0dc927b29 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -2647,6 +2647,18 @@ class Table(ABC):
         to check if the table is already using the new path style.
         """
 
+    # WAL-PK-FUSION: delete both hooks, here and on AsyncTable and RemoteTable.
+    def _hybrid_pk_fusion_learned(self) -> bool:
+        """Whether a hybrid query here has already been refused ``_rowid``.
+
+        Learned from a refusal, never probed, so asking is free. ``False`` for
+        table types that never refuse.
+        """
+        return False
+
+    def _note_hybrid_pk_fusion(self) -> None:
+        """Remember a ``_rowid`` refusal, so later hybrid queries skip it."""
+
 
 class LanceTable(Table):
     """
@@ -5294,6 +5306,15 @@ class AsyncTable:
         """
         return await self._inner.get_lsm_write_spec()
 
+    # WAL-PK-FUSION: delete both hooks.
+    def _hybrid_pk_fusion_learned(self) -> bool:
+        """See [`Table._hybrid_pk_fusion_learned`][lancedb.table.Table]."""
+        return self._inner.hybrid_pk_fusion_learned()
+
+    def _note_hybrid_pk_fusion(self) -> None:
+        """See [`Table._note_hybrid_pk_fusion`][lancedb.table.Table]."""
+        self._inner.note_hybrid_pk_fusion()
+
     async def checkpoint_lsm(self) -> None:
         """Converge this table's LSM write path into its base table.
 
diff --git a/python/python/tests/test_wal_hybrid.py b/python/python/tests/test_wal_hybrid.py
new file mode 100644
index 000000000..11a9755e0
--- /dev/null
+++ b/python/python/tests/test_wal_hybrid.py
@@ -0,0 +1,219 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+"""WAL-PK-FUSION: delete this file with `lancedb/_wal_hybrid.py`."""
+
+from unittest import mock
+
+import lancedb
+import pyarrow as pa
+import pytest
+from lancedb._wal_hybrid import WAL_ROW_ID_UNSUPPORTED, with_pk_fallback
+from lancedb.query import LanceHybridQueryBuilder
+from lancedb.rerankers import RRFReranker
+
+
+def _wal_table(schema: pa.Schema) -> mock.Mock:
+    """A MemWAL-backed table a previous query has already been refused
+    `_rowid` on, so the fusion goes straight to the primary key."""
+    table = mock.Mock()
+    table._hybrid_pk_fusion_learned.return_value = True
+    table.schema = schema
+    return table
+
+
+_PK_SCHEMA = pa.schema(
+    [
+        pa.field(
+            "id",
+            pa.string(),
+            metadata={b"lance-schema:unenforced-primary-key:position": b"1"},
+        ),
+        pa.field("text", pa.string()),
+    ]
+)
+
+
+def _wal_hybrid(**select):
+    builder = (
+        LanceHybridQueryBuilder(_wal_table(_PK_SCHEMA)).vector([0.1, 0.2]).text("puppy")
+    )
+    if "columns" in select:
+        builder = builder.select(select["columns"])
+    return builder
+
+
+def test_wal_hybrid_projects_the_key_instead_of_asking_for_row_ids():
+    builder = _wal_hybrid(columns=["text"])
+    builder._create_query_builders()
+
+    assert builder._pk_fusion.pk_columns == ["id"]
+    assert builder._pk_fusion.injected == ["id"]
+    for leg in (builder._vector_query, builder._fts_query):
+        assert leg._columns == ["text", "id"]
+        # A MemWAL table rejects with_row_id before it plans.
+        assert leg._with_row_id is not True
+
+
+def test_wal_hybrid_selecting_the_key_injects_nothing():
+    builder = _wal_hybrid(columns=["text", "id"])
+    builder._create_query_builders()
+    assert builder._pk_fusion.injected == []
+
+
+def test_wal_hybrid_without_a_projection_leaves_it_alone():
+    builder = _wal_hybrid()
+    builder._create_query_builders()
+    # Select-all already returns the key.
+    assert builder._pk_fusion.injected == []
+    assert builder._vector_query._columns is None
+
+
+def test_base_table_hybrid_still_joins_on_row_ids():
+    table = mock.Mock()
+    table._hybrid_pk_fusion_learned.return_value = False
+    builder = LanceHybridQueryBuilder(table).vector([0.1, 0.2]).text("puppy")
+    builder._create_query_builders()
+
+    # `to_arrow` is what turns row ids on for this path; all that matters here
+    # is that no key was resolved and no projection was touched.
+    assert builder._pk_fusion is None
+    assert builder._vector_query._columns is None
+
+
+def test_wal_hybrid_with_use_lsm_false_joins_on_row_ids():
+    builder = _wal_hybrid().use_lsm(False)
+    builder._create_query_builders()
+    assert builder._pk_fusion is None
+
+
+def test_wal_hybrid_refuses_an_explicit_with_row_id():
+    builder = _wal_hybrid().with_row_id(True)
+    with pytest.raises(NotImplementedError, match="use_lsm"):
+        builder._create_query_builders()
+
+
+def test_wal_hybrid_return_score_all_is_not_refused():
+    """`return_score="all"` turns row ids on for the reranker, not the caller."""
+    builder = _wal_hybrid().rerank(RRFReranker(return_score="all"))
+    builder._create_query_builders()
+    assert builder._pk_fusion.pk_columns == ["id"]
+
+
+def test_wal_hybrid_fuses_on_the_key_and_drops_it_again():
+    builder = _wal_hybrid(columns=["text"])
+    builder._create_query_builders()
+
+    vector_results = pa.table(
+        {"text": ["a", "b"], "id": ["a", "b"], "_distance": [0.1, 0.2]}
+    )
+    fts_results = pa.table({"text": ["b", "c"], "id": ["b", "c"], "_score": [1.0, 2.0]})
+    vector_results, fts_results = builder._pk_fusion.stamp(vector_results, fts_results)
+
+    combined = LanceHybridQueryBuilder._combine_hybrid_results(
+        fts_results=fts_results,
+        vector_results=vector_results,
+        norm="score",
+        fts_query="puppy",
+        reranker=RRFReranker(),
+        limit=10,
+        with_row_ids=True,
+    )
+    results = builder._finish_hybrid_results(combined)
+
+    assert "_rowid" not in results.column_names, "the surrogate is internal"
+    assert "id" not in results.column_names, "the caller selected only `text`"
+    assert sorted(results.column("text").to_pylist()) == ["a", "b", "c"]
+
+
+@pytest.mark.parametrize(
+    "row_id_first", [True, False], ids=["row_id_first", "rerank_first"]
+)
+def test_wal_hybrid_refuses_row_id_even_with_return_score_all(row_id_first: bool):
+    """`return_score="all"` sets the same flag the caller's request does, so
+    asking for both must still be refused — in either call order."""
+    builder = _wal_hybrid()
+    reranker = RRFReranker(return_score="all")
+    if row_id_first:
+        builder = builder.with_row_id(True).rerank(reranker)
+    else:
+        builder = builder.rerank(reranker).with_row_id(True)
+
+    with pytest.raises(NotImplementedError, match="use_lsm"):
+        builder._create_query_builders()
+
+
+def test_wal_hybrid_refuses_a_blob_projection_with_return_score_all():
+    """A projected blob needs a real row id to fetch through, which the
+    reranker turning `_with_row_id` on does not provide."""
+    schema = pa.schema(
+        [
+            pa.field(
+                "id",
+                pa.string(),
+                metadata={b"lance-schema:unenforced-primary-key:position": b"1"},
+            ),
+            pa.field("text", pa.string()),
+            lancedb.blob("blob"),
+        ]
+    )
+    builder = (
+        LanceHybridQueryBuilder(_wal_table(schema))
+        .vector([0.1, 0.2])
+        .text("puppy")
+        .select(["blob"])
+        .rerank(RRFReranker(return_score="all"))
+    )
+
+    with pytest.raises(NotImplementedError, match="blob"):
+        builder._create_query_builders()
+
+
+_REFUSAL = RuntimeError(
+    "Bad request: the MemWAL LSM scanner does not support with_row_id"
+)
+
+
+def test_fallback_retries_on_the_key_and_remembers_it():
+    table = mock.Mock()
+    attempts = []
+
+    def run():
+        attempts.append(len(attempts))
+        if len(attempts) == 1:
+            raise _REFUSAL
+        return "fused"
+
+    result = with_pk_fallback(
+        run, table, fused=lambda: False, caller_requested_row_id=False
+    )
+
+    assert result == "fused"
+    assert len(attempts) == 2
+    table._note_hybrid_pk_fusion.assert_called_once()
+
+
+def test_fallback_explains_the_refusal_to_a_caller_who_asked_for_row_ids():
+    def run():
+        raise _REFUSAL
+
+    with pytest.raises(NotImplementedError, match="use_lsm") as info:
+        with_pk_fallback(
+            run, mock.Mock(), fused=lambda: False, caller_requested_row_id=True
+        )
+    assert str(info.value) == WAL_ROW_ID_UNSUPPORTED
+
+
+@pytest.mark.parametrize(
+    "error, fused",
+    [(ValueError("unrelated"), False), (_REFUSAL, True)],
+    ids=["other_error", "already_fused"],
+)
+def test_fallback_reraises_what_it_cannot_retry(error, fused):
+    def run():
+        raise error
+
+    table = mock.Mock()
+    with pytest.raises(type(error)):
+        with_pk_fallback(run, table, fused=lambda: fused, caller_requested_row_id=False)
+    table._note_hybrid_pk_fusion.assert_not_called()
diff --git a/python/src/table.rs b/python/src/table.rs
index bf5250732..e7b6f1c6f 100644
--- a/python/src/table.rs
+++ b/python/src/table.rs
@@ -1637,6 +1637,21 @@ impl Table {
         })
     }
 
+    /// Whether a hybrid query on this table has already been refused `_rowid`.
+    /// Learned from a previous refusal, never probed, so this is free.
+    ///
+    /// WAL-PK-FUSION: delete this and `note_hybrid_pk_fusion`.
+    pub fn hybrid_pk_fusion_learned(self_: PyRef<'_, Self>) -> PyResult<bool> {
+        Ok(self_.inner_ref()?.base_table().hybrid_pk_fusion_learned())
+    }
+
+    /// Remember that this table refused `_rowid`, so later hybrid queries skip
+    /// straight to the primary-key fusion.
+    pub fn note_hybrid_pk_fusion(self_: PyRef<'_, Self>) -> PyResult<()> {
+        self_.inner_ref()?.base_table().note_hybrid_pk_fusion();
+        Ok(())
+    }
+
     /// Converge the table's LSM write path into its base table.
     ///
     /// Best-effort: with writes flowing, new rows may land after the last
diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs
index ab1bdfc4a..c8b19f013 100644
--- a/rust/lancedb/src/query.rs
+++ b/rust/lancedb/src/query.rs
@@ -48,6 +48,7 @@ use crate::{
 };
 
 mod hybrid;
+pub(crate) mod wal_fusion; // WAL-PK-FUSION: delete with the module.
 
 pub(crate) const DEFAULT_TOP_K: usize = 10;
 
@@ -1411,15 +1412,37 @@ impl VectorQuery {
     pub async fn execute_hybrid(
         &self,
         options: QueryExecutionOptions,
+    ) -> Result<SendableRecordBatchStream> {
+        // WAL-PK-FUSION: without the fallback, this body is `run_hybrid`'s,
+        // with its `pk_fusion` branches taken as `None`.
+        wal_fusion::with_pk_fallback(self, |pk_fusion| {
+            self.run_hybrid(pk_fusion, options.clone())
+        })
+        .await
+    }
+
+    async fn run_hybrid(
+        &self,
+        pk_fusion: Option<wal_fusion::PkFusion>,
+        options: QueryExecutionOptions,
     ) -> Result<SendableRecordBatchStream> {
         let max_batch_length = options.max_batch_length as usize;
         let internal_options = options.without_output_batch_length_limit();
         // clone query and specify we want to include row IDs, which can be needed for reranking
         let mut fts_query = Query::new(self.parent.clone());
         fts_query.request = self.request.base.clone();
-        fts_query = fts_query.with_row_id();
-
-        let mut vector_query = self.clone().with_row_id();
+        let mut vector_query = self.clone();
+        // WAL-PK-FUSION: without the fallback, keep only the `None` arm.
+        match &pk_fusion {
+            None => {
+                fts_query = fts_query.with_row_id();
+                vector_query = vector_query.with_row_id();
+            }
+            Some(pk_fusion) => {
+                pk_fusion.prepare_leg(&mut fts_query.request.select);
+                pk_fusion.prepare_leg(&mut vector_query.request.base.select);
+            }
+        }
 
         vector_query.request.base.full_text_search = None;
         let (fts_results, vec_results) = try_join!(
@@ -1440,6 +1463,11 @@ impl VectorQuery {
         let mut fts_results = concat_batches(&fts_schema, fts_results.iter())?;
         let mut vec_results = concat_batches(&vec_schema, vec_results.iter())?;
 
+        // WAL-PK-FUSION: delete.
+        if let Some(pk_fusion) = &pk_fusion {
+            (vec_results, fts_results) = pk_fusion.stamp(vec_results, fts_results)?;
+        }
+
         if matches!(self.request.base.norm, Some(NormalizeMethod::Rank)) {
             vec_results = hybrid::rank(vec_results, DIST_COL, None)?;
             fts_results = hybrid::rank(fts_results, SCORE_COL, None)?;
@@ -1475,8 +1503,14 @@ impl VectorQuery {
             results = results.slice(0, limit);
         }
 
-        if !self.request.base.with_row_id {
-            results = results.drop_column(ROW_ID)?;
+        // WAL-PK-FUSION: without the fallback, keep only the `None` arm.
+        match &pk_fusion {
+            None => {
+                if !self.request.base.with_row_id {
+                    results = results.drop_column(ROW_ID)?;
+                }
+            }
+            Some(pk_fusion) => results = pk_fusion.strip(results)?,
         }
 
         Ok(single_batch_stream(results, max_batch_length))
diff --git a/rust/lancedb/src/query/wal_fusion.rs b/rust/lancedb/src/query/wal_fusion.rs
new file mode 100644
index 000000000..b438f6150
--- /dev/null
+++ b/rust/lancedb/src/query/wal_fusion.rs
@@ -0,0 +1,523 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! WAL-PK-FUSION: hybrid search's primary-key fallback for MemWAL tables.
+//!
+//! Hybrid search joins its vector and full-text legs on `_rowid`, which a
+//! MemWAL table cannot supply: the fresh tier has no stable row id, so the
+//! server rejects `with_row_id` before it plans. When that happens the legs are
+//! joined on a surrogate `_rowid` built from the table's primary key instead.
+//!
+//! This is a stopgap until MemWAL supports `_rowid`, and everything it needs
+//! lives here so it can be removed in one pass:
+//!
+//! 1. Delete this file and its `mod wal_fusion;` line in `query.rs`.
+//! 2. Run `grep -rn WAL-PK-FUSION rust/ python/` and follow the note at each
+//!    marker; the Python side has its own copy in `lancedb/_wal_hybrid.py`.
+
+use std::collections::HashMap;
+use std::future::Future;
+use std::sync::{Arc, Mutex};
+use std::time::{Duration, Instant};
+
+use arrow::row::{RowConverter, SortField};
+use arrow_array::{RecordBatch, UInt64Array};
+use arrow_schema::{DataType, Field};
+use lance::dataset::ROW_ID;
+use lance_arrow::RecordBatchExt;
+use lance_core::datatypes::{LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION};
+use lance_core::utils::parse::str_is_truthy;
+
+use super::{Select, VectorQuery};
+use crate::error::{Error, Result};
+
+/// The part of the MemWAL `with_row_id` refusal that identifies it.
+///
+/// Both layers raise this same sentence verbatim — sophon's
+/// `LSM_WITH_ROW_ID_UNSUPPORTED` says so in as many words, and the OSS LSM
+/// scanner builds it from the same words — so this substring survives either
+/// path and the wrapping each adds. It is a coupling to a message rather than
+/// a code, because the refusal arrives as a generic 400; if the server ever
+/// rewords it the fallback stops firing and hybrid on a MemWAL table goes back
+/// to erroring visibly, which is loud rather than silently wrong.
+const WAL_ROW_ID_REFUSAL: &str = "does not support with_row_id";
+
+/// Whether `error` is the server declining `_rowid` because the table is
+/// MemWAL-backed, as opposed to any other failure the query might hit.
+fn is_wal_row_id_refusal(error: &Error) -> bool {
+    error.to_string().contains(WAL_ROW_ID_REFUSAL)
+}
+
+/// Why a hybrid query on a MemWAL table cannot hand back `_rowid`, raised
+/// whether the refusal was anticipated or came back from the server.
+fn wal_row_id_unsupported() -> Error {
+    Error::NotSupported {
+        message: "hybrid search on a MemWAL table cannot return _rowid: the fresh tier has no \
+                  stable row id, and the ids the fusion joins on are synthesized from the \
+                  primary key. Set use_lsm(false) to read the base table only (results will \
+                  exclude un-compacted MemWAL data)"
+            .to_string(),
+    }
+}
+
+/// Run a hybrid query, joining its legs on the primary key instead of `_rowid`
+/// when the table is MemWAL-backed.
+///
+/// `run(None)` must join the legs on `_rowid`; `run(Some(fusion))` must build
+/// them with `fusion` instead.
+///
+/// `_rowid` is asked for first and the primary key used only when the server
+/// refuses, rather than paying a round trip up front to find out. Only a
+/// MemWAL table refuses, so probing would tax every table to learn something
+/// almost none of them need — and the refusal has to be handled regardless,
+/// since a write spec installed elsewhere can arrive between any two queries.
+/// The refusal costs nothing server-side: it is raised before the query is
+/// planned.
+pub(super) async fn with_pk_fallback<T, F, Fut>(query: &VectorQuery, run: F) -> Result<T>
+where
+    F: Fn(Option<PkFusion>) -> Fut,
+    Fut: Future<Output = Result<T>>,
+{
+    // A previous query on this table was already refused, so skip straight to
+    // the key.
+    if query.request.base.use_lsm != Some(false) && query.parent.hybrid_pk_fusion_learned() {
+        return run(Some(PkFusion::resolve(query).await?)).await;
+    }
+    match run(None).await {
+        Err(e) if is_wal_row_id_refusal(&e) => {
+            // A caller who asked for `_rowid` gets the reason, not a bare 400
+            // — the same message the learned path raises up front.
+            if query.request.base.with_row_id {
+                return Err(wal_row_id_unsupported());
+            }
+            // Learned, so later queries on this table skip straight to it.
+            query.parent.note_hybrid_pk_fusion();
+            run(Some(PkFusion::resolve(query).await?)).await
+        }
+        other => other,
+    }
+}
+
+/// How long a table remembers that it refused `_rowid`.
+///
+/// Nothing fetches this — the refusal seeds it — so the bound exists only to
+/// stop the memory outliving a write spec someone removed through another
+/// handle. Being late to notice keeps the query on the primary key, which on a
+/// base table is still a valid read; the window is what stops that lasting.
+const MEMORY_TTL: Duration = Duration::from_secs(30);
+
+/// A table's memory of having refused `_rowid`, backing
+/// [`BaseTable::hybrid_pk_fusion_learned`](crate::table::BaseTable::hybrid_pk_fusion_learned)
+/// for table types that keep one.
+#[derive(Debug, Default)]
+pub struct PkFusionMemory(Mutex<Option<Instant>>);
+
+impl PkFusionMemory {
+    pub fn learned(&self) -> bool {
+        self.0
+            .lock()
+            .unwrap()
+            .is_some_and(|learned| learned.elapsed() < MEMORY_TTL)
+    }
+
+    pub fn note(&self) {
+        *self.0.lock().unwrap() = Some(Instant::now());
+    }
+
+    /// Forget the refusal, as when the write spec is removed through this handle.
+    pub fn forget(&self) {
+        *self.0.lock().unwrap() = None;
+    }
+}
+
+/// How the legs of a hybrid query on a MemWAL table are joined: on a surrogate
+/// `_rowid` derived client-side from the primary key.
+pub(super) struct PkFusion {
+    columns: Vec<String>,
+    /// Key columns the caller did not ask for, added to the projection so the
+    /// surrogate can be built and dropped again before returning.
+    injected: Vec<String>,
+}
+
+impl PkFusion {
+    /// Resolve the primary key the legs will be fused on. Needs the schema,
+    /// which the table has already fetched and cached for its own reasons.
+    async fn resolve(query: &VectorQuery) -> Result<Self> {
+        if query.request.base.with_row_id {
+            return Err(wal_row_id_unsupported());
+        }
+        let schema = query.parent.schema().await?;
+        let columns = pk_columns(&schema);
+        let injected = pk_columns_to_inject(&query.request.base.select, &columns);
+        Ok(Self { columns, injected })
+    }
+
+    /// Make a leg return the key columns, in place of the `_rowid` a MemWAL
+    /// table would refuse.
+    pub(super) fn prepare_leg(&self, select: &mut Select) {
+        inject_pk_columns(select, &self.columns);
+    }
+
+    /// Give both legs a surrogate `_rowid` built from the key.
+    pub(super) fn stamp(
+        &self,
+        vector: RecordBatch,
+        fts: RecordBatch,
+    ) -> Result<(RecordBatch, RecordBatch)> {
+        // Vector first: `merge_results` concatenates in that order and keeps
+        // the first occurrence, so first-seen ids preserve its tie break.
+        let mut legs = [vector, fts];
+        stamp_surrogate_row_ids(&mut legs, &self.columns)?;
+        let [vector, fts] = legs;
+        Ok((vector, fts))
+    }
+
+    /// Drop the surrogate and any injected key columns from the reranked
+    /// results. The surrogate is an internal join key, never an answer: it
+    /// goes whether or not the caller asked for `_rowid`, and `resolve` has
+    /// already refused the query if they did.
+    pub(super) fn strip(&self, results: RecordBatch) -> Result<RecordBatch> {
+        let mut results = results.drop_column(ROW_ID)?;
+        for column in &self.injected {
+            // A reranker is free to reshape its output, so only drop what
+            // actually came back.
+            if results.schema().column_with_name(column).is_some() {
+                results = results.drop_column(column)?;
+            }
+        }
+        Ok(results)
+    }
+}
+
+/// The key columns `select` does not already produce, and so would have to be
+/// added to it for the surrogate to be buildable.
+fn pk_columns_to_inject(select: &Select, pk_columns: &[String]) -> Vec<String> {
+    let produced: Vec<&String> = match select {
+        // Already every non-system column, the key among them.
+        Select::All => return Vec::new(),
+        Select::Columns(columns) => columns.iter().collect(),
+        Select::Dynamic(pairs) => pairs.iter().map(|(name, _)| name).collect(),
+        Select::Expr(pairs) => pairs.iter().map(|(name, _)| name).collect(),
+    };
+    pk_columns
+        .iter()
+        .filter(|pk| !produced.contains(pk))
+        .cloned()
+        .collect()
+}
+
+/// Add the key columns `select` is missing, as identity projections where the
+/// selection is expression-shaped.
+fn inject_pk_columns(select: &mut Select, pk_columns: &[String]) {
+    let missing = pk_columns_to_inject(select, pk_columns);
+    if missing.is_empty() {
+        return;
+    }
+    match select {
+        Select::All => {}
+        Select::Columns(columns) => columns.extend(missing),
+        Select::Dynamic(pairs) => pairs.extend(missing.into_iter().map(|pk| (pk.clone(), pk))),
+        Select::Expr(pairs) => {
+            pairs.extend(missing.into_iter().map(|pk| {
+                let expr = crate::expr::col(&pk);
+                (pk, expr)
+            }));
+        }
+    }
+}
+
+/// The unenforced primary key columns declared in `schema`, in key order.
+///
+/// Reads the field metadata directly rather than going through
+/// [`lance_core::datatypes::Schema::unenforced_primary_key`], because a remote
+/// table only ever has the Arrow schema `describe` returned — there is no Lance
+/// schema, and no field ids, on that side of the wire.
+///
+/// Ordering mirrors Lance's: fields with an explicit 1-based position first, in
+/// position order, then fields carrying only the legacy boolean marker, in
+/// declaration order. Only top-level fields are considered, which is every
+/// field a supported key dtype can live on.
+fn pk_columns(schema: &arrow_schema::Schema) -> Vec<String> {
+    let mut keyed: Vec<(bool, u32, usize, &str)> = schema
+        .fields()
+        .iter()
+        .enumerate()
+        .filter_map(|(idx, field)| {
+            let metadata = field.metadata();
+            let position = metadata
+                .get(LANCE_UNENFORCED_PRIMARY_KEY_POSITION)
+                .and_then(|value| value.parse::<u32>().ok())
+                .or_else(|| {
+                    metadata
+                        .get(LANCE_UNENFORCED_PRIMARY_KEY)
+                        .filter(|value| str_is_truthy(value))
+                        .map(|_| 0)
+                })?;
+            Some((position == 0, position, idx, field.name().as_str()))
+        })
+        .collect();
+    keyed.sort_unstable();
+    keyed
+        .into_iter()
+        .map(|(_, _, _, name)| name.to_string())
+        .collect()
+}
+
+/// Replace each batch's join column with a dense surrogate derived from
+/// `pk_columns`, in place and in the order given.
+///
+/// The fusion joins its two legs on [`ROW_ID`], which a MemWAL table cannot
+/// supply — the fresh tier has no stable row id. It only ever compares the
+/// column for equality, though (`merge_results` dedups, the rerankers group and
+/// the score restore looks up), so any injective mapping of the primary key
+/// serves. Ids are assigned in first-seen order across `batches`, so passing
+/// them in fusion order keeps the dedup's "first occurrence wins" and its tie
+/// break byte-identical to what a real row id would have produced.
+///
+/// Writing it under the [`ROW_ID`] name is what lets every reranker, including
+/// third-party implementations that hardcode the name, run unmodified. The
+/// column never reaches the caller: the hybrid query drops it before returning.
+fn stamp_surrogate_row_ids(batches: &mut [RecordBatch], pk_columns: &[String]) -> Result<()> {
+    if pk_columns.is_empty() {
+        return Err(Error::InvalidInput {
+            message: "hybrid search on a MemWAL table needs an unenforced primary key to \
+                      deduplicate on, and this table declares none"
+                .to_string(),
+        });
+    }
+
+    let mut converter: Option<RowConverter> = None;
+    let mut ids: HashMap<Vec<u8>, u64> = HashMap::new();
+
+    for batch in batches.iter_mut() {
+        // Both legs empty: `query_schemas` synthesizes a schema carrying only
+        // the score and join columns, so there is no key to read — and no row
+        // that would need one.
+        let row_ids: UInt64Array = if batch.num_rows() == 0 {
+            UInt64Array::from(Vec::<u64>::new())
+        } else {
+            let key_columns = pk_columns
+                .iter()
+                .map(|name| {
+                    batch
+                        .column_by_name(name)
+                        .cloned()
+                        .ok_or_else(|| Error::InvalidInput {
+                            message: format!(
+                                "hybrid search could not deduplicate: primary key column {} \
+                                 is missing from a result set with {} rows",
+                                name,
+                                batch.num_rows()
+                            ),
+                        })
+                })
+                .collect::<Result<Vec<_>>>()?;
+
+            let converter = match converter {
+                Some(ref converter) => converter,
+                None => converter.insert(RowConverter::new(
+                    key_columns
+                        .iter()
+                        .map(|column| SortField::new(column.data_type().clone()))
+                        .collect(),
+                )?),
+            };
+
+            let rows = converter.convert_columns(&key_columns)?;
+            let next = &mut ids;
+            UInt64Array::from_iter_values(rows.iter().map(|row| {
+                let id = next.len() as u64;
+                *next.entry(row.as_ref().to_vec()).or_insert(id)
+            }))
+        };
+
+        // `query_schemas` synthesizes a schema carrying `ROW_ID` when a leg came
+        // back empty, so the column can already be there — and it is ours to
+        // define either way.
+        let base = match batch.schema().column_with_name(ROW_ID) {
+            Some(_) => batch.drop_column(ROW_ID)?,
+            None => batch.clone(),
+        };
+        *batch = base.try_with_column(
+            Field::new(ROW_ID, DataType::UInt64, false),
+            Arc::new(row_ids),
+        )?;
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use arrow::array::downcast_array;
+    use arrow_array::{Float32Array, StringArray};
+    use arrow_schema::Schema;
+    use lance_index::{scalar::inverted::SCORE_COL, vector::DIST_COL};
+
+    use super::super::hybrid::{empty_fts_schema, empty_vec_schema};
+    use super::*;
+
+    fn field(name: &str, metadata: &[(&str, &str)]) -> Field {
+        Field::new(name, DataType::Utf8, false).with_metadata(
+            metadata
+                .iter()
+                .map(|(k, v)| (k.to_string(), v.to_string()))
+                .collect::<HashMap<_, _>>(),
+        )
+    }
+
+    #[test]
+    fn pk_columns_reads_the_position_marker() {
+        let schema = Schema::new(vec![
+            field("other", &[]),
+            field("id", &[(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "1")]),
+        ]);
+        assert_eq!(pk_columns(&schema), vec!["id".to_string()]);
+    }
+
+    /// Key order is the position, not the order the columns are declared in.
+    #[test]
+    fn pk_columns_orders_a_composite_key_by_position() {
+        let schema = Schema::new(vec![
+            field("id", &[(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "2")]),
+            field("tenant", &[(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "1")]),
+        ]);
+        assert_eq!(
+            pk_columns(&schema),
+            vec!["tenant".to_string(), "id".to_string()]
+        );
+    }
+
+    /// Datasets written before the position marker carry only the boolean.
+    #[test]
+    fn pk_columns_accepts_the_legacy_boolean_marker() {
+        let schema = Schema::new(vec![field("id", &[(LANCE_UNENFORCED_PRIMARY_KEY, "true")])]);
+        assert_eq!(pk_columns(&schema), vec!["id".to_string()]);
+    }
+
+    /// Mirrors Lance: an explicit position sorts ahead of a legacy field.
+    #[test]
+    fn pk_columns_sorts_positioned_fields_ahead_of_legacy_ones() {
+        let schema = Schema::new(vec![
+            field("legacy", &[(LANCE_UNENFORCED_PRIMARY_KEY, "yes")]),
+            field(
+                "positioned",
+                &[(LANCE_UNENFORCED_PRIMARY_KEY_POSITION, "1")],
+            ),
+        ]);
+        assert_eq!(
+            pk_columns(&schema),
+            vec!["positioned".to_string(), "legacy".to_string()]
+        );
+    }
+
+    #[test]
+    fn pk_columns_is_empty_without_a_key() {
+        let schema = Schema::new(vec![
+            field("a", &[]),
+            field("b", &[(LANCE_UNENFORCED_PRIMARY_KEY, "false")]),
+        ]);
+        assert!(pk_columns(&schema).is_empty());
+    }
+
+    fn batch(ids: Vec<&str>, score: &str) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Arc::new(Field::new("id", DataType::Utf8, false)),
+            Arc::new(Field::new(score, DataType::Float32, false)),
+        ]));
+        let scores = Float32Array::from(vec![0.0_f32; ids.len()]);
+        RecordBatch::try_new(
+            schema,
+            vec![Arc::new(StringArray::from(ids)), Arc::new(scores)],
+        )
+        .unwrap()
+    }
+
+    fn row_ids(batch: &RecordBatch) -> Vec<u64> {
+        let ids: UInt64Array = downcast_array(batch.column_by_name(ROW_ID).unwrap());
+        ids.values().to_vec()
+    }
+
+    /// The same key in both legs has to land on the same id or the fusion will
+    /// not dedup, and ids must ascend in fusion order so the tie break holds.
+    #[test]
+    fn test_surrogate_row_ids_are_shared_across_legs_in_first_seen_order() {
+        let mut batches = vec![
+            batch(vec!["a", "b", "c"], DIST_COL),
+            batch(vec!["b", "d"], SCORE_COL),
+        ];
+        stamp_surrogate_row_ids(&mut batches, &["id".to_string()]).unwrap();
+
+        assert_eq!(row_ids(&batches[0]), vec![0, 1, 2]);
+        assert_eq!(
+            row_ids(&batches[1]),
+            vec![1, 3],
+            "b keeps the id the vector leg gave it"
+        );
+    }
+
+    /// A string key is the common case and cannot be cast to the u64 the
+    /// rerankers read, which is the whole reason for the surrogate.
+    #[test]
+    fn test_surrogate_row_ids_repeat_within_a_leg() {
+        let mut batches = vec![batch(vec!["a", "a", "b"], DIST_COL)];
+        stamp_surrogate_row_ids(&mut batches, &["id".to_string()]).unwrap();
+        assert_eq!(row_ids(&batches[0]), vec![0, 0, 1]);
+    }
+
+    #[test]
+    fn test_surrogate_row_ids_support_a_composite_key() {
+        let schema = Arc::new(Schema::new(vec![
+            Arc::new(Field::new("tenant", DataType::Utf8, false)),
+            Arc::new(Field::new("id", DataType::Int32, false)),
+        ]));
+        let make = |tenants: Vec<&str>, ids: Vec<i32>| {
+            RecordBatch::try_new(
+                schema.clone(),
+                vec![
+                    Arc::new(StringArray::from(tenants)),
+                    Arc::new(arrow_array::Int32Array::from(ids)),
+                ],
+            )
+            .unwrap()
+        };
+        let mut batches = vec![make(vec!["x", "y"], vec![1, 1]), make(vec!["y"], vec![1])];
+        stamp_surrogate_row_ids(&mut batches, &["tenant".to_string(), "id".to_string()]).unwrap();
+
+        assert_eq!(
+            row_ids(&batches[0]),
+            vec![0, 1],
+            "same id, different tenant"
+        );
+        assert_eq!(row_ids(&batches[1]), vec![1]);
+    }
+
+    /// Both legs empty: `query_schemas` synthesizes a schema with no key column,
+    /// and there is no row that would need one.
+    #[test]
+    fn test_surrogate_row_ids_tolerate_the_both_empty_schema() {
+        let mut batches = vec![
+            RecordBatch::new_empty(Arc::new(empty_vec_schema())),
+            RecordBatch::new_empty(Arc::new(empty_fts_schema())),
+        ];
+        stamp_surrogate_row_ids(&mut batches, &["id".to_string()]).unwrap();
+        assert_eq!(row_ids(&batches[0]), Vec::<u64>::new());
+    }
+
+    /// Rows with no key would silently fuse into one bucket, so refuse.
+    #[test]
+    fn test_surrogate_row_ids_reject_a_missing_key_column() {
+        let mut batches = vec![batch(vec!["a"], DIST_COL)];
+        let err = stamp_surrogate_row_ids(&mut batches, &["nope".to_string()]).unwrap_err();
+        assert!(err.to_string().contains("nope"), "{err}");
+    }
+
+    #[test]
+    fn test_surrogate_row_ids_reject_a_table_with_no_primary_key() {
+        let mut batches = vec![batch(vec!["a"], DIST_COL)];
+        let err = stamp_surrogate_row_ids(&mut batches, &[]).unwrap_err();
+        assert!(err.to_string().contains("unenforced primary key"), "{err}");
+    }
+}
diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs
index 6bbb49adf..d6abfea9a 100644
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -20,6 +20,7 @@ use crate::job::Job;
 use crate::materialized_view::{
     MaterializedViewDefinition, MaterializedViewInfo, RefreshMaterializedViewResult, ViewProjection,
 };
+use crate::query::wal_fusion::PkFusionMemory; // WAL-PK-FUSION: delete.
 use crate::query::{QueryFilter, QueryRequest, Select, VectorQueryRequest};
 use crate::remote::job::RemoteJob;
 use crate::table::AddColumnsResult;
@@ -497,6 +498,8 @@ pub struct RemoteTable<S: HttpSend = Sender> {
     version: Arc<RwLock<Option<u64>>>,
     location: RwLock<Option<String>>,
     schema_cache: BackgroundCache<SchemaRef, Error>,
+    // WAL-PK-FUSION: delete, with its initializers below.
+    wal_pk_fusion: PkFusionMemory,
     freshness: Arc<Mutex<FreshnessState>>,
     /// The branch this handle is scoped to, or `None` for the main branch.
     /// Stamped onto every branch-accepting request so reads and writes resolve
@@ -665,6 +668,7 @@ impl<S: HttpSend> RemoteTable<S> {
             version: Arc::new(RwLock::new(None)),
             location: RwLock::new(None),
             schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
+            wal_pk_fusion: PkFusionMemory::default(),
             freshness: Arc::new(Mutex::new(FreshnessState::default())),
             branch: None,
         }
@@ -698,6 +702,7 @@ impl<S: HttpSend> RemoteTable<S> {
             version: Arc::new(RwLock::new(None)),
             location: RwLock::new(None),
             schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
+            wal_pk_fusion: PkFusionMemory::default(),
             freshness: Arc::new(Mutex::new(FreshnessState::default())),
             branch,
         }
@@ -1615,6 +1620,7 @@ mod test_utils {
                 version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
+                wal_pk_fusion: PkFusionMemory::default(),
                 freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
@@ -1639,6 +1645,7 @@ mod test_utils {
                 version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
+                wal_pk_fusion: PkFusionMemory::default(),
                 freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
@@ -1672,6 +1679,7 @@ mod test_utils {
                 version: Arc::new(RwLock::new(None)),
                 location: RwLock::new(None),
                 schema_cache: BackgroundCache::new(SCHEMA_CACHE_TTL, SCHEMA_CACHE_REFRESH_WINDOW),
+                wal_pk_fusion: PkFusionMemory::default(),
                 freshness: Arc::new(Mutex::new(FreshnessState::default())),
                 branch: None,
             }
@@ -3233,6 +3241,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         ));
         let (request_id, response) = self.send(request, true).await?;
         self.check_table_response(&request_id, response).await?;
+        self.wal_pk_fusion.forget(); // WAL-PK-FUSION: delete.
         Ok(())
     }
 
@@ -3299,6 +3308,15 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         Ok(Some(spec))
     }
 
+    // WAL-PK-FUSION: delete both.
+    fn hybrid_pk_fusion_learned(&self) -> bool {
+        self.wal_pk_fusion.learned()
+    }
+
+    fn note_hybrid_pk_fusion(&self) {
+        self.wal_pk_fusion.note();
+    }
+
     async fn tags(&self) -> Result<Box<dyn Tags + '_>> {
         Ok(Box::new(RemoteTags { inner: self }))
     }
@@ -13585,4 +13603,278 @@ mod tests {
             .unwrap();
         branch.stats().await.unwrap();
     }
+
+    // WAL-PK-FUSION: delete everything from here to the end of `mod tests`.
+
+    /// Verbatim from sophon's `LSM_WITH_ROW_ID_UNSUPPORTED`; the fallback keys
+    /// off this sentence, so a test that paraphrased it would prove nothing.
+    const WAL_ROW_ID_REFUSAL_BODY: &str = r#"{"code":13,"error":"Bad request: the MemWAL LSM scanner does not support with_row_id (the LSM scanner exposes _rowaddr, not a stable _rowid); set use_lsm(false) to read the base table only (results will exclude un-compacted MemWAL data)"}"#;
+
+    use lance::dataset::ROW_ID;
+
+    fn pk_schema() -> Schema {
+        use lance_core::datatypes::LANCE_UNENFORCED_PRIMARY_KEY_POSITION;
+        Schema::new(vec![
+            Field::new("id", DataType::Utf8, false).with_metadata(
+                [(
+                    LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(),
+                    "1".to_string(),
+                )]
+                .into_iter()
+                .collect(),
+            ),
+            Field::new("text", DataType::Utf8, true),
+        ])
+    }
+
+    /// One leg's worth of results, keyed so the two legs share a row.
+    /// `row_ids` mirrors a server answering a query that asked for `_rowid`.
+    fn leg(score_column: &str, ids: Vec<&str>, row_ids: bool) -> RecordBatch {
+        use arrow_array::{Float32Array, UInt64Array};
+        let mut fields = vec![
+            Field::new("id", DataType::Utf8, false),
+            Field::new("text", DataType::Utf8, true),
+            Field::new(score_column, DataType::Float32, false),
+        ];
+        let scores = Float32Array::from(vec![1.0_f32; ids.len()]);
+        let texts = StringArray::from(ids.clone());
+        let mut columns: Vec<arrow_array::ArrayRef> = vec![
+            Arc::new(StringArray::from(ids.clone())),
+            Arc::new(texts),
+            Arc::new(scores),
+        ];
+        if row_ids {
+            fields.push(Field::new(ROW_ID, DataType::UInt64, false));
+            columns.push(Arc::new(UInt64Array::from_iter_values(
+                ids.iter().map(|id| id.as_bytes()[0] as u64),
+            )));
+        }
+        RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap()
+    }
+
+    /// A server that refuses `_rowid` the way a MemWAL table does, recording
+    /// every query body so a test can see which key each attempt asked for.
+    fn wal_refusing_table(bodies: Arc<std::sync::Mutex<Vec<serde_json::Value>>>) -> Table {
+        Table::new_with_handler("my_table", move |request| match request.url().path() {
+            "/v1/table/my_table/describe/" => http::Response::builder()
+                .status(200)
+                .body(describe_response(&pk_schema()).into_bytes())
+                .unwrap(),
+            "/v1/table/my_table/query/" => {
+                let body = request_body_json(&request);
+                let asked_for_row_id = body["with_row_id"] == serde_json::Value::Bool(true);
+                let is_fts = body.get("full_text_query").is_some_and(|v| !v.is_null());
+                bodies.lock().unwrap().push(body);
+                if asked_for_row_id {
+                    return http::Response::builder()
+                        .status(400)
+                        .body(WAL_ROW_ID_REFUSAL_BODY.as_bytes().to_vec())
+                        .unwrap();
+                }
+                let batch = if is_fts {
+                    leg("_score", vec!["a", "b"], false)
+                } else {
+                    leg("_distance", vec!["a"], false)
+                };
+                http::Response::builder()
+                    .status(200)
+                    .body(write_ipc_file(&batch))
+                    .unwrap()
+            }
+            path => panic!("unexpected request path: {path}"),
+        })
+    }
+
+    fn hybrid(table: &Table) -> crate::query::VectorQuery {
+        table
+            .query()
+            .full_text_search(FullTextSearchQuery::new("a".to_string()))
+            .nearest_to(&[0.0, 0.0])
+            .unwrap()
+    }
+
+    /// The optimistic path: ask for `_rowid`, and when a MemWAL table refuses,
+    /// fall back to the primary key rather than failing.
+    #[tokio::test]
+    async fn test_hybrid_falls_back_to_the_primary_key_when_row_id_is_refused() {
+        let bodies = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let table = wal_refusing_table(bodies.clone());
+
+        let results = hybrid(&table)
+            .select(Select::columns(&["text"]))
+            .limit(10)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let bodies = bodies.lock().unwrap();
+        let (asked_row_id, asked_pk): (Vec<_>, Vec<_>) = bodies
+            .iter()
+            .partition(|b| b["with_row_id"] == serde_json::Value::Bool(true));
+        // The legs run concurrently under `try_join!`, which abandons the
+        // sibling as soon as one errors — so the refused attempt is one or two
+        // requests, never zero, and the retry always sends both.
+        assert!(
+            !asked_row_id.is_empty(),
+            "`_rowid` has to be tried before the fallback means anything"
+        );
+        assert_eq!(asked_pk.len(), 2, "both legs are retried on the key");
+        for body in asked_pk {
+            let columns = body["columns"].as_array().expect("a column projection");
+            assert!(
+                columns.iter().any(|c| c == "id"),
+                "the retry has to project the key: {body}"
+            );
+        }
+
+        let batch = &results[0];
+        assert!(batch.column_by_name(ROW_ID).is_none());
+        assert!(batch.column_by_name("id").is_none(), "injected key leaked");
+        assert_eq!(batch.num_rows(), 2, "`a` is in both legs and fuses");
+    }
+
+    /// A hybrid query that matches nothing must return nothing, not fail.
+    /// When both legs come back empty, `query_schemas` synthesizes a schema
+    /// that already carries `_rowid`, which the stamping has to tolerate.
+    #[tokio::test]
+    async fn test_hybrid_with_no_matches_returns_an_empty_result() {
+        let table = Table::new_with_handler("my_table", move |request| {
+            match request.url().path() {
+                "/v1/table/my_table/describe/" => http::Response::builder()
+                    .status(200)
+                    .body(describe_response(&pk_schema()).into_bytes())
+                    .unwrap(),
+                "/v1/table/my_table/query/" => {
+                    let body = request_body_json(&request);
+                    if body["with_row_id"] == serde_json::Value::Bool(true) {
+                        return http::Response::builder()
+                            .status(400)
+                            .body(WAL_ROW_ID_REFUSAL_BODY.as_bytes().to_vec())
+                            .unwrap();
+                    }
+                    // A leg that matched nothing: schema, no batches.
+                    let schema =
+                        Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
+                    let mut body = Vec::new();
+                    {
+                        let mut writer =
+                            arrow_ipc::writer::FileWriter::try_new(&mut body, &schema).unwrap();
+                        writer.finish().unwrap();
+                    }
+                    http::Response::builder().status(200).body(body).unwrap()
+                }
+                path => panic!("unexpected request path: {path}"),
+            }
+        });
+
+        let results = hybrid(&table)
+            .limit(10)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        assert_eq!(results.iter().map(|b| b.num_rows()).sum::<usize>(), 0);
+    }
+
+    /// The refusal is paid once: the second query goes straight to the key.
+    #[tokio::test]
+    async fn test_hybrid_remembers_the_refusal() {
+        let bodies = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let table = wal_refusing_table(bodies.clone());
+
+        for _ in 0..2 {
+            hybrid(&table)
+                .limit(10)
+                .execute()
+                .await
+                .unwrap()
+                .try_collect::<Vec<_>>()
+                .await
+                .unwrap();
+        }
+
+        let refused = bodies
+            .lock()
+            .unwrap()
+            .iter()
+            .filter(|b| b["with_row_id"] == serde_json::Value::Bool(true))
+            .count();
+        assert!(
+            (1..=2).contains(&refused),
+            "only the first query should pay the refusal, saw {refused} attempts"
+        );
+    }
+
+    /// A table that never refuses must never pay for the fallback existing.
+    #[tokio::test]
+    async fn test_hybrid_on_a_base_table_sends_no_extra_requests() {
+        let paths = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let seen = paths.clone();
+        let table = Table::new_with_handler("my_table", move |request| {
+            seen.lock().unwrap().push(request.url().path().to_string());
+            let batch = if request_body_json(&request)
+                .get("full_text_query")
+                .is_some_and(|v| !v.is_null())
+            {
+                leg("_score", vec!["a", "b"], true)
+            } else {
+                leg("_distance", vec!["a"], true)
+            };
+            http::Response::builder()
+                .status(200)
+                .body(write_ipc_file(&batch))
+                .unwrap()
+        });
+
+        hybrid(&table)
+            .limit(10)
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+
+        let paths = paths.lock().unwrap();
+        assert!(
+            paths.iter().all(|p| p.ends_with("/query/")),
+            "a table that never refuses must not be probed, got {paths:?}"
+        );
+        assert_eq!(paths.len(), 2, "one request per leg, got {paths:?}");
+    }
+
+    /// A caller who asked for `_rowid` gets the server's refusal, not a
+    /// surrogate dressed up as one.
+    #[tokio::test]
+    async fn test_hybrid_does_not_fall_back_when_the_caller_asked_for_row_id() {
+        let bodies = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let table = wal_refusing_table(bodies.clone());
+
+        let err = hybrid(&table)
+            .with_row_id()
+            .limit(10)
+            .execute()
+            .await
+            .err()
+            .expect("the refusal must reach the caller");
+
+        // Translated, not passed through: the caller gets the reason and the
+        // way out, which a bare 400 does not carry.
+        assert!(matches!(err, Error::NotSupported { .. }), "{err:?}");
+        assert!(err.to_string().contains("use_lsm(false)"), "{err}");
+        assert!(
+            bodies
+                .lock()
+                .unwrap()
+                .iter()
+                .all(|b| b["with_row_id"] == serde_json::Value::Bool(true)),
+            "no retry should have been attempted"
+        );
+    }
 }
diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs
index fbec73dfb..dcb3e1578 100644
--- a/rust/lancedb/src/table.rs
+++ b/rust/lancedb/src/table.rs
@@ -706,6 +706,31 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
             message: "get_lsm_write_spec is not supported on this table type".into(),
         })
     }
+    /// Whether a hybrid query on this table has already been told it cannot
+    /// join its legs on `_rowid`.
+    ///
+    /// WAL-PK-FUSION: delete this and `note_hybrid_pk_fusion`.
+    ///
+    /// Learned, never probed: hybrid optimistically asks for `_rowid` and only
+    /// a MemWAL table refuses, so paying a round trip up front would tax every
+    /// table to discover something almost none of them need. Synchronous and
+    /// free by construction — an implementation may only answer from what a
+    /// previous query already learned.
+    ///
+    /// The default is `false`, which keeps a table type that never refuses on
+    /// the `_rowid` path forever.
+    fn hybrid_pk_fusion_learned(&self) -> bool {
+        false
+    }
+
+    /// Record that this table refused `_rowid`, so later hybrid queries skip
+    /// straight to the primary-key fusion instead of paying the refusal again.
+    ///
+    /// Implementations should expire this the way they expire other table
+    /// metadata: a spec can be removed, after which `_rowid` works again and
+    /// the only cost of being late to notice is a base-only read that is still
+    /// correct.
+    fn note_hybrid_pk_fusion(&self) {}
     /// Seal every bucket's active memtable into L0.
     ///
     /// The default implementation returns `NotSupported`.

From d0fb43b0b45c959d44655f1471b469890787054f Mon Sep 17 00:00:00 2001
From: LanceDB Robot <robot@lancedb.com>
Date: Fri, 25 Sep 2026 07:02:00 -0700
Subject: [PATCH 02/11] chore: update lance dependency to v13.0.0-beta.14
 (#4324)

Update the Rust workspace Lance dependencies and Java lance-core from
v13.0.0-beta.12 to
[v13.0.0-beta.14](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.14),
including the refreshed Cargo lockfile.

No compatibility fixes were required; `cargo clippy --quiet --workspace
--tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and
`git diff --check` passed.
---
 Cargo.lock   | 88 ++++++++++++++++++++++++++--------------------------
 Cargo.toml   | 28 ++++++++---------
 java/pom.xml |  2 +-
 3 files changed, 59 insertions(+), 59 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 367ba7805..ed0fad6f3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5216,8 +5216,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5239,7 +5239,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5253,7 +5253,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5262,8 +5262,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -5273,8 +5273,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5311,8 +5311,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5343,8 +5343,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5361,8 +5361,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5371,8 +5371,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5405,8 +5405,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5438,8 +5438,8 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -5453,8 +5453,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5524,8 +5524,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5547,8 +5547,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5590,8 +5590,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5605,8 +5605,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5620,8 +5620,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5674,8 +5674,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5689,8 +5689,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5730,8 +5730,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5744,8 +5744,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "13.0.0-beta.12"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.12#5d2946fa452c61cb018144f3200892b6dd4a8b4f"
+version = "13.0.0-beta.14"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
 dependencies = [
  "frostem",
  "icu_segmenter",
diff --git a/Cargo.toml b/Cargo.toml
index c64305048..15770bebc 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=13.0.0-beta.12", default-features = false, "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=13.0.0-beta.12", "tag" = "v13.0.0-beta.12", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
diff --git a/java/pom.xml b/java/pom.xml
index 7fe65841f..752dacefd 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>13.0.0-beta.12</lance-core.version>
+        <lance-core.version>13.0.0-beta.14</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

From 139382b5f7cb9c299a4e9fc3bff6334fc17a17ac Mon Sep 17 00:00:00 2001
From: Ayush Chaurasia <ayush.chaurarsia@gmail.com>
Date: Fri, 25 Sep 2026 21:50:55 +0530
Subject: [PATCH 03/11] feat(python): pass batch_size and other CrossEncoder
 kwargs (#4328)

---
 python/python/lancedb/rerankers/cross_encoder.py | 13 ++++++++++++-
 python/python/tests/test_rerankers.py            |  8 ++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/python/python/lancedb/rerankers/cross_encoder.py b/python/python/lancedb/rerankers/cross_encoder.py
index 30ad9d382..7b16a6704 100644
--- a/python/python/lancedb/rerankers/cross_encoder.py
+++ b/python/python/lancedb/rerankers/cross_encoder.py
@@ -31,6 +31,12 @@ class CrossEncoderReranker(Reranker):
     trust_remote_code : bool, default True
         If True, will trust the remote code to be safe. If False, will not trust
         the remote code and will not run it
+    batch_size : int, default 32
+        The number of query-passage pairs scored per forward pass.
+    **kwargs
+        Additional keyword arguments passed to
+        ``sentence_transformers.CrossEncoder``, for example ``max_length`` or
+        ``model_kwargs={"torch_dtype": "bfloat16"}``.
     """
 
     def __init__(
@@ -40,6 +46,8 @@ class CrossEncoderReranker(Reranker):
         device: Union[str, None] = None,
         return_score="relevance",
         trust_remote_code: bool = True,
+        batch_size: int = 32,
+        **kwargs,
     ):
         super().__init__(return_score)
         torch = attempt_import_or_raise("torch")
@@ -47,6 +55,8 @@ class CrossEncoderReranker(Reranker):
         self.column = column
         self.device = device
         self.trust_remote_code = trust_remote_code
+        self.batch_size = batch_size
+        self.kwargs = kwargs
         if self.device is None:
             self.device = "cuda" if torch.cuda.is_available() else "cpu"
 
@@ -61,6 +71,7 @@ class CrossEncoderReranker(Reranker):
             self.model_name,
             device=self.device,
             trust_remote_code=self.trust_remote_code,
+            **self.kwargs,
         )
 
         return cross_encoder
@@ -71,7 +82,7 @@ class CrossEncoderReranker(Reranker):
             return result_set
         passages = result_set[self.column].to_pylist()
         cross_inp = [[query, passage] for passage in passages]
-        cross_scores = self.model.predict(cross_inp)
+        cross_scores = self.model.predict(cross_inp, batch_size=self.batch_size)
         result_set = result_set.append_column(
             "_relevance_score", pa.array(cross_scores, type=pa.float32())
         )
diff --git a/python/python/tests/test_rerankers.py b/python/python/tests/test_rerankers.py
index 33c0db488..b4b9bc9e2 100644
--- a/python/python/tests/test_rerankers.py
+++ b/python/python/tests/test_rerankers.py
@@ -481,6 +481,14 @@ def test_cross_encoder_reranker(tmp_path):
     _run_test_reranker(reranker, table, "single player experience", None, schema)
 
 
+def test_cross_encoder_reranker_kwargs(tmp_path):
+    pytest.importorskip("sentence_transformers")
+    reranker = CrossEncoderReranker(batch_size=4, max_length=64)
+    assert reranker.model.max_length == 64
+    table, schema = get_test_table(tmp_path)
+    _run_test_reranker(reranker, table, "single player experience", None, schema)
+
+
 def test_colbert_reranker(tmp_path):
     pytest.importorskip("rerankers")
     reranker = ColbertReranker()

From 0327618a62483feb6dca2336549845e5887cd48f Mon Sep 17 00:00:00 2001
From: Yang Cen <bubble-cal@outlook.com>
Date: Mon, 28 Sep 2026 18:06:55 +0800
Subject: [PATCH 04/11] feat(sql): add native vector dedup and partitioned mv
 refresh (#4303)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Summary

Materialize a source-preserving dedup result from an indexed, pinned
snapshot with a short declaration. The default greedy policy keeps the
smallest unassigned snapshot row ID and removes only its directly
matching neighbors: For A < B < C, an A–B–C chain without an A–C edge
keeps A and C.

```python
from lancedb import vector_dedup

view = db.create_materialized_view(
    "images_clean",
    vector_dedup("images", column="phash", distance_threshold=4),
)
cleaned = view.table
```

The Python source expression captures the source version once at
creation and
waits for the materialized result by default. Use `dataset_version=...`
for an
explicit snapshot, `with_no_data=True` for declaration only, or
`create_materialized_view_async(...)` for a job handle. AsyncConnection
supports
the same expression. It uses the existing MV REST endpoints and registry
jobs,
without opening a SQL query connection. Source names are exact
root-namespace
table names. Additional select/where/limit options are rejected for this
source.

The SQL declaration remains available:

```sql
CREATE MATERIALIZED VIEW images_clean AS
SELECT * FROM vector_dedup('images', 7, 'phash', 4);
REFRESH MATERIALIZED VIEW images_clean;
```

This is the host-facing SQL entry point. The MV builder now accepts a
typed native vector source; the Rust SDK also supports the same
definition through `MaterializedViewDefinition::from_sql` and
`prepare_definition`; `view.refresh().execute()` executes it locally.
Hosts distribute its units using their existing job infrastructure. The
raw `vector_duplicate_pairs` source remains available separately when
callers want the pair relation.

## Implementation and contract

- Use actual Rust Lance `13.0.0-beta.15` (`db53dcf367`); the native
partition primitive computes pairs from index representations without
source-vector aggregation or a Python pair loop.
- Extend partitioned MV refresh with two dependent waves. Physical
segment/partition tasks stream pairs through a bounded external sort,
select direct representatives and write per-source-fragment bitmap
artifacts. Source-fragment tasks start after every selection receipt
completes and stream retained original rows, including blob payloads,
into uncommitted output fragments.
- Reuse normal blob write coercion and row-mask scans, including stable
row-ID translation after compaction. Null vectors, isolated rows and
source deletions are preserved correctly. Conflicting representative
decisions across overlapping index scopes fail before publication.
- Expose `dependency_units()` and `write_refresh_partition_with_inputs`
alongside the existing plan/commit API. Receipts bind
snapshot/configuration/policy/destination generation. Only a complete
accepted set can publish; unfinished units restart, completed units can
be reused. There is no pair-position checkpoint.
- Use MV format 4 for dedup results and format 3 for native pairs;
Python and TypeScript metadata readers recognize both. No source
deletion or source row-ID migration occurs.

Only same-segment/same-partition edges are covered. Quantized scores use
symmetric represented-vector distances. Result materialization requires
V2 data storage and is initially a native MV source, not an ordinary
SELECT table function. Compressed vertex masks stay in memory; edge
sorting spills to local disk. Unreferenced artifacts use normal Lance
retention/cleanup and must be retained for the job's recovery window.

## Validation

AWS Linux validation:

- Native pair/result integration matrix passed: multiple
segments/partitions, exact thresholds, empty output, cross-batch data,
single-process pair parity, source preservation, lost receipts and
publication guards.
- Blob/null-vector/pinned-deletion tests passed with both row-ID modes,
including compaction that changes physical row addresses.
- Representative policy, forced external-sort spill and
conflicting/corrupt artifact tests passed.
- Full MV regression suite: 153 passed, 2 pre-existing ignored. Rust
formatting and Clippy passed.

- Python MV suite: 47 passed; TypeScript MV suite: 11 passed. Native
bindings were rebuilt in project-local environments; formatting, lint
and TypeScript docs passed.
- Rust declaration doctests: 3 passed, including the complete local
dedup example.
- Companion host validation: 41 MV job tests passed (1 pre-existing
ignored), full workspace Clippy passed, and the complete process-level
CLI E2E suite passed (3 tests). Recovery exercises both selection and
materialization waves.
- A real CIFAR-10 pHash run used five actual registry-worker owners,
completed 32 selection + 24 materialization tasks, and retained 49,846
of 50,000 rows with identical original values and image payloads.

Host workflow performance measurements are recorded in the companion
host PR. Cross-platform CI is still running. The previous remote-test
bootstrap failed before Cargo tests with `401 Bad credentials`; the
workflow fix described below is being validated by CI. No deployment or
merge is performed.


## Python source validation

AWS tests cover automatic and explicit snapshots, physical/stable row
IDs, local
and remote creation, sync/job/async-connection APIs, delayed refresh,
strict
options and identifier quoting. The native Python extension was rebuilt
through
the project's uv/maturin backend. Ruff, Pyright and the full MkDocs
API-reference
build passed; both new public symbols are rendered.

A real 50,000-image FEv2 run disabled both Python SQL entry points and
supplied an
unreachable SQL endpoint. The new API still materialized the same 49,846
retained
rows and original image payloads. Three recorded creation/refresh jobs
completed
56 tasks each on all five registry-worker owners, and remained on source
snapshot
3 after source latest advanced to 4. No worker or server wire-protocol
changes
were needed for this client entry point.

Previous CI authentication failure:
https://github.com/lancedb/lancedb/actions/runs/36334679276/job/108663151265.

## Review and CI fixes

- Validate all JSON merge source fields before projection, rejecting
unknown and duplicate names at every struct level. Missing/reordered
fields and ordinary JSON merges remain supported. A checked-in 12-case
integration matrix covers root fields, structs, lists, large lists,
fixed-size lists and map values; rejected writes leave the table version
and row count unchanged.
- Match namespace connection MV method signatures to
`MaterializedViewSource` in all four sync/async methods, fixing the
Python 3.10 import failure from runtime override checks.
- Restore the remote-test script's encrypted-CLI path: use the Actions
token for artifact download and pass `SOPHON_READ_TOKEN` under the
variable the decryption script consumes. The previous workflow omitted
that variable and attempted the private release path, failing with HTTP
401. On the final head, remote CI passes: 1,464 Rust unit tests, the
integration targets and doctests pass (3 unit tests ignored).

AWS validation for these fixes: both review reproducers failed before
their fixes and pass afterward; 55 merge tests, 29 cast tests and the
standalone 7-case admission regression pass. Python 3.10.21 imports
successfully; 90 MV/namespace tests, 14 JSON tests and four real
namespace dedup cases pass. After `794b97a0`, the native binding was
rebuilt again and the JSON suite rerun. Rust formatting/Clippy, Ruff and
Pyright pass. Sophon's locked test compilation passes against the final
SDK source for all seven affected crates. Extended runners, logs and
benchmark artifacts remain local.

Gatekeeper recommends approval on `794b97a0`, and the `breaking-change`
label is set. Final CI: **49 checks passed, 3 expected skips, no
failures or pending checks**, including remote tests, Python 3.10,
minimum dependencies and Windows ARM Rust tests. All wheel builds pass
after retrying the Linux matrix affected by transient AlmaLinux
repository-metadata 404s. All code-review threads are resolved. The PR
remains open for maintainer review and merge.

---------

Co-authored-by: Yang Cen <yangcen@Yangs-Mac-mini.local>
Co-authored-by: Lance Release <lance-dev@lancedb.com>
---
 .github/workflows/rust.yml                    |   3 +-
 Cargo.lock                                    |  90 +--
 Cargo.toml                                    |  28 +-
 docs/src/python/python.md                     |  27 +
 nodejs/__test__/materialized_view.test.ts     |  12 +-
 nodejs/lancedb/materialized_view.ts           |   5 +-
 python/python/lancedb/__init__.py             |   1 +
 python/python/lancedb/_lancedb.pyi            |   4 +
 python/python/lancedb/db.py                   |  55 +-
 python/python/lancedb/materialized_view.py    | 101 ++-
 python/python/lancedb/namespace.py            |   9 +-
 python/python/lancedb/remote/db.py            |   6 +-
 .../python/tests/test_materialized_views.py   | 158 ++++-
 python/src/connection.rs                      |  20 +-
 rust/lancedb/src/materialized_view.rs         | 211 +++++-
 .../src/materialized_view/duplicate_pairs.rs  | 478 +++++++++++++
 .../src/materialized_view/grouped_units.rs    |  50 +-
 .../src/materialized_view/partitioned.rs      | 192 ++++++
 rust/lancedb/src/materialized_view/query.rs   |  85 ++-
 rust/lancedb/src/materialized_view/refresh.rs |  24 +-
 .../src/materialized_view/vector_dedup.rs     | 636 ++++++++++++++++++
 rust/lancedb/src/remote/table.rs              |   1 +
 rust/lancedb/src/table/add_data.rs            |  18 +-
 rust/lancedb/src/table/datafusion/udtf.rs     |   2 +
 .../table/datafusion/udtf/duplicate_pairs.rs  | 586 ++++++++++++++++
 rust/lancedb/src/table/merge.rs               | 264 +++++++-
 rust/lancedb/tests/duplicate_pairs.rs         | 561 +++++++++++++++
 27 files changed, 3497 insertions(+), 130 deletions(-)
 create mode 100644 rust/lancedb/src/materialized_view/duplicate_pairs.rs
 create mode 100644 rust/lancedb/src/materialized_view/partitioned.rs
 create mode 100644 rust/lancedb/src/materialized_view/vector_dedup.rs
 create mode 100644 rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs
 create mode 100644 rust/lancedb/tests/duplicate_pairs.rs

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 4ac2c3070..585abf5ee 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -180,7 +180,8 @@ jobs:
     env:
       CC: clang-18
       CXX: clang++-18
-      GH_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
+      GH_TOKEN: ${{ github.token }}
+      SOPHON_READ_TOKEN: ${{ secrets.SOPHON_READ_TOKEN }}
     steps:
       - uses: actions/checkout@v6
         with:
diff --git a/Cargo.lock b/Cargo.lock
index ed0fad6f3..87bef3a25 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3588,8 +3588,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "rand 0.9.5",
@@ -5143,8 +5143,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5161,6 +5161,7 @@ dependencies = [
  "async-trait",
  "async_cell",
  "aws-sdk-dynamodb",
+ "blake3",
  "byteorder",
  "bytes",
  "chrono",
@@ -5216,8 +5217,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5239,7 +5240,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5253,7 +5254,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5262,8 +5263,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -5273,8 +5274,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5311,8 +5312,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5343,8 +5344,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5361,8 +5362,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5371,8 +5372,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5405,8 +5406,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5438,8 +5439,8 @@ dependencies = [
 
 [[package]]
 name = "lance-geo"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "datafusion",
  "geo-traits",
@@ -5453,8 +5454,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5524,8 +5525,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index-core"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5543,12 +5544,13 @@ dependencies = [
  "roaring",
  "serde",
  "serde_json",
+ "tokio",
 ]
 
 [[package]]
 name = "lance-io"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5590,8 +5592,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5605,8 +5607,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5620,8 +5622,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5674,8 +5676,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5689,8 +5691,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5730,8 +5732,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5744,8 +5746,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "13.0.0-beta.14"
-source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.14#b80ac1d9f3c24b47bd8e297d03053ea35d216a9f"
+version = "13.0.0-beta.15"
+source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.15#db53dcf367088b32ae3a7461407d43593759ae3e"
 dependencies = [
  "frostem",
  "icu_segmenter",
diff --git a/Cargo.toml b/Cargo.toml
index 15770bebc..1bf7ce86a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=13.0.0-beta.14", default-features = false, "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=13.0.0-beta.14", "tag" = "v13.0.0-beta.14", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=13.0.0-beta.15", default-features = false, "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=13.0.0-beta.15", "tag" = "v13.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
 lancedb = { path = "rust/lancedb", default-features = false }
 ahash = "0.8"
 # Note that this one does not include pyarrow
diff --git a/docs/src/python/python.md b/docs/src/python/python.md
index ccc44a55e..66fda4378 100644
--- a/docs/src/python/python.md
+++ b/docs/src/python/python.md
@@ -198,6 +198,33 @@ listing a storage directory.
 
 ::: lancedb.materialized_view.MaterializedViewDefinition
 
+::: lancedb.vector_dedup
+
+::: lancedb.materialized_view.VectorDedupSource
+
+For an existing indexed source, declare and materialize a dedup result without
+writing SQL. Creation captures the source version once and waits for the complete
+result; remote connections use the service's MV jobs.
+
+```python
+from lancedb import vector_dedup
+
+view = db.create_materialized_view(
+    "images_clean",
+    vector_dedup("images", column="phash", distance_threshold=4),
+)
+cleaned = view.table
+```
+
+To submit without waiting, call `db.create_materialized_view_async(...)` with the
+same source expression and wait on the returned job. To declare only, pass
+`with_no_data=True`, then use `view.refresh_async().wait()` when ready. Pass
+`dataset_version=...` to `vector_dedup` for an explicit snapshot. The default
+policy retains direct representatives (A-B-C without A-C keeps A and C), preserves
+the source, and materializes its original columns. Only within-segment/partition
+pairs are covered, using the index's distance representation.
+
+
 ## Views
 
 ::: lancedb.view.ViewDescription
diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts
index 5e5c08745..6739320aa 100644
--- a/nodejs/__test__/materialized_view.test.ts
+++ b/nodejs/__test__/materialized_view.test.ts
@@ -52,9 +52,19 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
+    for (const [format, fn] of [
+      [3, "vector_duplicate_pairs"],
+      [4, "vector_dedup"],
+    ]) {
+      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
+      expect(read(JSON.stringify({ format, query: native })).query).toBe(
+        native,
+      );
+    }
+
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":3,"query":${JSON.stringify(query)}}`,
+      `{"format":5,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);
diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts
index 51f8cfead..a59a3a5f6 100644
--- a/nodejs/lancedb/materialized_view.ts
+++ b/nodejs/lancedb/materialized_view.ts
@@ -9,9 +9,10 @@ export const DEFINITION_META_KEY = "mv.definition";
 
 /**
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
- * format 2 being a query with `GROUP BY`.
+ * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
+ * native dedup results.
  */
-export const DEFINITION_FORMAT = 2;
+export const DEFINITION_FORMAT = 4;
 
 /**
  * The query that defines a materialized view, as stored:
diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py
index d34b541a8..d3a678359 100644
--- a/python/python/lancedb/__init__.py
+++ b/python/python/lancedb/__init__.py
@@ -46,6 +46,7 @@ from .materialized_view import (
     AsyncMaterializedView,
     MaterializedView,
     MaterializedViewDefinition,
+    vector_dedup as vector_dedup,
 )
 from .view import ViewDescription as ViewDescription
 from .table import AsyncTable, Table
diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi
index 1f26ad6f4..cf63d7a9b 100644
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -247,6 +247,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        *,
+        vector_source_json: Optional[str] = None,
     ) -> Table: ...
     async def create_materialized_view_async(
         self,
@@ -256,6 +258,8 @@ class Connection(object):
         filter: Optional[str] = None,
         limit: Optional[int] = None,
         with_no_data: bool = False,
+        *,
+        vector_source_json: Optional[str] = None,
     ) -> Job: ...
     async def list_materialized_views(self) -> List[str]: ...
     async def drop_materialized_view(
diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py
index 02de75460..490987c3c 100644
--- a/python/python/lancedb/db.py
+++ b/python/python/lancedb/db.py
@@ -56,6 +56,8 @@ from .sql import QueryDescription
 from .materialized_view import (
     AsyncMaterializedView,
     MaterializedView,
+    MaterializedViewSource,
+    VectorDedupSource,
     SelectArg,
     normalize_select,
 )
@@ -555,7 +557,7 @@ class DBConnection(EnforceOverrides):
     def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -569,17 +571,21 @@ class DBConnection(EnforceOverrides):
         table. The view is a normal table: it can be queried, indexed and
         searched, and it appears in ``table_names``.
 
-        The source table must have stable row ids (create it with the
+        Ordinary source tables must have stable row ids (create them with the
         ``new_table_enable_stable_row_ids`` storage option): they keep the
         view's provenance valid across source compactions, and cannot be
-        enabled after a table exists.
+        enabled after a table exists. Native dedup sources also support physical
+        row IDs interpreted against their pinned snapshot.
 
         Parameters
         ----------
         name: str
             The name of the view.
-        source: str
-            The name of the source table, in this database.
+        source: str or VectorDedupSource
+            The name of the source table, or an indexed dedup source declared
+            with [vector_dedup][lancedb.vector_dedup]. Dedup sources capture a
+            fixed snapshot and preserve the source table. They accept no
+            additional select, where or limit options.
         select: list or dict, optional
             The view's columns: column names, ``(alias, SQL expression)``
             pairs, or a dict of the same. Omitting it selects every source
@@ -602,7 +608,7 @@ class DBConnection(EnforceOverrides):
     def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -1558,7 +1564,7 @@ class LanceDBConnection(DBConnection):
     def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -1603,7 +1609,7 @@ class LanceDBConnection(DBConnection):
     def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -2498,10 +2504,31 @@ class AsyncConnection(object):
             await tbl.checkout(version)
         return tbl
 
+    async def _materialized_view_source(
+        self,
+        source: MaterializedViewSource,
+        select: SelectArg,
+        where: Optional[str],
+        limit: Optional[int],
+    ) -> Tuple[str, Dict[str, str]]:
+        if not isinstance(source, VectorDedupSource):
+            return source, {}
+        if select is not None or where is not None or limit is not None:
+            raise ValueError(
+                "vector_dedup cannot be combined with select, where or limit"
+            )
+        version = source.dataset_version
+        if version is None:
+            table = await self.open_table(source.source)
+            version = await table.version()
+        return source.source, {
+            "vector_source_json": source._native_source_json(version)
+        }
+
     async def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -2512,6 +2539,9 @@ class AsyncConnection(object):
         See
         [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
         """
+        source, native_options = await self._materialized_view_source(
+            source, select, where, limit
+        )
         inner = await self._inner.create_materialized_view(
             name,
             source,
@@ -2519,13 +2549,14 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            **native_options,
         )
         return AsyncMaterializedView(AsyncTable(inner))
 
     async def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -2536,6 +2567,9 @@ class AsyncConnection(object):
 
         Wait for the returned job before opening or querying the view.
         """
+        source, native_options = await self._materialized_view_source(
+            source, select, where, limit
+        )
         inner = await self._inner.create_materialized_view_async(
             name,
             source,
@@ -2543,6 +2577,7 @@ class AsyncConnection(object):
             filter=where,
             limit=limit,
             with_no_data=with_no_data,
+            **native_options,
         )
         return AsyncJob(inner)
 
diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py
index a836210cb..b4d455870 100644
--- a/python/python/lancedb/materialized_view.py
+++ b/python/python/lancedb/materialized_view.py
@@ -7,6 +7,7 @@ maintained by refresh. See ``DBConnection.create_materialized_view``."""
 from __future__ import annotations
 
 import json
+import math
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
 
@@ -20,6 +21,7 @@ if TYPE_CHECKING:
     from .table import AsyncTable, LanceTable
 
 DEFINITION_META_KEY = b"mv.definition"
+_MAX_FLOAT32 = float.fromhex("0x1.fffffep+127")
 
 SelectArg = Union[
     str,
@@ -29,9 +31,104 @@ SelectArg = Union[
 ]
 
 
-DEFINITION_FORMAT = 2
+@dataclass(frozen=True)
+class VectorDedupSource:
+    """A declarative source of retained original rows from an indexed snapshot.
+
+    Construct with [vector_dedup][lancedb.vector_dedup] and pass it as the
+    ``source`` of ``create_materialized_view`` or
+    ``create_materialized_view_async``. An omitted ``dataset_version`` is
+    captured once when creation is submitted, not when this object is made.
+    Refreshes keep that captured version. The source table is never modified.
+    """
+
+    source: str
+    column: str
+    distance_threshold: float
+    dataset_version: Optional[int] = None
+
+    def __post_init__(self):
+        for name in ("source", "column"):
+            value = getattr(self, name)
+            if not isinstance(value, str) or not value:
+                raise ValueError(f"{name} must be a non-empty string")
+        if self.dataset_version is not None and (
+            type(self.dataset_version) is not int
+            or not 0 < self.dataset_version < 2**64
+        ):
+            raise ValueError("dataset_version must be a positive uint64 integer")
+        value = self.distance_threshold
+        if isinstance(value, bool) or not isinstance(value, (int, float)):
+            raise TypeError("distance_threshold must be a number")
+        try:
+            value = float(value)
+        except OverflowError as exc:
+            raise ValueError("distance_threshold must fit a finite float32") from exc
+        if not math.isfinite(value) or abs(value) > _MAX_FLOAT32:
+            raise ValueError("distance_threshold must fit a finite float32")
+        object.__setattr__(self, "distance_threshold", value)
+
+    def _native_source_json(self, version: int) -> str:
+        return json.dumps(
+            {
+                "kind": "Dedup",
+                "dataset_version": version,
+                "column": self.column,
+                "distance_threshold": repr(self.distance_threshold),
+            },
+            allow_nan=False,
+        )
+
+
+def vector_dedup(
+    source: str,
+    *,
+    column: str,
+    distance_threshold: float,
+    dataset_version: Optional[int] = None,
+) -> VectorDedupSource:
+    """Declare an indexed dedup source for a materialized view.
+
+    The default policy keeps the smallest unassigned snapshot row ID and removes
+    only its direct qualifying neighbors: an A-B-C chain without an A-C edge
+    retains A and C. Rows with no pair, including null vectors, survive.
+    Only same-segment/same-IVF-partition pairs are considered; quantized distances
+    use reconstructed index vectors. The index must cover the selected snapshot.
+
+    ``source`` is an exact table name in the connection's root namespace, just
+    as for ``create_materialized_view``. ``dataset_version=None`` captures the
+    source's version when creation is submitted. Pass an explicit positive
+    version to reproduce a previous snapshot. The result contains original
+    source columns and rows. Do not combine this source with ``select``,
+    ``where`` or ``limit``; query the resulting table for further transforms.
+
+    Creation follows the usual MV contract: by default it waits for the result.
+    Use ``with_no_data=True`` to declare only, or
+    ``create_materialized_view_async`` to get a job handle. On Cloud/Enterprise,
+    execution uses server-side MV registry jobs; local connections run in process.
+
+    Examples
+    --------
+    Given a connected ``db`` and an indexed ``images.phash`` column::
+
+        from lancedb import vector_dedup
+
+        view = db.create_materialized_view(
+            "images_clean",
+            vector_dedup("images", column="phash", distance_threshold=4),
+        )
+        cleaned = view.table
+    """
+    return VectorDedupSource(source, column, distance_threshold, dataset_version)
+
+
+MaterializedViewSource = Union[str, VectorDedupSource]
+
+
+DEFINITION_FORMAT = 4
 """The newest stored layout this version reads: ``{"format": N, "query": "<SQL>"}``,
-format 2 being a query with ``GROUP BY``. A ``kind`` key beside it is for
+format 2 being a query with ``GROUP BY``, format 3 native pairs and format 4
+native dedup results. A ``kind`` key beside it is for
 readers older than the format number."""
 
 
diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py
index fe38a47c7..cf1910c68 100644
--- a/python/python/lancedb/namespace.py
+++ b/python/python/lancedb/namespace.py
@@ -68,6 +68,7 @@ from lance_namespace import (
 from lancedb.materialized_view import (
     AsyncMaterializedView,
     MaterializedView,
+    MaterializedViewSource,
     SelectArg,
 )
 from lancedb.table import AsyncTable, LanceTable, Table
@@ -632,7 +633,7 @@ class LanceNamespaceDBConnection(DBConnection):
     def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: "SelectArg" = None,
         where: Optional[str] = None,
@@ -662,7 +663,7 @@ class LanceNamespaceDBConnection(DBConnection):
     def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: "SelectArg" = None,
         where: Optional[str] = None,
@@ -1242,7 +1243,7 @@ class AsyncLanceNamespaceDBConnection:
     async def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: "SelectArg" = None,
         where: Optional[str] = None,
@@ -1265,7 +1266,7 @@ class AsyncLanceNamespaceDBConnection:
     async def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: "SelectArg" = None,
         where: Optional[str] = None,
diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py
index 69c5a62b0..a1c71440c 100644
--- a/python/python/lancedb/remote/db.py
+++ b/python/python/lancedb/remote/db.py
@@ -39,7 +39,7 @@ from ..functions import FunctionVersion, UdfDefinition
 from ..job import AsyncJob, Job
 from ..sql import Query as SqlQuery
 from ..sql import QueryDescription
-from ..materialized_view import MaterializedView, SelectArg
+from ..materialized_view import MaterializedView, MaterializedViewSource, SelectArg
 from ..secrets import EnvVarSecret, SecretInfo
 from ..view import ViewDescription
 
@@ -713,7 +713,7 @@ class RemoteDBConnection(DBConnection):
     def create_materialized_view(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
@@ -745,7 +745,7 @@ class RemoteDBConnection(DBConnection):
     def create_materialized_view_async(
         self,
         name: str,
-        source: str,
+        source: MaterializedViewSource,
         *,
         select: SelectArg = None,
         where: Optional[str] = None,
diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py
index c030ac1df..1ee21fbac 100644
--- a/python/python/tests/test_materialized_views.py
+++ b/python/python/tests/test_materialized_views.py
@@ -520,10 +520,166 @@ def test_stored_queries_and_legacy_layouts_are_read():
         "SELECT * FROM people"
     )
 
+    for fmt, function in [(3, "vector_duplicate_pairs"), (4, "vector_dedup")]:
+        native = f"SELECT * FROM {function}('images', 3, 'phash', 4)"
+        assert read({"format": fmt, "query": native}).query == native
+
     # A newer writer's layout is reported, never guessed at.
     for newer in (
-        {"format": 3, "query": query},
+        {"format": 5, "query": query},
         {"kind": "select_v3", "source_table": "people"},
     ):
         with pytest.raises(NotImplementedError, match="cannot refresh"):
             read(newer)
+
+
+def indexed_dedup_source(db, name="images"):
+    import pyarrow as pa
+    from lancedb.index import IvfFlat
+
+    vectors = pa.FixedSizeListArray.from_arrays(
+        pa.array(
+            [v for i in range(256) for v in (float(i if i < 3 else i * 10), 0.0)],
+            type=pa.float32(),
+        ),
+        2,
+    )
+    table = db.create_table(name, pa.table({"id": range(256), "vector": vectors}))
+    table.create_index("vector", config=IvfFlat(num_partitions=1))
+    return table
+
+
+@pytest.mark.parametrize("stable", [False, True])
+def test_vector_dedup_source_captures_snapshot_and_preserves_source(tmp_path, stable):
+    db = lancedb.connect(
+        tmp_path,
+        storage_options={"new_table_enable_stable_row_ids": str(stable).lower()},
+    )
+    source = indexed_dedup_source(db, "images.with.dot")
+    version = source.version
+    expression = lancedb.vector_dedup(
+        source.name, column="vector", distance_threshold=1
+    )
+    view = db.create_materialized_view("clean", expression)
+    assert "vector_dedup(" in view.definition.query
+    assert f", {version}, 'vector', 1.0)" in view.definition.query
+    assert view.table.count_rows() == 255
+    assert [r["id"] for r in view.table.search().where("id < 3").to_list()] == [0, 2]
+    assert source.version == version
+    assert source.count_rows() == 256
+
+    source.delete("id = 0")
+    view.refresh_async().wait()
+    assert view.table.search().where("id = 0").to_list() == [
+        {"id": 0, "vector": [0.0, 0.0]}
+    ]
+    assert expression.dataset_version is None  # reusable immutable declaration
+
+
+def test_vector_dedup_create_job_and_explicit_snapshot(tmp_path):
+    db = lancedb.connect(tmp_path)
+    source = indexed_dedup_source(db)
+    version = source.version
+    source.delete("id = 0")
+    job = db.create_materialized_view_async(
+        "clean",
+        lancedb.vector_dedup(
+            "images", column="vector", distance_threshold=1, dataset_version=version
+        ),
+    )
+    assert job.wait() is None
+    view = db.open_materialized_view("clean")
+    assert view.table.count_rows() == 255
+    assert len(view.table.search().where("id = 0").to_list()) == 1
+
+
+@pytest.mark.asyncio
+async def test_async_vector_dedup_declaration_and_creation(tmp_path):
+    sync_db = lancedb.connect(tmp_path)
+    source = indexed_dedup_source(sync_db)
+    db = await lancedb.connect_async(tmp_path)
+    expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
+    view = await db.create_materialized_view("clean", expression, with_no_data=True)
+    assert await view.table.count_rows() == 0
+    # The snapshot is captured at declaration even if refresh comes later.
+    source.delete("id = 0")
+    job = await view.refresh_async()
+    await job.wait()
+    assert await view.table.count_rows() == 255
+    another = await db.create_materialized_view_async(
+        "later", expression, with_no_data=True
+    )
+    await another.wait()
+    later = await db.open_materialized_view("later")
+    assert (await later.definition()).query != (await view.definition()).query
+    assert await later.table.count_rows() == 0
+
+
+@pytest.mark.parametrize(
+    "options", [{"select": ["id"]}, {"select": []}, {"where": "id > 0"}, {"limit": 1}]
+)
+def test_vector_dedup_rejects_extra_clauses_before_opening_source(tmp_path, options):
+    db = lancedb.connect(tmp_path)
+    expression = lancedb.vector_dedup("missing", column="vector", distance_threshold=1)
+    with pytest.raises(ValueError, match="cannot be combined"):
+        db.create_materialized_view("bad", expression, **options)
+    assert db.list_tables().tables == []
+
+
+@pytest.mark.parametrize("version", [0, -1, True, 1.5, "2", 2**64])
+def test_vector_dedup_rejects_invalid_snapshot(version):
+    with pytest.raises(ValueError, match="dataset_version"):
+        lancedb.vector_dedup(
+            "images", column="vector", distance_threshold=1, dataset_version=version
+        )
+
+
+@pytest.mark.parametrize(
+    "threshold", [float("nan"), float("inf"), -float("inf"), 1e100, 10**1000, True, "4"]
+)
+def test_vector_dedup_rejects_invalid_threshold(threshold):
+    with pytest.raises((ValueError, TypeError), match="distance_threshold"):
+        lancedb.vector_dedup("images", column="vector", distance_threshold=threshold)
+
+
+def test_vector_dedup_descriptor_is_frozen():
+    from dataclasses import FrozenInstanceError
+
+    expression = lancedb.vector_dedup("images", column="vector", distance_threshold=1)
+    with pytest.raises(FrozenInstanceError):
+        expression.column = "other"
+
+
+@pytest.mark.parametrize("with_no_data", [False, True])
+def test_remote_vector_dedup_uses_mv_endpoint_without_sql_client(with_no_data):
+    with mock_remote_materialized_view_create() as (host, requests):
+        db = lancedb.connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            sql_host_override="grpc://127.0.0.1:1",
+            client_config={"retry_config": {"retries": 0}},
+        )
+        job = db.create_materialized_view_async(
+            "clean",
+            lancedb.vector_dedup(
+                "images.with'dot",
+                column="vector'field",
+                distance_threshold=4,
+                dataset_version=7,
+            ),
+            with_no_data=with_no_data,
+        )
+        assert job.id == "mv-create-123"
+    assert requests == [
+        (
+            "/v1/materialized_view/clean/create",
+            {
+                "query": (
+                    "SELECT * FROM vector_dedup('`images.with''dot`', "
+                    "7, 'vector''field', 4.0)"
+                ),
+                "with_no_data": with_no_data,
+            },
+        )
+    ]
diff --git a/python/src/connection.rs b/python/src/connection.rs
index 2959fe037..a0fc3b04b 100644
--- a/python/src/connection.rs
+++ b/python/src/connection.rs
@@ -407,7 +407,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
     pub fn create_materialized_view(
         self_: PyRef<'_, Self>,
         name: String,
@@ -416,10 +417,17 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        vector_source_json: Option<String>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(source) = vector_source_json {
+                builder = builder.vector_source(
+                    serde_json::from_str(&source)
+                        .map_err(|e| PyValueError::new_err(e.to_string()))?,
+                );
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }
@@ -435,7 +443,8 @@ impl Connection {
         })
     }
 
-    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))]
+    #[allow(clippy::too_many_arguments)]
+    #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false, *, vector_source_json=None))]
     pub fn create_materialized_view_async(
         self_: PyRef<'_, Self>,
         name: String,
@@ -444,10 +453,17 @@ impl Connection {
         filter: Option<String>,
         limit: Option<u64>,
         with_no_data: bool,
+        vector_source_json: Option<String>,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.get_inner()?.clone();
         future_into_py(self_.py(), async move {
             let mut builder = inner.create_materialized_view(name, source);
+            if let Some(source) = vector_source_json {
+                builder = builder.vector_source(
+                    serde_json::from_str(&source)
+                        .map_err(|e| PyValueError::new_err(e.to_string()))?,
+                );
+            }
             if let Some(projections) = projections {
                 builder = builder.select(projections);
             }
diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs
index 430750a00..616bd42d4 100644
--- a/rust/lancedb/src/materialized_view.rs
+++ b/rust/lancedb/src/materialized_view.rs
@@ -9,8 +9,12 @@
 //! query this version cannot maintain reads back as unrefreshable, not as a
 //! plain table. Queries, indexes and search work on the view unchanged.
 
+mod duplicate_pairs;
 mod grouped;
 mod grouped_units;
+mod partitioned;
+mod vector_dedup;
+pub use duplicate_pairs::{VectorSource, VectorSourceKind};
 pub use grouped::IVF_PARTITION;
 /// Refreshing a view grouped by [`IVF_PARTITION`] in units, one per index
 /// partition, so the work can be spread over several processes. The caller
@@ -74,6 +78,10 @@ pub use grouped_units::{
     GroupedRefreshPlan, WrittenUnit, commit_grouped_refresh, plan_grouped_refresh,
     write_grouped_unit,
 };
+pub use partitioned::{
+    PartitionedRefreshPlan, WrittenPartition, commit_partitioned_refresh, plan_partitioned_refresh,
+    write_refresh_partition, write_refresh_partition_with_inputs,
+};
 mod query;
 pub mod refresh;
 
@@ -146,18 +154,26 @@ const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions";
 
 /// The newest layout this version reads under [`DEFINITION_META_KEY`]:
 /// `{"format": N, "query": "<SQL>"}`, the query as
-/// [`MaterializedViewDefinition::to_sql`] renders it. Format 2 is a query
-/// with `GROUP BY`; every other query is written as format 1, so a reader
-/// that predates grouping reports a grouped view as unrefreshable rather
-/// than failing to parse it. A reader refuses a newer format rather than
+/// [`MaterializedViewDefinition::to_sql`] renders it. Format 3 identifies a
+/// native duplicate-pair source, format 4 a dedup result source, format 2 a query with `GROUP BY`, and
+/// format 1 an ordinary query. A reader refuses a newer format rather than
 /// guess at it. The layout also carries `"kind": "query"`, which readers
 /// older than the format number report as an unrefreshable view instead of
 /// failing to read the metadata.
-pub const DEFINITION_FORMAT: u64 = 2;
+pub const DEFINITION_FORMAT: u64 = 4;
 
 /// The format `definition` is written in; see [`DEFINITION_FORMAT`].
 fn format_of(definition: &MaterializedViewDefinition) -> u64 {
-    if definition.is_grouped() { 2 } else { 1 }
+    if let Some(source) = &definition.vector_source {
+        match source.kind {
+            VectorSourceKind::Pairs => 3,
+            VectorSourceKind::Dedup => 4,
+        }
+    } else if definition.is_grouped() {
+        2
+    } else {
+        1
+    }
 }
 
 /// Legacy `kind` tag of the structured layout written before
@@ -300,6 +316,8 @@ pub fn read_staging(metadata: &HashMap<String, String>) -> Result<Option<Staging
 /// for the shape.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct MaterializedViewDefinition {
+    /// A table-bound native vector source. It is never evaluated per source row.
+    pub vector_source: Option<Box<VectorSource>>,
     /// Name of the source table, in the same database as the view.
     pub source_table: String,
     /// Namespace path holding the source table; empty is the root namespace.
@@ -498,6 +516,7 @@ pub fn read_definition(metadata: &HashMap<String, String>) -> Result<Option<Stor
         )));
     }
     Ok(Some(StoredDefinition::Query(MaterializedViewDefinition {
+        vector_source: None,
         source_table: legacy.source_table,
         source_namespace: legacy.source_namespace,
         lateral: None,
@@ -554,6 +573,9 @@ pub(crate) fn plan(
     definition: &MaterializedViewDefinition,
     staging: Option<&StagingBinding>,
 ) -> Result<Planned> {
+    if definition.vector_source.is_some() {
+        return duplicate_pairs::plan(source_schema, definition);
+    }
     let filter = definition
         .filter
         .as_deref()
@@ -712,6 +734,7 @@ pub(crate) fn plan(
     }
 
     let definition = MaterializedViewDefinition {
+        vector_source: None,
         source_table: definition.source_table.clone(),
         source_namespace: definition.source_namespace.clone(),
         lateral: definition.lateral.clone(),
@@ -1145,6 +1168,11 @@ impl PreparedDeclaration {
     /// read: the column the view projects it to, if any, otherwise an
     /// internal projection added here, named by [`input_column_name`].
     pub fn input_column(&mut self, source_column: &str) -> Result<String> {
+        if self.definition.vector_source.is_some() {
+            return Err(Error::InvalidInput {
+                message: "native vector views have no per-source-row inputs".into(),
+            });
+        }
         // A grouped view's rows are groups; no source row carries a value into one.
         if self.definition.is_grouped() {
             return Err(Error::InvalidInput {
@@ -1245,6 +1273,11 @@ impl PreparedDeclaration {
         columns: Vec<(usize, ArrowField)>,
         bindings: &[FunctionBinding],
     ) -> Result<Self> {
+        if self.definition.vector_source.is_some() {
+            return Err(Error::InvalidInput {
+                message: "computed columns on native vector views are not supported".into(),
+            });
+        }
         let invalid = |message: String| Error::InvalidInput { message };
         if columns.is_empty() {
             return Err(invalid("at least one computed column is needed".into()));
@@ -1451,6 +1484,7 @@ pub async fn prepare_declaration(
     limit: Option<u64>,
 ) -> Result<PreparedDeclaration> {
     let definition = MaterializedViewDefinition {
+        vector_source: None,
         source_table: source.name().to_string(),
         source_namespace: source.namespace().to_vec(),
         lateral: None,
@@ -1512,6 +1546,27 @@ pub async fn prepare_declaration(
 /// # Ok(())
 /// # }
 /// ```
+///
+/// An indexed, pinned source can materialize retained original rows using the
+/// default greedy direct-representative policy. A row is removed only when it
+/// directly matches a retained row; an A-B-C chain keeps A and C. The source is
+/// unchanged. Hosts distribute this through [`plan_partitioned_refresh`] and
+/// [`write_refresh_partition_with_inputs`], with a barrier between selection
+/// tasks and source-fragment materialization tasks.
+///
+/// ```
+/// # #![recursion_limit = "256"]
+/// use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition};
+/// # async fn dedup(images: &lancedb::Table) -> Result<(), Box<dyn std::error::Error>> {
+/// let version = images.version().await?;
+/// let definition = MaterializedViewDefinition::from_sql(&format!(
+///     "SELECT * FROM vector_dedup('images', {version}, 'phash', 4)"
+/// ))?;
+/// let clean = prepare_definition(images, definition).await?.create("images_clean").await?;
+/// clean.refresh().execute().await?;
+/// # Ok(())
+/// # }
+/// ```
 pub async fn prepare_definition(
     source: &Table,
     definition: MaterializedViewDefinition,
@@ -1627,7 +1682,9 @@ async fn prepare_with(
             ),
         });
     }
-    if !native.dataset.get().await?.manifest.uses_stable_row_ids() {
+    if definition.vector_source.is_none()
+        && !native.dataset.get().await?.manifest.uses_stable_row_ids()
+    {
         return Err(Error::InvalidInput {
             message: format!(
                 "materialized views require stable row ids on the source table; \
@@ -1653,7 +1710,22 @@ async fn prepare_with(
             message: format!("view column name '{}' is reserved", reserved.output),
         });
     }
-    let source_schema = resolved.schema().await?;
+    let source_schema = if let Some(pairs) = &definition.vector_source {
+        let ds = native
+            .dataset
+            .get()
+            .await?
+            .checkout_version(pairs.dataset_version)
+            .await?;
+        crate::table::datafusion::udtf::duplicate_pairs::plan_duplicate_pairs(
+            Arc::new(ds.clone()),
+            &pairs.config()?,
+        )
+        .await?;
+        Arc::new(ArrowSchema::from(ds.schema()))
+    } else {
+        resolved.schema().await?
+    };
     let source_metadata = source_schema.metadata().clone();
     let Planned {
         definition,
@@ -1670,11 +1742,13 @@ async fn prepare_with(
         None => source_schema.clone(),
         Some(unnest) => flattened_schema(&source_schema, &unnest)?,
     };
-    fields.push(ArrowField::new(
-        SOURCE_ROW_ID_COLUMN,
-        DataType::UInt64,
-        false,
-    ));
+    if definition.vector_source.is_none() {
+        fields.push(ArrowField::new(
+            SOURCE_ROW_ID_COLUMN,
+            DataType::UInt64,
+            false,
+        ));
+    }
     // Only column-describing metadata comes along: structural declarations
     // describe how a table is written, and a view is written by refresh alone.
     let mut metadata: HashMap<String, String> = HashMap::new();
@@ -1718,6 +1792,7 @@ pub struct CreateMaterializedViewBuilder {
     namespace: Vec<String>,
     source: String,
     source_namespace: Vec<String>,
+    vector_source: Option<VectorSource>,
     projections: Vec<(String, String)>,
     filter: Option<String>,
     limit: Option<u64>,
@@ -1732,6 +1807,7 @@ impl CreateMaterializedViewBuilder {
             namespace: Vec::new(),
             source,
             source_namespace: Vec::new(),
+            vector_source: None,
             projections: Vec::new(),
             filter: None,
             limit: None,
@@ -1752,6 +1828,38 @@ impl CreateMaterializedViewBuilder {
         self
     }
 
+    /// Read native pairs or retained original rows from a fixed indexed snapshot.
+    /// This source selects its full output schema and cannot be combined with
+    /// projections, filters or limits. Remote creation uses the ordinary MV
+    /// endpoint and its registry job; it does not execute a SQL query client-side.
+    pub fn vector_source(mut self, source: VectorSource) -> Self {
+        self.vector_source = Some(source);
+        self
+    }
+
+    fn native_definition(&self) -> Result<Option<MaterializedViewDefinition>> {
+        let Some(source) = &self.vector_source else {
+            return Ok(None);
+        };
+        source.config()?;
+        if !self.projections.is_empty() || self.filter.is_some() || self.limit.is_some() {
+            return Err(Error::InvalidInput {
+                message: "native vector sources cannot be combined with select, where or limit"
+                    .into(),
+            });
+        }
+        Ok(Some(MaterializedViewDefinition {
+            vector_source: Some(Box::new(source.clone())),
+            source_table: self.source.clone(),
+            source_namespace: self.source_namespace.clone(),
+            projections: vec![ViewProjection::star()],
+            filter: None,
+            lateral: None,
+            group_by: Vec::new(),
+            limit: None,
+        }))
+    }
+
     /// The view's columns, as `(name, SQL expression)` pairs. Not calling
     /// this selects every source column, expanded at creation time.
     pub fn select(
@@ -1784,7 +1892,10 @@ impl CreateMaterializedViewBuilder {
         self
     }
 
-    fn query(&self) -> String {
+    fn query(&self) -> Result<String> {
+        if let Some(definition) = self.native_definition()? {
+            return Ok(definition.to_sql());
+        }
         fn quote(name: &str) -> String {
             format!("\"{}\"", name.replace('"', "\"\""))
         }
@@ -1813,12 +1924,13 @@ impl CreateMaterializedViewBuilder {
         if let Some(limit) = self.limit {
             query.push_str(&format!(" LIMIT {limit}"));
         }
-        query
+        Ok(query)
     }
 
     /// Submit creation and initial population, returning a [`Job`] that
-    /// settles when the view is ready. The source must keep stable row ids --
-    /// they hold provenance across compaction, and cannot be enabled later.
+    /// settles when the view is ready. Ordinary sources must keep stable row
+    /// ids for provenance across compaction. Native vector sources also support
+    /// physical row IDs because their definitions pin the dataset version.
     pub async fn execute_async(self) -> Result<Job> {
         if self.connection.uri().starts_with("db://") {
             return self
@@ -1827,7 +1939,7 @@ impl CreateMaterializedViewBuilder {
                 .create_materialized_view_async(CreateMaterializedViewRequest {
                     name: self.name.clone(),
                     namespace_path: self.namespace.clone(),
-                    query: self.query(),
+                    query: self.query()?,
                     with_no_data: self.with_no_data,
                 })
                 .await;
@@ -1855,19 +1967,24 @@ impl CreateMaterializedViewBuilder {
     }
 
     async fn execute_native(self) -> Result<MaterializedView> {
+        let native_definition = self.native_definition()?;
         let source = self
             .connection
             .open_table(&self.source)
             .namespace(self.source_namespace.clone())
             .execute()
             .await?;
-        let prepared = prepare_declaration(
-            &source,
-            (!self.projections.is_empty()).then_some(self.projections.as_slice()),
-            self.filter.as_deref(),
-            self.limit,
-        )
-        .await?;
+        let prepared = if let Some(definition) = native_definition {
+            prepare_definition(&source, definition).await?
+        } else {
+            prepare_declaration(
+                &source,
+                (!self.projections.is_empty()).then_some(self.projections.as_slice()),
+                self.filter.as_deref(),
+                self.limit,
+            )
+            .await?
+        };
         let view = prepared.create_in(&self.namespace, &self.name).await?;
         if !self.with_no_data {
             view.refresh().execute().await?;
@@ -2186,6 +2303,7 @@ mod tests {
         assert_eq!(
             view.definition(),
             &MaterializedViewDefinition {
+                vector_source: None,
                 source_table: "people".into(),
                 source_namespace: Vec::new(),
                 projections: vec![
@@ -2309,13 +2427,49 @@ mod tests {
             .select([("double\"age", "age * 2")])
             .only_if("age >= 18")
             .limit(10)
-            .query();
+            .query()
+            .unwrap();
         assert_eq!(
             query,
             "SELECT age * 2 AS \"double\"\"age\" FROM \"raw data\".\"odd\"\"source\" WHERE age >= 18 LIMIT 10"
         );
     }
 
+    #[tokio::test]
+    async fn native_vector_builder_preserves_source_and_rejects_extra_clauses() {
+        let conn = people_db().await;
+        let source = VectorSource {
+            kind: VectorSourceKind::Dedup,
+            dataset_version: 7,
+            column: "vector'field".into(),
+            distance_threshold: "4.0".into(),
+        };
+        let builder = || {
+            conn.create_materialized_view("clean", "odd.source'name")
+                .source_namespace(vec!["raw space".into()])
+                .vector_source(source.clone())
+        };
+        let query = builder().query().unwrap();
+        let definition = MaterializedViewDefinition::from_sql(&query).unwrap();
+        assert_eq!(definition.source_table, "odd.source'name");
+        assert_eq!(definition.source_namespace, ["raw space"]);
+        assert_eq!(definition.vector_source.as_deref(), Some(&source));
+        assert!(definition.selects_star());
+        assert!(builder().select([("id", "id")]).query().is_err());
+        assert!(builder().only_if("true").query().is_err());
+        assert!(builder().limit(1).query().is_err());
+        let invalid_source = VectorSource {
+            dataset_version: 0,
+            ..source
+        };
+        assert!(
+            conn.create_materialized_view("bad", "people")
+                .vector_source(invalid_source)
+                .query()
+                .is_err()
+        );
+    }
+
     /// No projection selects every source column, expanded now: the schema
     /// captured at creation is the definition.
     #[tokio::test]
@@ -3235,6 +3389,7 @@ mod tests {
 
     fn definition(source_namespace: Vec<String>) -> MaterializedViewDefinition {
         MaterializedViewDefinition {
+            vector_source: None,
             source_table: "people".to_string(),
             source_namespace,
             lateral: None,
@@ -3321,8 +3476,8 @@ mod tests {
     #[test]
     fn a_newer_format_is_reported_not_guessed() {
         assert_eq!(
-            read(r#"{"format":3,"query":"SELECT name FROM people"}"#).unwrap(),
-            Some(StoredDefinition::Newer { format: "3".into() })
+            read(r#"{"format":5,"query":"SELECT name FROM people"}"#).unwrap(),
+            Some(StoredDefinition::Newer { format: "5".into() })
         );
         assert_eq!(
             read(r#"{"kind":"join"}"#).unwrap(),
diff --git a/rust/lancedb/src/materialized_view/duplicate_pairs.rs b/rust/lancedb/src/materialized_view/duplicate_pairs.rs
new file mode 100644
index 000000000..d0dc219ad
--- /dev/null
+++ b/rust/lancedb/src/materialized_view/duplicate_pairs.rs
@@ -0,0 +1,478 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Native pair generation as partitioned materialized-view refresh work.
+
+use std::sync::Arc;
+
+use arrow_schema::{Schema, SchemaRef};
+use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+use datafusion_physical_plan::ExecutionPlan;
+use datafusion_sql::sqlparser::ast::{
+    Expr, FunctionArg, FunctionArgExpr, TableFunctionArgs, Value,
+};
+use futures::TryStreamExt;
+use lance::Dataset;
+use lance::dataset::transaction::Operation;
+use lance::dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams};
+use lance_table::format::Fragment;
+use serde::{Deserialize, Serialize};
+
+use super::grouped_units::GroupedRefreshPlan;
+use super::refresh::{
+    RefreshMaterializedViewResult, ensure_incarnation, ensure_no_mem_wal, open_source,
+};
+use super::{MaterializedViewDefinition, Planned, ViewProjection};
+use crate::table::Table;
+use crate::table::datafusion::udtf::duplicate_pairs::{
+    DUPLICATE_PAIRS_OPERATOR_VERSION, DuplicatePairTask, DuplicatePairsConfig, DuplicatePairsExec,
+    duplicate_pair_schema, plan_duplicate_pairs,
+};
+use crate::{Error, Result};
+
+pub(super) const FUNCTION_NAME: &str = "vector_duplicate_pairs";
+pub(super) const DEDUP_FUNCTION_NAME: &str = "vector_dedup";
+
+/// The result of a pinned native vector source.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+pub enum VectorSourceKind {
+    /// The qualifying pair relation.
+    Pairs,
+    /// Original source rows retained by greedy direct representatives.
+    Dedup,
+}
+
+/// A native table-function source, pinned for the lifetime of the view.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct VectorSource {
+    pub kind: VectorSourceKind,
+    pub dataset_version: u64,
+    pub column: String,
+    /// Validated numeric SQL spelling; execution and task identity use f32 bits.
+    pub distance_threshold: String,
+}
+
+impl VectorSource {
+    pub(super) fn function_name(&self) -> &'static str {
+        match self.kind {
+            VectorSourceKind::Pairs => FUNCTION_NAME,
+            VectorSourceKind::Dedup => DEDUP_FUNCTION_NAME,
+        }
+    }
+    pub(crate) fn config(&self) -> Result<DuplicatePairsConfig> {
+        let threshold: f32 = self
+            .distance_threshold
+            .parse()
+            .map_err(|_| invalid("invalid pair threshold"))?;
+        if self.dataset_version == 0 || self.column.is_empty() || !threshold.is_finite() {
+            return Err(invalid(
+                "duplicate pairs require a positive snapshot, a column and a finite threshold",
+            ));
+        }
+        Ok(DuplicatePairsConfig {
+            dataset_version: self.dataset_version,
+            column: self.column.clone(),
+            distance_threshold: threshold,
+        })
+    }
+}
+
+fn invalid(message: impl Into<String>) -> Error {
+    Error::InvalidInput {
+        message: message.into(),
+    }
+}
+
+pub(super) fn source(
+    args: &TableFunctionArgs,
+    kind: VectorSourceKind,
+) -> Result<(Vec<String>, String, VectorSource)> {
+    if args.settings.is_some() {
+        return Err(invalid("duplicate pairs do not accept function settings"));
+    }
+    let exprs = args
+        .args
+        .iter()
+        .map(|arg| match arg {
+            FunctionArg::Unnamed(FunctionArgExpr::Expr(expr)) => Ok(expr),
+            _ => Err(invalid(
+                "duplicate pairs require four positional literal arguments",
+            )),
+        })
+        .collect::<Result<Vec<_>>>()?;
+    let [table, version, column, threshold] = exprs.as_slice() else {
+        return Err(invalid(
+            "native vector sources require (table, dataset_version, column, distance_threshold)",
+        ));
+    };
+    let string = |expr: &Expr| match expr {
+        Expr::Value(value) => match &value.value {
+            Value::SingleQuotedString(value) => Ok(value.clone()),
+            _ => Err(invalid("table and column must be string literals")),
+        },
+        _ => Err(invalid("table and column must be string literals")),
+    };
+    let table = string(table)?;
+    let plain = MaterializedViewDefinition::from_sql(&format!("SELECT * FROM {table}"))?;
+    if plain.vector_source.is_some()
+        || plain.lateral.is_some()
+        || plain.filter.is_some()
+        || plain.is_grouped()
+        || plain.limit.is_some()
+    {
+        return Err(invalid("duplicate pairs must name one source table"));
+    }
+    let dataset_version = match version {
+        Expr::Value(value) => match &value.value {
+            Value::Number(number, _) => number
+                .parse()
+                .map_err(|_| invalid("invalid source version"))?,
+            _ => return Err(invalid("source version must be an integer literal")),
+        },
+        _ => return Err(invalid("source version must be an integer literal")),
+    };
+    let config = VectorSource {
+        kind,
+        dataset_version,
+        column: string(column)?,
+        distance_threshold: threshold.to_string(),
+    };
+    config.config()?;
+    Ok((plain.source_namespace, plain.source_table, config))
+}
+
+pub(super) fn check_shape(definition: &MaterializedViewDefinition) -> Result<()> {
+    if definition
+        .vector_source
+        .as_ref()
+        .is_some_and(|s| s.kind == VectorSourceKind::Dedup)
+    {
+        if !definition.selects_star()
+            || definition.filter.is_some()
+            || definition.limit.is_some()
+            || definition.lateral.is_some()
+            || definition.is_grouped()
+        {
+            return Err(invalid(
+                "a vector_dedup view must select * without other clauses",
+            ));
+        }
+        return Ok(());
+    }
+    let schema = duplicate_pair_schema();
+    let projections = schema
+        .fields()
+        .iter()
+        .map(|field| ViewProjection {
+            output: field.name().clone(),
+            expression: field.name().clone(),
+        })
+        .collect::<Vec<_>>();
+    if (!definition.selects_star() && definition.projections != projections)
+        || definition.filter.is_some()
+        || definition.limit.is_some()
+        || definition.lateral.is_some()
+        || definition.is_grouped()
+    {
+        return Err(invalid(
+            "a native pair view selects *, or row_id_a, row_id_b, distance, without other clauses",
+        ));
+    }
+    Ok(())
+}
+
+pub(super) fn plan(
+    source_schema: SchemaRef,
+    definition: &MaterializedViewDefinition,
+) -> Result<Planned> {
+    check_shape(definition)?;
+    let config = definition.vector_source.as_ref().expect("native source");
+    config.config()?;
+    source_schema.field_with_name(&config.column)?;
+    if config.kind == VectorSourceKind::Dedup {
+        return Ok(Planned {
+            definition: definition.clone(),
+            fields: source_schema
+                .fields()
+                .iter()
+                .map(|f| super::without_declarations(f))
+                .collect(),
+            lineage: source_schema
+                .fields()
+                .iter()
+                .map(|f| (f.name().clone(), vec![f.name().clone()]))
+                .collect(),
+            inputs: source_schema
+                .fields()
+                .iter()
+                .map(|f| f.name().clone())
+                .collect(),
+        });
+    }
+    let schema = duplicate_pair_schema();
+    let projections = schema
+        .fields()
+        .iter()
+        .map(|field| ViewProjection {
+            output: field.name().clone(),
+            expression: field.name().clone(),
+        })
+        .collect();
+    let mut definition = definition.clone();
+    let inputs = vec![config.column.clone()];
+    definition.projections = projections;
+    Ok(Planned {
+        definition,
+        fields: schema.fields().iter().map(|f| f.as_ref().clone()).collect(),
+        lineage: Default::default(),
+        inputs,
+    })
+}
+
+/// A saved native refresh plan, including every physical partition identity.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct DuplicatePairsRefreshPlan {
+    pub snapshot: GroupedRefreshPlan,
+    pub source_uri: String,
+    pub definition_sql: String,
+    pub operator_version: u32,
+    pub threshold_bits: u32,
+    pub tasks: Vec<DuplicatePairTask>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct WrittenPairsPartition {
+    pub(super) unit: u32,
+    pub(super) plan_id: String,
+    pub(super) task: DuplicatePairTask,
+    pub(super) fragments: Vec<Fragment>,
+    pub(super) rows: u64,
+}
+
+impl DuplicatePairsRefreshPlan {
+    pub(super) fn identity(&self) -> Result<String> {
+        use sha2::{Digest, Sha256};
+        let encoded = serde_json::to_vec(&(
+            &self.snapshot,
+            &self.source_uri,
+            &self.definition_sql,
+            self.operator_version,
+            self.threshold_bits,
+        ))
+        .map_err(|e| invalid(e.to_string()))?;
+        Ok(format!("{:x}", Sha256::digest(encoded)))
+    }
+}
+
+async fn open(view: &Table) -> Result<(Dataset, MaterializedViewDefinition, Arc<Dataset>)> {
+    let native = view
+        .as_native()
+        .ok_or_else(|| invalid("native pair refresh requires a local view"))?;
+    native.dataset.ensure_mutable()?;
+    native.dataset.reload().await?;
+    let view_ds = native.dataset.get().await?.as_ref().clone();
+    let Some(super::StoredDefinition::Query(definition)) =
+        super::read_definition(&view_ds.schema().metadata)?
+    else {
+        return Err(invalid("view does not carry a supported definition"));
+    };
+    let config = definition
+        .vector_source
+        .as_ref()
+        .ok_or_else(|| invalid("not a native pair view"))?
+        .config()?;
+    let source = Arc::new(
+        open_source(view, &definition, None)
+            .await?
+            .checkout_version(config.dataset_version)
+            .await?,
+    );
+    ensure_no_mem_wal(&source, "source table", &definition.source_table).await?;
+    let planned = plan(Arc::new(Schema::from(source.schema())), &definition)?;
+    let expected = Schema::new(planned.fields);
+    let physical = Schema::from(view_ds.schema());
+    if physical.fields().len() != expected.fields().len()
+        || physical
+            .fields()
+            .iter()
+            .zip(expected.fields())
+            .any(|(a, b)| {
+                a.name() != b.name()
+                    || a.data_type() != b.data_type()
+                    || a.is_nullable() != b.is_nullable()
+            })
+    {
+        return Err(invalid("native pair view schema changed since creation"));
+    }
+    Ok((view_ds, planned.definition, source))
+}
+
+pub(super) async fn plan_refresh(
+    view: &Table,
+    pinned: Option<u64>,
+) -> Result<DuplicatePairsRefreshPlan> {
+    let (view_ds, definition, source) = open(view).await?;
+    let config = definition
+        .vector_source
+        .as_ref()
+        .expect("native")
+        .config()?;
+    if pinned.is_some_and(|v| v != config.dataset_version) {
+        return Err(invalid(
+            "refresh source version differs from the native pair view's fixed snapshot",
+        ));
+    }
+    let tasks = plan_duplicate_pairs(source.clone(), &config).await?;
+    let units = u32::try_from(tasks.len()).map_err(|_| invalid("too many index partitions"))?;
+    let incarnation = view_ds
+        .schema()
+        .metadata
+        .get(super::INCARNATION_META_KEY)
+        .cloned()
+        .ok_or_else(|| invalid("native pair view has no incarnation"))?;
+    Ok(DuplicatePairsRefreshPlan {
+        snapshot: GroupedRefreshPlan {
+            units,
+            source_version: config.dataset_version,
+            view_version: view_ds.version().version,
+            incarnation,
+        },
+        source_uri: source.uri().to_string(),
+        definition_sql: definition.to_sql(),
+        operator_version: DUPLICATE_PAIRS_OPERATOR_VERSION,
+        threshold_bits: config.distance_threshold.to_bits(),
+        tasks,
+    })
+}
+
+pub(super) async fn open_planned(
+    view: &Table,
+    plan: &DuplicatePairsRefreshPlan,
+) -> Result<(Dataset, MaterializedViewDefinition, Arc<Dataset>)> {
+    let (view_ds, definition, source) = open(view).await?;
+    ensure_incarnation(&view_ds, Some(&plan.snapshot.incarnation), view.name()).await?;
+    let config = definition
+        .vector_source
+        .as_ref()
+        .expect("native")
+        .config()?;
+    if source.uri() != plan.source_uri
+        || source.version().version != plan.snapshot.source_version
+        || definition.to_sql() != plan.definition_sql
+        || config.distance_threshold.to_bits() != plan.threshold_bits
+        || plan.operator_version != DUPLICATE_PAIRS_OPERATOR_VERSION
+        || plan.tasks.len() != plan.snapshot.units as usize
+    {
+        return Err(invalid(
+            "native pair task belongs to a different snapshot, configuration or operator",
+        ));
+    }
+    Ok((view_ds, definition, source))
+}
+
+pub(super) async fn write_unit(
+    view: &Table,
+    unit: u32,
+    plan: &DuplicatePairsRefreshPlan,
+) -> Result<WrittenPairsPartition> {
+    let (view_ds, definition, source) = open_planned(view, plan).await?;
+    let task = plan
+        .tasks
+        .get(unit as usize)
+        .ok_or_else(|| invalid("pair partition is outside the plan"))?
+        .clone();
+    let config = definition
+        .vector_source
+        .as_ref()
+        .expect("native")
+        .config()?;
+    let exec = DuplicatePairsExec::try_new(source, config, vec![task.clone()])?;
+    let stream = exec.execute(0, Arc::new(TaskContext::default()))?;
+    let rows = Arc::new(std::sync::atomic::AtomicU64::new(0));
+    let count = rows.clone();
+    let stream = stream.inspect_ok(move |batch| {
+        count.fetch_add(
+            batch.num_rows() as u64,
+            std::sync::atomic::Ordering::Relaxed,
+        );
+    });
+    let stream: SendableRecordBatchStream = Box::pin(
+        datafusion_physical_plan::stream::RecordBatchStreamAdapter::new(
+            duplicate_pair_schema(),
+            stream,
+        ),
+    );
+    let txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds)))
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        })
+        .execute_uncommitted_stream(stream)
+        .await?;
+    let Operation::Append { fragments } = txn.operation else {
+        return Err(invalid(
+            "native pair partition did not stage append fragments",
+        ));
+    };
+    Ok(WrittenPairsPartition {
+        unit,
+        plan_id: plan.identity()?,
+        task,
+        fragments,
+        rows: rows.load(std::sync::atomic::Ordering::Relaxed),
+    })
+}
+
+pub(super) async fn commit(
+    view: &Table,
+    plan: &DuplicatePairsRefreshPlan,
+    units: Vec<WrittenPairsPartition>,
+    expected: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let (_, definition, source) = open_planned(view, plan).await?;
+    let config = definition
+        .vector_source
+        .as_ref()
+        .expect("native")
+        .config()?;
+    if plan_duplicate_pairs(source.clone(), &config).await? != plan.tasks {
+        return Err(invalid(
+            "pair plan does not cover the pinned index partitions exactly",
+        ));
+    }
+    let identity = plan.identity()?;
+    let mut ordered = vec![None; plan.snapshot.units as usize];
+    for written in units {
+        if written.plan_id != identity
+            || plan.tasks.get(written.unit as usize) != Some(&written.task)
+        {
+            return Err(invalid("pair partition receipt belongs to another plan"));
+        }
+        let slot = ordered
+            .get_mut(written.unit as usize)
+            .ok_or_else(|| invalid("unknown pair partition receipt"))?;
+        if slot.is_some() {
+            return Err(invalid("duplicate pair partition receipt"));
+        }
+        *slot = Some(written);
+    }
+    let units = ordered
+        .into_iter()
+        .collect::<Option<Vec<_>>>()
+        .ok_or_else(|| invalid("missing pair partition receipt"))?;
+    let rows = units.iter().map(|u| u.rows).sum();
+    let fragments = units.into_iter().flat_map(|u| u.fragments).collect();
+    super::grouped_units::commit_fragments(
+        view,
+        &plan.snapshot,
+        &definition,
+        source.manifest.timestamp_nanos,
+        fragments,
+        rows,
+        expected,
+    )
+    .await
+}
diff --git a/rust/lancedb/src/materialized_view/grouped_units.rs b/rust/lancedb/src/materialized_view/grouped_units.rs
index 44bbd7381..7f408be4d 100644
--- a/rust/lancedb/src/materialized_view/grouped_units.rs
+++ b/rust/lancedb/src/materialized_view/grouped_units.rs
@@ -53,6 +53,7 @@ const PROBE_ROWS: usize = 256;
 
 /// How a grouped view can be refreshed in units.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct GroupedRefreshPlan {
     /// Units `0..units`: unit `p` before the last holds index partition
     /// `p`'s groups; the last holds the rows the index cannot place (null or
@@ -177,6 +178,7 @@ pub async fn plan_grouped_refresh(
 /// accounted for has to be refusable. Travels to whoever commits, so it
 /// serializes, but its fields are not a caller's to write.
 #[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
 pub struct WrittenUnit {
     unit: u32,
     plan: GroupedRefreshPlan,
@@ -491,10 +493,38 @@ pub async fn commit_grouped_refresh(
     plan: &GroupedRefreshPlan,
     units: Vec<WrittenUnit>,
     expected_incarnation: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let grouped = open_planned(view, plan).await?;
+    let units = complete_set(view.name(), plan, units)?;
+    let rows_written = units.iter().map(|unit| unit.rows).sum();
+    let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect();
+    commit_fragments(
+        view,
+        plan,
+        &grouped.definition,
+        grouped.source.manifest.timestamp_nanos,
+        new_fragments,
+        rows_written,
+        expected_incarnation,
+    )
+    .await
+}
+
+/// The common all-or-nothing unit sink, shared by grouped and native views.
+#[allow(clippy::too_many_arguments)]
+pub(super) async fn commit_fragments(
+    view: &Table,
+    plan: &GroupedRefreshPlan,
+    definition: &MaterializedViewDefinition,
+    source_ts: u128,
+    new_fragments: Vec<Fragment>,
+    rows_written: u64,
+    expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
     let view_native = view.as_native().ok_or_else(|| Error::NotSupported {
         message: "materialized views are supported only on local tables".into(),
     })?;
+    view_native.dataset.ensure_mutable()?;
     let lock = refresh_lock(view_native.dataset.get().await?.uri());
     let _guard = lock.lock().await;
     if let Some(expected) = expected_incarnation
@@ -508,10 +538,17 @@ pub async fn commit_grouped_refresh(
             ),
         });
     }
-    let grouped = open_planned(view, plan).await?;
-    let source_ts = grouped.source.manifest.timestamp_nanos;
-
-    let ds = Arc::new(grouped.view_ds);
+    view_native.dataset.reload().await?;
+    let ds = view_native.dataset.get().await?.as_ref().clone();
+    ensure_incarnation(&ds, Some(&plan.incarnation), view.name()).await?;
+    if super::read_definition(&ds.schema().metadata)?
+        != Some(super::StoredDefinition::Query(definition.clone()))
+    {
+        return Err(Error::InvalidInput {
+            message: "view definition changed while its units were written".into(),
+        });
+    }
+    let ds = Arc::new(ds);
     if ds.version().version != plan.view_version {
         return Err(Error::Runtime {
             message: format!(
@@ -523,11 +560,8 @@ pub async fn commit_grouped_refresh(
             ),
         });
     }
-    let units = complete_set(view.name(), plan, units)?;
     let removed_fragment_ids: Vec<u64> = ds.get_fragments().iter().map(|f| f.id() as u64).collect();
     // Fragment ids are assigned at commit; the units' fragments all carry 0.
-    let rows_written = units.iter().map(|unit| unit.rows).sum();
-    let new_fragments = units.into_iter().flat_map(|unit| unit.fragments).collect();
     let transaction = Transaction::new(
         plan.view_version,
         Operation::Update {
@@ -566,7 +600,7 @@ pub async fn commit_grouped_refresh(
         committed,
         plan.source_version,
         source_ts,
-        Some(&grouped.definition),
+        Some(definition),
         expected_incarnation,
     )
     .await?;
diff --git a/rust/lancedb/src/materialized_view/partitioned.rs b/rust/lancedb/src/materialized_view/partitioned.rs
new file mode 100644
index 000000000..58192089b
--- /dev/null
+++ b/rust/lancedb/src/materialized_view/partitioned.rs
@@ -0,0 +1,192 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Shared plan/write/commit surface for materialized views with independent units.
+
+use serde::{Deserialize, Serialize};
+
+use super::duplicate_pairs::{
+    self, DuplicatePairsRefreshPlan, VectorSourceKind, WrittenPairsPartition,
+};
+use super::vector_dedup::{self, VectorDedupPlan, WrittenDedupUnit};
+use super::{GroupedRefreshPlan, RefreshMaterializedViewResult, StoredDefinition, WrittenUnit};
+use crate::table::Table;
+use crate::{Error, Result};
+
+/// Serializable work description consumed by a host's existing job workers.
+/// The grouped variant preserves the previous saved-plan representation.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum PartitionedRefreshPlan {
+    VectorDedup(VectorDedupPlan),
+    DuplicatePairs(DuplicatePairsRefreshPlan),
+    Grouped(GroupedRefreshPlan),
+}
+
+impl PartitionedRefreshPlan {
+    /// The shared source snapshot and destination generation fence.
+    pub fn snapshot(&self) -> &GroupedRefreshPlan {
+        match self {
+            Self::VectorDedup(plan) => &plan.snapshot,
+            Self::DuplicatePairs(plan) => &plan.snapshot,
+            Self::Grouped(plan) => plan,
+        }
+    }
+    /// Selection tasks that must complete before remaining tasks can start.
+    pub fn dependency_units(&self) -> u32 {
+        match self {
+            Self::VectorDedup(plan) => plan.dependencies(),
+            _ => 0,
+        }
+    }
+    /// Number of independent units in this immutable plan.
+    pub fn units(&self) -> u32 {
+        self.snapshot().units
+    }
+}
+
+/// One completed unit's uncommitted fragments. Failed attempts produce no receipt.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(untagged)]
+pub enum WrittenPartition {
+    VectorDedup(WrittenDedupUnit),
+    DuplicatePairs(WrittenPairsPartition),
+    Grouped(WrittenUnit),
+}
+
+impl WrittenPartition {
+    /// Number of pair or grouped rows staged by this unit.
+    pub fn rows(&self) -> u64 {
+        match self {
+            Self::VectorDedup(unit) => unit.rows,
+            Self::DuplicatePairs(unit) => unit.rows,
+            Self::Grouped(unit) => unit.rows(),
+        }
+    }
+    /// Uncommitted output files, retained only after task completion.
+    pub fn fragments(&self) -> &[lance_table::format::Fragment] {
+        match self {
+            Self::VectorDedup(unit) => &unit.fragments,
+            Self::DuplicatePairs(unit) => &unit.fragments,
+            Self::Grouped(unit) => unit.fragments(),
+        }
+    }
+}
+
+/// Plan native pair or indexed grouped work; return `None` for ordinary views.
+/// `pinned`, when supplied, must match a native view's declared source version.
+pub async fn plan_partitioned_refresh(
+    view: &Table,
+    pinned: Option<u64>,
+) -> Result<Option<PartitionedRefreshPlan>> {
+    if let Some(StoredDefinition::Query(definition)) =
+        super::read_definition(view.schema().await?.metadata())?
+        && definition.vector_source.is_some()
+    {
+        if definition
+            .vector_source
+            .as_ref()
+            .is_some_and(|s| s.kind == VectorSourceKind::Dedup)
+        {
+            return vector_dedup::plan_refresh(view, pinned)
+                .await
+                .map(|p| Some(PartitionedRefreshPlan::VectorDedup(p)));
+        }
+        return duplicate_pairs::plan_refresh(view, pinned)
+            .await
+            .map(|p| Some(PartitionedRefreshPlan::DuplicatePairs(p)));
+    }
+    super::plan_grouped_refresh(view, pinned)
+        .await
+        .map(|plan| plan.map(PartitionedRefreshPlan::Grouped))
+}
+
+/// Stream one unit into uncommitted fragments. Persist the returned receipt only
+/// on success. A failed or interrupted call must be retried from the unit's start.
+pub async fn write_refresh_partition(
+    view: &Table,
+    unit: u32,
+    plan: &PartitionedRefreshPlan,
+) -> Result<WrittenPartition> {
+    write_refresh_partition_with_inputs(view, unit, plan, &[]).await
+}
+
+/// Write a unit after its durable selection dependencies have completed.
+/// Supply exactly the first `plan.dependency_units()` receipts for a dependent
+/// unit. A missing, duplicate or foreign receipt fails before output is staged.
+pub async fn write_refresh_partition_with_inputs(
+    view: &Table,
+    unit: u32,
+    plan: &PartitionedRefreshPlan,
+    dependencies: &[WrittenPartition],
+) -> Result<WrittenPartition> {
+    match plan {
+        PartitionedRefreshPlan::VectorDedup(plan) => {
+            let dependencies = dependencies
+                .iter()
+                .map(|unit| match unit {
+                    WrittenPartition::VectorDedup(unit) => Ok(unit.clone()),
+                    _ => Err(Error::InvalidInput {
+                        message: "foreign dedup dependency".into(),
+                    }),
+                })
+                .collect::<Result<Vec<_>>>()?;
+            vector_dedup::write_unit(view, unit, plan, &dependencies)
+                .await
+                .map(WrittenPartition::VectorDedup)
+        }
+        PartitionedRefreshPlan::DuplicatePairs(plan) => {
+            duplicate_pairs::write_unit(view, unit, plan)
+                .await
+                .map(WrittenPartition::DuplicatePairs)
+        }
+        PartitionedRefreshPlan::Grouped(plan) => super::write_grouped_unit(view, unit, plan)
+            .await
+            .map(WrittenPartition::Grouped),
+    }
+}
+
+/// Publish a complete set of unit receipts through the common MV commit sink.
+/// `expected` fences the destination incarnation. No partial set is publishable.
+pub async fn commit_partitioned_refresh(
+    view: &Table,
+    plan: &PartitionedRefreshPlan,
+    units: Vec<WrittenPartition>,
+    expected: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let foreign = || Error::InvalidInput {
+        message: "refresh receipt does not match the plan kind".into(),
+    };
+    match plan {
+        PartitionedRefreshPlan::VectorDedup(plan) => {
+            let units = units
+                .into_iter()
+                .map(|unit| match unit {
+                    WrittenPartition::VectorDedup(unit) => Ok(unit),
+                    _ => Err(foreign()),
+                })
+                .collect::<Result<Vec<_>>>()?;
+            vector_dedup::commit(view, plan, units, expected).await
+        }
+        PartitionedRefreshPlan::DuplicatePairs(plan) => {
+            let units = units
+                .into_iter()
+                .map(|unit| match unit {
+                    WrittenPartition::DuplicatePairs(unit) => Ok(unit),
+                    _ => Err(foreign()),
+                })
+                .collect::<Result<Vec<_>>>()?;
+            duplicate_pairs::commit(view, plan, units, expected).await
+        }
+        PartitionedRefreshPlan::Grouped(plan) => {
+            let units = units
+                .into_iter()
+                .map(|unit| match unit {
+                    WrittenPartition::Grouped(unit) => Ok(unit),
+                    _ => Err(foreign()),
+                })
+                .collect::<Result<Vec<_>>>()?;
+            super::commit_grouped_refresh(view, plan, units, expected).await
+        }
+    }
+}
diff --git a/rust/lancedb/src/materialized_view/query.rs b/rust/lancedb/src/materialized_view/query.rs
index c1a987b2c..4bfd5542e 100644
--- a/rust/lancedb/src/materialized_view/query.rs
+++ b/rust/lancedb/src/materialized_view/query.rs
@@ -264,7 +264,36 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
     };
 
     let mut from = select.from.iter();
+    let mut vector_source = None;
     let (source_namespace, source_table) = match from.next().map(|f| &f.relation) {
+        Some(TableFactor::Table {
+            name,
+            args: Some(args),
+            alias: None,
+            with_hints,
+            version: None,
+            with_ordinality: false,
+            partitions,
+            json_path: None,
+            sample: None,
+            index_hints,
+            ..
+        }) if matches!(
+            name.to_string().as_str(),
+            super::duplicate_pairs::FUNCTION_NAME | super::duplicate_pairs::DEDUP_FUNCTION_NAME
+        ) && with_hints.is_empty()
+            && partitions.is_empty()
+            && index_hints.is_empty() =>
+        {
+            let kind = if name.to_string() == super::duplicate_pairs::FUNCTION_NAME {
+                super::VectorSourceKind::Pairs
+            } else {
+                super::VectorSourceKind::Dedup
+            };
+            let (namespace, table, config) = super::duplicate_pairs::source(args, kind)?;
+            vector_source = Some(Box::new(config));
+            (namespace, table)
+        }
         Some(TableFactor::Table { args: Some(_), .. }) => {
             return Err(invalid(
                 "a view reads a table; a Function in FROM position follows it: \
@@ -416,6 +445,7 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
     };
 
     let definition = MaterializedViewDefinition {
+        vector_source,
         source_table,
         source_namespace,
         lateral,
@@ -424,6 +454,9 @@ fn extract(query: &Query) -> Result<MaterializedViewDefinition> {
         group_by,
         limit,
     };
+    if definition.vector_source.is_some() {
+        super::duplicate_pairs::check_shape(&definition)?;
+    }
     check_grouping(&definition)?;
     Ok(definition)
 }
@@ -483,7 +516,18 @@ pub fn render(definition: &MaterializedViewDefinition) -> String {
         .chain(std::iter::once(&definition.source_table))
         .map(|part| ident_sql(part))
         .collect();
-    sql.push_str(&table.join("."));
+    if let Some(native) = &definition.vector_source {
+        sql.push_str(&format!(
+            "{}('{}', {}, '{}', {})",
+            native.function_name(),
+            table.join(".").replace('\'', "''"),
+            native.dataset_version,
+            native.column.replace('\'', "''"),
+            native.distance_threshold
+        ));
+    } else {
+        sql.push_str(&table.join("."));
+    }
     if let Some(lateral) = &definition.lateral {
         match &lateral.source {
             LateralSource::Unnest { column } => {
@@ -511,6 +555,45 @@ pub fn render(definition: &MaterializedViewDefinition) -> String {
 mod tests {
     use super::*;
 
+    #[test]
+    fn native_pairs_are_a_table_source_with_literal_snapshot_arguments() {
+        for threshold in ["4", "4.0", "-1", "0.05"] {
+            let sql =
+                format!("SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', {threshold})");
+            let definition = parse(&sql).unwrap();
+            assert_eq!(definition.source_table, "images");
+            assert!(definition.lateral.is_none());
+            assert_eq!(
+                definition.vector_source.as_ref().unwrap().dataset_version,
+                3
+            );
+            assert_eq!(parse(&render(&definition)).unwrap(), definition);
+            let metadata = std::collections::HashMap::from([(
+                super::super::DEFINITION_META_KEY.to_string(),
+                definition.to_json().unwrap(),
+            )]);
+            assert_eq!(
+                super::super::read_definition(&metadata).unwrap(),
+                Some(super::super::StoredDefinition::Query(definition))
+            );
+        }
+        for sql in [
+            "SELECT row_id_a FROM vector_duplicate_pairs('images', 3, 'phash', 4)",
+            "SELECT * FROM vector_duplicate_pairs('images', 0, 'phash', 4)",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 1e100)",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', cutoff)",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) LIMIT 1",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) WHERE distance < 2",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4) AS p",
+            "SELECT * FROM vector_duplicate_pairs('images', 3, 'phash', 4), f(phash) AS p",
+        ] {
+            assert!(
+                parse(sql).is_err(),
+                "accepted unsupported native view: {sql}"
+            );
+        }
+    }
+
     #[test]
     fn a_query_round_trips_through_its_canonical_form() {
         for (sql, canonical) in [
diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs
index 5e9fe13e3..9f0ea2646 100644
--- a/rust/lancedb/src/materialized_view/refresh.rs
+++ b/rust/lancedb/src/materialized_view/refresh.rs
@@ -117,6 +117,26 @@ pub(crate) async fn execute_refresh(
     pinned: Option<u64>,
     expected_incarnation: Option<&str>,
 ) -> Result<RefreshMaterializedViewResult> {
+    // Native table functions use the same partition writer and commit sink in
+    // local execution; Sophon drives these units through its job registry.
+    if let Some(super::StoredDefinition::Query(definition)) =
+        super::read_definition(view.schema().await?.metadata())?
+        && definition.vector_source.is_some()
+    {
+        let plan = super::plan_partitioned_refresh(view, pinned)
+            .await?
+            .ok_or_else(|| Error::Runtime {
+                message: "native pair view has no partition plan".into(),
+            })?;
+        let mut units = Vec::with_capacity(plan.units() as usize);
+        for unit in 0..plan.units() {
+            let dependencies = &units[..units.len().min(plan.dependency_units() as usize)];
+            units.push(
+                super::write_refresh_partition_with_inputs(view, unit, &plan, dependencies).await?,
+            );
+        }
+        return super::commit_partitioned_refresh(view, &plan, units, expected_incarnation).await;
+    }
     let view_native = view.as_native().ok_or_else(|| Error::NotSupported {
         message: "materialized views are supported only on local tables".into(),
     })?;
@@ -728,7 +748,7 @@ pub(super) async fn open_source(
         message: "materialized views are supported only on local tables".into(),
     })?;
     let dataset = native.dataset.get().await?.as_ref().clone();
-    if !dataset.manifest.uses_stable_row_ids() {
+    if definition.vector_source.is_none() && !dataset.manifest.uses_stable_row_ids() {
         return Err(Error::InvalidInput {
             message: format!(
                 "source table '{}' does not have stable row ids; it is not the \
@@ -3674,6 +3694,7 @@ pub(crate) mod tests {
         let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
 
         let replacement = crate::materialized_view::MaterializedViewDefinition {
+            vector_source: None,
             source_table: "src".into(),
             source_namespace: Vec::new(),
             projections: vec![
@@ -3715,6 +3736,7 @@ pub(crate) mod tests {
         let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await;
 
         let narrower = crate::materialized_view::MaterializedViewDefinition {
+            vector_source: None,
             source_table: "src".into(),
             source_namespace: Vec::new(),
             projections: vec![crate::materialized_view::ViewProjection {
diff --git a/rust/lancedb/src/materialized_view/vector_dedup.rs b/rust/lancedb/src/materialized_view/vector_dedup.rs
new file mode 100644
index 000000000..38c007076
--- /dev/null
+++ b/rust/lancedb/src/materialized_view/vector_dedup.rs
@@ -0,0 +1,636 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Two dependent waves in the ordinary refresh fleet: index partition selection,
+//! then source fragment materialization. Only the latter publishes data files.
+
+use std::collections::BTreeMap;
+use std::io::Cursor;
+use std::sync::Arc;
+
+use arrow_array::{Array, RecordBatch, UInt64Array};
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use datafusion::prelude::{SessionConfig, SessionContext, col};
+use datafusion_common::DataFusionError;
+use datafusion_execution::{
+    SendableRecordBatchStream, TaskContext, runtime_env::RuntimeEnvBuilder,
+};
+use datafusion_physical_plan::{ExecutionPlan, stream::RecordBatchStreamAdapter};
+use futures::TryStreamExt;
+use lance::Dataset;
+use lance::dataset::rowids::{get_row_id_index, load_row_id_sequence};
+use lance::dataset::scanner::{RowAddrMask, RowAddrTreeMap};
+use lance::dataset::transaction::Operation;
+use lance::dataset::{InsertBuilder, WriteDestination, WriteMode, WriteParams};
+use lance_core::datatypes::BlobHandling;
+use lance_core::utils::address::RowAddress;
+use lance_datafusion::exec::{OneShotExec, SessionContextExt};
+use lance_table::format::Fragment;
+use roaring::{RoaringBitmap, RoaringTreemap};
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+
+use super::duplicate_pairs::{self, DuplicatePairsRefreshPlan, VectorSourceKind};
+use super::{GroupedRefreshPlan, RefreshMaterializedViewResult};
+use crate::table::Table;
+use crate::table::datafusion::udtf::duplicate_pairs::{DuplicatePairsExec, plan_duplicate_pairs};
+use crate::{Error, Result};
+
+const POLICY_VERSION: u32 = 1;
+const SORT_MEMORY_BYTES: usize = 256 * 1024 * 1024;
+const MATERIALIZE_BATCH_ROWS: usize = 1024;
+const MASK_FILE: &str = "dedup.bitmap";
+const MASK_MAGIC: &[u8] = b"LANCEDB_DEDUP_1";
+
+fn invalid(message: impl Into<String>) -> Error {
+    Error::InvalidInput {
+        message: message.into(),
+    }
+}
+fn io_error(error: std::io::Error) -> Error {
+    invalid(format!("invalid dedup bitmap: {error}"))
+}
+
+/// Immutable snapshot, partition tasks and source fragments for both waves.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct VectorDedupPlan {
+    pub snapshot: GroupedRefreshPlan,
+    native: DuplicatePairsRefreshPlan,
+    source_fragments: Vec<u32>,
+    policy_version: u32,
+}
+impl VectorDedupPlan {
+    pub(super) fn dependencies(&self) -> u32 {
+        self.native.snapshot.units
+    }
+    fn identity(&self) -> Result<String> {
+        Ok(format!(
+            "{:x}",
+            Sha256::digest(serde_json::to_vec(self).map_err(|e| invalid(e.to_string()))?)
+        ))
+    }
+}
+
+/// Small receipt for a durable selection mask. Row IDs and pairs are not stored
+/// in the host's task registry. UUID directories use ordinary index-file GC.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct MaskFile {
+    source_fragment: u32,
+    object_id: Uuid,
+    bytes: usize,
+    digest: String,
+}
+
+/// A selection task produces masks; a materialization task produces data files.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct WrittenDedupUnit {
+    unit: u32,
+    plan_id: String,
+    masks: Vec<MaskFile>,
+    pub(super) fragments: Vec<Fragment>,
+    pub(super) rows: u64,
+}
+
+pub(super) async fn plan_refresh(view: &Table, pinned: Option<u64>) -> Result<VectorDedupPlan> {
+    let native = duplicate_pairs::plan_refresh(view, pinned).await?;
+    let (_, _, source) = duplicate_pairs::open_planned(view, &native).await?;
+    let source_fragments = source
+        .get_fragments()
+        .iter()
+        .map(|f| f.id() as u32)
+        .collect::<Vec<_>>();
+    let mut snapshot = native.snapshot.clone();
+    snapshot.units = snapshot
+        .units
+        .checked_add(
+            u32::try_from(source_fragments.len())
+                .map_err(|_| invalid("too many source fragments"))?,
+        )
+        .ok_or_else(|| invalid("too many dedup refresh units"))?;
+    Ok(VectorDedupPlan {
+        snapshot,
+        native,
+        source_fragments,
+        policy_version: POLICY_VERSION,
+    })
+}
+
+async fn open(
+    view: &Table,
+    plan: &VectorDedupPlan,
+) -> Result<(Dataset, super::MaterializedViewDefinition, Arc<Dataset>)> {
+    let (view_ds, definition, source) = duplicate_pairs::open_planned(view, &plan.native).await?;
+    let mut snapshot = plan.native.snapshot.clone();
+    snapshot.units = snapshot
+        .units
+        .checked_add(
+            u32::try_from(plan.source_fragments.len())
+                .map_err(|_| invalid("too many dedup fragments"))?,
+        )
+        .ok_or_else(|| invalid("too many dedup units"))?;
+    if plan.policy_version != POLICY_VERSION
+        || snapshot != plan.snapshot
+        || definition.vector_source.as_ref().map(|s| s.kind) != Some(VectorSourceKind::Dedup)
+        || source
+            .get_fragments()
+            .iter()
+            .map(|f| f.id() as u32)
+            .collect::<Vec<_>>()
+            != plan.source_fragments
+    {
+        return Err(invalid("dedup task belongs to another snapshot or policy"));
+    }
+    Ok((view_ds, definition, source))
+}
+
+fn edge_schema() -> SchemaRef {
+    Arc::new(Schema::new(vec![
+        Field::new("a", DataType::UInt64, false),
+        Field::new("b", DataType::UInt64, false),
+    ]))
+}
+
+/// Sorting canonical edges makes the first unremoved endpoint the smallest
+/// unassigned representative. The sort spills; only vertex masks stay in RAM.
+async fn select_representatives(
+    stream: SendableRecordBatchStream,
+    memory: usize,
+) -> Result<(RoaringTreemap, RoaringTreemap)> {
+    let schema = edge_schema();
+    let batch_schema = schema.clone();
+    let canonical = stream
+        .map_ok(move |batch| {
+            let a = batch
+                .column(0)
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .expect("native row id");
+            let b = batch
+                .column(1)
+                .as_any()
+                .downcast_ref::<UInt64Array>()
+                .expect("native row id");
+            RecordBatch::try_new(
+                batch_schema.clone(),
+                vec![
+                    Arc::new(UInt64Array::from_iter_values(
+                        a.values().iter().zip(b.values()).map(|(&a, &b)| a.min(b)),
+                    )),
+                    Arc::new(UInt64Array::from_iter_values(
+                        a.values().iter().zip(b.values()).map(|(&a, &b)| a.max(b)),
+                    )),
+                ],
+            )
+            .map_err(DataFusionError::from)
+        })
+        .and_then(futures::future::ready);
+    let canonical = Box::pin(RecordBatchStreamAdapter::new(schema, canonical));
+    let mut config = SessionConfig::default().with_batch_size(MATERIALIZE_BATCH_ROWS);
+    // Leave enough room to merge spilled runs while the final in-memory run
+    // is still resident. Small batches also bound each merge input buffer.
+    config.options_mut().execution.sort_spill_reservation_bytes = memory / 2;
+    let ctx = SessionContext::new_with_config_rt(
+        config,
+        RuntimeEnvBuilder::new()
+            .with_memory_limit(memory, 1.0)
+            .build_arc()?,
+    );
+    let mut sorted = ctx
+        .read_one_shot(canonical)?
+        .sort_by(vec![col("a"), col("b")])?
+        .execute_stream()
+        .await?;
+    let mut removed = RoaringTreemap::new();
+    let mut representatives = RoaringTreemap::new();
+    while let Some(batch) = sorted.try_next().await? {
+        let a = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .expect("sorted id");
+        let b = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .expect("sorted id");
+        for (&a, &b) in a.values().iter().zip(b.values()) {
+            if a != b && !removed.contains(a) && !removed.contains(b) {
+                representatives.insert(a);
+                removed.insert(b);
+            }
+        }
+    }
+    Ok((removed, representatives))
+}
+
+#[derive(Default)]
+struct Masks {
+    removed: RoaringBitmap,
+    representatives: RoaringBitmap,
+}
+
+async fn persist_masks(
+    view: &Dataset,
+    source: &Dataset,
+    removed: RoaringTreemap,
+    representatives: RoaringTreemap,
+) -> Result<Vec<MaskFile>> {
+    let row_index = get_row_id_index(source).await?;
+    let mut masks: BTreeMap<u32, Masks> = BTreeMap::new();
+    for (ids, is_removed) in [(removed, true), (representatives, false)] {
+        for id in ids {
+            let address = match &row_index {
+                Some(index) => index
+                    .get(id)?
+                    .ok_or_else(|| invalid("pair row id missing from pinned source"))?,
+                None => RowAddress::new_from_u64(id),
+            };
+            let mask = masks.entry(address.fragment_id()).or_default();
+            if is_removed {
+                mask.removed.insert(address.row_offset());
+            } else {
+                mask.representatives.insert(address.row_offset());
+            }
+        }
+    }
+    let store = view.object_store(None).await?;
+    let mut files = Vec::new();
+    for (source_fragment, mask) in masks {
+        if source.get_fragment(source_fragment as usize).is_none() {
+            return Err(invalid("pair row address is outside the pinned snapshot"));
+        }
+        let mut bytes = MASK_MAGIC.to_vec();
+        mask.removed.serialize_into(&mut bytes).map_err(io_error)?;
+        mask.representatives
+            .serialize_into(&mut bytes)
+            .map_err(io_error)?;
+        let object_id = Uuid::new_v4();
+        store
+            .put(
+                &view
+                    .indices_dir()
+                    .join(object_id.to_string())
+                    .join(MASK_FILE),
+                &bytes,
+            )
+            .await?;
+        files.push(MaskFile {
+            source_fragment,
+            object_id,
+            bytes: bytes.len(),
+            digest: format!("{:x}", Sha256::digest(&bytes)),
+        });
+    }
+    Ok(files)
+}
+
+fn validate_receipts<'a>(
+    plan: &VectorDedupPlan,
+    receipts: &'a [WrittenDedupUnit],
+    count: u32,
+) -> Result<Vec<&'a WrittenDedupUnit>> {
+    let identity = plan.identity()?;
+    let mut ordered = vec![None; count as usize];
+    for receipt in receipts {
+        if receipt.plan_id != identity {
+            return Err(invalid("dedup receipt belongs to another plan"));
+        }
+        let slot = ordered
+            .get_mut(receipt.unit as usize)
+            .ok_or_else(|| invalid("unexpected dedup receipt"))?;
+        if slot.is_some() {
+            return Err(invalid("duplicate dedup receipt"));
+        }
+        if receipt.unit < plan.dependencies() {
+            if !receipt.fragments.is_empty()
+                || receipt.rows != 0
+                || receipt
+                    .masks
+                    .iter()
+                    .any(|m| !plan.source_fragments.contains(&m.source_fragment))
+            {
+                return Err(invalid("invalid selection receipt"));
+            }
+        } else if !receipt.masks.is_empty() {
+            return Err(invalid("invalid materialization receipt"));
+        }
+        *slot = Some(receipt);
+    }
+    ordered
+        .into_iter()
+        .collect::<Option<Vec<_>>>()
+        .ok_or_else(|| invalid("missing dedup dependency receipt"))
+}
+
+async fn load_masks(
+    view: &Dataset,
+    source_fragment: u32,
+    dependencies: &[&WrittenDedupUnit],
+) -> Result<RoaringBitmap> {
+    let store = view.object_store(None).await?;
+    let mut masks = Masks::default();
+    for file in dependencies
+        .iter()
+        .flat_map(|u| &u.masks)
+        .filter(|m| m.source_fragment == source_fragment)
+    {
+        let reader = store
+            .open(
+                &view
+                    .indices_dir()
+                    .join(file.object_id.to_string())
+                    .join(MASK_FILE),
+            )
+            .await?;
+        if reader.size().await? != file.bytes {
+            return Err(invalid("dedup mask length mismatch"));
+        }
+        let bytes = reader.get_range(0..file.bytes).await?;
+        if !bytes.starts_with(MASK_MAGIC) || format!("{:x}", Sha256::digest(&bytes)) != file.digest
+        {
+            return Err(invalid("dedup mask checksum mismatch"));
+        }
+        let mut cursor = Cursor::new(&bytes[MASK_MAGIC.len()..]);
+        masks.removed |= RoaringBitmap::deserialize_from(&mut cursor).map_err(io_error)?;
+        masks.representatives |= RoaringBitmap::deserialize_from(&mut cursor).map_err(io_error)?;
+        if cursor.position() as usize != bytes.len() - MASK_MAGIC.len() {
+            return Err(invalid("dedup mask has trailing data"));
+        }
+    }
+    // Native index scopes normally have disjoint rows. Fail closed if an
+    // overlapping scope would delete a representative used by another scope.
+    if !masks.removed.is_disjoint(&masks.representatives) {
+        return Err(invalid(
+            "overlapping index scopes disagree on retained representatives",
+        ));
+    }
+    Ok(masks.removed)
+}
+
+pub(super) async fn write_unit(
+    view: &Table,
+    unit: u32,
+    plan: &VectorDedupPlan,
+    dependencies: &[WrittenDedupUnit],
+) -> Result<WrittenDedupUnit> {
+    let (view_ds, definition, source) = open(view, plan).await?;
+    let mut written = WrittenDedupUnit {
+        unit,
+        plan_id: plan.identity()?,
+        masks: Vec::new(),
+        fragments: Vec::new(),
+        rows: 0,
+    };
+    if unit < plan.dependencies() {
+        let config = definition
+            .vector_source
+            .as_ref()
+            .expect("dedup source")
+            .config()?;
+        let exec = DuplicatePairsExec::try_new(
+            source.clone(),
+            config,
+            vec![plan.native.tasks[unit as usize].clone()],
+        )?;
+        let stream = exec.execute(0, Arc::new(TaskContext::default()))?;
+        let (removed, representatives) = select_representatives(stream, SORT_MEMORY_BYTES).await?;
+        written.masks = persist_masks(&view_ds, &source, removed, representatives).await?;
+        return Ok(written);
+    }
+    let source_fragment = *plan
+        .source_fragments
+        .get((unit - plan.dependencies()) as usize)
+        .ok_or_else(|| invalid("dedup unit is outside the plan"))?;
+    let dependencies = validate_receipts(plan, dependencies, plan.dependencies())?;
+    let removed = load_masks(&view_ds, source_fragment, &dependencies).await?;
+    let fragment = source
+        .get_fragment(source_fragment as usize)
+        .ok_or_else(|| invalid("source fragment missing"))?;
+    // The scanner prefilter uses snapshot row IDs. Our persisted masks use
+    // physical offsets so each writer only loads its own fragment's artifacts.
+    let blocked = if source.manifest.uses_stable_row_ids() {
+        let sequence = load_row_id_sequence(&source, fragment.metadata()).await?;
+        if removed
+            .max()
+            .is_some_and(|offset| offset as u64 >= sequence.len())
+        {
+            return Err(invalid("dedup mask offset is outside the source fragment"));
+        }
+        RowAddrTreeMap::from_iter(sequence.select(removed.iter().map(|offset| offset as usize)))
+    } else {
+        RowAddrTreeMap::from_iter(
+            removed
+                .iter()
+                .map(|offset| u64::from(RowAddress::new_from_parts(source_fragment, offset))),
+        )
+    };
+    let mut scanner = source.scan();
+    scanner
+        .with_fragments(vec![fragment.metadata().clone()])
+        .with_row_addr_prefilter(RowAddrMask::from_block(blocked))
+        .blob_handling(BlobHandling::AllBinary)
+        .batch_size(MATERIALIZE_BATCH_ROWS)
+        .strict_batch_size(true);
+    let stream = scanner.try_into_stream().await?;
+    let schema = lance_io::stream::RecordBatchStream::schema(&stream);
+    let rows = Arc::new(std::sync::atomic::AtomicU64::new(0));
+    let count = rows.clone();
+    let stream = stream
+        .map_err(|e| DataFusionError::External(Box::new(e)))
+        .inspect_ok(move |batch| {
+            count.fetch_add(
+                batch.num_rows() as u64,
+                std::sync::atomic::Ordering::Relaxed,
+            );
+        });
+    let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new(schema, stream));
+    // Materialized binary payloads must become the destination's logical blob
+    // structs before the Lance writer encodes them. Reuse the normal add path's
+    // coercions, including nulls and nested fields.
+    let coerced = crate::table::datafusion::cast::cast_to_table_schema(
+        Arc::new(OneShotExec::new(stream)),
+        &Schema::from(view_ds.schema()),
+    )?;
+    let stream = coerced.execute(0, Arc::new(TaskContext::default()))?;
+    let txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds)))
+        .with_params(&WriteParams {
+            mode: WriteMode::Append,
+            ..Default::default()
+        })
+        .execute_uncommitted_stream(stream)
+        .await?;
+    let Operation::Append { fragments } = txn.operation else {
+        return Err(invalid(
+            "dedup materialization did not stage append fragments",
+        ));
+    };
+    written.fragments = fragments;
+    written.rows = rows.load(std::sync::atomic::Ordering::Relaxed);
+    Ok(written)
+}
+
+pub(super) async fn commit(
+    view: &Table,
+    plan: &VectorDedupPlan,
+    units: Vec<WrittenDedupUnit>,
+    expected: Option<&str>,
+) -> Result<RefreshMaterializedViewResult> {
+    let (_, definition, source) = open(view, plan).await?;
+    let config = definition
+        .vector_source
+        .as_ref()
+        .expect("dedup source")
+        .config()?;
+    if plan_duplicate_pairs(source.clone(), &config).await? != plan.native.tasks {
+        return Err(invalid("dedup plan does not cover the pinned index"));
+    }
+    validate_receipts(plan, &units, plan.snapshot.units)?;
+    let rows = units.iter().map(|u| u.rows).sum();
+    let fragments = units.into_iter().flat_map(|u| u.fragments).collect();
+    super::grouped_units::commit_fragments(
+        view,
+        &plan.snapshot,
+        &definition,
+        source.manifest.timestamp_nanos,
+        fragments,
+        rows,
+        expected,
+    )
+    .await
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn edges(values: &[(u64, u64)], batch_size: usize) -> SendableRecordBatchStream {
+        let schema = edge_schema();
+        let batches = values
+            .chunks(batch_size)
+            .map(|edges| {
+                Ok(RecordBatch::try_new(
+                    schema.clone(),
+                    vec![
+                        Arc::new(UInt64Array::from_iter_values(edges.iter().map(|e| e.0))),
+                        Arc::new(UInt64Array::from_iter_values(edges.iter().map(|e| e.1))),
+                    ],
+                )
+                .unwrap())
+            })
+            .collect::<Vec<_>>();
+        Box::pin(RecordBatchStreamAdapter::new(
+            schema,
+            futures::stream::iter(batches),
+        ))
+    }
+
+    #[tokio::test]
+    async fn direct_representatives_handle_chains_orientation_and_uint64() {
+        let high = u64::MAX - 2;
+        let pairs = [
+            (2, 1),
+            (1, 0),
+            (high + 2, high + 1),
+            (high, high + 1),
+            (1, 0),
+        ];
+        let (removed, representatives) =
+            select_representatives(edges(&pairs, 1), SORT_MEMORY_BYTES)
+                .await
+                .unwrap();
+        assert_eq!(removed.iter().collect::<Vec<_>>(), [1, high + 1]);
+        assert_eq!(representatives.iter().collect::<Vec<_>>(), [0, high]);
+    }
+
+    #[tokio::test]
+    async fn masks_reject_overlapping_representatives_and_corrupt_artifacts() {
+        let dir = tempfile::tempdir().unwrap();
+        let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::UInt64, false)]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![Arc::new(UInt64Array::from(vec![0, 1, 2]))],
+        )
+        .unwrap();
+        let ds = Dataset::write(
+            arrow_array::RecordBatchIterator::new(vec![Ok(batch)], schema),
+            dir.path().join("source.lance").to_str().unwrap(),
+            None,
+        )
+        .await
+        .unwrap();
+        let first = persist_masks(
+            &ds,
+            &ds,
+            [1].into_iter().collect(),
+            [0].into_iter().collect(),
+        )
+        .await
+        .unwrap();
+        let second = persist_masks(
+            &ds,
+            &ds,
+            [2].into_iter().collect(),
+            [1].into_iter().collect(),
+        )
+        .await
+        .unwrap();
+        let receipt = |masks| WrittenDedupUnit {
+            unit: 0,
+            plan_id: "test".into(),
+            masks,
+            fragments: vec![],
+            rows: 0,
+        };
+        let first = receipt(first);
+        let second = receipt(second);
+        assert_eq!(
+            load_masks(&ds, 0, &[&first]).await.unwrap(),
+            [1].into_iter().collect()
+        );
+        assert!(
+            load_masks(&ds, 0, &[&first, &second])
+                .await
+                .unwrap_err()
+                .to_string()
+                .contains("overlapping")
+        );
+        let file = &first.masks[0];
+        ds.object_store(None)
+            .await
+            .unwrap()
+            .put(
+                &ds.indices_dir()
+                    .join(file.object_id.to_string())
+                    .join(MASK_FILE),
+                &vec![0; file.bytes],
+            )
+            .await
+            .unwrap();
+        assert!(
+            load_masks(&ds, 0, &[&first])
+                .await
+                .unwrap_err()
+                .to_string()
+                .contains("checksum")
+        );
+    }
+
+    #[tokio::test]
+    async fn external_sort_handles_more_edges_than_its_budget() {
+        let pairs = (0..262144)
+            .rev()
+            .map(|i| ((i + 1) as u64, i as u64))
+            .collect::<Vec<_>>();
+        let (removed, reps) = select_representatives(edges(&pairs, 4096), 2 * 1024 * 1024)
+            .await
+            .unwrap();
+        assert_eq!(removed.len(), 131072);
+        assert_eq!(reps.len(), 131072);
+        assert!(removed.iter().all(|id| id % 2 == 1));
+        assert!(reps.iter().all(|id| id % 2 == 0));
+    }
+}
diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs
index d6abfea9a..cabddbe7b 100644
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -2140,6 +2140,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
         let definition = match response.query {
             Some(query) => MaterializedViewDefinition::from_sql(&query)?,
             None => MaterializedViewDefinition {
+                vector_source: None,
                 source_table: response.source_table,
                 source_namespace: response.source_namespace,
                 lateral: None,
diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs
index 0bfec9d68..c82b85971 100644
--- a/rust/lancedb/src/table/add_data.rs
+++ b/rust/lancedb/src/table/add_data.rs
@@ -1019,14 +1019,23 @@ mod tests {
     #[case::utf8(DataType::Utf8)]
     #[case::large_utf8(DataType::LargeUtf8)]
     #[tokio::test]
-    async fn test_add_arrow_json_into_lance_json_table(#[case] input_type: DataType) {
+    async fn test_add_arrow_json_into_lance_json_table(
+        #[case] input_type: DataType,
+        #[values(false, true)] table_has_serialized_metadata: bool,
+        #[values(false, true)] input_has_serialized_metadata: bool,
+    ) {
         use arrow_array::{Array, cast::AsArray};
         use lance_arrow::ARROW_EXT_NAME_KEY;
         use lance_arrow::json::{ARROW_JSON_EXT_NAME, JSON_EXT_NAME};
 
         // Build a table whose "data" column is lance.json (LargeBinary +
         // ARROW:extension:name = "lance.json").
-        let lance_json_field = lance_arrow::json::json_field("data", true);
+        let mut lance_json_field = lance_arrow::json::json_field("data", true);
+        if table_has_serialized_metadata {
+            let mut metadata = lance_json_field.metadata().clone();
+            metadata.insert(lance_arrow::ARROW_EXT_META_KEY.to_string(), String::new());
+            lance_json_field = lance_json_field.with_metadata(metadata);
+        }
         let table_schema = Arc::new(Schema::new(vec![lance_json_field]));
 
         let db = connect("memory://").execute().await.unwrap();
@@ -1050,10 +1059,13 @@ mod tests {
 
         // Build an arrow.json input field (Utf8/LargeUtf8 + arrow.json extension).
         // This is what PyArrow produces for pa.json_() arrays.
-        let arrow_json_metadata = std::collections::HashMap::from([(
+        let mut arrow_json_metadata = std::collections::HashMap::from([(
             ARROW_EXT_NAME_KEY.to_string(),
             ARROW_JSON_EXT_NAME.to_string(),
         )]);
+        if input_has_serialized_metadata {
+            arrow_json_metadata.insert(lance_arrow::ARROW_EXT_META_KEY.to_string(), String::new());
+        }
         let arrow_json_field =
             Field::new("data", input_type.clone(), true).with_metadata(arrow_json_metadata);
         let arrow_json_schema = Arc::new(Schema::new(vec![arrow_json_field]));
diff --git a/rust/lancedb/src/table/datafusion/udtf.rs b/rust/lancedb/src/table/datafusion/udtf.rs
index 11f0cc61b..a7a72f800 100644
--- a/rust/lancedb/src/table/datafusion/udtf.rs
+++ b/rust/lancedb/src/table/datafusion/udtf.rs
@@ -4,3 +4,5 @@
 //! User-Defined Table Functions (UDTFs) for DataFusion integration
 
 pub mod fts;
+
+pub mod duplicate_pairs;
diff --git a/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs b/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs
new file mode 100644
index 000000000..47a35cbd2
--- /dev/null
+++ b/rust/lancedb/src/table/datafusion/udtf/duplicate_pairs.rs
@@ -0,0 +1,586 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+//! Index-partition duplicate pairs. No source-vector scan or shuffle is needed.
+
+use std::collections::BTreeSet;
+use std::sync::Arc;
+
+use arrow_schema::{DataType, Field, Schema, SchemaRef};
+use async_trait::async_trait;
+use datafusion::catalog::TableFunctionImpl;
+use datafusion_catalog::{Session, TableProvider};
+use datafusion_common::{DataFusionError, Result, ScalarValue, plan_err};
+use datafusion_execution::{SendableRecordBatchStream, TaskContext};
+use datafusion_expr::{Expr, TableType};
+use datafusion_physical_expr::EquivalenceProperties;
+use datafusion_physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+    execution_plan::{Boundedness, EmissionType},
+    metrics::{ExecutionPlanMetricsSet, MetricBuilder, MetricsSet},
+    projection::ProjectionExec,
+    stream::RecordBatchStreamAdapter,
+};
+use futures::{StreamExt, TryStreamExt, stream};
+use lance::{
+    Dataset,
+    index::{DatasetIndexExt, DatasetIndexInternalExt},
+};
+use lance_index::metrics::NoOpMetricsCollector;
+use uuid::Uuid;
+
+use super::super::BaseTableAdapter;
+use crate::table::NativeTableExt;
+
+// Admission estimates for Lance 13's bounded pairwise kernel. Keep these with
+// the minimum dependency when the kernel's buffering contract changes.
+/// Version of the native task descriptor and its execution semantics.
+pub const DUPLICATE_PAIRS_OPERATOR_VERSION: u32 = 2;
+
+const CODE_STAGING_BYTES: usize = 16 * 1024 * 1024;
+const INDEX_READ_ROWS: usize = 8192;
+const VECTOR_BATCH_ROWS: usize = 8192;
+const VECTOR_BATCH_BYTES: usize = 16 * 1024 * 1024;
+const MIN_VECTOR_BATCH_ROWS: usize = 32;
+const ANCHOR_BLOCK_ROWS: usize = 32;
+const SCORING_CONCURRENCY: usize = 4;
+const MAX_COORDINATE_BYTES: usize = 8;
+const STAGED_ROW_OVERHEAD_BYTES: usize = 64;
+const PAIR_ROW_BYTES: usize = 20;
+
+/// Configuration belongs to one immutable source snapshot. Threshold uses
+/// Lance's float32 index distance, including symmetric reconstructed distances
+/// for quantized indexes; it is not an exact source-embedding distance.
+#[derive(Debug, Clone)]
+pub struct DuplicatePairsConfig {
+    pub dataset_version: u64,
+    pub column: String,
+    pub distance_threshold: f32,
+}
+
+/// One worker input, never a row of source vectors. The complete task identity
+/// also includes the dataset URI/version, column, threshold and operator version.
+#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
+pub struct DuplicatePairTask {
+    pub segment_id: Uuid,
+    pub partition_id: usize,
+}
+
+/// Resolve an authorized source table and preserve its catalog identity.
+pub trait DuplicatePairsResolver: std::fmt::Debug + Send + Sync {
+    fn resolve(
+        &self,
+        table: &str,
+        config: DuplicatePairsConfig,
+        tasks_only: bool,
+    ) -> Result<Arc<dyn TableProvider>>;
+}
+
+/// `vector_duplicate_pairs('table', version, 'column', threshold)`.
+/// Registration is explicit so the host controls catalog access and execution.
+#[derive(Debug)]
+pub struct DuplicatePairsTableFunction {
+    resolver: Arc<dyn DuplicatePairsResolver>,
+    tasks_only: bool,
+}
+
+impl DuplicatePairsTableFunction {
+    pub fn new(resolver: Arc<dyn DuplicatePairsResolver>) -> Self {
+        Self {
+            resolver,
+            tasks_only: false,
+        }
+    }
+}
+
+impl DuplicatePairsTableFunction {
+    /// Register as `vector_duplicate_pair_tasks` to inspect or persist exactly
+    /// the descriptors used by the pair operator, without producing any pairs.
+    pub fn task_manifest(resolver: Arc<dyn DuplicatePairsResolver>) -> Self {
+        Self {
+            resolver,
+            tasks_only: true,
+        }
+    }
+}
+
+impl TableFunctionImpl for DuplicatePairsTableFunction {
+    fn call(&self, args: &[Expr]) -> Result<Arc<dyn TableProvider>> {
+        if args.len() != 4 {
+            return plan_err!(
+                "vector_duplicate_pairs requires (table, dataset_version, column, distance_threshold)"
+            );
+        }
+        let string = |expr: &Expr| match expr {
+            Expr::Literal(ScalarValue::Utf8(Some(s)) | ScalarValue::LargeUtf8(Some(s)), _) => {
+                Ok(s.clone())
+            }
+            _ => plan_err!("table and column must be string literals"),
+        };
+        let dataset_version = match &args[1] {
+            Expr::Literal(ScalarValue::Int64(Some(v)), _) if *v > 0 => *v as u64,
+            Expr::Literal(ScalarValue::UInt64(Some(v)), _) if *v > 0 => *v,
+            _ => return plan_err!("dataset_version must be a positive integer literal"),
+        };
+        let distance_threshold = match &args[3] {
+            Expr::Literal(value, _) if value.data_type().is_numeric() => {
+                value.cast_to(&DataType::Float32)?
+            }
+            _ => return plan_err!("distance_threshold must be a finite numeric literal"),
+        };
+        let ScalarValue::Float32(Some(distance_threshold)) = distance_threshold else {
+            return plan_err!("distance_threshold cannot be null");
+        };
+        if !distance_threshold.is_finite() {
+            return plan_err!("distance_threshold must be finite");
+        }
+        self.resolver.resolve(
+            &string(&args[0])?,
+            DuplicatePairsConfig {
+                dataset_version,
+                column: string(&args[2])?,
+                distance_threshold,
+            },
+            self.tasks_only,
+        )
+    }
+}
+
+/// Three non-null columns; pair direction is storage traversal order, not row-ID order.
+pub fn duplicate_pair_schema() -> SchemaRef {
+    Arc::new(Schema::new(vec![
+        Field::new("row_id_a", DataType::UInt64, false),
+        Field::new("row_id_b", DataType::UInt64, false),
+        Field::new("distance", DataType::Float32, false),
+    ]))
+}
+
+#[derive(Debug)]
+pub struct DuplicatePairsTable {
+    source: Arc<dyn crate::table::BaseTable>,
+    config: DuplicatePairsConfig,
+    tasks_only: bool,
+}
+
+impl BaseTableAdapter {
+    /// Bind the builtin to this authorized table; only metadata is read while
+    /// planning. The source snapshot must remain retained until execution ends.
+    pub fn duplicate_pairs(
+        &self,
+        config: DuplicatePairsConfig,
+        tasks_only: bool,
+    ) -> DuplicatePairsTable {
+        DuplicatePairsTable {
+            source: self.table.clone(),
+            config,
+            tasks_only,
+        }
+    }
+}
+
+#[async_trait]
+impl TableProvider for DuplicatePairsTable {
+    fn schema(&self) -> SchemaRef {
+        if self.tasks_only {
+            duplicate_pair_task_schema()
+        } else {
+            duplicate_pair_schema()
+        }
+    }
+    fn table_type(&self) -> TableType {
+        TableType::View
+    }
+    async fn scan(
+        &self,
+        _state: &dyn Session,
+        projection: Option<&Vec<usize>>,
+        _filters: &[Expr],
+        _limit: Option<usize>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        let native = self.source.as_native().ok_or_else(|| {
+            DataFusionError::Plan("duplicate pairs requires a native indexed table".into())
+        })?;
+        let current = native
+            .dataset
+            .get()
+            .await
+            .map_err(|e| DataFusionError::External(Box::new(e)))?;
+        let dataset = Arc::new(
+            current
+                .checkout_version(self.config.dataset_version)
+                .await?,
+        );
+        let tasks = plan_duplicate_pairs(dataset.clone(), &self.config).await?;
+        if self.tasks_only {
+            let batch = duplicate_pair_task_batch(&dataset, &self.config, &tasks)?;
+            let table =
+                datafusion::datasource::MemTable::try_new(batch.schema(), vec![vec![batch]])?;
+            return table.scan(_state, projection, _filters, _limit).await;
+        }
+        let plan: Arc<dyn ExecutionPlan> = Arc::new(DuplicatePairsExec::try_new(
+            dataset,
+            self.config.clone(),
+            tasks,
+        )?);
+        match projection {
+            Some(indices) => {
+                let schema = plan.schema();
+                let exprs = indices
+                    .iter()
+                    .map(|&i| {
+                        (
+                            Arc::new(datafusion_physical_expr::expressions::Column::new(
+                                schema.field(i).name(),
+                                i,
+                            )) as _,
+                            schema.field(i).name().clone(),
+                        )
+                    })
+                    .collect::<Vec<_>>();
+                Ok(Arc::new(ProjectionExec::try_new(exprs, plan)?))
+            }
+            None => Ok(plan),
+        }
+    }
+}
+
+/// Enumerate segment-local IVF partitions using metadata only. Live-row masks
+/// and index codes are opened by the scoped worker, never by a source-vector
+/// scan on the coordinator. The high-level pair API is only a test oracle.
+pub async fn plan_duplicate_pairs(
+    dataset: Arc<Dataset>,
+    config: &DuplicatePairsConfig,
+) -> Result<Vec<DuplicatePairTask>> {
+    if dataset.version().version != config.dataset_version || !config.distance_threshold.is_finite()
+    {
+        return plan_err!("duplicate pairs source snapshot or threshold is invalid");
+    }
+    let field = dataset.schema().field_id(&config.column)?;
+    let indices = dataset.load_indices().await?;
+    let segments = indices
+        .iter()
+        .filter(|meta| {
+            meta.keyed_fields() == [field]
+                && meta.index_details.as_ref().map_or_else(
+                    || {
+                        meta.files.as_ref().is_some_and(|files| {
+                            files
+                                .iter()
+                                .any(|file| file.path == lance_index::INDEX_FILE_NAME)
+                        })
+                    },
+                    |details| details.type_url.ends_with("VectorIndexDetails"),
+                )
+        })
+        .collect::<Vec<_>>();
+    let names = segments
+        .iter()
+        .map(|meta| meta.name.as_str())
+        .collect::<BTreeSet<_>>();
+    if names.len() != 1 {
+        return plan_err!(
+            "column '{}' requires exactly one current-format vector index, found {}",
+            config.column,
+            names.len()
+        );
+    }
+    let name = names.into_iter().next().expect("one index name");
+    if !dataset.unindexed_fragments(name).await?.is_empty() {
+        return plan_err!(
+            "vector index on column '{}' does not cover all fragments; optimize the index first",
+            config.column
+        );
+    }
+    let mut tasks = Vec::new();
+    for meta in segments {
+        if dataset.fragments().iter().any(|fragment| {
+            meta.fragment_bitmap
+                .as_ref()
+                .is_none_or(|ids| ids.contains(fragment.id as u32))
+                && fragment.overlays.iter().any(|overlay| {
+                    overlay.committed_version > meta.dataset_version
+                        && overlay.data_file.fields.contains(&field)
+                })
+        }) {
+            return plan_err!(
+                "column '{}' has stale index values in segment {}; rebuild the index",
+                config.column,
+                meta.uuid
+            );
+        }
+        let index = dataset
+            .open_vector_index(&config.column, &meta.uuid, &NoOpMetricsCollector)
+            .await?;
+        if !index.supports_pairwise_vectors() {
+            return plan_err!(
+                "segment {} requires a current-format pairwise vector index; rebuild the index",
+                meta.uuid
+            );
+        }
+        for partition_id in 0..index.ivf_model().num_partitions() {
+            tasks.push(DuplicatePairTask {
+                segment_id: meta.uuid,
+                partition_id,
+            });
+        }
+    }
+    if tasks.is_empty() {
+        return plan_err!("no vector index partitions found");
+    }
+    Ok(tasks)
+}
+
+/// Streaming native operator over partition descriptors, never source-vector
+/// rows. Assigned descriptors are consumed sequentially; distributed hosts send
+/// one descriptor per worker request. No Python or per-vector ANN calls.
+/// Distributed merging deliberately advertises no global row ordering.
+#[derive(Debug)]
+pub struct DuplicatePairsExec {
+    dataset: Arc<Dataset>,
+    config: DuplicatePairsConfig,
+    tasks: Vec<DuplicatePairTask>,
+    properties: Arc<PlanProperties>,
+    metrics: ExecutionPlanMetricsSet,
+}
+
+impl DuplicatePairsExec {
+    pub fn try_new(
+        dataset: Arc<Dataset>,
+        config: DuplicatePairsConfig,
+        tasks: Vec<DuplicatePairTask>,
+    ) -> Result<Self> {
+        if dataset.version().version != config.dataset_version
+            || !config.distance_threshold.is_finite()
+            || tasks.is_empty()
+        {
+            return plan_err!("invalid duplicate-pair snapshot, threshold, or empty task list");
+        }
+        let unique = tasks
+            .iter()
+            .map(|t| (t.segment_id, t.partition_id))
+            .collect::<BTreeSet<_>>();
+        if unique.len() != tasks.len() {
+            return plan_err!("duplicate partition tasks");
+        }
+        let properties = Arc::new(PlanProperties::new(
+            EquivalenceProperties::new(duplicate_pair_schema()),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        ));
+        Ok(Self {
+            dataset,
+            config,
+            tasks,
+            properties,
+            metrics: ExecutionPlanMetricsSet::new(),
+        })
+    }
+    pub fn dataset(&self) -> &Arc<Dataset> {
+        &self.dataset
+    }
+    pub fn config(&self) -> &DuplicatePairsConfig {
+        &self.config
+    }
+    pub fn tasks(&self) -> &[DuplicatePairTask] {
+        &self.tasks
+    }
+}
+
+impl DisplayAs for DuplicatePairsExec {
+    fn fmt_as(&self, _: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "DuplicatePairsExec: source_version={}, column={}, threshold={}, tasks={}",
+            self.config.dataset_version,
+            self.config.column,
+            self.config.distance_threshold,
+            self.tasks.len()
+        )
+    }
+}
+
+impl ExecutionPlan for DuplicatePairsExec {
+    fn name(&self) -> &str {
+        "DuplicatePairsExec"
+    }
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.properties
+    }
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        vec![]
+    }
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if !children.is_empty() {
+            return plan_err!("DuplicatePairsExec has no children");
+        }
+        Ok(self)
+    }
+    fn metrics(&self) -> Option<MetricsSet> {
+        Some(self.metrics.clone_inner())
+    }
+    fn execute(
+        &self,
+        partition: usize,
+        context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        if partition != 0 {
+            return plan_err!("duplicate-pair output partition out of bounds");
+        }
+        let tasks = self.tasks.clone();
+        let dataset = self.dataset.clone();
+        let config = self.config.clone();
+        let output_rows = MetricBuilder::new(&self.metrics).counter("output_rows", partition);
+        let completed =
+            MetricBuilder::new(&self.metrics).counter("completed_partitions", partition);
+        // Beta.14 stages native codes and scores bounded tiles concurrently.
+        // Keep the host staging budget explicit rather than inheriting Lance's
+        // larger default or multiplying CPU-pool concurrency across PE tasks.
+        let dimension = match dataset
+            .schema()
+            .field(&config.column)
+            .map(|f| f.data_type())
+        {
+            Some(DataType::FixedSizeList(_, dimension)) if dimension > 0 => dimension as usize,
+            _ => return plan_err!("duplicate pairs requires a fixed-size vector column"),
+        };
+        let batch_bytes = VECTOR_BATCH_BYTES.max(
+            MIN_VECTOR_BATCH_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES),
+        );
+        // Allow staging/preparation overlap, at least three live spill batches,
+        // source input, and in-flight output plus the batch being encoded.
+        // Quantizer models and caches remain outside this admission estimate.
+        let budget = 2 * CODE_STAGING_BYTES
+            + 3 * batch_bytes
+            + INDEX_READ_ROWS * (dimension * MAX_COORDINATE_BYTES + STAGED_ROW_OVERHEAD_BYTES)
+            + (SCORING_CONCURRENCY + 1) * ANCHOR_BLOCK_ROWS * VECTOR_BATCH_ROWS * PAIR_ROW_BYTES;
+        let reservation =
+            datafusion_execution::memory_pool::MemoryConsumer::new("DuplicatePairsExec")
+                .register(context.memory_pool());
+        reservation.try_grow(budget)?;
+        let reservation = Arc::new(reservation);
+        // Sequential local work prevents one query from reserving a kernel for
+        // every IVF partition at once. Distributed hosts split these descriptors
+        // into one-task PE requests before execution.
+        let stream = stream::iter(tasks)
+            .then(move |task| {
+                let dataset = dataset.clone();
+                let config = config.clone();
+                let reservation = reservation.clone();
+                let output_rows = output_rows.clone();
+                let completed = completed.clone();
+                async move {
+                    let reader = lance::index::vector::dedup::find_duplicate_pairs_in_partition_with_options(
+                        dataset,
+                        &config.column,
+                        task.segment_id,
+                        task.partition_id,
+                        config.distance_threshold,
+                        lance::index::vector::dedup::DuplicatePairsOptions::default()
+                            .with_memory_limit(CODE_STAGING_BYTES)
+                            .with_max_concurrency(SCORING_CONCURRENCY),
+                    )
+                    .await?;
+                    let measured = stream::try_unfold(
+                        (reader, reservation),
+                        move |(mut reader, reservation)| {
+                            let output_rows = output_rows.clone();
+                            let completed = completed.clone();
+                            async move {
+                                match reader.try_next().await? {
+                                    Some(batch) => {
+                                        output_rows.add(batch.num_rows());
+                                        Ok::<_, DataFusionError>(Some((
+                                            batch,
+                                            (reader, reservation),
+                                        )))
+                                    }
+                                    None => {
+                                        completed.add(1);
+                                        Ok(None)
+                                    }
+                                }
+                            }
+                        },
+                    );
+                    Ok::<_, DataFusionError>(Box::pin(RecordBatchStreamAdapter::new(
+                        duplicate_pair_schema(),
+                        measured,
+                    )) as SendableRecordBatchStream)
+                }
+            })
+            .try_flatten();
+        Ok(Box::pin(RecordBatchStreamAdapter::new(
+            duplicate_pair_schema(),
+            stream,
+        )))
+    }
+}
+
+/// Inspectable immutable task manifest. Credentials are never included.
+pub fn duplicate_pair_task_schema() -> SchemaRef {
+    Arc::new(Schema::new(vec![
+        Field::new("task_id", DataType::Utf8, false),
+        Field::new("dataset_uri", DataType::Utf8, false),
+        Field::new("dataset_version", DataType::UInt64, false),
+        Field::new("column", DataType::Utf8, false),
+        Field::new("segment_id", DataType::Utf8, false),
+        Field::new("partition_id", DataType::UInt64, false),
+        Field::new("distance_threshold", DataType::Float32, false),
+        Field::new("operator_version", DataType::UInt32, false),
+    ]))
+}
+
+pub fn duplicate_pair_task_batch(
+    dataset: &Dataset,
+    config: &DuplicatePairsConfig,
+    tasks: &[DuplicatePairTask],
+) -> Result<arrow_array::RecordBatch> {
+    use arrow_array::{Float32Array, StringArray, UInt32Array, UInt64Array};
+    use sha2::{Digest, Sha256};
+    let ids = tasks
+        .iter()
+        .map(|task| {
+            // Length-delimited strings prevent ambiguous concatenation; threshold
+            // bits preserve the exact float32 value sent to the native primitive.
+            let bytes = serde_json::to_vec(&(
+                DUPLICATE_PAIRS_OPERATOR_VERSION,
+                dataset.uri(),
+                config.dataset_version,
+                &config.column,
+                task.segment_id.to_string(),
+                task.partition_id,
+                config.distance_threshold.to_bits(),
+            ))
+            .expect("task fields are serializable");
+            format!("{:x}", Sha256::digest(bytes))
+        })
+        .collect::<Vec<_>>();
+    let n = tasks.len();
+    Ok(arrow_array::RecordBatch::try_new(
+        duplicate_pair_task_schema(),
+        vec![
+            Arc::new(StringArray::from(ids)),
+            Arc::new(StringArray::from(vec![dataset.uri(); n])),
+            Arc::new(UInt64Array::from(vec![config.dataset_version; n])),
+            Arc::new(StringArray::from(vec![config.column.as_str(); n])),
+            Arc::new(StringArray::from(
+                tasks
+                    .iter()
+                    .map(|t| t.segment_id.to_string())
+                    .collect::<Vec<_>>(),
+            )),
+            Arc::new(UInt64Array::from(
+                tasks
+                    .iter()
+                    .map(|t| t.partition_id as u64)
+                    .collect::<Vec<_>>(),
+            )),
+            Arc::new(Float32Array::from(vec![config.distance_threshold; n])),
+            Arc::new(UInt32Array::from(vec![DUPLICATE_PAIRS_OPERATOR_VERSION; n])),
+        ],
+    )?)
+}
diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs
index 8f5585829..982818ff6 100644
--- a/rust/lancedb/src/table/merge.rs
+++ b/rust/lancedb/src/table/merge.rs
@@ -1,15 +1,18 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+use std::collections::HashSet;
 use std::sync::Arc;
 use std::time::Duration;
 
 use arrow_array::RecordBatchReader;
+use arrow_schema::{DataType, Fields};
 use futures::future::Either;
 use futures::{FutureExt, TryFutureExt};
 use lance::dataset::{
     MergeInsertBuilder as LanceMergeInsertBuilder, WhenMatched, WhenNotMatchedBySource,
 };
+use lance_datafusion::utils::StreamingWriteSource;
 use serde::{Deserialize, Serialize};
 
 use crate::error::{Error, Result};
@@ -248,6 +251,59 @@ fn canonicalize_merge_filter(filter: Option<MergeFilter>) -> Result<Option<Merge
         .transpose()
 }
 
+// The JSON projection iterates target fields, so reject unknown source fields
+// before it can erase them. Missing and reordered fields remain valid merge
+// inputs; type compatibility is still checked by the cast and Lance writer.
+fn validate_merge_source_fields(input: &Fields, target: &Fields, parent: &str) -> Result<()> {
+    let mut names = HashSet::with_capacity(input.len());
+    for field in input {
+        let path = if parent.is_empty() {
+            field.name().clone()
+        } else {
+            format!("{parent}.{}", field.name())
+        };
+        if !names.insert(field.name()) {
+            return Err(Error::InvalidInput {
+                message: format!("merge source field '{path}' is specified more than once"),
+            });
+        }
+        let target_field = target
+            .iter()
+            .find(|candidate| candidate.name() == field.name())
+            .ok_or_else(|| Error::InvalidInput {
+                message: format!("merge source field '{path}' is not present in the table schema"),
+            })?;
+        validate_merge_source_type(field.data_type(), target_field.data_type(), &path)?;
+    }
+    Ok(())
+}
+
+fn validate_merge_source_type(input: &DataType, target: &DataType, path: &str) -> Result<()> {
+    match (input, target) {
+        (DataType::Struct(input), DataType::Struct(target)) => {
+            validate_merge_source_fields(input, target, path)
+        }
+        (
+            DataType::List(input)
+            | DataType::LargeList(input)
+            | DataType::FixedSizeList(input, _)
+            | DataType::ListView(input)
+            | DataType::LargeListView(input),
+            DataType::List(target)
+            | DataType::LargeList(target)
+            | DataType::FixedSizeList(target, _)
+            | DataType::ListView(target)
+            | DataType::LargeListView(target),
+        )
+        | (DataType::Map(input, _), DataType::Map(target, _)) => {
+            validate_merge_source_type(input.data_type(), target.data_type(), path)
+        }
+        (DataType::Dictionary(_, input), _) => validate_merge_source_type(input, target, path),
+        (_, DataType::Dictionary(_, target)) => validate_merge_source_type(input, target, path),
+        _ => Ok(()),
+    }
+}
+
 /// Internal implementation of the merge insert logic
 ///
 /// This logic was moved from NativeTable::merge_insert to keep table.rs clean.
@@ -279,6 +335,27 @@ pub(crate) async fn execute_merge_insert(
     }
 
     let dataset = table.dataset.get().await?;
+    let schema = arrow_schema::Schema::from(dataset.schema());
+    // JSON source fields must carry the stored extension metadata just as on
+    // append. Keep arrow.json text labelled until Lance encodes it as JSONB.
+    let source = if schema
+        .fields()
+        .iter()
+        .any(|field| lance_arrow::json::has_json_fields(field))
+    {
+        validate_merge_source_fields(new_data.schema().fields(), schema.fields(), "")?;
+        let plan = Arc::new(super::datafusion::scannable_exec::ScannableExec::new(
+            Box::new(new_data),
+            None,
+        ));
+        let plan = super::datafusion::cast::cast_to_table_schema(plan, &schema)?;
+        datafusion_physical_plan::execute_stream(
+            plan,
+            Arc::new(datafusion_execution::TaskContext::default()),
+        )?
+    } else {
+        new_data.into_stream()
+    };
     let mut builder = LanceMergeInsertBuilder::try_new(dataset.clone(), params.on)?;
     match (
         params.when_matched_update_all,
@@ -313,10 +390,7 @@ pub(crate) async fn execute_merge_insert(
     builder.use_index(params.use_index);
 
     let future = if let Some(timeout) = params.timeout {
-        let future = builder
-            .retry_timeout(timeout)
-            .try_build()?
-            .execute_reader(new_data);
+        let future = builder.retry_timeout(timeout).try_build()?.execute(source);
         Either::Left(tokio::time::timeout(timeout, future).map(|res| match res {
             Ok(Ok((new_dataset, stats))) => Ok((new_dataset, stats)),
             Ok(Err(e)) => Err(e.into()),
@@ -326,7 +400,7 @@ pub(crate) async fn execute_merge_insert(
         }))
     } else {
         let job = builder.try_build()?;
-        Either::Right(job.execute_reader(new_data).map_err(|e| e.into()))
+        Either::Right(job.execute(source).map_err(|e| e.into()))
     };
     let (new_dataset, stats) = future.await?;
     let version = new_dataset.manifest().version;
@@ -353,6 +427,186 @@ mod tests {
 
     use crate::connect;
 
+    #[rstest::rstest]
+    #[case(None)]
+    #[case(Some("struct"))]
+    #[case(Some("list"))]
+    #[case(Some("large_list"))]
+    #[case(Some("fixed_size_list"))]
+    #[case(Some("map"))]
+    #[tokio::test]
+    async fn merge_json_rejects_ambiguous_source_fields(
+        #[case] nested: Option<&str>,
+        #[values(false, true)] duplicate: bool,
+    ) {
+        let mut target = vec![
+            Field::new("id", DataType::Int32, false),
+            lance_arrow::json::json_field("payload", true),
+        ];
+        let mut input = vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("payload", DataType::Utf8, true),
+        ];
+        let mut columns: Vec<Arc<dyn arrow_array::Array>> = vec![
+            Arc::new(Int32Array::from(vec![1])),
+            Arc::new(StringArray::from(vec![r#"{"x":1}"#])),
+        ];
+        if let Some(nested) = nested {
+            let wrap = |children: Vec<Field>| {
+                let structure = DataType::Struct(children.into());
+                let item = Arc::new(Field::new("item", structure.clone(), true));
+                match nested {
+                    "struct" => structure,
+                    "list" => DataType::List(item),
+                    "large_list" => DataType::LargeList(item),
+                    "fixed_size_list" => DataType::FixedSizeList(item, 2),
+                    "map" => DataType::Map(
+                        Arc::new(Field::new(
+                            "entries",
+                            DataType::Struct(
+                                vec![
+                                    Field::new("key", DataType::Utf8, false),
+                                    Field::new("value", structure, true),
+                                ]
+                                .into(),
+                            ),
+                            false,
+                        )),
+                        false,
+                    ),
+                    _ => unreachable!(),
+                }
+            };
+            let known = Field::new("value", DataType::Int32, true);
+            let unexpected = if duplicate { "value" } else { "extra" };
+            let target_type = wrap(vec![known.clone()]);
+            let input_type = wrap(vec![known, Field::new(unexpected, DataType::Int32, true)]);
+            columns.push(arrow_array::new_null_array(&input_type, 1));
+            target.push(Field::new("details", target_type, true));
+            input.push(Field::new("details", input_type, true));
+        } else {
+            let unexpected = if duplicate { "id" } else { "extra" };
+            input.push(Field::new(unexpected, DataType::Int32, true));
+            columns.push(Arc::new(Int32Array::from(vec![99])));
+        }
+        let db = connect("memory://").execute().await.unwrap();
+        let table = db
+            .create_empty_table("ambiguous_json", Arc::new(Schema::new(target)))
+            .execute()
+            .await
+            .unwrap();
+        let version = table.version().await.unwrap();
+        let schema = Arc::new(Schema::new(input));
+        let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();
+        let mut merge = table.merge_insert(&["id"]);
+        merge.when_not_matched_insert_all();
+        let error = merge
+            .execute(Box::new(RecordBatchIterator::new(vec![Ok(batch)], schema)))
+            .await
+            .expect_err("JSON alignment must not drop source fields");
+        let expected = if duplicate {
+            "more than once"
+        } else {
+            "not present"
+        };
+        assert!(error.to_string().contains(expected), "{error}");
+        assert_eq!(table.version().await.unwrap(), version);
+        assert_eq!(table.count_rows(None).await.unwrap(), 0);
+    }
+
+    #[tokio::test]
+    async fn merge_json_preserves_empty_extension_metadata() {
+        use crate::query::{ExecutableQuery, QueryBase, Select};
+        use futures::TryStreamExt;
+        let mut stored_json = lance_arrow::json::json_field("payload", true);
+        let mut metadata = stored_json.metadata().clone();
+        metadata.insert(lance_arrow::ARROW_EXT_META_KEY.into(), String::new());
+        stored_json = stored_json.with_metadata(metadata);
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            stored_json,
+        ]));
+        let db = connect("memory://").execute().await.unwrap();
+        let table = db
+            .create_empty_table("json_merge_metadata", schema)
+            .execute()
+            .await
+            .unwrap();
+        let input_schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int32, false),
+            Field::new("payload", DataType::Utf8, true).with_metadata(
+                std::collections::HashMap::from([(
+                    lance_arrow::ARROW_EXT_NAME_KEY.into(),
+                    lance_arrow::json::ARROW_JSON_EXT_NAME.into(),
+                )]),
+            ),
+        ]));
+        let seed = RecordBatch::try_new(
+            input_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1])),
+                Arc::new(StringArray::from(vec![r#"{"old":true}"#])),
+            ],
+        )
+        .unwrap();
+        table.add(seed).execute().await.unwrap();
+        let batch = RecordBatch::try_new(
+            input_schema.clone(),
+            vec![
+                Arc::new(Int32Array::from(vec![1, 2])),
+                Arc::new(StringArray::from(vec![
+                    r#"{"updated":true}"#,
+                    r#"{"inserted":true}"#,
+                ])),
+            ],
+        )
+        .unwrap();
+        let mut merge = table.merge_insert(&["id"]);
+        merge
+            .when_matched_update_all(None)
+            .when_not_matched_insert_all();
+        let result = merge
+            .execute(Box::new(RecordBatchIterator::new(
+                vec![Ok(batch)],
+                input_schema,
+            )))
+            .await
+            .unwrap();
+        assert_eq!(result.num_updated_rows, 1);
+        assert_eq!(result.num_inserted_rows, 1);
+        let output = table
+            .query()
+            .select(Select::columns(&["payload"]))
+            .execute()
+            .await
+            .unwrap()
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        let values = output
+            .iter()
+            .flat_map(|b| {
+                b.column(0)
+                    .as_any()
+                    .downcast_ref::<StringArray>()
+                    .unwrap()
+                    .iter()
+            })
+            .flatten()
+            .collect::<Vec<_>>();
+        assert_eq!(values.len(), 2);
+        assert!(
+            values
+                .iter()
+                .any(|v| serde_json::from_str::<serde_json::Value>(v).unwrap()["updated"] == true)
+        );
+        assert!(
+            values
+                .iter()
+                .any(|v| serde_json::from_str::<serde_json::Value>(v).unwrap()["inserted"] == true)
+        );
+    }
+
     fn merge_insert_test_batches(offset: i32, age: i32) -> Box<dyn RecordBatchReader + Send> {
         let schema = Arc::new(Schema::new(vec![
             Field::new("i", DataType::Int32, false),
diff --git a/rust/lancedb/tests/duplicate_pairs.rs b/rust/lancedb/tests/duplicate_pairs.rs
new file mode 100644
index 000000000..910088bc8
--- /dev/null
+++ b/rust/lancedb/tests/duplicate_pairs.rs
@@ -0,0 +1,561 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+use arrow_array::{
+    Array, FixedSizeListArray, Float32Array, Int64Array, RecordBatch, RecordBatchIterator,
+    UInt64Array,
+};
+use arrow_schema::{DataType, Field, Schema};
+use datafusion::prelude::SessionContext;
+use datafusion_catalog::TableProvider;
+use datafusion_common::Result;
+use futures::TryStreamExt;
+use lance::{
+    Dataset,
+    dataset::WriteParams,
+    index::{DatasetIndexExt, vector::VectorIndexParams},
+};
+use lance_index::{IndexType, vector::ivf::IvfBuildParams};
+use lance_linalg::distance::MetricType;
+use lancedb::table::datafusion::{
+    BaseTableAdapter,
+    udtf::duplicate_pairs::{
+        DuplicatePairsConfig, DuplicatePairsExec, DuplicatePairsResolver,
+        DuplicatePairsTableFunction, duplicate_pair_task_batch, plan_duplicate_pairs,
+    },
+};
+use std::sync::Arc;
+
+#[derive(Debug)]
+struct Resolver(Arc<BaseTableAdapter>);
+impl DuplicatePairsResolver for Resolver {
+    fn resolve(
+        &self,
+        table: &str,
+        config: DuplicatePairsConfig,
+        tasks_only: bool,
+    ) -> Result<Arc<dyn TableProvider>> {
+        assert_eq!(table, "source");
+        Ok(Arc::new(self.0.duplicate_pairs(config, tasks_only)))
+    }
+}
+
+fn pairs(batches: &[RecordBatch]) -> Vec<(u64, u64, u32)> {
+    let mut result = Vec::new();
+    for batch in batches {
+        let a = batch
+            .column(0)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+        let b = batch
+            .column(1)
+            .as_any()
+            .downcast_ref::<UInt64Array>()
+            .unwrap();
+        let d = batch
+            .column(2)
+            .as_any()
+            .downcast_ref::<Float32Array>()
+            .unwrap();
+        for i in 0..batch.num_rows() {
+            result.push((a.value(i), b.value(i), d.value(i).to_bits()));
+        }
+    }
+    result.sort_unstable();
+    result
+}
+
+#[tokio::test]
+async fn sql_pairs_match_native_snapshot_and_partition_tasks() -> anyhow::Result<()> {
+    for partitions in [1, 8] {
+        let dir = tempfile::tempdir()?;
+        let uri = dir
+            .path()
+            .join("source.lance")
+            .to_str()
+            .unwrap()
+            .to_string();
+        let n = 1025;
+        let vectors = (0..n * 2)
+            .flat_map(|i| {
+                let i = i % n;
+                let x = if i == n - 1 {
+                    0.0
+                } else if i < 3 {
+                    i as f32
+                } else {
+                    i as f32 * 10.0
+                };
+                [x, 0.0]
+            })
+            .collect::<Vec<_>>();
+        let vectors = FixedSizeListArray::try_new(
+            Arc::new(Field::new("item", DataType::Float32, true)),
+            2,
+            Arc::new(Float32Array::from(vectors)),
+            None,
+        )?;
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("vector", vectors.data_type().clone(), false),
+        ]));
+        let batch = RecordBatch::try_new(
+            schema.clone(),
+            vec![
+                Arc::new(Int64Array::from_iter_values(0..(n * 2) as i64)),
+                Arc::new(vectors),
+            ],
+        )?;
+        let mut ds = Dataset::write(
+            RecordBatchIterator::new(vec![Ok(batch)], schema),
+            &uri,
+            Some(WriteParams {
+                max_rows_per_file: n,
+                ..Default::default()
+            }),
+        )
+        .await?;
+        let fragments = ds
+            .fragments()
+            .iter()
+            .map(|f| f.id as u32)
+            .collect::<Vec<_>>();
+        assert_eq!(fragments.len(), 2);
+        let mut segments = Vec::new();
+        for (i, fragment) in fragments.into_iter().enumerate() {
+            let centroids = FixedSizeListArray::try_new(
+                Arc::new(Field::new("item", DataType::Float32, true)),
+                2,
+                Arc::new(Float32Array::from(
+                    (0..partitions)
+                        .flat_map(|p| [p as f32 * 1400.0 + i as f32 * 0.125, 0.0])
+                        .collect::<Vec<_>>(),
+                )),
+                None,
+            )?;
+            let ivf = IvfBuildParams {
+                centroids: Some(Arc::new(centroids)),
+                ..IvfBuildParams::new(partitions)
+            };
+            let params = VectorIndexParams::with_ivf_flat_params(MetricType::L2, ivf);
+            segments.push(
+                ds.create_index_builder(&["vector"], IndexType::Vector, &params)
+                    .name("vector_idx".into())
+                    .fragments(vec![fragment])
+                    .execute_uncommitted()
+                    .await?,
+            );
+        }
+        ds.commit_existing_index_segments("vector_idx", "vector", segments)
+            .await?;
+        let version = ds.version().version;
+        let ds = Arc::new(ds);
+        let conn = lancedb::connect(dir.path().to_str().unwrap())
+            .execute()
+            .await?;
+        let table = conn.open_table("source").execute().await?;
+        let resolver = Arc::new(Resolver(Arc::new(
+            BaseTableAdapter::try_new(table.base_table().clone()).await?,
+        )));
+        // More partitions than this pool could hold concurrently: local
+        // execution must consume scoped readers sequentially.
+        let runtime = Arc::new(
+            datafusion_execution::runtime_env::RuntimeEnvBuilder::new()
+                .with_memory_limit(128 * 1024 * 1024, 1.0)
+                .build()?,
+        );
+        let ctx =
+            SessionContext::new_with_config_rt(datafusion::prelude::SessionConfig::new(), runtime);
+        ctx.register_udtf(
+            "vector_duplicate_pairs",
+            Arc::new(DuplicatePairsTableFunction::new(resolver.clone())),
+        );
+        ctx.register_udtf(
+            "vector_duplicate_pair_tasks",
+            Arc::new(DuplicatePairsTableFunction::task_manifest(resolver)),
+        );
+        for threshold in [-1.0, 0.0, 1.0] {
+            let config = DuplicatePairsConfig {
+                dataset_version: version,
+                column: "vector".into(),
+                distance_threshold: threshold,
+            };
+            let tasks = plan_duplicate_pairs(ds.clone(), &config).await?;
+            assert_eq!(tasks.len(), partitions * 2);
+            let manifest = duplicate_pair_task_batch(&ds, &config, &tasks)?;
+            let sql_manifest = ctx.sql(&format!("SELECT * FROM vector_duplicate_pair_tasks('source', {version}, 'vector', {threshold})")).await?.collect().await?;
+            assert_eq!(sql_manifest, vec![manifest]);
+            let expected =
+                lance::index::vector::dedup::find_duplicate_pairs(ds.clone(), "vector", threshold)
+                    .await?
+                    .try_collect::<Vec<_>>()
+                    .await?;
+            let sql = format!(
+                "SELECT * FROM vector_duplicate_pairs('source', {version}, 'vector', {threshold})"
+            );
+            let actual = ctx.sql(&sql).await?.collect().await?;
+            assert_eq!(pairs(&actual), pairs(&expected));
+            assert_eq!(
+                pairs(&actual).len(),
+                match threshold {
+                    -1.0 => 0,
+                    0.0 => 2,
+                    _ => 8,
+                }
+            );
+            let retry = ctx.sql(&sql).await?.collect().await?;
+            assert_eq!(pairs(&retry), pairs(&actual));
+            // The same table function can be maintained through the MV unit
+            // lifecycle. Staging attempts do not make any rows visible.
+            use lancedb::materialized_view::{
+                MaterializedViewDefinition, commit_partitioned_refresh, plan_partitioned_refresh,
+                prepare_definition, write_refresh_partition,
+            };
+            let view_name = format!("pairs_{}", threshold.to_bits());
+            let view = prepare_definition(&table, MaterializedViewDefinition::from_sql(&sql)?)
+                .await?
+                .create(&view_name)
+                .await?;
+            assert_eq!(view.table().schema().await?.fields().len(), 3);
+            let plan = plan_partitioned_refresh(view.table(), Some(version))
+                .await?
+                .unwrap();
+            assert_eq!(plan.units() as usize, tasks.len());
+            let plan = serde_json::from_slice(&serde_json::to_vec(&plan)?)?;
+            let first = write_refresh_partition(view.table(), 0, &plan).await?;
+            assert_eq!(view.table().count_rows(None).await?, 0);
+            assert!(
+                commit_partitioned_refresh(
+                    view.table(),
+                    &plan,
+                    vec![first.clone(), first.clone()],
+                    view.incarnation()
+                )
+                .await
+                .is_err()
+            );
+            // Lost receipt: its files remain unreferenced; the retry writes
+            // another attempt, of which only one receipt is ever committed.
+            drop(write_refresh_partition(view.table(), 0, &plan).await?);
+            let mut units = vec![first];
+            for unit in 1..plan.units() {
+                units.push(write_refresh_partition(view.table(), unit, &plan).await?);
+            }
+            assert_eq!(view.table().count_rows(None).await?, 0);
+            let outcome =
+                commit_partitioned_refresh(view.table(), &plan, units.clone(), view.incarnation())
+                    .await?;
+            assert_eq!(outcome.rows_written as usize, pairs(&expected).len());
+            assert!(
+                commit_partitioned_refresh(view.table(), &plan, units, view.incarnation())
+                    .await
+                    .is_err()
+            );
+            use lancedb::query::ExecutableQuery;
+            let written = view
+                .table()
+                .query()
+                .execute()
+                .await?
+                .try_collect::<Vec<_>>()
+                .await?;
+            assert_eq!(pairs(&written), pairs(&expected));
+            // Repeating a refresh replaces the view rather than appending.
+            view.refresh().execute().await?;
+            assert_eq!(
+                view.table().count_rows(None).await? as usize,
+                pairs(&expected).len()
+            );
+            // The complete declarative pipeline keeps direct representatives
+            // and materializes original rows, including isolated source rows.
+            use lancedb::materialized_view::write_refresh_partition_with_inputs;
+            use lancedb::query::QueryBase;
+            let definition = MaterializedViewDefinition::from_sql(&format!(
+                "SELECT * FROM vector_dedup('source', {version}, 'vector', {threshold})"
+            ))?;
+            let clean = prepare_definition(&table, definition)
+                .await?
+                .create(&format!("clean_{}", threshold.to_bits()))
+                .await?;
+            let clean_plan = plan_partitioned_refresh(clean.table(), Some(version))
+                .await?
+                .unwrap();
+            assert_eq!(clean_plan.dependency_units() as usize, tasks.len());
+            assert_eq!(clean_plan.units() as usize, tasks.len() + 2);
+            let clean_plan: lancedb::materialized_view::PartitionedRefreshPlan =
+                serde_json::from_slice(&serde_json::to_vec(&clean_plan)?)?;
+            assert!(
+                write_refresh_partition(clean.table(), tasks.len() as u32, &clean_plan)
+                    .await
+                    .is_err()
+            );
+            let mut receipts = Vec::new();
+            for unit in 0..clean_plan.units() {
+                let dependencies = &receipts[..receipts.len().min(tasks.len())];
+                if unit == 0 || unit == tasks.len() as u32 {
+                    // Both selection and materialization can lose a receipt.
+                    drop(
+                        write_refresh_partition_with_inputs(
+                            clean.table(),
+                            unit,
+                            &clean_plan,
+                            dependencies,
+                        )
+                        .await?,
+                    );
+                }
+                let receipt = write_refresh_partition_with_inputs(
+                    clean.table(),
+                    unit,
+                    &clean_plan,
+                    dependencies,
+                )
+                .await?;
+                receipts.push(serde_json::from_slice(&serde_json::to_vec(&receipt)?)?);
+            }
+            assert_eq!(clean.table().count_rows(None).await?, 0);
+            assert!(
+                commit_partitioned_refresh(
+                    clean.table(),
+                    &clean_plan,
+                    receipts[..receipts.len() - 1].to_vec(),
+                    clean.incarnation()
+                )
+                .await
+                .is_err()
+            );
+            commit_partitioned_refresh(clean.table(), &clean_plan, receipts, clean.incarnation())
+                .await?;
+            let mut edges = pairs(&expected)
+                .into_iter()
+                .map(|(a, b, _)| (a.min(b), a.max(b)))
+                .collect::<Vec<_>>();
+            edges.sort_unstable();
+            let mut removed = std::collections::BTreeSet::new();
+            for (a, b) in edges {
+                if !removed.contains(&a) {
+                    removed.insert(b);
+                }
+            }
+            let source_ids = ds
+                .scan()
+                .project(&["id"])?
+                .with_row_id()
+                .try_into_stream()
+                .await?
+                .try_collect::<Vec<_>>()
+                .await?;
+            let mut expected_ids = Vec::new();
+            for batch in source_ids {
+                let ids = batch
+                    .column_by_name("id")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<Int64Array>()
+                    .unwrap();
+                let rowids = batch
+                    .column_by_name("_rowid")
+                    .unwrap()
+                    .as_any()
+                    .downcast_ref::<UInt64Array>()
+                    .unwrap();
+                for i in 0..batch.num_rows() {
+                    if !removed.contains(&rowids.value(i)) {
+                        expected_ids.push(ids.value(i));
+                    }
+                }
+            }
+            let kept = clean
+                .table()
+                .query()
+                .select(lancedb::query::Select::columns(&["id"]))
+                .execute()
+                .await?
+                .try_collect::<Vec<_>>()
+                .await?;
+            let mut actual_ids = kept
+                .iter()
+                .flat_map(|b| {
+                    b.column(0)
+                        .as_any()
+                        .downcast_ref::<Int64Array>()
+                        .unwrap()
+                        .values()
+                        .iter()
+                        .copied()
+                })
+                .collect::<Vec<_>>();
+            actual_ids.sort_unstable();
+            expected_ids.sort_unstable();
+            assert_eq!(actual_ids, expected_ids);
+            if threshold == 1.0 {
+                assert!(actual_ids.contains(&0) && actual_ids.contains(&2));
+                assert!(!actual_ids.contains(&1), "A-B-C retains A and C");
+            }
+            clean.refresh().execute().await?;
+            assert_eq!(clean.table().count_rows(None).await?, expected_ids.len());
+            assert_eq!(table.count_rows(None).await?, n * 2);
+            assert!(
+                DuplicatePairsExec::try_new(
+                    ds.clone(),
+                    config.clone(),
+                    vec![tasks[0].clone(), tasks[0].clone()]
+                )
+                .is_err()
+            );
+        }
+        // Latest no longer contains B, but pinned queries must retain the same pairs.
+        table.delete("id = 1").await?;
+        let actual = ctx
+            .sql(&format!(
+                "SELECT * FROM vector_duplicate_pairs('source', {version}, 'vector', 1.0)"
+            ))
+            .await?
+            .collect()
+            .await?;
+        assert_eq!(pairs(&actual).len(), 8);
+        assert!(
+            ctx.sql(&format!(
+                "SELECT * FROM vector_duplicate_pairs('source', {version}, 'missing', 1.0)"
+            ))
+            .await?
+            .collect()
+            .await
+            .is_err()
+        );
+    }
+    Ok(())
+}
+
+#[tokio::test]
+async fn dedup_materializes_blob_payloads_null_vectors_and_snapshot_deletions() -> anyhow::Result<()>
+{
+    use arrow_array::{LargeBinaryArray, StringArray, StructArray, types::Float32Type};
+    use lancedb::materialized_view::{MaterializedViewDefinition, prepare_definition};
+    for stable in [false, true] {
+        let dir = tempfile::tempdir()?;
+        let conn = lancedb::connect(dir.path().to_str().unwrap())
+            .execute()
+            .await?;
+        let vectors = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+            [
+                Some(vec![Some(0.0), Some(0.0)]),
+                Some(vec![Some(1.0), Some(0.0)]),
+                Some(vec![Some(2.0), Some(0.0)]),
+                None,
+                Some(vec![Some(30.0), Some(0.0)]),
+                Some(vec![Some(40.0), Some(0.0)]),
+            ],
+            2,
+        );
+        let blob = lancedb::blob::blob("image", true);
+        let DataType::Struct(fields) = blob.data_type().clone() else {
+            unreachable!()
+        };
+        let images = StructArray::new(
+            fields,
+            vec![
+                Arc::new(LargeBinaryArray::from_iter_values(
+                    (0..6).map(|id| vec![id; 4096]),
+                )),
+                Arc::new(StringArray::from(vec![None::<&str>; 6])),
+            ],
+            None,
+        );
+        let batch = RecordBatch::try_new(
+            Arc::new(Schema::new(vec![
+                Field::new("id", DataType::Int64, false),
+                Field::new("vector", vectors.data_type().clone(), true),
+                blob,
+            ])),
+            vec![
+                Arc::new(Int64Array::from_iter_values(0..6)),
+                Arc::new(vectors),
+                Arc::new(images),
+            ],
+        )?;
+        let table = conn
+            .create_table("source", batch)
+            .storage_option("new_table_enable_stable_row_ids", stable.to_string())
+            .execute()
+            .await?;
+        table.delete("id = 5").await?;
+        table
+            .optimize(lancedb::table::OptimizeAction::Compact {
+                options: lancedb::table::CompactionOptions::default(),
+                remap_options: None,
+            })
+            .await?;
+        let uri = table.uri().await?;
+        let mut ds = Dataset::open(&uri).await?;
+        assert!(
+            ds.fragments().iter().all(|f| f.id > 0),
+            "compaction must change physical row addresses"
+        );
+        let centroids = FixedSizeListArray::from_iter_primitive::<Float32Type, _, _>(
+            [Some(vec![Some(0.0), Some(0.0)])],
+            2,
+        );
+        let ivf = IvfBuildParams {
+            centroids: Some(Arc::new(centroids)),
+            ..IvfBuildParams::new(1)
+        };
+        ds.create_index(
+            &["vector"],
+            IndexType::Vector,
+            Some("vector_idx".into()),
+            &VectorIndexParams::with_ivf_flat_params(MetricType::L2, ivf),
+            true,
+        )
+        .await?;
+        let version = ds.version().version;
+        let definition = MaterializedViewDefinition::from_sql(&format!(
+            "SELECT * FROM vector_dedup('source', {version}, 'vector', 1)"
+        ))?;
+        let clean = prepare_definition(&table, definition)
+            .await?
+            .create("clean")
+            .await?;
+        // A later source delete does not change the declared result snapshot.
+        table.delete("id = 0").await?;
+        clean.refresh().execute().await?;
+        assert_eq!(clean.table().count_rows(None).await?, 4);
+        let output = Dataset::open(&clean.table().uri().await?).await?;
+        let batches = output
+            .scan()
+            .blob_handling(lance_core::datatypes::BlobHandling::AllBinary)
+            .try_into_stream()
+            .await?
+            .try_collect::<Vec<_>>()
+            .await?;
+        let mut ids = Vec::new();
+        for batch in batches {
+            let row_ids = batch
+                .column_by_name("id")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<Int64Array>()
+                .unwrap();
+            let images = batch
+                .column_by_name("image")
+                .unwrap()
+                .as_any()
+                .downcast_ref::<LargeBinaryArray>()
+                .unwrap();
+            for i in 0..batch.num_rows() {
+                let id = row_ids.value(i);
+                assert_eq!(images.value(i), vec![id as u8; 4096]);
+                ids.push(id);
+            }
+        }
+        ids.sort_unstable();
+        assert_eq!(ids, [0, 2, 3, 4]);
+        assert_eq!(
+            table.count_rows(None).await?,
+            4,
+            "refresh must not mutate the source"
+        );
+    }
+    Ok(())
+}

From d8063545d9da6ebb55ffe523534c267bd9d1d548 Mon Sep 17 00:00:00 2001
From: breken <support@brekfuz.com>
Date: Mon, 28 Sep 2026 18:11:39 -0700
Subject: [PATCH 05/11] fix(node): write NaN and Infinity as SQL literals in
 table.update (#4327)

## What's wrong

`toSQL` renders numbers with `toString()`, so `NaN` becomes `NaN` and
`Infinity` becomes `Infinity`. SQL reads those as column names:

```ts
await table.update({ values: { price: NaN }, where: "id = 1" });
// Error: Invalid input, Schema error: No field named "NaN". Valid fields are id, price.
```

`Infinity` and `-Infinity` fail the same way. The only workaround today
is `valuesSql`.

## Fix

Emit `CAST('NaN' AS DOUBLE)`, `CAST('Infinity' AS DOUBLE)` and
`CAST('-Infinity' AS DOUBLE)` for these values. Finite numbers render as
before.

## Tests

- `__test__/util.test.ts` adds `toSQL` cases for `NaN`, `Infinity` and
`-Infinity`.
- `__test__/table.test.ts` adds "should let me update float values to
NaN and Infinity", which updates a float column and reads the values
back.
- On `main`, the new cases fail with `No field named "NaN"`.
- With the fix, all of `util.test.ts` and `table.test.ts` pass (344
tests). These were run against the published
`@lancedb/lancedb@0.40.0-beta.11` native binary.
- `biome check` is clean on the changed files.

The Python SDK has the same issue in `value_to_sql`. That fix is in
#4326.

The fix was found and written by an AI agent (Breken) and checked
against the tests above.

Co-authored-by: breken-ai <312387581+breken-ai@users.noreply.github.com>
---
 nodejs/__test__/table.test.ts | 20 ++++++++++++++++++++
 nodejs/__test__/util.test.ts  |  3 +++
 nodejs/lancedb/util.ts        |  8 ++++++++
 3 files changed, 31 insertions(+)

diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts
index ac8904c16..7fe854438 100644
--- a/nodejs/__test__/table.test.ts
+++ b/nodejs/__test__/table.test.ts
@@ -416,6 +416,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(await table.countRows("id == 10")).toBe(1);
     });
 
+    it("should let me update float values to NaN and Infinity", async () => {
+      const db = await connect(tmpDir.name);
+      const floats = await db.createTable("floats", [
+        { id: 1, price: 1.5 },
+        { id: 2, price: 2.5 },
+        { id: 3, price: 3.5 },
+        { id: 4, price: 4.5 },
+      ]);
+      await floats.update({ values: { price: NaN }, where: "id = 1" });
+      await floats.update({ values: { price: Infinity }, where: "id = 2" });
+      await floats.update({ values: { price: -Infinity }, where: "id = 3" });
+
+      const rows = await floats.query().toArray();
+      const price = (id: number) => rows.find((row) => row.id === id).price;
+      expect(price(1)).toBeNaN();
+      expect(price(2)).toBe(Infinity);
+      expect(price(3)).toBe(-Infinity);
+      expect(price(4)).toBe(4.5);
+    });
+
     // https://github.com/lancedb/lancedb/issues/1293
     test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
       "can create empty table with non default float type: %s",
diff --git a/nodejs/__test__/util.test.ts b/nodejs/__test__/util.test.ts
index 802f8ed4d..d7e489f73 100644
--- a/nodejs/__test__/util.test.ts
+++ b/nodejs/__test__/util.test.ts
@@ -14,6 +14,9 @@ test.each([
   [new ArrayBuffer(8), "X'0000000000000000'"],
   [Buffer.from("hello"), "X'68656c6c6f'"],
   ["Hello 'world'", "'Hello ''world'''"],
+  [NaN, "CAST('NaN' AS DOUBLE)"],
+  [Infinity, "CAST('Infinity' AS DOUBLE)"],
+  [-Infinity, "CAST('-Infinity' AS DOUBLE)"],
 ])("toSQL(%p) === %p", (value, expected) => {
   expect(toSQL(value)).toBe(expected);
 });
diff --git a/nodejs/lancedb/util.ts b/nodejs/lancedb/util.ts
index 9219f3d1d..5127a1c7f 100644
--- a/nodejs/lancedb/util.ts
+++ b/nodejs/lancedb/util.ts
@@ -15,6 +15,14 @@ export function toSQL(value: IntoSql): string {
   if (typeof value === "string") {
     return `'${value.replace(/'/g, "''")}'`;
   } else if (typeof value === "number") {
+    // toString() gives "NaN" / "Infinity", which SQL reads as column names.
+    if (Number.isNaN(value)) {
+      return "CAST('NaN' AS DOUBLE)";
+    } else if (!Number.isFinite(value)) {
+      return value > 0
+        ? "CAST('Infinity' AS DOUBLE)"
+        : "CAST('-Infinity' AS DOUBLE)";
+    }
     return value.toString();
   } else if (typeof value === "boolean") {
     return value ? "TRUE" : "FALSE";

From 1a463f3633c01f8ebe36abed174d7da7497f9754 Mon Sep 17 00:00:00 2001
From: Jay Zhou <zhoujunbai123@gmail.com>
Date: Tue, 29 Sep 2026 09:11:47 +0800
Subject: [PATCH 06/11] fix(python): warn when optimize cleans up all old
 versions (#4337)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Why

`optimize(cleanup_older_than=timedelta(0))` deletes the data files of
every version except the latest. Any other handle still on an older
version then fails with a missing-file error, and nothing tells the
caller this can happen. #2470 asks for a docs note and a runtime
warning.

Fixes #2470

## Scope

- `AsyncTable.optimize` emits a `UserWarning` when `cleanup_older_than`
rounds to 0 ms or less. `LanceTable.optimize` runs through it, so sync
callers get the warning too.
- The `cleanup_older_than` docstring on `Table.optimize`,
`LanceTable.optimize`, and `AsyncTable.optimize` gets a `.. warning::`
block, in the same style as the existing `delete_unverified` one.
- Removed the function-local `import warnings` in `AsyncTable.optimize`.
`warnings` is already imported at module level, and the local import
would shadow it for the new check (`UnboundLocalError`).
- `test_optimize` and `test_optimize_delete_unverified` intentionally
use a 0 cleanup, so they now assert the warning with `pytest.warns`.

Nodejs `cleanupOlderThan` is out of scope, since the issue is filed
against the Python SDK.

## Tradeoffs

Like the existing `retrain` deprecation warning, the warning is raised
inside the async coroutine. For sync callers it is attributed to
`asyncio/events.py` rather than the caller's line. Warning from both the
sync and async entry points would duplicate the check for a small gain.

## Verification

- Repro: a second handle checked out at an older version, then
`optimize(cleanup_older_than=timedelta(0))` on the first. The second
handle's read fails with `LanceError(IO) ... Not found:
.../data/<file>.lance`. Before this change no warning is emitted. After
it, the `UserWarning` fires.
- New tests `test_optimize_warns_on_zero_cleanup` (async, and checks
that `timedelta(days=1)` and the default don't warn) and
`test_optimize_warns_on_zero_cleanup_sync` fail on main (`DID NOT WARN`)
and pass with the fix. The failing tests are in their own commit before
the fix.
- `pytest python/tests/test_table.py -m "not slow and not s3_test"`: all
pass except `test_merge_insert`. That test's `timeout=timedelta(0)`
assertion fails the same way on unmodified main on my machine.
- `ruff format --check` and `ruff check` are clean on the changed files.

Tests ran against the source tree with the `_lancedb` extension from the
matching `0.40.0b11` wheel. The change touches no Rust.

This PR was written by an AI coding agent (Claude Code) and reviewed
before submission.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Jay Zhou <190699450+jayzhou2309@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
---
 python/python/lancedb/table.py    | 76 ++++++++++++++++++++++++-------
 python/python/tests/test_table.py | 32 +++++++++++--
 2 files changed, 87 insertions(+), 21 deletions(-)

diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index 0dc927b29..8068e73d2 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -134,6 +134,34 @@ _MODEL_BACKED_TOKENIZER_ERRORS = (
 )
 
 
+def _optimize_cleanup_since_ms(
+    cleanup_older_than: Optional[timedelta], retrain: bool
+) -> Optional[int]:
+    # Called directly by both the sync and async optimize so stacklevel=3
+    # names the user's call site rather than the background event loop.
+    cleanup_since_ms: Optional[int] = None
+    if cleanup_older_than is not None:
+        cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
+        if cleanup_since_ms <= 0:
+            warnings.warn(
+                "optimize(cleanup_older_than=0) removes every version except "
+                "the latest. Any concurrent reader or writer still using an "
+                "older version will fail. Use a longer cleanup_older_than "
+                "unless no other process is working on this table.",
+                UserWarning,
+                stacklevel=3,
+            )
+
+    if retrain:
+        warnings.warn(
+            "The 'retrain' parameter is deprecated and will be removed in a "
+            "future version.",
+            DeprecationWarning,
+            stacklevel=3,
+        )
+    return cleanup_since_ms
+
+
 def _add_unique_note(exception: BaseException, note: str) -> None:
     existing_notes = getattr(exception, "__notes__", ()) or ()
     message = (
@@ -2217,6 +2245,13 @@ class Table(ABC):
             All files belonging to versions older than this will be removed.  Set
             to 0 days to remove all versions except the latest.  The latest version
             is never removed.
+
+            .. warning::
+
+                Setting this to 0 deletes the data files of every older
+                version, so any other reader or writer still using an older
+                version of the table will fail. Only set it to 0 if no other
+                process is working on this dataset.
         delete_unverified: bool, default False
             Files leftover from a failed transaction may appear to be part of an
             in-progress operation (e.g. appending new data) and these files will not
@@ -4423,6 +4458,13 @@ class LanceTable(Table):
             All files belonging to versions older than this will be removed.  Set
             to 0 days to remove all versions except the latest.  The latest version
             is never removed.
+
+            .. warning::
+
+                Setting this to 0 deletes the data files of every older
+                version, so any other reader or writer still using an older
+                version of the table will fail. Only set it to 0 if no other
+                process is working on this dataset.
         delete_unverified: bool, default False
             Files leftover from a failed transaction may appear to be part of an
             in-progress operation (e.g. appending new data) and these files will not
@@ -4447,10 +4489,9 @@ class LanceTable(Table):
         modification operations.
         """
         LOOP.run(
-            self._table.optimize(
-                cleanup_older_than=cleanup_older_than,
-                delete_unverified=delete_unverified,
-                retrain=retrain,
+            self._table._do_optimize(
+                _optimize_cleanup_since_ms(cleanup_older_than, retrain),
+                delete_unverified,
             )
         )
 
@@ -6958,6 +6999,13 @@ class AsyncTable:
             All files belonging to versions older than this will be removed.  Set
             to 0 days to remove all versions except the latest.  The latest version
             is never removed.
+
+            .. warning::
+
+                Setting this to 0 deletes the data files of every older
+                version, so any other reader or writer still using an older
+                version of the table will fail. Only set it to 0 if no other
+                process is working on this dataset.
         delete_unverified: bool, default False
             Files leftover from a failed transaction may appear to be part of an
             in-progress operation (e.g. appending new data) and these files will not
@@ -6981,20 +7029,14 @@ class AsyncTable:
         you have added or modified 100,000 or more records or run more than 20 data
         modification operations.
         """
-        cleanup_since_ms: Optional[int] = None
-        if cleanup_older_than is not None:
-            cleanup_since_ms = round(cleanup_older_than.total_seconds() * 1000)
-
-        if retrain:
-            import warnings
-
-            warnings.warn(
-                "The 'retrain' parameter is deprecated and will be removed in a "
-                "future version.",
-                DeprecationWarning,
-                stacklevel=2,
-            )
+        return await self._do_optimize(
+            _optimize_cleanup_since_ms(cleanup_older_than, retrain),
+            delete_unverified,
+        )
 
+    async def _do_optimize(
+        self, cleanup_since_ms: Optional[int], delete_unverified: bool
+    ) -> OptimizeStats:
         return await self._inner.optimize(
             cleanup_since_ms=cleanup_since_ms,
             delete_unverified=delete_unverified,
diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py
index 5c56fea38..fa1a494b0 100644
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -4307,7 +4307,8 @@ async def test_optimize(mem_db_async: AsyncConnection):
     assert stats.prune.bytes_removed == 0
     assert stats.prune.old_versions_removed == 0
 
-    stats = await table.optimize(cleanup_older_than=timedelta(seconds=0))
+    with pytest.warns(UserWarning, match="concurrent"):
+        stats = await table.optimize(cleanup_older_than=timedelta(seconds=0))
     assert stats.prune.bytes_removed > 0
     assert stats.prune.old_versions_removed == 3
 
@@ -4335,12 +4336,35 @@ async def test_optimize_delete_unverified(tmp_db_async: AsyncConnection, tmp_pat
 
     stats = await table.optimize(delete_unverified=False)
     assert stats.prune.old_versions_removed == 0
-    stats = await table.optimize(
-        cleanup_older_than=timedelta(seconds=0), delete_unverified=True
-    )
+    with pytest.warns(UserWarning, match="concurrent"):
+        stats = await table.optimize(
+            cleanup_older_than=timedelta(seconds=0), delete_unverified=True
+        )
     assert stats.prune.old_versions_removed == 2
 
 
+@pytest.mark.asyncio
+async def test_optimize_warns_on_zero_cleanup(mem_db_async: AsyncConnection):
+    table = await mem_db_async.create_table("test", data=[{"x": [1]}])
+    with pytest.warns(UserWarning, match="concurrent"):
+        await table.optimize(cleanup_older_than=timedelta(0))
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        await table.optimize(cleanup_older_than=timedelta(days=1))
+        await table.optimize()
+
+
+def test_optimize_warns_on_zero_cleanup_sync(mem_db: DBConnection):
+    table = mem_db.create_table("test", data=[{"x": [1]}])
+    with warnings.catch_warnings(record=True) as seen:
+        warnings.simplefilter("default")
+        table.optimize(cleanup_older_than=timedelta(0))
+        table.optimize(cleanup_older_than=timedelta(0))
+    assert [w.filename for w in seen] == [__file__, __file__]
+    assert all("concurrent" in str(w.message) for w in seen)
+
+
 def test_replace_field_metadata(tmp_path):
     db = lancedb.connect(tmp_path)
     table = db.create_table("my_table", data=[{"x": 0}])

From 580240278fe205ee72d498976a7c8755ce7491eb Mon Sep 17 00:00:00 2001
From: Mohammad Hijjawi <51020382+MohammadHijjawi97@users.noreply.github.com>
Date: Tue, 29 Sep 2026 02:12:00 +0100
Subject: [PATCH 07/11] fix(python): apply the default limit of 10 to sync
 hybrid queries (#4344)

## Summary

A synchronous hybrid query without an explicit `.limit()` can return up
to 20 rows. The documented default is 10, and `AsyncHybridQuery` already
falls back to `DEFAULT_HYBRID_LIMIT` (10).

In `LanceHybridQueryBuilder`, each sub-query (vector and FTS) falls back
to its own default of 10 rows. The fused result was then sliced with
`limit=None`, i.e. not truncated at all. When the two legs match
different rows, the caller gets the union.

t = db.create_table("t", [{"text": ("dog" if i >= 20 else "cat") + f"
{i}", "vector": [float(i), 0.0]} for i in range(40)])
    t.create_index("text", config=FTS())
len(t.search(query_type="hybrid").vector([0.0,
0.0]).text("dog").to_arrow())
    # before: 20, after: 10

For the same reason, `.offset(n)` without `.limit()` did not raise the
sub-query limits to cover the skipped prefix; that only happened inside
`if self._limit:`.

## Changes

- Use `DEFAULT_HYBRID_LIMIT` when no limit is set, both for the
sub-query limits (`limit + offset`) and for the final slice of the
reranked results, matching `AsyncHybridQuery`.

## Test plan

- [x] New `test_hybrid_query_default_limit` in `test_hybrid_query.py`.
It is the sync counterpart of `test_async_hybrid_query_default_limit`,
plus an `offset()`-without-`limit()` check. It fails before the fix (20
rows) and passes after.
- [x] `pytest python/tests/test_hybrid_query.py
python/tests/test_rerankers.py`
- [x] `ruff format --check` / `ruff check`

I have no local Rust toolchain, so I ran the tests against the released
0.39.0 native module with the changed Python files overlaid. The changed
functions are identical between 0.39.0 and main.
---
 python/python/lancedb/query.py           | 16 +++++++-------
 python/python/tests/test_hybrid_query.py | 27 ++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py
index c42344ad3..db213ac71 100644
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -2296,7 +2296,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
             norm=self._norm,
             fts_query=self._fts_query._query,
             reranker=self._reranker,
-            limit=self._limit,
+            limit=self._limit or DEFAULT_HYBRID_LIMIT,
             with_row_ids=True,
             offset=self._offset,
         )
@@ -2748,13 +2748,13 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
         )
 
         # Apply common configurations
-        if self._limit:
-            # The final offset/limit window is sliced out of the combined,
-            # reranked results, so each sub-query must fetch enough rows to
-            # cover the skipped prefix as well as the window itself.
-            sub_query_limit = self._limit + (self._offset or 0)
-            self._vector_query.limit(sub_query_limit)
-            self._fts_query.limit(sub_query_limit)
+        # The final offset/limit window is sliced out of the combined,
+        # reranked results, so each sub-query must fetch enough rows to
+        # cover the skipped prefix as well as the window itself.
+        limit = self._limit or DEFAULT_HYBRID_LIMIT
+        sub_query_limit = limit + (self._offset or 0)
+        self._vector_query.limit(sub_query_limit)
+        self._fts_query.limit(sub_query_limit)
         # WAL-PK-FUSION: without the fallback, select `self._columns` as is.
         self._pk_fusion = None
         columns = self._columns
diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py
index 61b8001cf..c5f58308d 100644
--- a/python/python/tests/test_hybrid_query.py
+++ b/python/python/tests/test_hybrid_query.py
@@ -315,6 +315,33 @@ def test_hybrid_query_offset(sync_table: Table):
     assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:]
 
 
+def test_hybrid_query_default_limit(sync_table: Table):
+    # The vector and FTS legs match disjoint rows, so fusing their results
+    # yields more rows than the default limit.
+    new_rows = []
+    for i in range(20):
+        new_rows.append({"text": "close_vec", "vector": [0.1, 0.1]})
+        new_rows.append({"text": "dog", "vector": [50.0 + i, 50.0 + i]})
+    sync_table.add(new_rows)
+
+    def query():
+        return (
+            sync_table.search(query_type="hybrid")
+            .vector([0.1, 0.1])
+            .text("dog")
+            .with_row_id(True)
+        )
+
+    # Like the async hybrid query, the default limit is 10.
+    result = query().to_arrow()
+    assert len(result) == 10
+
+    # The offset window is taken after the default limit is applied.
+    full = query().limit(15).to_arrow()
+    offset_result = query().offset(5).to_arrow()
+    assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[5:]
+
+
 def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
     # minimum_nprobes(0) must raise the same validation error a plain vector
     # query raises, not silently no-op because 0 is falsy.

From 70a11234048f7d61c5e18f31dd1694142531fafc Mon Sep 17 00:00:00 2001
From: breken <support@brekfuz.com>
Date: Mon, 28 Sep 2026 18:16:26 -0700
Subject: [PATCH 08/11] fix(python): write NaN and infinity as SQL literals in
 table.update (#4326)

## What's wrong

`value_to_sql` renders floats with `str()`, so `float("nan")` becomes
`nan` and `float("inf")` becomes `inf`. SQL reads those as column names:

```python
table.update(where="id = 1", values={"price": float("nan")})
# ValueError: Invalid input, Schema error: No field named nan. Valid fields are id, price.
```

The same happens for `math.inf`, `-math.inf`, `np.nan` and
`np.float32("inf")`, which go through the same float path. The only
workaround today is `values_sql`.

## Fix

Emit `CAST('NaN' AS DOUBLE)`, `CAST('Infinity' AS DOUBLE)` and
`CAST('-Infinity' AS DOUBLE)` for these values. Finite floats render as
before. The cast also lands correctly in `float32` columns.

## Tests

`test_value_to_sql_nan_and_infinity` in
`python/python/tests/test_util.py` updates `float64` and `float32`
columns to NaN, +inf and -inf, then reads them back:

- On `main`, it fails with `ValueError: Invalid input, Schema error: No
field named nan`.
- With the fix, it passes. All of `test_util.py` passes (75 passed, 1
skipped), and so do the update tests in `test_table.py` (`-k update`, 8
passed).
- `ruff format --check` and `ruff check` are clean.

NaN inside a list value (for example a vector) still fails, because
Lance only accepts literals inside array values. This PR covers scalar
columns only.

The fix was found and written by an AI agent (Breken) and checked
against the tests above.

Co-authored-by: breken-ai <312387581+breken-ai@users.noreply.github.com>
---
 python/python/lancedb/util.py    |  8 ++++++++
 python/python/tests/test_util.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/python/python/lancedb/util.py b/python/python/lancedb/util.py
index dbc52bff6..3cf5460a5 100644
--- a/python/python/lancedb/util.py
+++ b/python/python/lancedb/util.py
@@ -5,6 +5,7 @@
 import binascii
 import functools
 import importlib
+import math
 import os
 import pathlib
 import warnings
@@ -351,6 +352,13 @@ def _(value: int):
 
 @value_to_sql.register(float)
 def _(value: float):
+    # str() gives "nan" / "inf", which SQL would read as column names.
+    if math.isnan(value):
+        return "CAST('NaN' AS DOUBLE)"
+    if math.isinf(value):
+        return (
+            "CAST('Infinity' AS DOUBLE)" if value > 0 else "CAST('-Infinity' AS DOUBLE)"
+        )
     return str(value)
 
 
diff --git a/python/python/tests/test_util.py b/python/python/tests/test_util.py
index 29cfd7f0b..d9c574eb9 100644
--- a/python/python/tests/test_util.py
+++ b/python/python/tests/test_util.py
@@ -213,6 +213,35 @@ def test_value_to_sql_numpy_scalars():
     assert value_to_sql(np.bool_(False)) == "FALSE"
 
 
+def test_value_to_sql_nan_and_infinity(tmp_path):
+    # str(float("nan")) is "nan", which SQL reads as a column named nan, so
+    # table.update(values={"x": float("nan")}) failed with "No field named nan".
+    import math
+
+    import numpy as np
+
+    db = lancedb.connect(tmp_path)
+    table = db.create_table(
+        "test",
+        pa.table(
+            {
+                "id": [1, 2, 3, 4],
+                "f64": [1.0, 2.0, 3.0, 4.0],
+                "f32": pa.array([1.0, 2.0, 3.0, 4.0], pa.float32()),
+            }
+        ),
+    )
+    table.update(where="id = 1", values={"f64": float("nan"), "f32": np.nan})
+    table.update(where="id = 2", values={"f64": math.inf, "f32": np.float32("inf")})
+    table.update(where="id = 3", values={"f64": -math.inf, "f32": -math.inf})
+
+    rows = {row["id"]: row for row in table.to_arrow().to_pylist()}
+    assert math.isnan(rows[1]["f64"]) and math.isnan(rows[1]["f32"])
+    assert rows[2]["f64"] == math.inf and rows[2]["f32"] == math.inf
+    assert rows[3]["f64"] == -math.inf and rows[3]["f32"] == -math.inf
+    assert rows[4]["f64"] == 4.0 and rows[4]["f32"] == 4.0
+
+
 def test_append_vector_columns():
     registry = EmbeddingFunctionRegistry.get_instance()
     registry.register("test")(MockTextEmbeddingFunction)

From ef11e44adc252e048337a40bdc21d7cc6918a9f9 Mon Sep 17 00:00:00 2001
From: ningsh7 <ningsh7@outlook.com>
Date: Tue, 29 Sep 2026 09:19:36 +0800
Subject: [PATCH 09/11] feat(node): expose allowExternalBlobOutsideBases in
 Table.add (#4317)

## Summary

- Add `allowExternalBlobOutsideBases` to the Node `AddDataOptions` and
forward it through N-API to the Rust `AddDataBuilder`.
- Document its default and local-only external blob behavior.
- Cover URI strings and URI objects, option scoping, progress callbacks,
overwrite mode, and remote rejection.

## Validation

- `pnpm test --runInBand` (21 suites passed, 1 skipped; 908 tests
passed, 6 skipped)
- `pnpm lint-ci`
- `pnpm tsc`
- `pnpm run docs`
- `cargo fmt --all -- --check`
- `CC=clang cargo clippy --profile ci -p lancedb-nodejs --all-features
-- -D warnings`

Fixes #4267
---
 docs/src/js/interfaces/AddDataOptions.md |  16 +++
 nodejs/__test__/remote.test.ts           |  63 +++++++++++
 nodejs/__test__/table.test.ts            | 127 +++++++++++++++++++++++
 nodejs/lancedb/table.ts                  |  18 +++-
 nodejs/src/table.rs                      |   6 +-
 5 files changed, 228 insertions(+), 2 deletions(-)

diff --git a/docs/src/js/interfaces/AddDataOptions.md b/docs/src/js/interfaces/AddDataOptions.md
index 6976787f0..eb10fdb0f 100644
--- a/docs/src/js/interfaces/AddDataOptions.md
+++ b/docs/src/js/interfaces/AddDataOptions.md
@@ -10,6 +10,22 @@ Options for adding data to a table.
 
 ## Properties
 
+### allowExternalBlobOutsideBases?
+
+```ts
+optional allowExternalBlobOutsideBases: boolean;
+```
+
+Whether blob URIs outside registered external bases may be written.
+
+Defaults to `false`. This option is supported only for local/native
+tables; remote tables return an error when it is enabled. An enabled write
+stores the absolute URI reference without registering a base or copying
+the external object into the database. The object must remain accessible
+when the blob is read later.
+
+***
+
 ### mode
 
 ```ts
diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts
index cb0047f37..898ba533e 100644
--- a/nodejs/__test__/remote.test.ts
+++ b/nodejs/__test__/remote.test.ts
@@ -85,6 +85,69 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it("rejects the external blob opt-in without blocking regular adds", async () => {
+    let insertRequests = 0;
+    const describeRequests: string[] = [];
+
+    await withMockDatabase(
+      (req, res) => {
+        const requestPath = req.url ?? "";
+        if (requestPath.endsWith("/describe/")) {
+          describeRequests.push(requestPath);
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "items",
+              version: 1,
+              schema: {
+                fields: [
+                  {
+                    name: "id",
+                    type: { type: "int64" },
+                    nullable: true,
+                  },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+        if (requestPath.endsWith("/insert/")) {
+          insertRequests++;
+          req.resume();
+          req.on("end", () => {
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(JSON.stringify({ version: insertRequests + 1 }));
+          });
+          return;
+        }
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("items");
+
+        await expect(
+          table.add([{ id: 1n }], {
+            allowExternalBlobOutsideBases: true,
+          }),
+        ).rejects.toThrow("only supported on local tables");
+        expect(insertRequests).toBe(0);
+
+        await expect(table.add([{ id: 2n }])).resolves.toMatchObject({
+          version: 2,
+        });
+        await expect(
+          table.add([{ id: 3n }], {
+            allowExternalBlobOutsideBases: false,
+          }),
+        ).resolves.toMatchObject({ version: 3 });
+      },
+    );
+
+    expect(describeRequests.length).toBeGreaterThan(0);
+    expect(insertRequests).toBe(2);
+  });
+
   it("lists materialized views through the namespace route", async () => {
     await withMockDatabase(
       (req, res) => {
diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts
index 7fe854438..ae257455d 100644
--- a/nodejs/__test__/table.test.ts
+++ b/nodejs/__test__/table.test.ts
@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import * as fs from "fs";
+import { pathToFileURL } from "node:url";
 import * as path from "path";
 import * as tmp from "tmp";
 
@@ -2449,6 +2450,118 @@ describe("when dealing with blob columns", () => {
     await expect(table.countRows()).resolves.toBe(0);
   });
 
+  it("requires an explicit opt-in for blob URIs outside registered bases", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(0);
+
+    await expect(
+      table.add([{ id: 2n, payload: uri }], {
+        allowExternalBlobOutsideBases: false,
+      }),
+    ).rejects.toThrow("allow_external_blob_outside_bases");
+    await expect(table.countRows()).resolves.toBe(0);
+  });
+
+  it.each([
+    ["URI string", (uri: string) => uri],
+    ["URI struct", (uri: string) => ({ uri })],
+  ])(
+    "round-trips an external blob from a %s after reopening",
+    async (_label, blobValue) => {
+      const { db, table, uri, payload } = await openExternalBlobTable();
+
+      const result = await table.add([{ id: 1n, payload: blobValue(uri) }], {
+        allowExternalBlobOutsideBases: true,
+      });
+      expect(result.version).toBeGreaterThan(0);
+      await expect(table.countRows()).resolves.toBe(1);
+
+      table.close();
+      const reopened = await db.openTable("external_blobs");
+      const rows = await reopened.query().withRowId().toArray();
+      expect(rows).toHaveLength(1);
+      expect(rows[0].payload.blob_uri).toBe(uri);
+      const bytes = await reopened.fetchBlobs("payload", [
+        rows[0]._rowid as bigint,
+      ]);
+      expect(bytes[0]).toEqual(payload);
+    },
+  );
+
+  it("applies the external blob opt-in to Arrow table input", async () => {
+    const { table, schema, uri, payload } = await openExternalBlobTable();
+    const input = makeArrowTable([{ id: 1n, payload: uri }], { schema });
+
+    const result = await table.add(input, {
+      allowExternalBlobOutsideBases: true,
+    });
+
+    expect(result.version).toBeGreaterThan(0);
+    const rows = await table.query().withRowId().toArray();
+    expect(rows[0].payload.blob_uri).toBe(uri);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
+  it("applies the external blob opt-in only to the current add", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await table.add([{ id: 1n, payload: uri }], {
+      allowExternalBlobOutsideBases: true,
+    });
+    await expect(table.countRows()).resolves.toBe(1);
+
+    await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(1);
+  });
+
+  it("supports progress and overwrite with the external blob opt-in", async () => {
+    const { table, uri, payload } = await openExternalBlobTable();
+    await table.add([{ id: 1n, payload: Buffer.from("old") }]);
+
+    let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void;
+    let rejectFinal!: (error: Error) => void;
+    const finalProgress = new Promise<import("../lancedb").WriteProgress>(
+      (resolve, reject) => {
+        resolveFinal = resolve;
+        rejectFinal = reject;
+      },
+    );
+    const timeout = setTimeout(
+      () => rejectFinal(new Error("timed out waiting for final progress")),
+      5_000,
+    );
+    try {
+      const result = await table.add([{ id: 2n, payload: uri }], {
+        mode: "overwrite",
+        allowExternalBlobOutsideBases: true,
+        progress: (progress) => {
+          if (progress.done) {
+            resolveFinal(progress);
+          }
+        },
+      });
+      expect(result.version).toBeGreaterThan(0);
+      const progress = await finalProgress;
+      expect(progress.outputRows).toBe(1);
+      expect(progress.totalRows).toBe(1);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    const rows = await table.query().withRowId().toArray();
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe(2n);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
   it("discovers blob columns", async () => {
     const { table } = await openBlobTable();
     expect(await table.blobColumns()).toEqual(["image"]);
@@ -2708,6 +2821,20 @@ describe("when dealing with blob columns", () => {
     const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
     return { table, rowIds, alpha, beta };
   }
+
+  async function openExternalBlobTable() {
+    const db = await connect(path.join(tmpDir.name, "db"));
+    const payload = Buffer.from("external-payload");
+    const payloadPath = path.join(tmpDir.name, "payload.bin");
+    fs.writeFileSync(payloadPath, payload);
+    const uri = pathToFileURL(payloadPath).href;
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("payload"),
+    ]);
+    const table = await db.createEmptyTable("external_blobs", schema);
+    return { db, table, schema, uri, payload };
+  }
 });
 
 describe("when dealing with tags", () => {
diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts
index a098322a9..cabcf88aa 100644
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -124,6 +124,17 @@ export interface AddDataOptions {
    * ```
    */
   progress: (progress: WriteProgress) => void;
+
+  /**
+   * Whether blob URIs outside registered external bases may be written.
+   *
+   * Defaults to `false`. This option is supported only for local/native
+   * tables; remote tables return an error when it is enabled. An enabled write
+   * stores the absolute URI reference without registering a base or copying
+   * the external object into the database. The object must remain accessible
+   * when the blob is read later.
+   */
+  allowExternalBlobOutsideBases?: boolean;
 }
 
 export interface UpdateOptions {
@@ -1091,7 +1102,12 @@ export class LocalTable extends Table {
           }
         }
       : undefined;
-    return await this.inner.add(buffer, mode, progress);
+    return await this.inner.add(
+      buffer,
+      mode,
+      progress,
+      options?.allowExternalBlobOutsideBases,
+    );
   }
 
   async update(
diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs
index fa74c20dc..d381d98e5 100644
--- a/nodejs/src/table.rs
+++ b/nodejs/src/table.rs
@@ -74,13 +74,14 @@ impl Table {
 
     #[napi(
         catch_unwind,
-        ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void"
+        ts_args_type = "buf: Buffer, mode: string, progressCallback?: (progress: WriteProgressInfo) => void, allowExternalBlobOutsideBases?: boolean"
     )]
     pub async fn add(
         &self,
         buf: Buffer,
         mode: String,
         progress_callback: Option<ProgressFn>,
+        allow_external_blob_outside_bases: Option<bool>,
     ) -> napi::Result<AddResult> {
         let batches = ipc_file_to_batches(buf.to_vec())
             .map_err(|e| napi::Error::from_reason(format!("Failed to read IPC file: {}", e)))?;
@@ -105,6 +106,9 @@ impl Table {
             return Err(napi::Error::from_reason(format!("Invalid mode: {}", mode)));
         };
 
+        op = op
+            .allow_external_blob_outside_bases(allow_external_blob_outside_bases.unwrap_or(false));
+
         if let Some(tsfn) = progress_callback {
             op = op.progress(move |p| {
                 // NonBlocking: dispatch onto the JS event loop without

From c119f0c8e5ff617f670e6a8dde95378187d1e69a Mon Sep 17 00:00:00 2001
From: breken <support@brekfuz.com>
Date: Mon, 28 Sep 2026 18:19:51 -0700
Subject: [PATCH 10/11] fix(python): rank each result list separately in RRF
 multivector rerank (#4325)

## What's wrong

`RRFReranker.rerank_multivector` concatenates every vector result list
and passes the concatenation to `rerank_hybrid` as a single list.
`rerank_hybrid` then scores each row by its position in that
concatenation, so a row's rank depends on how many rows came before it
in *earlier* lists.

With two searches of 3 rows each (`K=60`):

| row | rank in its own list | score today | RRF score |
|---|---|---|---|
| 1 (top of list 1) | 1 | 1/61 | 1/61 |
| 3 (third of list 1) | 3 | 1/63 | 1/63 |
| 4 (top of list 2) | 1 | **1/64** | 1/61 |
| 5 (second of list 2) | 2 | **1/65** | 1/62 |

So the best hit of the second search ranks below the worst hit of the
first, and the order of the lists passed in decides the fused ranking.
`MRRReranker.rerank_multivector` already scores each list on its own
("MRR semantics require treating each vector result as a separate
ranking system"). RRF is defined the same way.

## Fix

Score each result list with its own ranks (`1 / (rank + K)`, summed
across lists for rows found by more than one), then dedupe the
concatenation by `_rowid`, attach `_relevance_score`, and sort. The
`return_score` handling is unchanged.

## Tests

`test_rrf_multivector_ranks_each_result_list` in
`python/python/tests/test_rerankers.py`:

- On `main`, it fails with `assert 0.015625 == 0.01639344262295082` (the
top hit of list 2 got `1/64` instead of `1/61`).
- With the fix, it passes, and so do the other tests in
`test_rerankers.py` (16 passed, 12 skipped for missing API keys or
models).
- `ruff format --check` and `ruff check` are clean.

The fix was found and written by an AI agent (Breken) and checked
against the tests above.

Co-authored-by: breken-ai <312387581+breken-ai@users.noreply.github.com>
---
 python/python/lancedb/rerankers/rrf.py | 24 +++++++++++++++----
 python/python/tests/test_rerankers.py  | 32 ++++++++++++++++++++++++++
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/python/python/lancedb/rerankers/rrf.py b/python/python/lancedb/rerankers/rrf.py
index 6d5b9a2ba..9c835a3ea 100644
--- a/python/python/lancedb/rerankers/rrf.py
+++ b/python/python/lancedb/rerankers/rrf.py
@@ -107,8 +107,24 @@ class RRFReranker(Reranker):
                     `search().with_row_id(True)`"
             )
 
-        combined = pa.concat_tables(vector_results, **self._concat_tables_args)
-        empty_table = pa.Table.from_arrays([], names=[])
-        reranked = self.rerank_hybrid(query, combined, empty_table)
+        # RRF ranks each result list on its own, so a document's rank is its
+        # position within the list it came from, not within the concatenation.
+        rrf_score_map = defaultdict(float)
+        for result in vector_results:
+            for i, result_id in enumerate(result["_rowid"].to_pylist(), 1):
+                rrf_score_map[result_id] += 1 / (i + self.K)
 
-        return reranked
+        combined = pa.concat_tables(vector_results, **self._concat_tables_args)
+        combined = self._deduplicate(combined)
+        relevance_scores = [
+            rrf_score_map[row_id] for row_id in combined["_rowid"].to_pylist()
+        ]
+        combined = combined.append_column(
+            "_relevance_score", pa.array(relevance_scores, type=pa.float32())
+        )
+        combined = combined.sort_by([("_relevance_score", "descending")])
+
+        if self.score == "relevance":
+            combined = self._keep_relevance_score(combined)
+
+        return combined
diff --git a/python/python/tests/test_rerankers.py b/python/python/tests/test_rerankers.py
index b4b9bc9e2..850b477a0 100644
--- a/python/python/tests/test_rerankers.py
+++ b/python/python/tests/test_rerankers.py
@@ -395,6 +395,38 @@ def test_mrr_multivector_rewards_consensus():
     assert result["_rowid"].to_pylist()[0] == 2
 
 
+def test_rrf_multivector_ranks_each_result_list():
+    # RRF scores a document by its rank *within each* result list. The top hit
+    # of the second list must score the same as the top hit of the first list,
+    # not as if it were ranked after every row of the first list.
+    reranker = RRFReranker(K=60)
+
+    def ranking(row_ids):
+        return pa.table({"_rowid": pa.array(row_ids, type=pa.int64())})
+
+    rs1 = ranking([1, 2, 3])
+    rs2 = ranking([4, 5, 6])
+
+    result = reranker.rerank_multivector([rs1, rs2])
+    scores = dict(
+        zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist())
+    )
+
+    assert scores[1] == pytest.approx(1 / 61)
+    assert scores[4] == pytest.approx(1 / 61)
+    assert scores[5] == pytest.approx(1 / 62)
+    # the second-ranked hit of the second list beats the third hit of the first
+    assert scores[5] > scores[3]
+
+    # A document found by both lists sums its reciprocal ranks from each list.
+    result = reranker.rerank_multivector([ranking([1, 2]), ranking([2, 3])])
+    scores = dict(
+        zip(result["_rowid"].to_pylist(), result["_relevance_score"].to_pylist())
+    )
+    assert scores[2] == pytest.approx(1 / 62 + 1 / 61)
+    assert result["_rowid"].to_pylist() == [2, 1, 3]
+
+
 def test_rrf_reranker_distance():
     data = pa.table(
         {

From 8e3606fbe31a83e895210d202d4982a0d36efbc3 Mon Sep 17 00:00:00 2001
From: "lancedb-gatefixer[bot]"
 <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Date: Mon, 28 Sep 2026 20:31:36 -0500
Subject: [PATCH 11/11] fix: handle remote blob fetch request caps (#4309)

## Cause

Remote `fetch_blobs` sent every row ID in one Cloud request, exceeding
the server's 1024-row or 64 MiB byte cap. A single blob over the byte
cap could not be read through this method.

## Fix

- Send at most 1024 row IDs per request and split batches further when
the server reports the blob-byte cap, preserving input order and nulls.
- Resolve one exact dataset version before multi-request reads. If an
initial single request hits the byte cap, resolve the version before any
successful split or Range read; use that version for every resulting
request.
- Read an individual oversized blob through the existing Range route.
- Document the per-request caps and transparent handling in the Rust,
Python, and TypeScript APIs and generated JS docs.

## Validation

- Focused Rust `fetch_blobs` tests: 26 passed, including row-count,
byte-cap, Range fallback, and exact-version assertions.
- Rust clippy and formatting passed.
- Focused Python blob tests: 13 passed; Ruff lint and formatting passed.
- Node build, lint, docs generation, and focused `fetchBlobs` test
passed.

Fixes #4275

<!-- lance-gatekeeper-fix:v1 agent=6ae94d49bf1d4cae057bba72584e7b90
generation=1 -->

---------

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
---
 docs/src/js/classes/Table.md           |   4 +
 nodejs/lancedb/table.ts                |   4 +
 python/python/lancedb/table.py         |   6 +-
 rust/lancedb/src/remote/table.rs       | 186 +++++++++++++++++++++++++
 rust/lancedb/src/remote/table/blobs.rs |  84 ++++++++++-
 rust/lancedb/src/table.rs              |   5 +-
 6 files changed, 284 insertions(+), 5 deletions(-)

diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md
index a8c382c27..17e2d4f34 100644
--- a/docs/src/js/classes/Table.md
+++ b/docs/src/js/classes/Table.md
@@ -548,6 +548,10 @@ Bytes for `column` at row IDs from [Query.withRowId](Query.md#withrowid).
 Reads the table's current checkout. IDs from another version can fail after
 compaction unless stable row ids are enabled. Results keep input order and
 duplicates. Null blobs are `null`. Empty blobs are empty buffers.
+Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
+The client splits requests automatically and reads an individual larger
+blob through the Range route. This method still materializes all bytes in
+memory; use [Table.fetchBlobFiles](Table.md#fetchblobfiles) for large values.
 
 #### Parameters
 
diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts
index cabcf88aa..132e8bc20 100644
--- a/nodejs/lancedb/table.ts
+++ b/nodejs/lancedb/table.ts
@@ -537,6 +537,10 @@ export abstract class Table {
    * Reads the table's current checkout. IDs from another version can fail after
    * compaction unless stable row ids are enabled. Results keep input order and
    * duplicates. Null blobs are `null`. Empty blobs are empty buffers.
+   * Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
+   * The client splits requests automatically and reads an individual larger
+   * blob through the Range route. This method still materializes all bytes in
+   * memory; use {@link Table.fetchBlobFiles} for large values.
    */
   abstract fetchBlobs(
     column: string,
diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py
index 8068e73d2..f4e835547 100644
--- a/python/python/lancedb/table.py
+++ b/python/python/lancedb/table.py
@@ -1963,8 +1963,10 @@ class Table(ABC):
         ``_rowid`` values stay valid after compaction when the table has stable
         row ids.
 
-        Convenience for small payloads. For large values use
-        :meth:`fetch_blob_files`.
+        Remote servers limit each request to 1024 row IDs and 64 MiB of blob
+        bytes. The client splits requests automatically and reads an individual
+        larger blob through the Range route. This method still materializes all
+        bytes in memory; for large values use :meth:`fetch_blob_files`.
         """
 
     @abstractmethod
diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs
index cabddbe7b..195c18ea5 100644
--- a/rust/lancedb/src/remote/table.rs
+++ b/rust/lancedb/src/remote/table.rs
@@ -5187,6 +5187,192 @@ mod tests {
         assert_eq!(blobs.value(2), b"gamma");
     }
 
+    #[tokio::test]
+    async fn test_fetch_blobs_splits_row_ids_at_one_version_and_preserves_order() {
+        let request_sizes = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let seen = request_sizes.clone();
+        let table = Table::new_with_handler_version(
+            "my_table",
+            semver::Version::new(0, 5, 0),
+            move |request| {
+                if request.url().path() == "/v1/table/my_table/describe/" {
+                    return http::Response::builder()
+                        .status(200)
+                        .body(br#"{"version":7,"schema":{"fields":[]}}"#.to_vec())
+                        .unwrap();
+                }
+                assert_eq!(request.url().path(), "/v1/table/my_table/fetch_blobs/");
+                let body = request_body_json(&request);
+                assert_eq!(body["version"], 7);
+                let ids = body["row_ids"].as_array().unwrap();
+                seen.lock().unwrap().push(ids.len());
+                if ids.len() > 1024 {
+                    return http::Response::builder()
+                        .status(400)
+                        .body(b"fetch_blobs accepts at most 1024 row IDs".to_vec())
+                        .unwrap();
+                }
+                let mut builder = LargeBinaryBuilder::new();
+                for id in ids {
+                    let id = id.as_u64().unwrap();
+                    if id == 1023 {
+                        builder.append_null();
+                    } else {
+                        builder.append_value(id.to_string().as_bytes());
+                    }
+                }
+                let batch = RecordBatch::try_new(
+                    Arc::new(Schema::new(vec![Field::new(
+                        "image",
+                        DataType::LargeBinary,
+                        true,
+                    )])),
+                    vec![Arc::new(builder.finish())],
+                )
+                .unwrap();
+                http::Response::builder()
+                    .status(200)
+                    .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE)
+                    .body(write_ipc_stream_uncompressed(&batch))
+                    .unwrap()
+            },
+        );
+
+        let ids: Vec<u64> = (0..1024).chain([42]).collect();
+        let blobs = table.fetch_blobs("image", &ids).await.unwrap();
+        assert_eq!(blobs.len(), 1025);
+        assert_eq!(blobs.value(0), b"0");
+        assert_eq!(blobs.value(1022), b"1022");
+        assert!(blobs.is_null(1023));
+        assert_eq!(blobs.value(1024), b"42");
+        assert_eq!(request_sizes.lock().unwrap().as_slice(), &[1024, 1]);
+    }
+
+    #[tokio::test]
+    async fn test_fetch_blobs_splits_byte_limited_requests_and_reads_large_blob_by_range() {
+        // Simulate a lower byte cap so this test exercises the same 400 response
+        // without allocating 64 MiB of blob data.
+        let requests = Arc::new(std::sync::Mutex::new(Vec::new()));
+        let seen = requests.clone();
+        let table = Table::new_with_handler_version(
+            "my_table",
+            semver::Version::new(0, 5, 0),
+            move |request| {
+                let path = request.url().path();
+                if path == "/v1/table/my_table/describe/" {
+                    return http::Response::builder()
+                        .status(200)
+                        .body(br#"{"version":42,"schema":{"fields":[]}}"#.to_vec())
+                        .unwrap();
+                }
+                if path == "/v1/table/my_table/fetch_blobs/" {
+                    let body = request_body_json(&request);
+                    let ids = body["row_ids"].as_array().unwrap();
+                    if body["version"].is_null() {
+                        // Only the initial failed request may read live latest.
+                        assert_eq!(ids.len(), 5);
+                    } else {
+                        assert_eq!(body["version"], 42);
+                    }
+                    seen.lock().unwrap().push(format!("POST {}", ids.len()));
+                    let mut builder = LargeBinaryBuilder::new();
+                    let mut total_bytes = 0;
+                    for id in ids {
+                        let value: Option<&[u8]> = match id.as_u64().unwrap() {
+                            10 => Some(b"aaaa"),
+                            20 => Some(b"bbb"),
+                            30 => None,
+                            40 => Some(b"0123456789"),
+                            id => panic!("unexpected row id {id}"),
+                        };
+                        if let Some(value) = value {
+                            total_bytes += value.len();
+                            builder.append_value(value);
+                        } else {
+                            builder.append_null();
+                        }
+                    }
+                    if total_bytes > 6 {
+                        return http::Response::builder()
+                            .status(400)
+                            .body(br#"{"error":"Bad request: fetch_blobs accepts at most 67108864 total blob bytes"}"#.to_vec())
+                            .unwrap();
+                    }
+                    let batch = RecordBatch::try_new(
+                        Arc::new(Schema::new(vec![Field::new(
+                            "image",
+                            DataType::LargeBinary,
+                            true,
+                        )])),
+                        vec![Arc::new(builder.finish())],
+                    )
+                    .unwrap();
+                    return http::Response::builder()
+                        .status(200)
+                        .header(CONTENT_TYPE, ARROW_STREAM_CONTENT_TYPE)
+                        .body(write_ipc_stream_uncompressed(&batch))
+                        .unwrap();
+                }
+                assert_eq!(path, "/v1/table/my_table/blob/image/40/bytes");
+                assert!(request.url().query().unwrap().contains("version=42"));
+                let range = request
+                    .headers()
+                    .get(reqwest::header::RANGE)
+                    .unwrap()
+                    .to_str()
+                    .unwrap();
+                seen.lock().unwrap().push(format!("GET {range}"));
+                match range {
+                    "bytes=0-0" => http::Response::builder()
+                        .status(206)
+                        .header(reqwest::header::CONTENT_RANGE, "bytes 0-0/10")
+                        .body(b"0".to_vec())
+                        .unwrap(),
+                    "bytes=0-" => http::Response::builder()
+                        .status(206)
+                        .header(reqwest::header::CONTENT_RANGE, "bytes 0-9/10")
+                        .body(b"0123456789".to_vec())
+                        .unwrap(),
+                    _ => panic!("unexpected range: {range}"),
+                }
+            },
+        );
+
+        let blobs = table
+            .fetch_blobs("image", &[10, 20, 30, 40, 20])
+            .await
+            .unwrap();
+        assert_eq!(blobs.len(), 5);
+        assert_eq!(blobs.value(0), b"aaaa");
+        assert_eq!(blobs.value(1), b"bbb");
+        assert!(blobs.is_null(2));
+        assert_eq!(blobs.value(3), b"0123456789");
+        assert_eq!(blobs.value(4), b"bbb");
+        let requests = requests.lock().unwrap();
+        assert!(requests.contains(&"GET bytes=0-0".to_string()));
+        assert!(requests.contains(&"GET bytes=0-".to_string()));
+    }
+
+    #[tokio::test]
+    async fn test_fetch_blobs_does_not_split_unrelated_bad_requests() {
+        let requests = Arc::new(AtomicUsize::new(0));
+        let seen = requests.clone();
+        let table =
+            Table::new_with_handler_version("my_table", semver::Version::new(0, 5, 0), move |_| {
+                seen.fetch_add(1, Ordering::SeqCst);
+                http::Response::builder()
+                    .status(400)
+                    .body(b"unknown blob column".to_vec())
+                    .unwrap()
+            });
+
+        assert_fetch_blobs_http_error(
+            table.fetch_blobs("missing", &[10, 20]).await.unwrap_err(),
+            "unknown blob column",
+        );
+        assert_eq!(requests.load(Ordering::SeqCst), 1);
+    }
+
     fn table_with_fetch_blobs_response(body: Vec<u8>) -> Table {
         table_with_fetch_blobs_content_type(Some(ARROW_STREAM_CONTENT_TYPE), body)
     }
diff --git a/rust/lancedb/src/remote/table/blobs.rs b/rust/lancedb/src/remote/table/blobs.rs
index 597b41782..29023eeed 100644
--- a/rust/lancedb/src/remote/table/blobs.rs
+++ b/rust/lancedb/src/remote/table/blobs.rs
@@ -21,7 +21,22 @@ use crate::error::Result;
 use crate::remote::client::{HttpSend, RequestResultExt, RestfulLanceDbClient};
 use crate::table::BaseTable;
 
-use super::{FreshnessHeaders, FreshnessState, RemoteTable, freshness_headers_snapshot};
+use super::{
+    FreshnessHeaders, FreshnessState, ReadSnapshot, RemoteTable, freshness_headers_snapshot,
+};
+
+// The Cloud route rejects larger row-id lists. Its separate 64 MiB byte limit
+// is handled by splitting a rejected request below.
+const MAX_FETCH_BLOBS_ROW_IDS: usize = 1024;
+
+fn is_fetch_blobs_byte_limit_error(error: &Error) -> bool {
+    matches!(error, Error::Http {
+        source,
+        status_code: Some(StatusCode::BAD_REQUEST),
+        ..
+    } if source.to_string().contains("fetch_blobs accepts at most")
+        && source.to_string().contains("total blob bytes"))
+}
 
 #[derive(Debug, Clone, Copy)]
 enum RangeRequestMode {
@@ -369,7 +384,62 @@ impl<S: HttpSend> RemoteTable<S> {
                 message: "fetch_blobs is not supported on this LanceDB Cloud server".into(),
             });
         }
-        let read_snapshot = self.snapshot_read_state().await;
+        let mut read_snapshot = self.snapshot_read_state().await;
+        // A selection spanning requests must use one exact dataset version.
+        // Resolve latest before the first chunk; a checked-out version is exact already.
+        if row_ids.len() > MAX_FETCH_BLOBS_ROW_IDS && read_snapshot.version.is_none() {
+            read_snapshot.version = Some(self.describe_read_snapshot(read_snapshot).await?.version);
+        }
+        let mut pending: Vec<&[u64]> = row_ids.chunks(MAX_FETCH_BLOBS_ROW_IDS).rev().collect();
+        let mut chunks = Vec::new();
+        while let Some(ids) = pending.pop() {
+            match self.fetch_blobs_chunk(column, ids, read_snapshot).await {
+                Ok(blobs) => chunks.push(blobs),
+                Err(error) if is_fetch_blobs_byte_limit_error(&error) => {
+                    // A single-request call may turn into several requests after a
+                    // byte-cap error. No bytes from the failed request were used.
+                    if read_snapshot.version.is_none() {
+                        read_snapshot.version =
+                            Some(self.describe_read_snapshot(read_snapshot).await?.version);
+                    }
+                    if ids.len() == 1 {
+                        // The whole-byte route cannot serve this blob. The Range route
+                        // has no aggregate response limit and preserves null alignment.
+                        let mut files = self
+                            .fetch_blob_files_with_snapshot(column, ids, read_snapshot)
+                            .await?;
+                        let blob = match files.pop().unwrap() {
+                            Some(file) => Some(file.read().await?),
+                            None => None,
+                        };
+                        chunks.push(LargeBinaryArray::from(vec![blob.as_deref()]));
+                    } else {
+                        let mid = ids.len() / 2;
+                        pending.push(&ids[mid..]);
+                        pending.push(&ids[..mid]);
+                    }
+                }
+                Err(error) => return Err(error),
+            }
+        }
+
+        if chunks.len() == 1 {
+            return Ok(chunks.pop().unwrap());
+        }
+        let chunk_refs: Vec<&dyn Array> = chunks.iter().map(|chunk| chunk as &dyn Array).collect();
+        Ok(arrow::compute::concat(&chunk_refs)?
+            .as_any()
+            .downcast_ref::<LargeBinaryArray>()
+            .expect("concatenating LargeBinary arrays returns LargeBinary")
+            .clone())
+    }
+
+    async fn fetch_blobs_chunk(
+        &self,
+        column: &str,
+        row_ids: &[u64],
+        read_snapshot: ReadSnapshot,
+    ) -> Result<LargeBinaryArray> {
         let mut body = serde_json::json!({
             "version": read_snapshot.version,
             "column": column,
@@ -460,6 +530,16 @@ impl<S: HttpSend> RemoteTable<S> {
         }
 
         let read_snapshot = self.snapshot_read_state().await;
+        self.fetch_blob_files_with_snapshot(column, row_ids, read_snapshot)
+            .await
+    }
+
+    async fn fetch_blob_files_with_snapshot(
+        &self,
+        column: &str,
+        row_ids: &[u64],
+        read_snapshot: ReadSnapshot,
+    ) -> Result<Vec<Option<BlobFile>>> {
         let encoded_column = urlencoding::encode(column);
         let requesters = row_ids
             .iter()
diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs
index dcb3e1578..f6455b290 100644
--- a/rust/lancedb/src/table.rs
+++ b/rust/lancedb/src/table.rs
@@ -1251,7 +1251,10 @@ impl Table {
     /// Materialize blob bytes for the given row ids.
     ///
     /// Output matches `row_ids` in length and order. Null blobs are null;
-    /// valid empty blobs contain empty byte strings. Prefer
+    /// valid empty blobs contain empty byte strings. Cloud limits individual
+    /// requests to 1024 row ids and 64 MiB of blob bytes; the remote client
+    /// splits requests and reads a single larger blob through the Range route.
+    /// This method still materializes all bytes in memory, so prefer
     /// [`Self::fetch_blob_files`] for large selections.
     ///
     /// `_rowid` values stay valid after compaction when the table has stable