From 123c921c4f6cf6ff350922d9236215d408efecb6 Mon Sep 17 00:00:00 2001
From: "lancedb-gatefixer[bot]"
 <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Date: Thu, 6 Aug 2026 16:40:41 +0800
Subject: [PATCH] test(python): cover sliced nullable table search (#3875)

## Summary

- add a Python regression for vector search over a sliced Arrow table
with nullable scalar columns
- verify the nearest row retains its non-null score values after the
table is written

## Root cause

Lance 0.19.2 deep-copied a validity bitmap without preserving its
non-zero bit offset. For a sliced nullable table, scalar values and
vectors began at the slice while the copied validity bitmap began at the
parent table's first row. That made valid score values appear null even
though the corresponding vector stayed intact. The upstream Lance repair
is already present in the current dependency; this adds a LanceDB-level
guard for the reported create/search path.

## Validation

- reproduced on Python 3.12 with LanceDB 0.16.0, pylance 0.19.2, PyArrow
18.0.0, and Polars 1.14.0
- `uv run --project python --extra dev ruff format --check
python/python/tests/test_table.py`
- `uv run --project python --extra dev ruff check .`
- `cd python && uv run --extra tests pytest
python/tests/test_table.py::test_search_preserves_nulls_from_sliced_arrow_table
-q`

Fixes #1879

<!-- lance-gatekeeper-fix:v1 agent=bfa0551793f8e3cf3980cf64ad89908a
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
---
 python/python/tests/test_table.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py
index 1f2e14742..ef59502d7 100644
--- a/python/python/tests/test_table.py
+++ b/python/python/tests/test_table.py
@@ -99,6 +99,30 @@ def test_basic(mem_db: DBConnection):
     assert table.to_arrow() == expected_data
 
 
+def test_search_preserves_nulls_from_sliced_arrow_table(mem_db: DBConnection):
+    data = pa.table(
+        {
+            "id": [0, 1, 2, 3, 4],
+            "score_cn": [None, 22, None, 5, 8],
+            "score_mt": [None, 42, None, 5, 8],
+            "vector": [
+                [20, 19, -1, -1],
+                [41, 38, 22, 42],
+                [10, 10, -1, -1],
+                [5, 5, 5, 5],
+                [8, 8, 8, 8],
+            ],
+        }
+    ).slice(1)
+
+    table = mem_db.create_table("sliced_nullable", data=data)
+    result = table.search([41, 38, 22, 42]).limit(1).to_arrow()
+
+    assert result["id"].to_pylist() == [1]
+    assert result["score_cn"].to_pylist() == [22]
+    assert result["score_mt"].to_pylist() == [42]
+
+
 def test_table_to_pandas_default_matches_arrow(tmp_db: DBConnection):
     pd = pytest.importorskip("pandas")
     data = pa.table({"id": [1, 2], "text": ["one", "two"]})