mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 20:18:37 +00:00
fix(python): cover nullable list v2.2 decoding (#3853)
## Summary - add a minimized regression for mostly-null `list<float32>` data at the v2.2 structural page boundary - verify scans preserve all 64,885 rows, including 64,668 null list values ## Root cause Lance 3.0.0 sliced repetition/definition state using top-level row offsets in the complex all-null decoder. At this page boundary, the list and validity children were materialized at different lengths. The current Lance dependency contains the upstream decoder repair; this test locks that behavior into the LanceDB Python suite without duplicating decoder logic. ## Validation - reproduced the attached 1,892,466-row case on `lancedb==0.30.0` with `expected 1024 got 285` - verified the full attachment reads on the current branch - `python/.venv/bin/ruff format --check python/python/tests/test_table.py` - `python/.venv/bin/ruff check .` - `cd python && uv run --extra tests pytest python/tests/test_table.py::test_read_mostly_null_list_v2_2_page_boundary -q` - `cd python && uv run --extra tests pytest python/tests/test_table.py -q` (137 passed) Fixes #3194 <!-- lance-gatekeeper-fix:v1 agent=0445adc5303a3302152cea3d2110bed1 generation=1 --> Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
3af51541a0
commit
0ba82873c5
@@ -1884,6 +1884,33 @@ def test_add_nullable_struct_with_none(mem_db: DBConnection):
|
||||
assert result.column("data").to_pylist() == [{"x": 1.0}, None]
|
||||
|
||||
|
||||
def test_read_mostly_null_list_v2_2_page_boundary(tmp_path):
|
||||
# Regression test for #3194. This row/value count crosses a v2.2 structural
|
||||
# encoding page boundary where Lance 3.0.0 sliced repetition/definition
|
||||
# levels by row offset and decoded child arrays at different lengths.
|
||||
num_rows = 64_885
|
||||
num_values = 217
|
||||
list_type = pa.list_(pa.float32())
|
||||
source = pa.table(
|
||||
{
|
||||
"id": np.arange(num_rows, dtype=np.int64),
|
||||
"coords": pa.array(
|
||||
[[1.0, 2.0, 3.0, 4.0]] * num_values + [None] * (num_rows - num_values),
|
||||
type=list_type,
|
||||
),
|
||||
}
|
||||
)
|
||||
db = lancedb.connect(
|
||||
tmp_path,
|
||||
storage_options={"new_table_data_storage_version": "2.2"},
|
||||
)
|
||||
table = db.create_table("test_sparse_nullable_list", data=source)
|
||||
|
||||
result = table.search().select(["id", "coords"]).limit(num_rows).to_arrow()
|
||||
|
||||
assert result.equals(source)
|
||||
|
||||
|
||||
def test_add_with_integer_embeddings_preserves_casting(mem_db: DBConnection):
|
||||
class Schema(LanceModel):
|
||||
text: str
|
||||
|
||||
Reference in New Issue
Block a user