mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 20:18:37 +00:00
fix(python): preserve repeated indexed merge inserts (#3850)
## Summary - add a Python regression test for two partial-schema merge inserts against the same BTree-indexed rows - verify repeated updates retain one copy of every row and the final update values ## Root cause Lance 4.0, used by LanceDB 0.30.2, removed a rewritten fragment from the index bitmap while stale BTree entries for that fragment remained searchable. The next merge found each target through both the stale index and the unindexed-fragment scan, producing the ambiguous-match error. Lance fixed the root cause in lance-format/lance#6563 by applying the fragment-bitmap allow-list to index results, and the Lance release pinned by current LanceDB includes that fix. This test preserves the corrected behavior through the Python API. ## Validation - `cd python && uv run --extra tests pytest python/tests/test_table.py -k merge_insert -q` (9 passed) - `cd python && uv run --extra tests --extra dev ruff format --check python/tests/test_table.py` - `cd python && uv run --extra tests --extra dev ruff check python/tests/test_table.py` Repository-wide Ruff also reports 20 pre-existing violations in untouched CI and plugin scripts. Fixes #3280 <!-- lance-gatekeeper-fix:v1 agent=ee6b9565f9780712026076930566f116 generation=1 --> Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
0ba82873c5
commit
c1a3fa7f51
@@ -2598,6 +2598,36 @@ def test_merge_insert_subschema(mem_db: DBConnection, data_format):
|
||||
assert table.to_arrow().sort_by("id") == expected
|
||||
|
||||
|
||||
def test_repeated_partial_merge_insert_with_scalar_index(mem_db: DBConnection):
|
||||
def make_batch(start: int) -> pa.Table:
|
||||
return pa.table(
|
||||
{
|
||||
"id": [f"id-{i:04}" for i in range(start, start + 100)],
|
||||
"category": ["A"] * 100,
|
||||
"value_a": [float(i) for i in range(start, start + 100)],
|
||||
"value_b": [float(i) / 10 for i in range(100)],
|
||||
}
|
||||
)
|
||||
|
||||
table = mem_db.create_table("my_table", data=make_batch(0))
|
||||
table.add(make_batch(100))
|
||||
table.add(make_batch(200))
|
||||
table.create_index("id", config=BTree())
|
||||
|
||||
ids = [f"id-{i:04}" for i in range(100, 200)]
|
||||
for value in (999.0, 888.0):
|
||||
result = (
|
||||
table.merge_insert("id")
|
||||
.when_matched_update_all()
|
||||
.execute(pa.table({"id": ids, "value_a": [value] * 100}))
|
||||
)
|
||||
assert result.num_updated_rows == 100
|
||||
|
||||
actual = table.to_arrow().sort_by("id")
|
||||
assert actual.num_rows == 300
|
||||
assert actual["value_a"].to_pylist()[100:200] == [888.0] * 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_merge_insert_async(mem_db_async: AsyncConnection):
|
||||
data = pa.table({"a": [1, 2, 3], "b": ["a", "b", "c"]})
|
||||
|
||||
Reference in New Issue
Block a user