mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-14 07:42:28 +00:00
XY Zhan
723a2394e5
**The maintained set is an intent, not a snapshot.** An unnamed set is stored as the intent to maintain every index, which lance re-resolves whenever it builds a MemTable, so an index created after the spec installs is maintained too. A named set is checked here, where the caller can be told which names exist; an index of a kind the MemWAL cannot mirror fails a named set and is skipped by an unnamed one. **A newly created index reaches a writer that is already open.** lancedb caches a `ShardWriter` per shard and reuses it, so `create_index` installs the current set on the cached writer. A failure there is reported rather than propagated: the index is committed durably by then, and the writer this process holds may be fenced or poisoned, which is no reason to fail creating an index. **A table carries one spec.** `set_lsm_write_spec` fails while one is installed and names `unset_lsm_write_spec` in the error, because the generations already written were homed under that spec. Accepting a repeat that changed only the maintained set would hand a caller who also passed a different sharding a success that changed nothing they asked for. **The read path asks what is actually carried.** An unnamed set leaves the stored list empty, so the list alone cannot decide whether an index is maintained. The read asks the spec *and* every resident MemTable whether it carries the index, so a MemTable built before the index joined the set refuses the read rather than answering without its rows. Both spec routes report an unnamed set as `null` rather than `[]`: maintain whatever the table gains, and maintain nothing, are different settings. ## Known gap The guard above covers resident MemTables. Once a writer flushes and closes none is resident, so a retained generation written before the index existed still gets an indexed read that omits its rows. This window is **wider than on main**: a set resolved at install time excluded an index created later, so the read was refused; an intent includes it, and the refusal only holds while a MemTable is resident. Closing it needs the generation's own coverage, which this layer cannot see — its snapshot carries a generation and a path — so it belongs in lance's scanner or flush path. ## Testing `cargo nextest run -p lancedb --lib --features remote` — **1479 passed, 3 skipped**. `cargo fmt --all --check`, and `cargo clippy --features remote --lib --tests --examples -- -D warnings`, both clean. Covered: the round trip of an unnamed set, a set over an installed spec refused, a spec replaced through unset, a read refused while a resident MemTable predates the index, and an index created mid-write reaching the open writer. Two Python binding tests move with the semantics: an unnamed set now reads back as `None` rather than the list it resolves to, and the refusal is matched on `already set`. Neither was run locally — no Python build here — so CI is the check on them.
225 lines
7.4 KiB
Python
225 lines
7.4 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
|
|
"""Tests for installing and clearing an LsmWriteSpec via
|
|
`Table.set_lsm_write_spec` / `Table.unset_lsm_write_spec`.
|
|
"""
|
|
|
|
from datetime import timedelta
|
|
|
|
import lancedb
|
|
import pyarrow as pa
|
|
import pytest
|
|
from lancedb._lancedb import LsmWriteSpec
|
|
from lancedb.index import BTree
|
|
|
|
SCHEMA = pa.schema(
|
|
[
|
|
pa.field("id", pa.utf8(), nullable=False),
|
|
pa.field("v", pa.int32(), nullable=False),
|
|
]
|
|
)
|
|
|
|
|
|
def test_lsm_write_spec_module_metadata():
|
|
assert lancedb.LsmWriteSpec is LsmWriteSpec
|
|
assert LsmWriteSpec.__module__ == "lancedb._lancedb"
|
|
|
|
|
|
def _batch(ids, vs):
|
|
return pa.RecordBatch.from_arrays(
|
|
[pa.array(ids, type=pa.utf8()), pa.array(vs, type=pa.int32())],
|
|
schema=SCHEMA,
|
|
)
|
|
|
|
|
|
def _reader(ids, vs):
|
|
return pa.RecordBatchReader.from_batches(SCHEMA, [_batch(ids, vs)])
|
|
|
|
|
|
def _make_table(tmp_path):
|
|
db = lancedb.connect(tmp_path, read_consistency_interval=timedelta(seconds=0))
|
|
table = db.create_table("t", _reader(["seed"], [0]))
|
|
return db, table
|
|
|
|
|
|
def test_set_lsm_write_spec_validates(tmp_path):
|
|
_db, table = _make_table(tmp_path)
|
|
|
|
# Out-of-range num_buckets.
|
|
with pytest.raises(Exception, match="num_buckets"):
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 0))
|
|
with pytest.raises(Exception, match="num_buckets"):
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 1025))
|
|
|
|
# Happy path, then a set over the installed spec is refused.
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
|
|
with pytest.raises(Exception, match="already set"):
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
|
|
|
|
|
|
def test_unset_lsm_write_spec(tmp_path):
|
|
_db, table = _make_table(tmp_path)
|
|
|
|
# unset errors when no spec is set.
|
|
with pytest.raises(Exception, match="no LSM write spec"):
|
|
table.unset_lsm_write_spec()
|
|
|
|
# Install a spec, then remove it; afterwards a fresh spec can be set.
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
|
|
table.unset_lsm_write_spec()
|
|
# A second unset errors — there is no spec left to remove.
|
|
with pytest.raises(Exception, match="no LSM write spec"):
|
|
table.unset_lsm_write_spec()
|
|
table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 8))
|
|
|
|
|
|
def test_set_unsharded_spec(tmp_path):
|
|
_db, table = _make_table(tmp_path)
|
|
# Lance MemWAL still requires a primary key on the dataset; Unsharded
|
|
# just skips per-row hashing.
|
|
table.set_unenforced_primary_key("id")
|
|
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
|
|
table.unset_lsm_write_spec()
|
|
|
|
|
|
def test_lsm_write_spec_repr():
|
|
s = LsmWriteSpec.bucket("id", 4)
|
|
assert s.spec_type == "bucket"
|
|
assert s.column == "id"
|
|
assert s.num_buckets == 4
|
|
# A fresh spec defers its maintained set to install time.
|
|
assert s.maintained_indexes is None
|
|
assert s.with_maintained_indexes([]).maintained_indexes == []
|
|
assert "bucket" in repr(s)
|
|
assert "id" in repr(s)
|
|
assert "4" in repr(s)
|
|
|
|
u = LsmWriteSpec.unsharded()
|
|
assert u.spec_type == "unsharded"
|
|
assert u.column is None
|
|
assert u.num_buckets is None
|
|
assert "unsharded" in repr(u)
|
|
|
|
|
|
def test_lsm_write_spec_with_maintained_indexes():
|
|
s = LsmWriteSpec.bucket("id", 4).with_maintained_indexes(["idx_a", "idx_b"])
|
|
assert s.maintained_indexes == ["idx_a", "idx_b"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_set_unset_lsm_write_spec(tmp_path):
|
|
db = await lancedb.connect_async(
|
|
tmp_path, read_consistency_interval=timedelta(seconds=0)
|
|
)
|
|
table = await db.create_table(
|
|
"t",
|
|
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
|
|
)
|
|
|
|
await table.set_unenforced_primary_key("id")
|
|
await table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 4))
|
|
await table.unset_lsm_write_spec()
|
|
# A second unset errors.
|
|
with pytest.raises(Exception, match="no LSM write spec"):
|
|
await table.unset_lsm_write_spec()
|
|
|
|
|
|
def test_set_identity_spec(tmp_path):
|
|
_db, table = _make_table(tmp_path)
|
|
# Identity sharding still requires an unenforced primary key on the
|
|
# table; it shards by the raw value of the given column.
|
|
table.set_unenforced_primary_key("id")
|
|
table.set_lsm_write_spec(LsmWriteSpec.identity("v"))
|
|
table.unset_lsm_write_spec()
|
|
|
|
|
|
def test_lsm_write_spec_identity_and_writer_config_defaults():
|
|
s = LsmWriteSpec.identity("v")
|
|
assert s.spec_type == "identity"
|
|
assert s.column == "v"
|
|
assert s.num_buckets is None
|
|
assert "identity" in repr(s)
|
|
|
|
s = s.with_writer_config_defaults({"durable_write": "false"})
|
|
assert s.writer_config_defaults == {"durable_write": "false"}
|
|
assert "durable_write" in repr(s)
|
|
|
|
|
|
def test_get_lsm_write_spec(tmp_path):
|
|
_db, table = _make_table(tmp_path)
|
|
table.set_unenforced_primary_key("id")
|
|
|
|
# None when nothing is installed.
|
|
assert table.get_lsm_write_spec() is None
|
|
|
|
# A real scalar index is needed to name it as a maintained index.
|
|
table.create_index("id", config=BTree())
|
|
idx_name = table.list_indices()[0].name
|
|
|
|
# Bucket spec round-trips, including maintained indexes and writer config
|
|
# defaults.
|
|
table.set_lsm_write_spec(
|
|
LsmWriteSpec.bucket("id", 4)
|
|
.with_maintained_indexes([idx_name])
|
|
.with_writer_config_defaults({"durable_write": "false"})
|
|
)
|
|
spec = table.get_lsm_write_spec()
|
|
assert spec is not None
|
|
assert spec.spec_type == "bucket"
|
|
assert spec.column == "id"
|
|
assert spec.num_buckets == 4
|
|
assert spec.maintained_indexes == [idx_name]
|
|
assert spec.writer_config_defaults == {"durable_write": "false"}
|
|
|
|
# After unset, None again.
|
|
table.unset_lsm_write_spec()
|
|
assert table.get_lsm_write_spec() is None
|
|
|
|
# Identity round-trips (column recovered from the schema). An unnamed
|
|
# maintained set reads back as None: what is stored is the intent to
|
|
# maintain every index, not the set it resolves to now.
|
|
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
|
|
spec = table.get_lsm_write_spec()
|
|
assert spec.spec_type == "identity"
|
|
assert spec.column == "id"
|
|
assert spec.maintained_indexes is None
|
|
table.unset_lsm_write_spec()
|
|
|
|
# Unsharded round-trips (no routing column). Opting out is distinct from
|
|
# the unnamed default.
|
|
table.set_lsm_write_spec(LsmWriteSpec.unsharded().with_maintained_indexes([]))
|
|
spec = table.get_lsm_write_spec()
|
|
assert spec.spec_type == "unsharded"
|
|
assert spec.column is None
|
|
assert spec.maintained_indexes == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_get_lsm_write_spec(tmp_path):
|
|
db = await lancedb.connect_async(
|
|
tmp_path, read_consistency_interval=timedelta(seconds=0)
|
|
)
|
|
table = await db.create_table(
|
|
"t",
|
|
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
|
|
)
|
|
|
|
assert await table.get_lsm_write_spec() is None
|
|
|
|
# A real scalar index is needed to name it as a maintained index.
|
|
await table.create_index("id", config=BTree())
|
|
idx_name = (await table.list_indices())[0].name
|
|
|
|
await table.set_lsm_write_spec(
|
|
LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
|
|
)
|
|
spec = await table.get_lsm_write_spec()
|
|
assert spec is not None
|
|
assert spec.spec_type == "bucket"
|
|
assert spec.column == "id"
|
|
assert spec.num_buckets == 8
|
|
assert spec.maintained_indexes == [idx_name]
|
|
await table.unset_lsm_write_spec()
|
|
assert await table.get_lsm_write_spec() is None
|