mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-19 04:28:35 +00:00
feat: add get_lsm_write_spec to read the installed LSM write spec (#3631)
## Summary Adds `Table::get_lsm_write_spec` returning `Option<LsmWriteSpec>` — the read counterpart to the existing `set_lsm_write_spec` / `unset_lsm_write_spec`. Returns `None` when the MemWAL LSM write path is not enabled; otherwise reconstructs the spec (mode, shard column, `num_buckets`, `maintained_indexes`, `writer_config_defaults`) exactly as installed. ## Changes - **Rust core (`NativeTable`)** — reconstructs the spec from `mem_wal_index_details()`, resolving the shard column from its Lance field id via the dataset schema. This is a raw metadata read, so it is unaffected by `describe_indices` system-index filtering. - **Remote (`RemoteTable`)** — reads the `__lance_mem_wal` system index through `index/list` with `include_system: true` (so the curated `list_indices` surface stays unchanged), then parses the index `details` JSON. It matches the index by name and ignores `index_type`, so no client `IndexType` variant is needed. It uses the **server-resolved `column` name** from the details (Lance field ids do not travel to the remote client). - **Python + TypeScript bindings** — sync and async, mirroring `set`/`unset`, with round-trip tests (bucket / identity / unsharded, plus `None` when unset). ## Tests - Rust: native round-trip unit test + remote mock-endpoint tests (present + absent). All green (`cargo test --features remote -p lancedb`). - Python/TS: round-trip tests added; binding-runtime execution runs in CI. ## Dependencies for the remote path The remote path is complete on the client side but depends on two out-of-repo pieces to work end-to-end: 1. **lance** — emit the server-resolved shard **`column`** name in the MemWAL index `details` JSON (field ids can't reach the client). See lance-format/lance#7667. 2. **server** — honor `include_system` on `index/list` so the `__lance_mem_wal` entry is returned for this read. Against an older server (no `include_system`), the remote getter degrades gracefully to `Ok(None)` rather than erroring. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import lancedb
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
from lancedb._lancedb import LsmWriteSpec
|
||||
from lancedb.index import BTree
|
||||
|
||||
SCHEMA = pa.schema(
|
||||
[
|
||||
@@ -136,3 +137,76 @@ def test_lsm_write_spec_identity_and_writer_config_defaults():
|
||||
s = s.with_writer_config_defaults({"durable_write": "false"})
|
||||
assert s.writer_config_defaults == {"durable_write": "false"}
|
||||
assert "durable_write" in repr(s)
|
||||
|
||||
|
||||
def test_get_lsm_write_spec(tmp_path):
|
||||
_db, table = _make_table(tmp_path)
|
||||
table.set_unenforced_primary_key("id")
|
||||
|
||||
# None when nothing is installed.
|
||||
assert table.get_lsm_write_spec() is None
|
||||
|
||||
# A real scalar index is needed to name it as a maintained index.
|
||||
table.create_index("id", config=BTree())
|
||||
idx_name = table.list_indices()[0].name
|
||||
|
||||
# Bucket spec round-trips, including maintained indexes and writer config
|
||||
# defaults.
|
||||
table.set_lsm_write_spec(
|
||||
LsmWriteSpec.bucket("id", 4)
|
||||
.with_maintained_indexes([idx_name])
|
||||
.with_writer_config_defaults({"durable_write": "false"})
|
||||
)
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec is not None
|
||||
assert spec.spec_type == "bucket"
|
||||
assert spec.column == "id"
|
||||
assert spec.num_buckets == 4
|
||||
assert spec.maintained_indexes == [idx_name]
|
||||
assert spec.writer_config_defaults == {"durable_write": "false"}
|
||||
|
||||
# After unset, None again.
|
||||
table.unset_lsm_write_spec()
|
||||
assert table.get_lsm_write_spec() is None
|
||||
|
||||
# Identity round-trips (column recovered from the schema).
|
||||
table.set_lsm_write_spec(LsmWriteSpec.identity("id"))
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec.spec_type == "identity"
|
||||
assert spec.column == "id"
|
||||
table.unset_lsm_write_spec()
|
||||
|
||||
# Unsharded round-trips (no routing column).
|
||||
table.set_lsm_write_spec(LsmWriteSpec.unsharded())
|
||||
spec = table.get_lsm_write_spec()
|
||||
assert spec.spec_type == "unsharded"
|
||||
assert spec.column is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_get_lsm_write_spec(tmp_path):
|
||||
db = await lancedb.connect_async(
|
||||
tmp_path, read_consistency_interval=timedelta(seconds=0)
|
||||
)
|
||||
table = await db.create_table(
|
||||
"t",
|
||||
pa.RecordBatchReader.from_batches(SCHEMA, [_batch(["seed"], [0])]),
|
||||
)
|
||||
|
||||
assert await table.get_lsm_write_spec() is None
|
||||
|
||||
# A real scalar index is needed to name it as a maintained index.
|
||||
await table.create_index("id", config=BTree())
|
||||
idx_name = (await table.list_indices())[0].name
|
||||
|
||||
await table.set_lsm_write_spec(
|
||||
LsmWriteSpec.bucket("id", 8).with_maintained_indexes([idx_name])
|
||||
)
|
||||
spec = await table.get_lsm_write_spec()
|
||||
assert spec is not None
|
||||
assert spec.spec_type == "bucket"
|
||||
assert spec.column == "id"
|
||||
assert spec.num_buckets == 8
|
||||
assert spec.maintained_indexes == [idx_name]
|
||||
await table.unset_lsm_write_spec()
|
||||
assert await table.get_lsm_write_spec() is None
|
||||
|
||||
Reference in New Issue
Block a user