feat: add get_lsm_write_spec to read the installed LSM write spec (#3631)

## Summary

Adds `Table::get_lsm_write_spec` returning `Option<LsmWriteSpec>` — the
read counterpart to the existing `set_lsm_write_spec` /
`unset_lsm_write_spec`. Returns `None` when the MemWAL LSM write path is
not enabled; otherwise reconstructs the spec (mode, shard column,
`num_buckets`, `maintained_indexes`, `writer_config_defaults`) exactly
as installed.

## Changes

- **Rust core (`NativeTable`)** — reconstructs the spec from
`mem_wal_index_details()`, resolving the shard column from its Lance
field id via the dataset schema. This is a raw metadata read, so it is
unaffected by `describe_indices` system-index filtering.
- **Remote (`RemoteTable`)** — reads the `__lance_mem_wal` system index
through `index/list` with `include_system: true` (so the curated
`list_indices` surface stays unchanged), then parses the index `details`
JSON. It matches the index by name and ignores `index_type`, so no
client `IndexType` variant is needed. It uses the **server-resolved
`column` name** from the details (Lance field ids do not travel to the
remote client).
- **Python + TypeScript bindings** — sync and async, mirroring
`set`/`unset`, with round-trip tests (bucket / identity / unsharded,
plus `None` when unset).

## Tests

- Rust: native round-trip unit test + remote mock-endpoint tests
(present + absent). All green (`cargo test --features remote -p
lancedb`).
- Python/TS: round-trip tests added; binding-runtime execution runs in
CI.

## Dependencies for the remote path

The remote path is complete on the client side but depends on two
out-of-repo pieces to work end-to-end:
1. **lance** — emit the server-resolved shard **`column`** name in the
MemWAL index `details` JSON (field ids can't reach the client). See
lance-format/lance#7667.
2. **server** — honor `include_system` on `index/list` so the
`__lance_mem_wal` entry is returned for this read.

Against an older server (no `include_system`), the remote getter
degrades gracefully to `Ok(None)` rather than erroring.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dan Rammer
2026-07-08 14:05:41 -05:00
committed by GitHub
parent f428c6a76c
commit 6c066530e5
12 changed files with 573 additions and 3 deletions

View File

@@ -2992,6 +2992,56 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
}),
).rejects.toThrow();
});
it("reads back the installed spec via getLsmWriteSpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getLsmWriteSpec()).toBeUndefined();
// A real scalar index is needed to name it as a maintained index.
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
await table.createIndex("id");
const indexName = (await table.listIndices())[0].name;
// Bucket spec round-trips, including maintained indexes and writer config
// defaults. Lance writer-config keys are canonically snake_case.
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
const writerConfigDefaults = { durable_write: "false" };
await table.setLsmWriteSpec({
specType: "bucket",
column: "id",
numBuckets: 4,
maintainedIndexes: [indexName],
writerConfigDefaults,
});
const spec = await table.getLsmWriteSpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
expect(spec?.numBuckets).toBe(4);
expect(spec?.maintainedIndexes).toEqual([indexName]);
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
// After unset, undefined again.
await table.unsetLsmWriteSpec();
expect(await table.getLsmWriteSpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
const identity = await table.getLsmWriteSpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.unsetLsmWriteSpec();
// Unsharded round-trips (no routing column).
await table.setLsmWriteSpec({ specType: "unsharded" });
const unsharded = await table.getLsmWriteSpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
});
describe("LSM merge insert", () => {

View File

@@ -585,6 +585,17 @@ export abstract class Table {
* @returns {Promise<void>}
*/
abstract unsetLsmWriteSpec(): Promise<void>;
/**
* Read the {@link LsmWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* The returned spec — including its `maintainedIndexes` and
* `writerConfigDefaults` — mirrors what was passed to
* {@link Table#setLsmWriteSpec}.
* @returns {Promise<LsmWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
*
@@ -1091,6 +1102,15 @@ export class LocalTable extends Table {
return await this.inner.unsetLsmWriteSpec();
}
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
// The native binding types `specType` as a plain `string`; narrow it back
// to the public union. The Rust `From` impl only ever emits one of the
// three valid values, so the cast is safe.
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
| LsmWriteSpec
| undefined;
}
async closeLsmWriters(): Promise<void> {
return await this.inner.closeLsmWriters();
}

View File

@@ -411,6 +411,16 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
let spec = self
.inner_ref()?
.get_lsm_write_spec()
.await
.default_error()?;
Ok(spec.map(LsmWriteSpec::from))
}
#[napi(catch_unwind)]
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_lsm_writers().await.default_error()
@@ -728,6 +738,47 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
use lancedb::table::LsmWriteSpec as Native;
match spec {
Native::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
maintained_indexes,
writer_config_defaults,
} => Self {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
writer_config_defaults: Some(writer_config_defaults),
},
}
}
}
/// Statistics about a compaction operation.
#[napi(object)]
#[derive(Clone, Debug)]