refactor!: rename the LSM table APIs to MemWAL

The write path these APIs drive is Lance's MemWAL, and "LSM" described
its shape rather than naming it. The surface now says MemWAL, and turning
it on and off reads as enabling a feature rather than installing a spec.

| Before | After |
|---|---|
| `set_lsm_write_spec` | `enable_memwal` |
| `unset_lsm_write_spec` | `disable_memwal` |
| `get_lsm_write_spec` | `get_memwal_spec` |
| `flush_lsm` / `compact_lsm` / `checkpoint_lsm` | `flush_memwal` / `compact_memwal` / `checkpoint_memwal` |
| `get_lsm_stats` | `get_memwal_stats` |
| `close_lsm_writers` | `close_memwal_writers` |
| `use_lsm` (query, merge insert) | `use_memwal` |
| `LsmWriteSpec` / `LsmStats` | `MemWalWriteSpec` / `MemWalStats` |
| Java `LanceDbTableLsm` | `LanceDbTableMemWal` |

TypeScript and Java use camelCase (`enableMemWal`, `getMemWalStats`,
...); the napi methods set `js_name` so the generated names keep the
`MemWal` casing. The flush/compact/checkpoint methods keep a `_memwal`
suffix so `compact_memwal` is not confused with `compact_files` and
`optimize`, which compact the base table.

## Wire

The remote routes and JSON envelopes move with the SDKs and must ship
together with the matching server change:

- `/set_lsm_write_spec/` -> `/enable_memwal/`, `/unset_lsm_write_spec/`
  -> `/disable_memwal/`, `/get_lsm_write_spec/` -> `/get_memwal_spec/`,
  `/flush_lsm/`, `/compact_lsm/`, `/get_lsm_stats/` -> `*_memwal/`
- response keys `lsm_write_spec` -> `memwal_spec`, `lsm_stats` ->
  `memwal_stats`; query and merge-insert body field `use_lsm` ->
  `use_memwal`

## Also fixed

The previous rename turned the Java `bucket` sharding mode into
`tableShard` in the javadoc example and tests, and the test fixtures
sent `tableShards` where the parser reads `table_shards`, so the Java
tests did not compile. Both are restored.

Internal names (`LsmMode`, `LsmScanner`, the `merge::lsm` and
`query::lsm` modules) are unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Daniel Rammer
2026-10-12 15:54:03 -05:00
co-authored by Claude Opus 5.5
parent 06d7f82fa2
commit d7e921dbec
62 changed files with 1349 additions and 1360 deletions
+48 -48
View File
@@ -640,11 +640,11 @@ describe.each(arrowVersions)(
);
});
it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => {
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
// useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray();
expect(res.map((r) => r.id)).toEqual([1, 3]);
});
@@ -4606,7 +4606,7 @@ describe("setUnenforcedPrimaryKey", () => {
});
});
describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
describe("enableMemWal / disableMemWal", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -4626,16 +4626,16 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 4,
});
await table.unsetLsmWriteSpec();
await table.disableMemWal();
// A second unset errors — there is no spec left to remove.
await expect(table.unsetLsmWriteSpec()).rejects.toThrow();
await expect(table.disableMemWal()).rejects.toThrow();
// A fresh spec can be installed after unset.
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 8,
@@ -4647,8 +4647,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "unsharded" });
await table.unsetLsmWriteSpec();
await table.enableMemWal({ specType: "unsharded" });
await table.disableMemWal();
});
it("installs an identity spec", async () => {
@@ -4656,8 +4656,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
await table.unsetLsmWriteSpec();
await table.enableMemWal({ specType: "identity", column: "id" });
await table.disableMemWal();
});
it("rejects an invalid spec", async () => {
@@ -4667,7 +4667,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
await table.setUnenforcedPrimaryKey("id");
// num_buckets out of range.
await expect(
table.setLsmWriteSpec({
table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 0,
@@ -4675,7 +4675,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
).rejects.toThrow();
// Column mismatch.
await expect(
table.setLsmWriteSpec({
table.enableMemWal({
specType: "bucket",
column: "missing",
numBuckets: 4,
@@ -4683,13 +4683,13 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
).rejects.toThrow();
});
it("reads back the installed spec via getLsmWriteSpec", async () => {
it("reads back the installed spec via getMemWalSpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getLsmWriteSpec()).toBeUndefined();
expect(await table.getMemWalSpec()).toBeUndefined();
// A real scalar index is needed to name it as a maintained index.
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
@@ -4700,14 +4700,14 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
// defaults. Lance writer-config keys are canonically snake_case.
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
const writerConfigDefaults = { durable_write: "false" };
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 4,
maintainedIndexes: [indexName],
writerConfigDefaults,
});
const spec = await table.getLsmWriteSpec();
const spec = await table.getMemWalSpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
@@ -4716,19 +4716,19 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
// After unset, undefined again.
await table.unsetLsmWriteSpec();
expect(await table.getLsmWriteSpec()).toBeUndefined();
await table.disableMemWal();
expect(await table.getMemWalSpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
const identity = await table.getLsmWriteSpec();
await table.enableMemWal({ specType: "identity", column: "id" });
const identity = await table.getMemWalSpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.unsetLsmWriteSpec();
await table.disableMemWal();
// Unsharded round-trips (no routing column).
await table.setLsmWriteSpec({ specType: "unsharded" });
const unsharded = await table.getLsmWriteSpec();
await table.enableMemWal({ specType: "unsharded" });
const unsharded = await table.getMemWalSpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
@@ -4757,7 +4757,7 @@ describe("LSM merge insert", () => {
]);
await table.setUnenforcedPrimaryKey("id");
// numBuckets = 1: every row routes to the single bucket.
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 1,
@@ -4782,17 +4782,17 @@ describe("LSM merge insert", () => {
expect(res.version).toBe(0);
expect(res.numInsertedRows).toBe(0);
await table.closeLsmWriters();
await table.closeMemWalWriters();
});
it("falls back to the standard path with useLsm(false)", async () => {
it("falls back to the standard path with useMemWal(false)", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
const res = await table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useLsm(false)
.useMemWal(false)
.execute([
{ id: "b", value: 9 },
{ id: "e", value: 5 },
@@ -4815,7 +4815,7 @@ describe("LSM merge insert", () => {
expect(res.numRows).toBe(1);
});
it("rejects a non-upsert merge under an LSM spec", async () => {
it("rejects a non-upsert merge under a MemWAL spec", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
@@ -4841,22 +4841,22 @@ describe("LSM merge insert", () => {
const lsm = await table.query().toArray();
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
// useLsm(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useLsm(false).toArray();
// useMemWal(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useMemWal(false).toArray();
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
});
it("reads the base table when no LSM spec is installed", async () => {
it("reads the base table when no MemWAL spec is installed", async () => {
const conn = await connect(tmpDir.name);
const table = await conn.createEmptyTable(
"plain",
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
// No spec: default read and useLsm(false) both succeed against the base table.
// No spec: default read and useMemWal(false) both succeed against the base table.
await expect(table.query().toArray()).resolves.toBeDefined();
await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
// useLsm(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
await expect(table.query().useMemWal(false).toArray()).resolves.toBeDefined();
// useMemWal(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useMemWal(true).toArray()).rejects.toThrow();
});
});
@@ -4874,7 +4874,7 @@ describe("LSM convergence and stats", () => {
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "unsharded" });
await table.enableMemWal({ specType: "unsharded" });
return table;
}
@@ -4882,34 +4882,34 @@ describe("LSM convergence and stats", () => {
// rejects them rather than answering. What is asserted here is that the
// bindings reach the core at all; the behavior against a real endpoint is
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
it("rejects flushLsm on a local table", async () => {
it("rejects flushMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
await expect(table.flushMemWal()).rejects.toThrow(/not supported/i);
});
it("rejects compactLsm on a local table", async () => {
it("rejects compactMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
await expect(table.compactMemWal()).rejects.toThrow(/not supported/i);
});
it("rejects getLsmStats on a local table", async () => {
it("rejects getMemWalStats on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i);
await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i);
});
it("rejects checkpointLsm on a local table", async () => {
it("rejects checkpointMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
// checkpointLsm freezes first, so it surfaces flushLsm's rejection.
await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
// checkpointMemWal freezes first, so it surfaces flushMemWal's rejection.
await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i);
});
});
+2 -2
View File
@@ -167,8 +167,8 @@ export {
WriteProgress,
FtsToken,
TokenizeTableOptions,
LsmWriteSpec,
LsmStats,
MemWalWriteSpec,
MemWalStats,
TableShardStats,
SsTableStats,
MemtableStats,
+6 -6
View File
@@ -90,25 +90,25 @@ export class MergeInsertBuilder {
/**
* Control MemWAL routing for this merge.
*
* By default (unset), a `mergeInsert` on a table with an LSM write spec is
* By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
* routed through Lance's MemWAL shard writer, and a table without one uses the
* standard path.
*
* @param enable - `true` forces MemWAL routing and errors if the table has no
* LSM write spec. `false` forces the standard write path even when a spec is set.
* MemWAL write spec. `false` forces the standard write path even when a spec is set.
*/
useLsm(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
useMemWal(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema);
}
/**
* Controls how an LSM merge checks that its input targets a single shard.
*
* When a table has an LSM write spec, every row in a `mergeInsert` call must
* When a table has a MemWAL write spec, every row in a `mergeInsert` call must
* route to the same shard. When `true` (the default), every row is inspected
* to verify this. When `false`, only the first row is inspected and the
* shard it routes to is used for the whole input — a faster path for callers
* that have already pre-sharded their input. Has no effect on tables without
* an LSM write spec.
* a MemWAL write spec.
*
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
*/
+6 -6
View File
@@ -464,7 +464,7 @@ export class StandardQueryBase<
* Control MemWAL read routing for this query.
*
* By default (unset), when the table carries a MemWAL write spec (see
* {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
* {@link Table#enableMemWal}), reads are routed through the LSM scanner so
* they also return data written via the `mergeInsert` LSM path that has not yet
* been compacted into the base table (the active/frozen in-memory memtables and
* the SSTables), deduplicated by primary key; a table without a spec
@@ -476,11 +476,11 @@ export class StandardQueryBase<
*
* Note: the LSM scanner does not support every query shape (e.g. reranking,
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
* `useLsm(false)` is set, because a base-only read would silently exclude
* `useMemWal(false)` is set, because a base-only read would silently exclude
* un-compacted MemWAL data.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
useMemWal(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useMemWal(enable));
return this;
}
}
@@ -871,8 +871,8 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
*
* @param enable - `false` reads the base table only.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
useMemWal(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable));
return this;
}
}
+49 -49
View File
@@ -35,7 +35,7 @@ import {
FunctionErrorsOptions,
IndexConfig,
IndexStatistics,
LsmStats,
MemWalStats,
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
@@ -61,7 +61,7 @@ export { IndexConfig } from "./native";
export {
TableShardStats,
SsTableStats,
LsmStats,
MemWalStats,
MemtableStats,
} from "./native";
@@ -217,7 +217,7 @@ export type TokenizeTableOptions =
* `column` value, or upserts of that key can land in different shards and a
* stale version can win).
*/
export interface LsmWriteSpec {
export interface MemWalWriteSpec {
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
specType: "bucket" | "identity" | "unsharded";
/** Bucket and identity variants: the sharding column. */
@@ -751,10 +751,10 @@ export abstract class Table {
*/
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
/**
* Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL
* Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
* LSM-style write path for future `mergeInsert` calls.
*
* `LsmWriteSpec` chooses one of three sharding strategies via `specType`:
* `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
*
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
* key (`column` and `numBuckets` required).
@@ -769,12 +769,12 @@ export abstract class Table {
* including ones created later, and skips a kind the MemWAL cannot maintain.
* Naming them pins an exact set, and a still-building index is rejected
* rather than quietly omitted.
* @param {LsmWriteSpec} spec The sharding spec to install.
* @param {MemWalWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
* ```ts
* await table.setUnenforcedPrimaryKey("id");
* await table.setLsmWriteSpec({
* await table.enableMemWal({
* specType: "bucket",
* column: "id",
* numBuckets: 16,
@@ -782,36 +782,36 @@ export abstract class Table {
* });
* ```
*/
abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
/**
* Remove the {@link LsmWriteSpec} from this table, reverting to the standard
* Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
* `mergeInsert` write path.
*
* Errors if no spec is currently set.
* @returns {Promise<void>}
*/
abstract unsetLsmWriteSpec(): Promise<void>;
abstract disableMemWal(): Promise<void>;
/**
* Read the {@link LsmWriteSpec} currently installed on this table.
* Read the {@link MemWalWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* Resolves to `undefined` when the MemWAL write path is not enabled (no
* spec has been set, or it was removed with {@link Table#disableMemWal}).
* The spec is the one installed, including its maintained-index selection:
* an absent `maintainedIndexes` for every index the table has, an empty
* array for none.
* @returns {Promise<LsmWriteSpec | undefined>}
* @returns {Promise<MemWalWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
*
* When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL
* When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
* shard writers and caches them for reuse across calls. This closes them,
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
* It is a no-op when no writers are cached.
* @returns {Promise<void>}
*/
abstract closeLsmWriters(): Promise<void>;
abstract closeMemWalWriters(): Promise<void>;
/**
* Freeze every table shard's active memtable into a new SSTable.
*
@@ -819,18 +819,18 @@ export abstract class Table {
* so this is safe to call repeatedly.
* @returns {Promise<void>}
*/
abstract flushLsm(): Promise<void>;
abstract flushMemWal(): Promise<void>;
/**
* Trigger a background SSTable compaction pass per table shard.
*
* Returns once the passes are *dispatched*, not once they finish — watch
* {@link Table#getLsmStats} for progress, or use
* {@link Table#checkpointLsm} to wait for convergence.
* {@link Table#getMemWalStats} for progress, or use
* {@link Table#checkpointMemWal} to wait for convergence.
* @returns {Promise<void>}
*/
abstract compactLsm(): Promise<void>;
abstract compactMemWal(): Promise<void>;
/**
* Converge this table's LSM write path into its base table.
* Converge this table's MemWAL write path into its base table.
*
* Freezes once, then triggers compaction and polls until the SSTables that existed
* at the start is gone. The target set is fixed at the start, so
@@ -845,26 +845,26 @@ export abstract class Table {
* @returns {Promise<void>}
* @example
* ```ts
* const before = await table.getLsmStats();
* await table.checkpointLsm();
* const after = await table.getLsmStats();
* const before = await table.getMemWalStats();
* await table.checkpointMemWal();
* const after = await table.getMemWalStats();
* ```
*/
abstract checkpointLsm(): Promise<void>;
abstract checkpointMemWal(): Promise<void>;
/**
* Read live per-table-shard LSM state.
* Read live per-table-shard MemWAL state.
*
* Answers "how far behind is my fresh tier", "which table shard is hot", and
* "why is my fresh-tier vector search brute-force". Mutates no table state.
*
* Resolves to `undefined` only when the LSM write path is not enabled.
* Resolves to `undefined` only when the MemWAL write path is not enabled.
* @param {boolean} includeSstableRows Also count rows per SSTable.
* Off by default because each count opens an uncached Lance dataset.
* @returns {Promise<LsmStats | undefined>}
* @returns {Promise<MemWalStats | undefined>}
*/
abstract getLsmStats(
abstract getMemWalStats(
includeSstableRows?: boolean,
): Promise<LsmStats | undefined>;
): Promise<MemWalStats | undefined>;
/** Retrieve the version of the table */
abstract version(): Promise<number>;
@@ -1467,43 +1467,43 @@ export class LocalTable extends Table {
return await this.inner.setUnenforcedPrimaryKey(cols);
}
async setLsmWriteSpec(spec: LsmWriteSpec): Promise<void> {
return await this.inner.setLsmWriteSpec(spec);
async enableMemWal(spec: MemWalWriteSpec): Promise<void> {
return await this.inner.enableMemWal(spec);
}
async unsetLsmWriteSpec(): Promise<void> {
return await this.inner.unsetLsmWriteSpec();
async disableMemWal(): Promise<void> {
return await this.inner.disableMemWal();
}
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
async getMemWalSpec(): Promise<MemWalWriteSpec | undefined> {
// The native binding types `specType` as a plain `string`; narrow it back
// to the public union. The Rust `From` impl only ever emits one of the
// three valid values, so the cast is safe.
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
| LsmWriteSpec
return ((await this.inner.getMemWalSpec()) ?? undefined) as
| MemWalWriteSpec
| undefined;
}
async closeLsmWriters(): Promise<void> {
return await this.inner.closeLsmWriters();
async closeMemWalWriters(): Promise<void> {
return await this.inner.closeMemWalWriters();
}
async flushLsm(): Promise<void> {
return await this.inner.flushLsm();
async flushMemWal(): Promise<void> {
return await this.inner.flushMemWal();
}
async compactLsm(): Promise<void> {
return await this.inner.compactLsm();
async compactMemWal(): Promise<void> {
return await this.inner.compactMemWal();
}
async checkpointLsm(): Promise<void> {
return await this.inner.checkpointLsm();
async checkpointMemWal(): Promise<void> {
return await this.inner.checkpointMemWal();
}
async getLsmStats(
async getMemWalStats(
includeSstableRows: boolean = false,
): Promise<LsmStats | undefined> {
return (await this.inner.getLsmStats(includeSstableRows)) ?? undefined;
): Promise<MemWalStats | undefined> {
return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined;
}
async version(): Promise<number> {
+3 -3
View File
@@ -50,10 +50,10 @@ impl NativeMergeInsertBuilder {
this
}
#[napi]
pub fn use_lsm(&self, enable: bool) -> Self {
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&self, enable: bool) -> Self {
let mut this = self.clone();
this.inner.use_lsm(enable);
this.inner.use_memwal(enable);
this
}
+9 -9
View File
@@ -168,9 +168,9 @@ impl Query {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi]
@@ -394,9 +394,9 @@ impl VectorQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi]
@@ -504,9 +504,9 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi(catch_unwind)]
+46 -50
View File
@@ -580,64 +580,60 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::LsmWriteSpec::try_from(spec)?;
#[napi(catch_unwind, js_name = "enableMemWal")]
pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?;
self.inner_ref()?
.set_lsm_write_spec(native_spec)
.enable_memwal(native_spec)
.await
.default_error()
}
#[napi(catch_unwind)]
pub async fn unset_lsm_write_spec(&self) -> napi::Result<()> {
#[napi(catch_unwind, js_name = "disableMemWal")]
pub async fn disable_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.disable_memwal().await.default_error()
}
#[napi(catch_unwind, js_name = "getMemWalSpec")]
pub async fn get_memwal_spec(&self) -> napi::Result<Option<MemWalWriteSpec>> {
let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?;
Ok(spec.map(MemWalWriteSpec::from))
}
#[napi(catch_unwind, js_name = "closeMemWalWriters")]
pub async fn close_memwal_writers(&self) -> napi::Result<()> {
self.inner_ref()?
.unset_lsm_write_spec()
.close_memwal_writers()
.await
.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
let spec = self
.inner_ref()?
.get_lsm_write_spec()
.await
.default_error()?;
Ok(spec.map(LsmWriteSpec::from))
#[napi(catch_unwind, js_name = "flushMemWal")]
pub async fn flush_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.flush_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_lsm_writers().await.default_error()
#[napi(catch_unwind, js_name = "compactMemWal")]
pub async fn compact_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.compact_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn flush_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.flush_lsm().await.default_error()
#[napi(catch_unwind, js_name = "checkpointMemWal")]
pub async fn checkpoint_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn compact_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.compact_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_stats(
#[napi(catch_unwind, js_name = "getMemWalStats")]
pub async fn get_memwal_stats(
&self,
include_sstable_rows: bool,
) -> napi::Result<Option<LsmStats>> {
) -> napi::Result<Option<MemWalStats>> {
let stats = self
.inner_ref()?
.get_lsm_stats(include_sstable_rows)
.get_memwal_stats(include_sstable_rows)
.await
.default_error()?;
Ok(stats.map(LsmStats::from))
Ok(stats.map(MemWalStats::from))
}
#[napi(catch_unwind)]
@@ -958,7 +954,7 @@ impl From<LanceDbFtsToken> for FtsToken {
/// `column` is required.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct LsmWriteSpec {
pub struct MemWalWriteSpec {
/// One of `"bucket"`, `"identity"`, or `"unsharded"`.
pub spec_type: String,
/// Bucket and identity variants: the sharding column.
@@ -972,31 +968,31 @@ pub struct LsmWriteSpec {
pub writer_config_defaults: Option<HashMap<String, String>>,
}
impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
type Error = napi::Error;
fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
fn try_from(value: MemWalWriteSpec) -> napi::Result<Self> {
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec bucket requires `column`")
napi::Error::from_reason("MemWalWriteSpec bucket requires `column`")
})?;
let num_buckets = value.num_buckets.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec bucket requires `numBuckets`")
napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`")
})?;
Self::bucket(column, num_buckets)
}
"identity" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec identity requires `column`")
napi::Error::from_reason("MemWalWriteSpec identity requires `column`")
})?;
Self::identity(column)
}
"unsharded" => Self::unsharded(),
other => {
return Err(napi::Error::from_reason(format!(
"LsmWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
"MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
other
)));
}
@@ -1007,9 +1003,9 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
use lancedb::table::LsmWriteSpec as Native;
impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
fn from(spec: lancedb::table::MemWalWriteSpec) -> Self {
use lancedb::table::MemWalWriteSpec as Native;
match spec {
Native::Bucket {
column,
@@ -1152,19 +1148,19 @@ impl From<lancedb::table::TableShardStats> for TableShardStats {
}
}
/// Live per-table-shard LSM state, as returned by `Table#getLsmStats`.
/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
///
/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
/// the caller's to compute.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct LsmStats {
pub struct MemWalStats {
/// One entry per table shard backing this table.
pub table_shards: Vec<TableShardStats>,
}
impl From<lancedb::table::LsmStats> for LsmStats {
fn from(stats: lancedb::table::LsmStats) -> Self {
impl From<lancedb::table::MemWalStats> for MemWalStats {
fn from(stats: lancedb::table::MemWalStats) -> Self {
Self {
table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
}