rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-19 02:02:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'merge-insert-source-stats' into merge-insert-source-stats-bindings

Browse Source
This commit is contained in:
Will Jones
2026-10-16 21:18:09 +00:00
parent 2c2bb9a974 e7d060dca8
commit 3c78cb4980
312 changed files with 21498 additions and 7261 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.41.0-beta.3"
version = "0.41.0-beta.7"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/catalog.test.ts
+36 -3
View File
@@ -266,8 +266,41 @@ describe("remote catalog", () => {
"limit",
);
}
await expect(catalog.connectDatabase("a$b")).rejects.toThrow(
"Invalid database name",
);
});
it.each([
"",
".",
"..",
"/db",
"db/",
"a//b",
"a/./b",
"a/../b",
"a b",
"a:b",
"a$b",
"a%b",
"a?b",
"a#b",
"a\\b",
"café",
"a\nb",
])(
"rejects invalid database name %j before sending requests",
async (name) => {
await withCatalog([], async (catalog, requests) => {
for (const method of [
"createDatabase",
"connectDatabase",
"dropDatabase",
] as const) {
await expect(catalog[method](name)).rejects.toThrow(
"Invalid database name",
);
}
expect(requests).toEqual([]);
});
},
);
});
nodejs/__test__/query.test.ts
+103
View File
@@ -527,3 +527,106 @@ describe("Query orderBy", () => {
]);
});
});
describe("IVF search effort", () => {
let directory: tmp.DirResult;
let table: Table;
const vector = Array.from({ length: 16 }, (_, i) => (i === 0 ? 1 : 0));
beforeEach(async () => {
directory = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(directory.name);
const rows = Array.from({ length: 512 }, (_, id) => ({
id,
vector: Array.from(
{ length: 16 },
(_, j) =>
(Math.floor(id / 32) === j ? 1 : 0) + Math.sin(id * 17 + j) * 0.001,
),
}));
table = await db.createTable("effort", rows.slice(0, 256));
await table.add(rows.slice(256));
await table.createIndex("vector", {
config: Index.ivfFlat({ numPartitions: 16 }),
});
});
afterEach(() => {
directory.removeCallback();
});
it.each([
[null, 6],
[0, 1],
[0.25, 3],
[0.5, 6],
[0.75, 10],
[1, 16],
])(
"preserves effort %s through execution and probe precedence",
async (effort, expected) => {
const partitions = async (query: ReturnType<Table["vectorSearch"]>) => {
const plan = await query.limit(8).analyzePlan();
const matches = [...plan.matchAll(/partitions_searched=(\d+)/g)];
expect(matches.length).toBeGreaterThan(0);
return matches.reduce((total, match) => total + Number(match[1]), 0);
};
expect(
await partitions(table.vectorSearch(vector).searchEffort(effort)),
).toBe(expected);
expect(
await partitions(
table.vectorSearch(vector).nprobes(1).searchEffort(effort),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).nprobes(1),
),
).toBe(effort === null ? 1 : expected);
expect(
await partitions(
table
.vectorSearch(vector)
.searchEffort(effort)
.nprobes(1)
.searchEffort(null),
),
).toBe(1);
expect(
await partitions(
table.vectorSearch(vector).searchEffort(effort).maximumNprobes(2),
),
).toBeLessThanOrEqual(2);
const actual = await table
.vectorSearch(vector)
.searchEffort(effort)
.limit(8)
.toArray();
const exact = await table
.vectorSearch(vector)
.bypassVectorIndex()
.limit(8)
.toArray();
const ids = new Set(exact.map((row) => row.id));
expect(
actual.filter((row) => ids.has(row.id)).length / 8,
).toBeGreaterThanOrEqual(0.5);
},
);
it.each([
-0.1,
1.1,
Number.NaN,
Number.POSITIVE_INFINITY,
Number.NEGATIVE_INFINITY,
])("rejects invalid effort %s before execution", (effort) => {
expect(() => table.vectorSearch(vector).searchEffort(effort)).toThrow(
"search_effort must be finite and in [0, 1]",
);
expect(() =>
table.vectorSearch(vector).fullTextSearch("dog").searchEffort(effort),
).toThrow("search_effort must be finite and in [0, 1]");
});
});
nodejs/__test__/remote.test.ts
+9 -4
View File
@@ -125,10 +125,6 @@ describe("remote connection", () => {
const cases: [() => Promise<unknown>, string][] = [
[() => table.usesV2ManifestPaths(), "uses_v2_manifest_paths"],
[() => table.migrateManifestPathsV2(), "migrate_manifest_paths_v2"],
[
() => table.setUnenforcedPrimaryKey("id"),
"set_unenforced_primary_key",
],
[() => table.optimize(), "optimize"],
[
() => table.createIndex("vector", { config: Index.hnswPq() }),
@@ -480,6 +476,15 @@ describe("remote connection", () => {
);
});
it("rejects a zero blob request concurrency limit", async () => {
await expect(
connect("db://dev", {
apiKey: "fake",
clientConfig: { blobRequestConcurrency: 0 },
}),
).rejects.toThrow("blob_request_concurrency");
});
it("shows the full error messages on retry errors", async () => {
await withMockDatabase(
(_req, res) => {
nodejs/__test__/table.test.ts
+257 -50
View File
@@ -10,6 +10,7 @@ import * as tmp from "tmp";
import {
AutoQuery,
type BlobInput,
type BlobMode,
Connection,
MatchQuery,
PhraseQuery,
@@ -18,6 +19,7 @@ import {
VectorQuery,
blob,
connect,
isBlobField,
tokenize,
} from "../lancedb";
import {
@@ -640,11 +642,11 @@ describe.each(arrowVersions)(
);
});
it("should expose useLsm on takeRowIds as the base-only escape hatch", async () => {
it("should expose useMemWal on takeRowIds as the base-only escape hatch", async () => {
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
// useLsm(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// useMemWal(false) is reachable on TakeQuery (the escape hatch for MemWAL tables,
// where take-by-row-id auto-routes to the LSM scanner and is rejected).
const res = await table.takeRowIds([0, 2]).useLsm(false).toArray();
const res = await table.takeRowIds([0, 2]).useMemWal(false).toArray();
expect(res.map((r) => r.id)).toEqual([1, 3]);
});
@@ -2827,6 +2829,18 @@ describe("when dealing with blob columns", () => {
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("rejects a whole blob read batch when one row ID was deleted", async () => {
const { table, rowIds } = await openBlobTable();
await table.delete("id = 2");
await expect(table.fetchBlobs("image", rowIds)).rejects.toThrow(
`first missing row ids: [${rowIds[1]}]`,
);
await expect(table.fetchBlobFiles("image", rowIds)).rejects.toThrow(
`first missing row ids: [${rowIds[1]}]`,
);
});
it("adds a LargeBinary Arrow column to a blob table", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
@@ -3014,6 +3028,161 @@ describe("when dealing with blob columns", () => {
expect(await handle!.readRange(0n, 4n)).toEqual(payload.subarray(0, 4));
});
it("toArrow returns blob bytes with blobMode bytes", async () => {
const { table, alpha, beta } = await openBlobTable();
const result = await table.query().toArrow({ blobMode: "bytes" });
const field = result.schema.fields.find((f) => f.name === "image")!;
expect(arrow.DataType.isLargeBinary(field.type)).toBe(true);
expect(isBlobField(field)).toBe(false);
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
const byId = new Map(
result.toArray().map((row) => [Number(row.id), row.image]),
);
expect(Buffer.from(byId.get(1)!)).toEqual(alpha);
expect(Buffer.from(byId.get(2)!)).toEqual(beta);
expect(byId.get(3)).toBeNull();
});
it("toArray returns blob bytes and keeps _rowid only when asked", async () => {
const { table, rowIds, alpha } = await openBlobTable();
const [plain] = await table
.query()
.where("id = 1")
.select(["id", "image"])
.toArray({ blobMode: "bytes" });
expect(Object.keys(plain)).toEqual(["id", "image"]);
expect(Buffer.from(plain.image)).toEqual(alpha);
const [withRowId] = await table
.query()
.where("id = 1")
.withRowId()
.toArray({ blobMode: "bytes" });
expect(withRowId._rowid).toBe(rowIds[0]);
expect(Buffer.from(withRowId.image)).toEqual(alpha);
});
it("fills a renamed blob column with bytes", async () => {
const { table, beta } = await openBlobTable();
const [row] = await table
.query()
.where("id = 2")
.select(
new Map([
["key", "id"],
["picture", "image"],
]),
)
.toArray({ blobMode: "bytes" });
expect(Number(row.key)).toBe(2);
expect(Buffer.from(row.picture)).toEqual(beta);
});
it("returns blob bytes from a take query", async () => {
const { table, rowIds, alpha, beta } = await openBlobTable();
const rows = await table
.takeRowIds([rowIds[1], rowIds[0]])
.toArray({ blobMode: "bytes" });
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(Buffer.from(byId.get(1)!.image)).toEqual(alpha);
expect(Buffer.from(byId.get(2)!.image)).toEqual(beta);
expect(Object.keys(rows[0])).not.toContain("_rowid");
});
it("does not change later executions of the same query", async () => {
const { table } = await openBlobTable();
const query = table.query().where("id = 1");
await query.toArray({ blobMode: "bytes" });
const [row] = await query.toArray();
expect(Object.keys(row)).not.toContain("_rowid");
expect(Number(row.image.size)).toBe(5);
});
it("skips row ids when no blob column is selected", async () => {
const { table } = await openBlobTable();
const result = await table
.query()
.select(["id"])
.toArrow({ blobMode: "bytes" });
expect(result.schema.fields.map((f) => f.name)).toEqual(["id"]);
expect(result.numRows).toBe(3);
});
it("returns an empty bytes column for an empty result", async () => {
const { table } = await openBlobTable();
const result = await table
.query()
.where("id > 100")
.toArrow({ blobMode: "bytes" });
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
const field = result.schema.fields.find((f) => f.name === "image")!;
expect(arrow.DataType.isLargeBinary(field.type)).toBe(true);
expect(isBlobField(field)).toBe(false);
});
it("keeps the descriptor schema for an empty result", async () => {
const { table } = await openBlobTable();
const result = await table.query().where("id > 100").toArrow();
expect(result.numRows).toBe(0);
expect(result.schema.fields.map((f) => f.name)).toEqual(["id", "image"]);
const field = result.schema.fields.find((f) => f.name === "image")!;
expect(arrow.DataType.isStruct(field.type)).toBe(true);
});
it("rejects an unknown blobMode", async () => {
const { table } = await openBlobTable();
await expect(
table.query().toArray({ blobMode: "lazy" as BlobMode }),
).rejects.toThrow(/blobMode must be/);
});
it("reruns the query when the table changes before the version check", async () => {
const { table, native, update } = await openVersionedBlobTable();
const liveVersion = native.version.bind(native);
native.version = async () => {
native.version = liveVersion;
await update(2n, "new");
return liveVersion();
};
const [row] = await table.query().toArray({ blobMode: "bytes" });
expect(row.generation).toBe(2n);
expect(Buffer.from(row.image).toString()).toBe("new");
});
it("reads bytes from the query's version when a write lands before the fetch", async () => {
const { table, native, update } = await openVersionedBlobTable();
const liveVersion = native.version.bind(native);
native.version = async () => {
native.version = liveVersion;
const version = await liveVersion();
await update(2n, "new");
return version;
};
const [row] = await table.query().toArray({ blobMode: "bytes" });
expect(row.generation).toBe(1n);
expect(Buffer.from(row.image).toString()).toBe("old");
});
it("fails when the table changes during every attempt", async () => {
const { table, native, update } = await openVersionedBlobTable();
const liveVersion = native.version.bind(native);
let generation = 1n;
native.version = async () => {
generation += 1n;
await update(generation, `generation ${generation}`);
return liveVersion();
};
try {
await expect(
table.query().toArray({ blobMode: "bytes" }),
).rejects.toThrow(/changed during each of 3 attempts/);
expect(generation).toBe(4n);
} finally {
native.version = liveVersion;
}
});
it("rejects fetchBlobs on a non-blob column", async () => {
const { table, rowIds } = await openBlobTable();
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
@@ -3039,7 +3208,7 @@ describe("when dealing with blob columns", () => {
expect(bytes[0]!.equals(payload)).toBe(true);
});
it("creates and adds list blob columns", async () => {
it("creates list blobs without advertising unfetchable elements", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), true),
@@ -3058,7 +3227,16 @@ describe("when dealing with blob columns", () => {
{ id: 3n, images: [gamma, null] },
{ id: 4n, images: [] },
]);
expect(await table.blobColumns()).toEqual(["images.image"]);
expect(await table.blobColumns()).toEqual([]);
const [first] = await table.query().withRowId().toArray();
for (const column of ["images", "images.image"]) {
await expect(
table.fetchBlobs(column, [first._rowid as bigint]),
).rejects.toThrow(/blobs inside lists cannot be fetched/);
await expect(
table.fetchBlobFiles(column, [first._rowid as bigint]),
).rejects.toThrow(/blobs inside lists cannot be fetched/);
}
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(descriptorSizes(byId.get(1)!.images)).toEqual([
@@ -3104,6 +3282,7 @@ describe("when dealing with blob columns", () => {
],
},
]);
expect(await table.blobColumns()).toEqual([]);
const rows = await table.query().toArray();
const byId = new Map(rows.map((row) => [Number(row.id), row]));
expect(
@@ -3186,6 +3365,32 @@ describe("when dealing with blob columns", () => {
).map((value) => (value == null ? null : Number(value.size)));
}
async function openVersionedBlobTable() {
const db = await connect(tmpDir.name, {
storageOptions: { newTableEnableStableRowIds: "true" },
});
const schema = new Schema([
new Field("id", new Int64(), false),
new Field("generation", new Int64(), true),
blob("image"),
]);
const table = await db.createTable(
"versions",
[{ id: 1n, generation: 1n, image: Buffer.from("old") }],
{ schema },
);
const native = (
table as unknown as { inner: { version: () => Promise<number> } }
).inner;
const update = async (generation: bigint, image: string) => {
await table
.mergeInsert("id")
.whenMatchedUpdateAll()
.execute([{ id: 1n, generation, image: Buffer.from(image) }]);
};
return { table, native, update };
}
async function openBlobTable() {
const db = await connect(tmpDir.name);
const schema = new Schema([
@@ -4616,7 +4821,7 @@ describe("setUnenforcedPrimaryKey", () => {
});
});
describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
describe("enableMemWal / disableMemWal", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
@@ -4636,16 +4841,16 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 4,
});
await table.unsetLsmWriteSpec();
await table.disableMemWal();
// A second unset errors — there is no spec left to remove.
await expect(table.unsetLsmWriteSpec()).rejects.toThrow();
await expect(table.disableMemWal()).rejects.toThrow();
// A fresh spec can be installed after unset.
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 8,
@@ -4657,8 +4862,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "unsharded" });
await table.unsetLsmWriteSpec();
await table.enableMemWal({ specType: "unsharded" });
await table.disableMemWal();
});
it("installs an identity spec", async () => {
@@ -4666,8 +4871,8 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
await table.unsetLsmWriteSpec();
await table.enableMemWal({ specType: "identity", column: "id" });
await table.disableMemWal();
});
it("rejects an invalid spec", async () => {
@@ -4677,7 +4882,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
await table.setUnenforcedPrimaryKey("id");
// num_buckets out of range.
await expect(
table.setLsmWriteSpec({
table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 0,
@@ -4685,7 +4890,7 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
).rejects.toThrow();
// Column mismatch.
await expect(
table.setLsmWriteSpec({
table.enableMemWal({
specType: "bucket",
column: "missing",
numBuckets: 4,
@@ -4693,13 +4898,13 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
).rejects.toThrow();
});
it("reads back the installed spec via getLsmWriteSpec", async () => {
it("reads back the installed spec via getMemWalSpec", async () => {
const conn = await connect(tmpDir.name);
const table = await makeTable(conn);
await table.setUnenforcedPrimaryKey("id");
// Nothing installed yet.
expect(await table.getLsmWriteSpec()).toBeUndefined();
expect(await table.getMemWalSpec()).toBeUndefined();
// A real scalar index is needed to name it as a maintained index.
await table.add([{ id: 1 }, { id: 2 }, { id: 3 }]);
@@ -4710,14 +4915,14 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
// defaults. Lance writer-config keys are canonically snake_case.
// biome-ignore lint/style/useNamingConvention: Lance writer-config keys are snake_case
const writerConfigDefaults = { durable_write: "false" };
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 4,
maintainedIndexes: [indexName],
writerConfigDefaults,
});
const spec = await table.getLsmWriteSpec();
const spec = await table.getMemWalSpec();
expect(spec).toBeDefined();
expect(spec?.specType).toBe("bucket");
expect(spec?.column).toBe("id");
@@ -4726,19 +4931,19 @@ describe("setLsmWriteSpec / unsetLsmWriteSpec", () => {
expect(spec?.writerConfigDefaults).toEqual(writerConfigDefaults);
// After unset, undefined again.
await table.unsetLsmWriteSpec();
expect(await table.getLsmWriteSpec()).toBeUndefined();
await table.disableMemWal();
expect(await table.getMemWalSpec()).toBeUndefined();
// Identity round-trips (column recovered from the schema).
await table.setLsmWriteSpec({ specType: "identity", column: "id" });
const identity = await table.getLsmWriteSpec();
await table.enableMemWal({ specType: "identity", column: "id" });
const identity = await table.getMemWalSpec();
expect(identity?.specType).toBe("identity");
expect(identity?.column).toBe("id");
await table.unsetLsmWriteSpec();
await table.disableMemWal();
// Unsharded round-trips (no routing column).
await table.setLsmWriteSpec({ specType: "unsharded" });
const unsharded = await table.getLsmWriteSpec();
await table.enableMemWal({ specType: "unsharded" });
const unsharded = await table.getMemWalSpec();
expect(unsharded?.specType).toBe("unsharded");
expect(unsharded?.column).toBeFalsy();
});
@@ -4767,7 +4972,7 @@ describe("LSM merge insert", () => {
]);
await table.setUnenforcedPrimaryKey("id");
// numBuckets = 1: every row routes to the single bucket.
await table.setLsmWriteSpec({
await table.enableMemWal({
specType: "bucket",
column: "id",
numBuckets: 1,
@@ -4792,17 +4997,17 @@ describe("LSM merge insert", () => {
expect(res.version).toBe(0);
expect(res.numInsertedRows).toBe(0);
await table.closeLsmWriters();
await table.closeMemWalWriters();
});
it("falls back to the standard path with useLsm(false)", async () => {
it("falls back to the standard path with useMemWal(false)", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
const res = await table
.mergeInsert("id")
.whenNotMatchedInsertAll()
.useLsm(false)
.useMemWal(false)
.execute([
{ id: "b", value: 9 },
{ id: "e", value: 5 },
@@ -4825,7 +5030,7 @@ describe("LSM merge insert", () => {
expect(res.numRows).toBe(1);
});
it("rejects a non-upsert merge under an LSM spec", async () => {
it("rejects a non-upsert merge under a MemWAL spec", async () => {
const conn = await connect(tmpDir.name);
const table = await bucketTable(conn);
@@ -4851,22 +5056,24 @@ describe("LSM merge insert", () => {
const lsm = await table.query().toArray();
expect(lsm.map((r) => r.id).sort()).toEqual(["a", "b", "c"]);
// useLsm(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useLsm(false).toArray();
// useMemWal(false) bypasses the MemWAL and reads the base table only.
const baseOnly = await table.query().useMemWal(false).toArray();
expect(baseOnly.map((r) => r.id).sort()).toEqual(["a", "b"]);
});
it("reads the base table when no LSM spec is installed", async () => {
it("reads the base table when no MemWAL spec is installed", async () => {
const conn = await connect(tmpDir.name);
const table = await conn.createEmptyTable(
"plain",
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
// No spec: default read and useLsm(false) both succeed against the base table.
// No spec: default read and useMemWal(false) both succeed against the base table.
await expect(table.query().toArray()).resolves.toBeDefined();
await expect(table.query().useLsm(false).toArray()).resolves.toBeDefined();
// useLsm(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
await expect(
table.query().useMemWal(false).toArray(),
).resolves.toBeDefined();
// useMemWal(true) demands MemWAL routing; without a spec it errors.
await expect(table.query().useMemWal(true).toArray()).rejects.toThrow();
});
});
@@ -4884,7 +5091,7 @@ describe("LSM convergence and stats", () => {
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "unsharded" });
await table.enableMemWal({ specType: "unsharded" });
return table;
}
@@ -4892,34 +5099,34 @@ describe("LSM convergence and stats", () => {
// rejects them rather than answering. What is asserted here is that the
// bindings reach the core at all; the behavior against a real endpoint is
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
it("rejects flushLsm on a local table", async () => {
it("rejects flushMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
await expect(table.flushMemWal()).rejects.toThrow(/not supported/i);
});
it("rejects compactLsm on a local table", async () => {
it("rejects compactMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
await expect(table.compactMemWal()).rejects.toThrow(/not supported/i);
});
it("rejects getLsmStats on a local table", async () => {
it("rejects getMemWalStats on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
await expect(table.getMemWalStats()).rejects.toThrow(/not supported/i);
await expect(table.getMemWalStats(true)).rejects.toThrow(/not supported/i);
});
it("rejects checkpointLsm on a local table", async () => {
it("rejects checkpointMemWal on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
// checkpointLsm seals first, so it surfaces flushLsm's rejection.
await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
// checkpointMemWal freezes first, so it surfaces flushMemWal's rejection.
await expect(table.checkpointMemWal()).rejects.toThrow(/not supported/i);
});
});
nodejs/lancedb/blob.ts
+2
View File
@@ -81,6 +81,8 @@ export type BlobOptions = {
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
* Blobs inside lists can be written and queried as descriptors, but cannot
* currently be fetched because a row ID does not identify a list element.
*
* @example
* ```ts
nodejs/lancedb/index.ts
+5 -4
View File
@@ -122,6 +122,7 @@ export {
VectorQuery,
TakeQuery,
AnalyzePlanDistributedMetrics,
BlobMode,
QueryExecutionOptions,
ColumnOrdering,
FullTextSearchOptions,
@@ -167,10 +168,10 @@ export {
WriteProgress,
FtsToken,
TokenizeTableOptions,
LsmWriteSpec,
LsmStats,
BucketStats,
GenerationStats,
MemWalWriteSpec,
MemWalStats,
TableShardStats,
SsTableStats,
MemtableStats,
ColumnAlteration,
FieldMetadataUpdate,
nodejs/lancedb/merge.ts
+6 -6
View File
@@ -90,25 +90,25 @@ export class MergeInsertBuilder {
/**
* Control MemWAL routing for this merge.
*
* By default (unset), a `mergeInsert` on a table with an LSM write spec is
* By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
* routed through Lance's MemWAL shard writer, and a table without one uses the
* standard path.
*
* @param enable - `true` forces MemWAL routing and errors if the table has no
* LSM write spec. `false` forces the standard write path even when a spec is set.
* MemWAL write spec. `false` forces the standard write path even when a spec is set.
*/
useLsm(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
useMemWal(enable: boolean): MergeInsertBuilder {
return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema);
}
/**
* Controls how an LSM merge checks that its input targets a single shard.
*
* When a table has an LSM write spec, every row in a `mergeInsert` call must
* When a table has a MemWAL write spec, every row in a `mergeInsert` call must
* route to the same shard. When `true` (the default), every row is inspected
* to verify this. When `false`, only the first row is inspected and the
* shard it routes to is used for the whole input — a faster path for callers
* that have already pre-sharded their input. Has no effect on tables without
* an LSM write spec.
* a MemWAL write spec.
*
* @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
*/
nodejs/lancedb/query.ts
+266 -24
View File
@@ -3,12 +3,18 @@
import {
Table as ArrowTable,
Field,
type IntoVector,
LargeBinary,
RecordBatch,
Schema,
Struct,
extractVectorBuffer,
fromBufferToRecordBatch,
fromRecordBatchToBuffer,
makeData,
tableFromIPC,
vectorFromArray,
} from "./arrow";
import { type IvfPqOptions } from "./indices";
import {
@@ -41,6 +47,112 @@ export async function* RecordBatchIterator(
}
}
/**
* How a query returns blob v2 columns. See {@link QueryExecutionOptions.blobMode}.
*/
export type BlobMode = "descriptions" | "bytes";
/**
* The table a query reads, and the parts of the query that decide which blob
* columns `blobMode: "bytes"` fills in.
*
* @hidden
*/
export interface QueryBlobContext {
table: NativeTable;
withRowId: boolean;
projection?: [string, string][];
}
const BLOB_BYTES_ATTEMPTS = 3;
function blobSources(
blobColumns: string[],
projection: [string, string][] | undefined,
): Map<string, string> {
const columns = new Set(blobColumns);
const pairs =
projection ??
blobColumns.map((column): [string, string] => [column, column]);
return new Map(pairs.filter(([, source]) => columns.has(source)));
}
async function readBlobBytes(
table: ArrowTable,
sources: Map<string, string>,
snapshot: NativeTable,
dropRowId: boolean,
): Promise<ArrowTable> {
const rowIdColumn = table.getChild("_rowid");
if (table.numRows > 0 && rowIdColumn === null) {
throw new Error('blobMode "bytes" needs _rowid in the query result');
}
const rowIds =
rowIdColumn === null
? []
: Array.from(rowIdColumn.toArray() as BigUint64Array);
const replacements = new Map<number, (Buffer | null)[]>();
for (const [output, source] of sources) {
const index = table.schema.fields.findIndex(
(field) => field.name === output,
);
if (index < 0) {
continue;
}
const values =
rowIds.length === 0 ? [] : await snapshot.fetchBlobs(source, rowIds);
replacements.set(
index,
values.map((value) => value ?? null),
);
}
const result = withBinaryColumns(table, replacements);
if (!dropRowId) {
return result;
}
return result.select(
result.schema.fields
.map((field) => field.name)
.filter((name) => name !== "_rowid"),
);
}
// A fresh field, not a clone: the descriptor field's metadata must not follow
// the bytes.
function withBinaryColumns(
table: ArrowTable,
replacements: Map<number, (Buffer | null)[]>,
): ArrowTable {
if (replacements.size === 0) {
return table;
}
const fields = table.schema.fields.map((field, index) =>
replacements.has(index)
? new Field(field.name, new LargeBinary(), true)
: field,
);
const schema = new Schema(fields, table.schema.metadata);
let offset = 0;
const batches = table.batches.map((batch) => {
const children = batch.data.children.map((child, index) => {
const values = replacements.get(index);
if (values === undefined) {
return child;
}
return vectorFromArray(
values.slice(offset, offset + batch.numRows),
new LargeBinary(),
).data[0];
});
offset += batch.numRows;
return new RecordBatch(
schema,
makeData({ type: new Struct(fields), length: batch.numRows, children }),
);
});
return new ArrowTable(schema, batches);
}
/**
* Options that control the behavior of a particular query execution
*/
@@ -57,6 +169,32 @@ export interface QueryExecutionOptions {
* Timeout for query execution in milliseconds
*/
timeoutMs?: number;
/**
* How to return blob v2 columns. Applies to `toArrow()` and `toArray()`.
*
* - `"descriptions"` (default): each blob is a descriptor. Read the bytes
* with {@link Table.fetchBlobs} or {@link Table.fetchBlobFiles}.
* - `"bytes"`: each top-level blob column holds the blob bytes as
* `LargeBinary`. The query also reads `_rowid` to fetch the bytes, and
* leaves it out of the result unless {@link QueryBase.withRowId} was
* called. All the bytes are held in memory.
*
* Blobs nested in a struct or a list keep their descriptors.
*
* The rows and the bytes come from the same table version. If the table
* changes while the query runs, the query runs again, up to three times.
*
* @example
* ```ts
* const rows = await table
* .query()
* .select(["id", "image"])
* .toArray({ blobMode: "bytes" });
* const image: Uint8Array | null = rows[0].image;
* ```
*/
blobMode?: BlobMode;
}
export type AnalyzePlanDistributedMetrics = "aggregate" | "per_worker" | "full";
@@ -119,10 +257,19 @@ export class QueryBase<
/**
* @hidden
*/
protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
protected blobContext?: QueryBlobContext;
/**
* @hidden
*/
protected constructor(
inner?: NativeQueryType | Promise<NativeQueryType>,
blobContext?: QueryBlobContext,
) {
if (inner !== undefined) {
this.inner = inner;
}
this.blobContext = blobContext;
}
// call a function on the inner (either a promise or the actual object)
@@ -193,14 +340,22 @@ export class QueryBase<
});
};
let projection: [string, string][];
if (typeof columns === "string") {
selectColumns([columns]);
projection = [[columns, columns]];
} else if (Array.isArray(columns)) {
selectColumns(columns);
projection = columns.map((column) => [column, column]);
} else if (columns instanceof Map) {
selectMapping(Array.from(columns.entries()));
projection = Array.from(columns.entries());
selectMapping(projection);
} else {
selectMapping(Object.entries(columns));
projection = Object.entries(columns);
selectMapping(projection);
}
if (this.blobContext !== undefined) {
this.blobContext = { ...this.blobContext, projection };
}
return this;
@@ -215,6 +370,9 @@ export class QueryBase<
*/
withRowId(): this {
this.doCall((inner: NativeQueryType) => inner.withRowId());
if (this.blobContext !== undefined) {
this.blobContext = { ...this.blobContext, withRowId: true };
}
return this;
}
@@ -223,9 +381,14 @@ export class QueryBase<
*/
protected async nativeExecute(
options?: Partial<QueryExecutionOptions>,
withRowId = false,
): Promise<NativeBatchIterator> {
const inner = await this.getInner();
return inner.execute(options?.maxBatchLength, options?.timeoutMs);
return inner.execute(
options?.maxBatchLength,
options?.timeoutMs,
withRowId,
);
}
/**
@@ -257,8 +420,41 @@ export class QueryBase<
* Empty results retain the query's output schema.
*/
async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
const context = this.blobBytesContext(options?.blobMode);
if (context === undefined) {
return this.collect(options, false);
}
for (let attempt = 1; ; attempt++) {
// The snapshot pins the bytes. The query runs on the live table, so an
// unchanged version afterwards proves it read the snapshot's version.
const snapshot = await context.table.querySnapshot();
const version = await snapshot.version();
const sources = blobSources(
await snapshot.blobColumns(),
context.projection,
);
if (sources.size === 0) {
return this.collect(options, false);
}
const addRowId = !context.withRowId;
const table = await this.collect(options, addRowId);
if ((await context.table.version()) === version) {
return readBlobBytes(table, sources, snapshot, addRowId);
}
if (attempt === BLOB_BYTES_ATTEMPTS) {
throw new Error(
`blobMode "bytes": the table changed during each of ${BLOB_BYTES_ATTEMPTS} attempts to read it`,
);
}
}
}
private async collect(
options: Partial<QueryExecutionOptions> | undefined,
withRowId: boolean,
): Promise<ArrowTable> {
const batches = [];
const iterator = this.nativeExecute(options);
const iterator = this.nativeExecute(options, withRowId);
for await (const batch of RecordBatchIterator(iterator)) {
batches.push(batch);
}
@@ -268,6 +464,23 @@ export class QueryBase<
return new ArrowTable(batches);
}
private blobBytesContext(
mode: BlobMode | undefined,
): QueryBlobContext | undefined {
if (mode === undefined || mode === "descriptions") {
return undefined;
}
if (mode !== "bytes") {
throw new Error(
`blobMode must be "descriptions" or "bytes", got "${String(mode)}"`,
);
}
if (this.blobContext === undefined) {
throw new Error('blobMode "bytes" is not supported for this query');
}
return this.blobContext;
}
/** Collect the results as an array of objects. */
// biome-ignore lint/suspicious/noExplicitAny: arrow.toArrow() returns any[]
async toArray(options?: Partial<QueryExecutionOptions>): Promise<any[]> {
@@ -355,8 +568,11 @@ export class StandardQueryBase<
extends QueryBase<NativeQueryType>
implements ExecutableQuery
{
constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
super(inner);
constructor(
inner?: NativeQueryType | Promise<NativeQueryType>,
blobContext?: QueryBlobContext,
) {
super(inner, blobContext);
}
/**
@@ -464,10 +680,10 @@ export class StandardQueryBase<
* Control MemWAL read routing for this query.
*
* By default (unset), when the table carries a MemWAL write spec (see
* {@link Table#setLsmWriteSpec}), reads are routed through the LSM scanner so
* {@link Table#enableMemWal}), reads are routed through the LSM scanner so
* they also return data written via the `mergeInsert` LSM path that has not yet
* been compacted into the base table (the active/frozen in-memory memtables and
* the flushed generations), deduplicated by primary key; a table without a spec
* the SSTables), deduplicated by primary key; a table without a spec
* reads the base table.
*
* @param enable - `true` forces the LSM scanner and errors if the table has no
@@ -476,11 +692,11 @@ export class StandardQueryBase<
*
* Note: the LSM scanner does not support every query shape (e.g. reranking,
* hybrid search, `orderBy`). On a MemWAL table those shapes error unless
* `useLsm(false)` is set, because a base-only read would silently exclude
* `useMemWal(false)` is set, because a base-only read would silently exclude
* un-compacted MemWAL data.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useLsm(enable));
useMemWal(enable: boolean): this {
this.doCall((inner: NativeQueryType) => inner.useMemWal(enable));
return this;
}
}
@@ -505,8 +721,11 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
/**
* @hidden
*/
constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>) {
super(inner);
constructor(
inner: NativeVectorQuery | Promise<NativeVectorQuery>,
blobContext?: QueryBlobContext,
) {
super(inner, blobContext);
}
/**
@@ -549,6 +768,27 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
return this;
}
/**
* Set the initial IVF search effort to a finite value in [0, 1].
*
* The default, null, uses Auto (internally 0.5). Zero starts at the caller
* minimum; one searches all partitions within the caller maximum. Later
* probing may continue when too few candidates are found.
* Any explicit effort, including 0.5, overrides nprobes in either call order.
* Explicit minimum and maximum bounds still apply. This is not a recall
* guarantee and does not change refinement or HNSW ef. The MemWAL LSM
* scanner does not yet support this option.
*
* @example
* ```ts
* table.vectorSearch([0.1, 0.2]).searchEffort(0.75).limit(10);
* ```
*/
searchEffort(searchEffort: number | null): VectorQuery {
this.doVectorCall((inner) => inner.searchEffort(searchEffort));
return this;
}
/**
* Set the minimum number of probes used.
*
@@ -739,7 +979,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
addQueryVectorToNative(inner, outcome.value);
return inner;
})();
return new VectorQuery(res);
return new VectorQuery(res, this.blobContext);
} else {
this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
return this;
@@ -828,7 +1068,7 @@ export function createAutoQuery(
return nearestToNative(route.table.query(), vector);
};
return new AutoQuery(createInner);
return new AutoQuery(createInner, { table, withRowId: false });
}
/**
@@ -837,8 +1077,8 @@ export function createAutoQuery(
* @hideconstructor
*/
export class TakeQuery extends QueryBase<NativeTakeQuery> {
constructor(inner: NativeTakeQuery) {
super(inner);
constructor(inner: NativeTakeQuery, blobContext?: QueryBlobContext) {
super(inner, blobContext);
}
/**
@@ -850,8 +1090,8 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
*
* @param enable - `false` reads the base table only.
*/
useLsm(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useLsm(enable));
useMemWal(enable: boolean): this {
this.doCall((inner: NativeTakeQuery) => inner.useMemWal(enable));
return this;
}
}
@@ -877,8 +1117,9 @@ export class AutoQuery extends StandardQueryBase<
private readonly createInner: () => Promise<
NativeQuery | NativeVectorQuery
>,
blobContext?: QueryBlobContext,
) {
super();
super(undefined, blobContext);
}
/** @hidden */
@@ -912,7 +1153,7 @@ export class Query extends StandardQueryBase<NativeQuery> {
* @hidden
*/
constructor(tbl: NativeTable) {
super(tbl.query());
super(tbl.query(), { table: tbl, withRowId: false });
}
/**
@@ -958,14 +1199,15 @@ export class Query extends StandardQueryBase<NativeQuery> {
const nativeQuery = inner.then(async (resolvedInner) =>
nearestToNative(resolvedInner, await vector),
);
return new VectorQuery(nativeQuery);
return new VectorQuery(nativeQuery, this.blobContext);
}
if (vector instanceof Promise) {
return new VectorQuery(
vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
this.blobContext,
);
}
return new VectorQuery(nearestToNative(inner, vector));
return new VectorQuery(nearestToNative(inner, vector), this.blobContext);
}
nearestToText(query: string | FullTextQuery, columns?: string[]): Query {
nodejs/lancedb/table.ts
+76 -63
View File
@@ -35,7 +35,7 @@ import {
FunctionErrorsOptions,
IndexConfig,
IndexStatistics,
LsmStats,
MemWalStats,
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
@@ -59,9 +59,9 @@ import { sanitizeType } from "./sanitize";
import { IntoSql, toSQL } from "./util";
export { IndexConfig } from "./native";
export {
BucketStats,
GenerationStats,
LsmStats,
TableShardStats,
SsTableStats,
MemWalStats,
MemtableStats,
} from "./native";
@@ -217,7 +217,7 @@ export type TokenizeTableOptions =
* `column` value, or upserts of that key can land in different shards and a
* stale version can win).
*/
export interface LsmWriteSpec {
export interface MemWalWriteSpec {
/** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
specType: "bucket" | "identity" | "unsharded";
/** Bucket and identity variants: the sharding column. */
@@ -526,7 +526,8 @@ export abstract class Table {
abstract takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
/**
* Blob v2 columns, including nested dotted paths.
* Blob v2 columns addressable by row ID, including struct-nested dotted paths.
* Blobs inside lists are omitted because a row ID does not identify an element.
*/
abstract blobColumns(): Promise<string[]>;
@@ -536,6 +537,10 @@ export abstract class Table {
* Reads the table's current checkout. IDs from another version can fail after
* compaction unless stable row ids are enabled. Results keep input order and
* duplicates. Null blobs are `null`. Empty blobs are empty buffers.
* One missing or deleted row ID rejects the entire batch, including on remote
* tables; no partial results are returned.
* Blobs inside lists cannot be fetched by row ID.
*
* Remote servers limit each request to 1024 row IDs and 64 MiB of blob bytes.
* The client splits requests automatically and reads an individual larger
* blob through the Range route. This method still materializes all bytes in
@@ -553,6 +558,8 @@ export abstract class Table {
*
* Preserves input order, duplicates, and nulls. Use this for large payloads.
* See {@link Table.fetchBlobs} for row-ID validity across versions.
* One missing or deleted row ID rejects the entire batch, including on remote
* tables; no partial handles are returned.
*/
abstract fetchBlobFiles(
column: string,
@@ -751,10 +758,10 @@ export abstract class Table {
*/
abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
/**
* Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL
* Install an {@link MemWalWriteSpec} on this table, selecting Lance's MemWAL
* LSM-style write path for future `mergeInsert` calls.
*
* `LsmWriteSpec` chooses one of three sharding strategies via `specType`:
* `MemWalWriteSpec` chooses one of three sharding strategies via `specType`:
*
* - `"bucket"` — hash-bucket writes by the single-column unenforced primary
* key (`column` and `numBuckets` required).
@@ -769,12 +776,12 @@ export abstract class Table {
* including ones created later, and skips a kind the MemWAL cannot maintain.
* Naming them pins an exact set, and a still-building index is rejected
* rather than quietly omitted.
* @param {LsmWriteSpec} spec The sharding spec to install.
* @param {MemWalWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
* ```ts
* await table.setUnenforcedPrimaryKey("id");
* await table.setLsmWriteSpec({
* await table.enableMemWal({
* specType: "bucket",
* column: "id",
* numBuckets: 16,
@@ -782,59 +789,59 @@ export abstract class Table {
* });
* ```
*/
abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
abstract enableMemWal(spec: MemWalWriteSpec): Promise<void>;
/**
* Remove the {@link LsmWriteSpec} from this table, reverting to the standard
* Remove the {@link MemWalWriteSpec} from this table, reverting to the standard
* `mergeInsert` write path.
*
* Errors if no spec is currently set.
* @returns {Promise<void>}
*/
abstract unsetLsmWriteSpec(): Promise<void>;
abstract disableMemWal(): Promise<void>;
/**
* Read the {@link LsmWriteSpec} currently installed on this table.
* Read the {@link MemWalWriteSpec} currently installed on this table.
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* Resolves to `undefined` when the MemWAL write path is not enabled (no
* spec has been set, or it was removed with {@link Table#disableMemWal}).
* The spec is the one installed, including its maintained-index selection:
* an absent `maintainedIndexes` for every index the table has, an empty
* array for none.
* @returns {Promise<LsmWriteSpec | undefined>}
* @returns {Promise<MemWalWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
abstract getMemWalSpec(): Promise<MemWalWriteSpec | undefined>;
/**
* Drain and close any cached MemWAL shard writers held for this table.
*
* When an {@link LsmWriteSpec} is installed, `mergeInsert` opens MemWAL
* When an {@link MemWalWriteSpec} is installed, `mergeInsert` opens MemWAL
* shard writers and caches them for reuse across calls. This closes them,
* flushing pending data; writers reopen lazily on the next `mergeInsert`.
* It is a no-op when no writers are cached.
* @returns {Promise<void>}
*/
abstract closeLsmWriters(): Promise<void>;
abstract closeMemWalWriters(): Promise<void>;
/**
* Seal every bucket's active memtable into a new L0 generation.
* Freeze every table shard's active memtable into a new SSTable.
*
* Returns once the seal is committed. Sealing an empty memtable is a no-op,
* Returns once the freeze is committed. Freezing an empty memtable is a no-op,
* so this is safe to call repeatedly.
* @returns {Promise<void>}
*/
abstract flushLsm(): Promise<void>;
abstract flushMemWal(): Promise<void>;
/**
* Trigger a background L0 → base compaction pass per bucket.
* Trigger a background SSTable compaction pass per table shard.
*
* Returns once the passes are *dispatched*, not once they finish — watch
* {@link Table#getLsmStats} for progress, or use
* {@link Table#checkpointLsm} to wait for convergence.
* {@link Table#getMemWalStats} for progress, or use
* {@link Table#checkpointMemWal} to wait for convergence.
* @returns {Promise<void>}
*/
abstract compactLsm(): Promise<void>;
abstract compactMemWal(): Promise<void>;
/**
* Converge this table's LSM write path into its base table.
* Converge this table's MemWAL write path into its base table.
*
* Seals once, then triggers compaction and polls until the L0 that existed
* Freezes once, then triggers compaction and polls until the SSTables that existed
* at the start is gone. The target set is fixed at the start, so
* generations created *during* the checkpoint are ignored — that is what
* SSTables created *during* the checkpoint are ignored — that is what
* lets it terminate under write load, and what makes it best-effort: it
* converges the fresh tier as of some instant. Idempotent, abandonable at
* any point, and safe to run on a cadence.
@@ -845,26 +852,26 @@ export abstract class Table {
* @returns {Promise<void>}
* @example
* ```ts
* const before = await table.getLsmStats();
* await table.checkpointLsm();
* const after = await table.getLsmStats();
* const before = await table.getMemWalStats();
* await table.checkpointMemWal();
* const after = await table.getMemWalStats();
* ```
*/
abstract checkpointLsm(): Promise<void>;
abstract checkpointMemWal(): Promise<void>;
/**
* Read live per-bucket LSM state.
* Read live per-table-shard MemWAL state.
*
* Answers "how far behind is my fresh tier", "which bucket is hot", and
* Answers "how far behind is my fresh tier", "which table shard is hot", and
* "why is my fresh-tier vector search brute-force". Mutates no table state.
*
* Resolves to `undefined` only when the LSM write path is not enabled.
* @param {boolean} includeGenerationRows Also count rows per L0 generation.
* Resolves to `undefined` only when the MemWAL write path is not enabled.
* @param {boolean} includeSstableRows Also count rows per SSTable.
* Off by default because each count opens an uncached Lance dataset.
* @returns {Promise<LsmStats | undefined>}
* @returns {Promise<MemWalStats | undefined>}
*/
abstract getLsmStats(
includeGenerationRows?: boolean,
): Promise<LsmStats | undefined>;
abstract getMemWalStats(
includeSstableRows?: boolean,
): Promise<MemWalStats | undefined>;
/** Retrieve the version of the table */
abstract version(): Promise<number>;
@@ -1233,11 +1240,17 @@ export class LocalTable extends Table {
}
takeOffsets(offsets: number[]): TakeQuery {
return new TakeQuery(this.inner.takeOffsets(offsets));
return new TakeQuery(this.inner.takeOffsets(offsets), {
table: this.inner,
withRowId: false,
});
}
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery {
return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)));
return new TakeQuery(this.inner.takeRowIds(rowIdsToBigInts(rowIds)), {
table: this.inner,
withRowId: false,
});
}
blobColumns(): Promise<string[]> {
@@ -1467,43 +1480,43 @@ export class LocalTable extends Table {
return await this.inner.setUnenforcedPrimaryKey(cols);
}
async setLsmWriteSpec(spec: LsmWriteSpec): Promise<void> {
return await this.inner.setLsmWriteSpec(spec);
async enableMemWal(spec: MemWalWriteSpec): Promise<void> {
return await this.inner.enableMemWal(spec);
}
async unsetLsmWriteSpec(): Promise<void> {
return await this.inner.unsetLsmWriteSpec();
async disableMemWal(): Promise<void> {
return await this.inner.disableMemWal();
}
async getLsmWriteSpec(): Promise<LsmWriteSpec | undefined> {
async getMemWalSpec(): Promise<MemWalWriteSpec | undefined> {
// The native binding types `specType` as a plain `string`; narrow it back
// to the public union. The Rust `From` impl only ever emits one of the
// three valid values, so the cast is safe.
return ((await this.inner.getLsmWriteSpec()) ?? undefined) as
| LsmWriteSpec
return ((await this.inner.getMemWalSpec()) ?? undefined) as
| MemWalWriteSpec
| undefined;
}
async closeLsmWriters(): Promise<void> {
return await this.inner.closeLsmWriters();
async closeMemWalWriters(): Promise<void> {
return await this.inner.closeMemWalWriters();
}
async flushLsm(): Promise<void> {
return await this.inner.flushLsm();
async flushMemWal(): Promise<void> {
return await this.inner.flushMemWal();
}
async compactLsm(): Promise<void> {
return await this.inner.compactLsm();
async compactMemWal(): Promise<void> {
return await this.inner.compactMemWal();
}
async checkpointLsm(): Promise<void> {
return await this.inner.checkpointLsm();
async checkpointMemWal(): Promise<void> {
return await this.inner.checkpointMemWal();
}
async getLsmStats(
includeGenerationRows: boolean = false,
): Promise<LsmStats | undefined> {
return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
async getMemWalStats(
includeSstableRows: boolean = false,
): Promise<MemWalStats | undefined> {
return (await this.inner.getMemWalStats(includeSstableRows)) ?? undefined;
}
async version(): Promise<number> {
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.41.0-beta.3",
"version": "0.41.0-beta.7",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/merge.rs
+3 -3
View File
@@ -51,10 +51,10 @@ impl NativeMergeInsertBuilder {
this
}
#[napi]
pub fn use_lsm(&self, enable: bool) -> Self {
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&self, enable: bool) -> Self {
let mut this = self.clone();
this.inner.use_lsm(enable);
this.inner.use_memwal(enable);
this
}
nodejs/src/query.rs
+40 -15
View File
@@ -168,9 +168,9 @@ impl Query {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi]
@@ -196,6 +196,7 @@ impl Query {
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -204,8 +205,12 @@ impl Query {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
@@ -298,6 +303,16 @@ impl VectorQuery {
Ok(())
}
#[napi]
pub fn search_effort(&mut self, search_effort: Option<f64>) -> napi::Result<()> {
self.inner = self
.inner
.clone()
.search_effort(search_effort)
.default_error()?;
Ok(())
}
#[napi]
pub fn minimum_nprobes(&mut self, minimum_nprobe: u32) -> napi::Result<()> {
self.inner = self
@@ -384,9 +399,9 @@ impl VectorQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi]
@@ -422,6 +437,7 @@ impl VectorQuery {
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -430,8 +446,12 @@ impl VectorQuery {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
@@ -494,9 +514,9 @@ impl TakeQuery {
self.inner = self.inner.clone().with_row_id();
}
#[napi]
pub fn use_lsm(&mut self, enable: bool) {
self.inner = self.inner.clone().use_lsm(enable);
#[napi(js_name = "useMemWal")]
pub fn use_memwal(&mut self, enable: bool) {
self.inner = self.inner.clone().use_memwal(enable);
}
#[napi(catch_unwind)]
@@ -510,6 +530,7 @@ impl TakeQuery {
&self,
max_batch_length: Option<u32>,
timeout_ms: Option<u32>,
with_row_id: Option<bool>,
) -> napi::Result<RecordBatchIterator> {
let mut execution_opts = QueryExecutionOptions::default();
if let Some(max_batch_length) = max_batch_length {
@@ -518,8 +539,12 @@ impl TakeQuery {
if let Some(timeout_ms) = timeout_ms {
execution_opts.timeout = Some(std::time::Duration::from_millis(timeout_ms as u64))
}
let inner_stream = self
.inner
let query = if with_row_id == Some(true) {
self.inner.clone().with_row_id()
} else {
self.inner.clone()
};
let inner_stream = query
.execute_with_options(execution_opts)
.await
.map_err(|e| {
nodejs/src/remote.rs
+5
View File
@@ -92,6 +92,10 @@ pub struct ClientConfig {
pub user_agent: Option<String>,
pub retry_config: Option<RetryConfig>,
pub timeout_config: Option<TimeoutConfig>,
/// Maximum number of concurrent blob HTTP requests across all tables and
/// blob handles on this connection. Defaults to 8. Can also be set with
/// `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
pub blob_request_concurrency: Option<u32>,
pub extra_headers: Option<HashMap<String, String>>,
/// The delimiter joining a namespace path and a name into one object
/// identifier. `"$"` is the only supported value, and leaving this unset is
@@ -428,6 +432,7 @@ impl From<ClientConfig> for lancedb::remote::ClientConfig {
.unwrap_or(concat!("LanceDB-Node-Client/", env!("CARGO_PKG_VERSION")).to_string()),
retry_config: config.retry_config.map(Into::into).unwrap_or_default(),
timeout_config: config.timeout_config.map(Into::into).unwrap_or_default(),
blob_request_concurrency: config.blob_request_concurrency.map(|value| value as usize),
extra_headers: config.extra_headers.unwrap_or_default(),
id_delimiter: config.id_delimiter,
tls_config: config.tls_config.map(Into::into),
nodejs/src/table.rs
+70 -74
View File
@@ -580,64 +580,60 @@ impl Table {
.default_error()
}
#[napi(catch_unwind)]
pub async fn set_lsm_write_spec(&self, spec: LsmWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::LsmWriteSpec::try_from(spec)?;
#[napi(catch_unwind, js_name = "enableMemWal")]
pub async fn enable_memwal(&self, spec: MemWalWriteSpec) -> napi::Result<()> {
let native_spec = lancedb::table::MemWalWriteSpec::try_from(spec)?;
self.inner_ref()?
.set_lsm_write_spec(native_spec)
.enable_memwal(native_spec)
.await
.default_error()
}
#[napi(catch_unwind)]
pub async fn unset_lsm_write_spec(&self) -> napi::Result<()> {
#[napi(catch_unwind, js_name = "disableMemWal")]
pub async fn disable_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.disable_memwal().await.default_error()
}
#[napi(catch_unwind, js_name = "getMemWalSpec")]
pub async fn get_memwal_spec(&self) -> napi::Result<Option<MemWalWriteSpec>> {
let spec = self.inner_ref()?.get_memwal_spec().await.default_error()?;
Ok(spec.map(MemWalWriteSpec::from))
}
#[napi(catch_unwind, js_name = "closeMemWalWriters")]
pub async fn close_memwal_writers(&self) -> napi::Result<()> {
self.inner_ref()?
.unset_lsm_write_spec()
.close_memwal_writers()
.await
.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_write_spec(&self) -> napi::Result<Option<LsmWriteSpec>> {
let spec = self
.inner_ref()?
.get_lsm_write_spec()
.await
.default_error()?;
Ok(spec.map(LsmWriteSpec::from))
#[napi(catch_unwind, js_name = "flushMemWal")]
pub async fn flush_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.flush_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn close_lsm_writers(&self) -> napi::Result<()> {
self.inner_ref()?.close_lsm_writers().await.default_error()
#[napi(catch_unwind, js_name = "compactMemWal")]
pub async fn compact_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.compact_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn flush_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.flush_lsm().await.default_error()
#[napi(catch_unwind, js_name = "checkpointMemWal")]
pub async fn checkpoint_memwal(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_memwal().await.default_error()
}
#[napi(catch_unwind)]
pub async fn compact_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.compact_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_stats(
#[napi(catch_unwind, js_name = "getMemWalStats")]
pub async fn get_memwal_stats(
&self,
include_generation_rows: bool,
) -> napi::Result<Option<LsmStats>> {
include_sstable_rows: bool,
) -> napi::Result<Option<MemWalStats>> {
let stats = self
.inner_ref()?
.get_lsm_stats(include_generation_rows)
.get_memwal_stats(include_sstable_rows)
.await
.default_error()?;
Ok(stats.map(LsmStats::from))
Ok(stats.map(MemWalStats::from))
}
#[napi(catch_unwind)]
@@ -958,7 +954,7 @@ impl From<LanceDbFtsToken> for FtsToken {
/// `column` is required.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct LsmWriteSpec {
pub struct MemWalWriteSpec {
/// One of `"bucket"`, `"identity"`, or `"unsharded"`.
pub spec_type: String,
/// Bucket and identity variants: the sharding column.
@@ -972,31 +968,31 @@ pub struct LsmWriteSpec {
pub writer_config_defaults: Option<HashMap<String, String>>,
}
impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
impl TryFrom<MemWalWriteSpec> for lancedb::table::MemWalWriteSpec {
type Error = napi::Error;
fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
fn try_from(value: MemWalWriteSpec) -> napi::Result<Self> {
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec bucket requires `column`")
napi::Error::from_reason("MemWalWriteSpec bucket requires `column`")
})?;
let num_buckets = value.num_buckets.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec bucket requires `numBuckets`")
napi::Error::from_reason("MemWalWriteSpec bucket requires `numBuckets`")
})?;
Self::bucket(column, num_buckets)
}
"identity" => {
let column = value.column.ok_or_else(|| {
napi::Error::from_reason("LsmWriteSpec identity requires `column`")
napi::Error::from_reason("MemWalWriteSpec identity requires `column`")
})?;
Self::identity(column)
}
"unsharded" => Self::unsharded(),
other => {
return Err(napi::Error::from_reason(format!(
"LsmWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
"MemWalWriteSpec `specType` must be 'bucket', 'identity', or 'unsharded', got '{}'",
other
)));
}
@@ -1007,9 +1003,9 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
}
impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
fn from(spec: lancedb::table::LsmWriteSpec) -> Self {
use lancedb::table::LsmWriteSpec as Native;
impl From<lancedb::table::MemWalWriteSpec> for MemWalWriteSpec {
fn from(spec: lancedb::table::MemWalWriteSpec) -> Self {
use lancedb::table::MemWalWriteSpec as Native;
match spec {
Native::Bucket {
column,
@@ -1048,21 +1044,21 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
}
}
/// One flushed L0 generation.
/// One SSTable.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct GenerationStats {
/// The generation number. Increases as memtables are sealed into L0.
pub struct SsTableStats {
/// The generation number. Increases as memtables are frozen into SSTables.
pub generation: i64,
/// On-disk size of the generation.
/// On-disk size of the SSTable.
pub bytes: i64,
/// Present only when `includeGenerationRows` was requested. Off by default
/// Present only when `includeSstableRows` was requested. Off by default
/// because each count opens an uncached Lance dataset.
pub rows: Option<i64>,
}
impl From<lancedb::table::GenerationStats> for GenerationStats {
fn from(g: lancedb::table::GenerationStats) -> Self {
impl From<lancedb::table::SsTableStats> for SsTableStats {
fn from(g: lancedb::table::SsTableStats) -> Self {
Self {
generation: g.generation as i64,
bytes: g.bytes as i64,
@@ -1075,7 +1071,7 @@ impl From<lancedb::table::GenerationStats> for GenerationStats {
#[napi(object)]
#[derive(Clone, Debug)]
pub struct MemtableStats {
/// The generation this memtable will become once sealed.
/// The generation this memtable will become once frozen.
pub generation: i64,
/// Rows currently buffered.
pub rows: i64,
@@ -1100,13 +1096,13 @@ impl From<lancedb::table::MemtableStats> for MemtableStats {
}
}
/// Live state of one bucket. A table is N buckets on one node; flattening to a
/// single number hides the one hot bucket that is usually why someone opened
/// Live state of one table shard. A table is N table shards on one node; flattening to a
/// single number hides the one hot table shard that is usually why someone opened
/// this endpoint.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct BucketStats {
/// The shard this bucket writes.
pub struct TableShardStats {
/// The id of this table shard.
pub shard_id: String,
/// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
pub status: String,
@@ -1121,20 +1117,20 @@ pub struct BucketStats {
/// Highest WAL position the writer has seen. The difference against
/// `replayAfterWalEntryPosition` is the WAL lag.
pub wal_entry_position_last_seen: i64,
/// Flushed L0 generations not yet merged into the base table.
pub generations: Vec<GenerationStats>,
/// Whether a pass owns this bucket's compaction latch right now. Says *a*
/// SSTables not yet merged into the base table.
pub sstables: Vec<SsTableStats>,
/// Whether a pass owns this table shard's compaction latch right now. Says *a*
/// driver is running, not *whose*, and the latch is held from dispatch —
/// including while the pass queues for a pod-wide compactor permit. Read it
/// as "do not pile on", never as "mine is progressing".
pub compacting: bool,
/// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
/// Oldest first, active last. Absent for a `"Sealed"` table shard, whose
/// in-memory state is torn down.
pub memtables: Option<Vec<MemtableStats>>,
}
impl From<lancedb::table::BucketStats> for BucketStats {
fn from(b: lancedb::table::BucketStats) -> Self {
impl From<lancedb::table::TableShardStats> for TableShardStats {
fn from(b: lancedb::table::TableShardStats) -> Self {
Self {
shard_id: b.shard_id,
status: b.status,
@@ -1143,7 +1139,7 @@ impl From<lancedb::table::BucketStats> for BucketStats {
current_generation: b.current_generation as i64,
replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
generations: b.generations.into_iter().map(Into::into).collect(),
sstables: b.sstables.into_iter().map(Into::into).collect(),
compacting: b.compacting,
memtables: b
.memtables
@@ -1152,21 +1148,21 @@ impl From<lancedb::table::BucketStats> for BucketStats {
}
}
/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
/// Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
///
/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
/// Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
/// the caller's to compute.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct LsmStats {
/// One entry per bucket backing this table.
pub buckets: Vec<BucketStats>,
pub struct MemWalStats {
/// One entry per table shard backing this table.
pub table_shards: Vec<TableShardStats>,
}
impl From<lancedb::table::LsmStats> for LsmStats {
fn from(stats: lancedb::table::LsmStats) -> Self {
impl From<lancedb::table::MemWalStats> for MemWalStats {
fn from(stats: lancedb::table::MemWalStats) -> Self {
Self {
buckets: stats.buckets.into_iter().map(Into::into).collect(),
table_shards: stats.table_shards.into_iter().map(Into::into).collect(),
}
}
}