rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-21 21:48:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge origin/main into gatekeeper/fix-2057-1

Browse Source
This commit is contained in:
Gatefixer
2026-08-21 09:25:17 +00:00
parent 8af2c26d54 685cb01d6d
commit c821cf39fd
173 changed files with 16513 additions and 1899 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/Cargo.toml
+5 -5
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.37.1-beta.0"
version = "0.38.0-beta.2"
publish = false
license.workspace = true
description.workspace = true
@@ -16,12 +16,12 @@ crate-type = ["cdylib"]
async-trait.workspace = true
arrow-ipc.workspace = true
arrow-array.workspace = true
arrow-buffer = "58.0.0"
arrow-buffer.workspace = true
half.workspace = true
arrow-schema.workspace = true
env_logger.workspace = true
futures.workspace = true
lancedb = { path = "../rust/lancedb", default-features = false }
lancedb.workspace = true
lance-namespace.workspace = true
napi = { version = "3.8.3", default-features = false, features = [
"napi9",
@@ -29,8 +29,8 @@ napi = { version = "3.8.3", default-features = false, features = [
"chrono_date",
"serde-json",
] }
chrono = { version = "0.4", default-features = false, features = ["clock"] }
serde_json = "1"
chrono.workspace = true
serde_json.workspace = true
napi-derive = "3.5.2"
# Prevent dynamic linking of lzma, which comes from datafusion
lzma-sys = { version = "0.1", features = ["static"] }
nodejs/__test__/arrow.test.ts
+144
View File
@@ -6,7 +6,9 @@ import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import {
Vector as CurrentVector,
convertToTable,
tableFromIPC as currentTableFromIPC,
fromBufferToRecordBatch,
fromDataToBuffer,
fromRecordBatchToBuffer,
@@ -19,6 +21,7 @@ import {
FunctionOptions,
} from "../lancedb/embedding/embedding_function";
import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
import { sanitizeTable } from "../lancedb/sanitize";
// biome-ignore lint/suspicious/noExplicitAny: skip
function sampleRecords(): Array<Record<string, any>> {
@@ -64,7 +67,11 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
tableFromIPC,
DataType,
Dictionary,
RecordBatch: ArrowRecordBatch,
Table: ArrowTable,
Uint8: ArrowUint8,
makeData: arrowMakeData,
vectorFromArray,
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
} = <any>arrow;
type Schema = ApacheArrow["Schema"];
@@ -197,6 +204,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]);
});
it("will use a provided FixedSizeList schema with typed array values", function () {
const schema = new Schema([
new Field("text", new Utf8(), false),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float32(), false)),
false,
),
]);
const table = makeArrowTable(
[
{
text: "foo",
vector: new Float32Array([1, 2, 3]),
},
],
{ schema },
);
expect(table.getChild("text")?.toJSON()).toEqual(["foo"]);
expect(
table
.getChild("vector")
?.toJSON()
.map((value) => value.toJSON()),
).toEqual([[1, 2, 3]]);
});
it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
const schema = new Schema([
new Field("a", new Float(Precision.DOUBLE), true),
@@ -1025,6 +1061,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
});
describe("when using two versions of arrow", function () {
it("preserves a dictionary shared by multiple fields", async function () {
const values = ["alpha", "beta", "alpha"];
const dictionaryVector = vectorFromArray(values);
const batch = new ArrowRecordBatch({
first: dictionaryVector.data[0],
second: dictionaryVector.data[0],
});
const table = new ArrowTable([batch]);
const sanitized = sanitizeTable(table);
expect([...sanitized.getChild("first")!]).toEqual(values);
expect([...sanitized.getChild("second")!]).toEqual(values);
const firstType = sanitized.schema.fields[0].type as {
dictionary: unknown;
};
const secondType = sanitized.schema.fields[1].type as {
dictionary: unknown;
};
expect(secondType.dictionary).toBe(firstType.dictionary);
expect(sanitized.batches[0].data.children[1].dictionary).toBe(
sanitized.batches[0].data.children[0].dictionary,
);
const buf = await fromDataToBuffer(table);
const actual = currentTableFromIPC(buf);
expect([...actual.getChild("first")!]).toEqual(values);
expect([...actual.getChild("second")!]).toEqual(values);
});
it("preserves shared dictionary data from another Arrow version", async function () {
const values = ["alpha", "beta", "alpha"];
const dictionaryVector = vectorFromArray(values);
const firstBatch = new ArrowRecordBatch({
label: dictionaryVector.slice(0, 2).data[0],
});
const secondBatch = new ArrowRecordBatch({
label: dictionaryVector.slice(2).data[0],
});
const table = new ArrowTable([firstBatch, secondBatch]);
const sanitized = sanitizeTable(table);
expect([...sanitized.getChild("label")!]).toEqual(values);
const dictionaries = sanitized.batches.map(
(batch) => batch.data.children[0].dictionary,
);
expect(dictionaries[0]).toBeInstanceOf(CurrentVector);
expect(dictionaries[1]).toBe(dictionaries[0]);
const buf = await fromDataToBuffer(table);
const actual = currentTableFromIPC(buf);
expect([...actual.getChild("label")!]).toEqual(values);
});
it("preserves shared chunks in growing dictionaries", async function () {
const type = new Dictionary(new Utf8(), new Int32(), 42, false);
const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8());
const secondDictionary = firstDictionary.concat(
vectorFromArray(["gamma"], new Utf8()),
);
const firstData = arrowMakeData({
type,
data: Int32Array.from([0, 1]),
dictionary: firstDictionary,
});
const secondData = arrowMakeData({
type,
data: Int32Array.from([2]),
dictionary: secondDictionary,
});
const table = new ArrowTable([
new ArrowRecordBatch({ label: firstData }),
new ArrowRecordBatch({ label: secondData }),
]);
const sanitized = sanitizeTable(table);
const expected = ["alpha", "beta", "gamma"];
expect([...sanitized.getChild("label")!]).toEqual(expected);
const firstLocalDictionary =
sanitized.batches[0].data.children[0].dictionary!;
const secondLocalDictionary =
sanitized.batches[1].data.children[0].dictionary!;
expect(secondLocalDictionary.data[0]).toBe(
firstLocalDictionary.data[0],
);
const buf = await fromTableToBuffer(sanitized);
const actual = currentTableFromIPC(buf);
expect([...actual.getChild("label")!]).toEqual(expected);
});
it("can serialize list data from another Arrow version", async function () {
const values = [["anime", "action"], [], null];
const vector = vectorFromArray(
values,
new List(new Field("item", new Utf8(), true)),
);
const table = new ArrowTable({ tags: vector });
const buf = await fromDataToBuffer(table);
const actual = currentTableFromIPC(buf);
const actualTags = actual.getChild("tags");
expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]);
expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]);
expect(actualTags?.get(2)).toBeNull();
});
it("can still import data", async function () {
const schema = new arrow15.Schema([
new arrow15.Field("id", new arrow15.Int32()),
nodejs/__test__/connection.test.ts
+10
View File
@@ -89,6 +89,16 @@ describe("given a connection", () => {
await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
});
it("should return a completed job when dropping a local table", async () => {
await db.createTable("async-drop", [{ id: 1 }]);
const job = await db.dropTableAsync("async-drop");
expect(job.id).toBeNull();
await expect(job.status()).resolves.toBe("finished");
await job.wait();
await expect(db.tableNames()).resolves.toEqual([]);
});
it("should fail if creating table twice, unless overwrite is true", async () => {
let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
await expect(tbl.countRows()).resolves.toBe(2);
nodejs/__test__/embedding.test.ts
+60
View File
@@ -11,8 +11,11 @@ import {
Float16,
Float32,
Float64,
Int32,
Schema,
Utf8,
fromDataToBuffer,
tableFromIPC,
} from "../lancedb/arrow";
import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -184,6 +187,63 @@ describe("embedding functions", () => {
const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
expect(vector0).toEqual([1, 2, 3]);
});
it("should append generated vectors to a non-nullable schema", async () => {
@register("non_nullable_schema_test")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float64();
}
async computeSourceEmbeddings(data: string[]) {
return data.map(() => [1, 2, 3]);
}
}
const schema = new Schema([
new Field("id", new Int32()),
new Field("text", new Utf8()),
new Field("type", new Utf8()),
new Field(
"vector",
new FixedSizeList(3, new Field("item", new Float64())),
),
]);
const func = new MockEmbeddingFunction();
const db = await connect(tmpDir.name);
const table = await db.createEmptyTable("test_non_nullable", schema, {
embeddingFunction: {
function: func,
sourceColumn: "text",
},
});
const data = [
{ id: 1, text: "Carrot", type: "vegetable" },
{ id: 2, text: "Apple", type: "fruit" },
];
const buffer = await fromDataToBuffer(
data,
undefined,
await table.schema(),
);
const generatedTable = tableFromIPC(buffer);
const vectorField = generatedTable.schema.fields.find(
(field) => field.name === "vector",
);
expect(vectorField?.nullable).toBe(false);
await table.add(data);
const rows = await table.query().toArray();
expect(rows).toHaveLength(2);
for (const row of rows) {
expect([...row.vector]).toEqual([1, 2, 3]);
}
});
it("should error when appending to a table with an unregistered embedding function", async () => {
@register("mock")
class MockEmbeddingFunction extends EmbeddingFunction<string> {
nodejs/__test__/package.test.ts
+14
View File
@@ -0,0 +1,14 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import packageJson = require("../package.json");
describe("package metadata", () => {
it("requires Node.js type declarations compatible with the runtime", () => {
expect(packageJson.engines.node).toBe(">= 18");
expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
optional: true,
});
});
});
nodejs/__test__/query.test.ts
+75
View File
@@ -110,6 +110,81 @@ describe("Query outputSchema", () => {
});
});
describe("Search pagination", () => {
let tmpDir: tmp.DirResult;
let table: Table;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const schema = new Schema([
new Field("id", new Int64(), false),
new Field("text", new Utf8(), false),
new Field(
"vector",
new FixedSizeList(2, new Field("item", new Float32())),
false,
),
]);
const data = makeArrowTable(
[
{ id: 1n, text: "common", vector: [0, 0] },
{ id: 2n, text: "common common", vector: [1, 1] },
{ id: 3n, text: "common common common", vector: [2, 2] },
{ id: 4n, text: "common common common common", vector: [3, 3] },
],
{ schema },
);
table = await db.createTable("test", data);
});
afterEach(() => {
tmpDir.removeCallback();
});
it("applies offset after the vector search limit", async () => {
const allResults = await table
.vectorSearch([0, 0])
.select(["id"])
.limit(4)
.toArray();
const secondPage = await table
.vectorSearch([0, 0])
.select(["id"])
.limit(2)
.offset(2)
.toArray();
expect(allResults).toHaveLength(4);
expect(secondPage).toHaveLength(2);
expect(secondPage.map((row) => row.id)).toEqual(
allResults.slice(2, 4).map((row) => row.id),
);
});
it("applies offset after the full-text search limit", async () => {
await table.createIndex("text", { config: Index.fts() });
const allResults = await table
.search("common", "fts")
.select(["id"])
.limit(4)
.toArray();
const secondPage = await table
.search("common", "fts")
.select(["id"])
.limit(2)
.offset(2)
.toArray();
expect(allResults).toHaveLength(4);
expect(secondPage).toHaveLength(2);
expect(secondPage.map((row) => row.id)).toEqual(
allResults.slice(2, 4).map((row) => row.id),
);
});
});
describe("Query orderBy", () => {
let tmpDir: tmp.DirResult;
let table: Table;
nodejs/__test__/remote.test.ts
+32
View File
@@ -170,6 +170,38 @@ describe("remote connection", () => {
);
});
it("surfaces JSON server errors from remote table operations", async () => {
await withMockDatabase(
(req, res) => {
const path = req.url ?? "";
if (path.endsWith("/describe/")) {
res.writeHead(200, { "Content-Type": "application/json" }).end(
JSON.stringify({
name: "broken_table",
version: 1,
schema: { fields: [] },
}),
);
return;
}
if (path.endsWith("/count_rows/")) {
res
.writeHead(400, { "Content-Type": "application/json" })
.end(JSON.stringify({ error: "count rows failed" }));
return;
}
res.writeHead(404).end();
},
async (db) => {
const table = await db.openTable("broken_table");
await expect(table.countRows()).rejects.toThrow("count rows failed");
},
);
});
it("should pass on requested extra headers", async () => {
await withMockDatabase(
(req, res) => {
nodejs/__test__/table.test.ts
+164 -1
View File
@@ -86,6 +86,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await expect(table.countRows()).resolves.toBe(3);
});
it("should support a foreign Float64 vector schema end to end", async () => {
const conn = await connect(tmpDir.name);
const schema = new arrow.Schema([
new arrow.Field("resource_id", new arrow.Int32(), false),
new arrow.Field(
"vector",
new arrow.FixedSizeList(
3,
new arrow.Field("value", new arrow.Float64(), true),
),
false,
),
]);
const data = [
{
// biome-ignore lint/style/useNamingConvention: matches the reported schema
resource_id: 0,
vector: [0.1, 0.1, 0.1],
},
];
const resources = await conn.createTable("resources", data, { schema });
const existing = await resources
.query()
.where("resource_id = 0")
.limit(1)
.toArray();
expect(existing).toHaveLength(1);
const matched = await resources
.search(Float64Array.from(data[0].vector))
.limit(1)
.toArray();
expect(matched).toHaveLength(1);
expect(matched[0]["resource_id"]).toBe(0);
});
it("should support branches", async () => {
await table.add([{ id: 1 }]);
expect(await table.countRows()).toBe(1);
@@ -239,8 +277,16 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
},
numIndices: 0,
numRows: 3,
totalBytes: 44,
// Full on-disk size of the two data files, footers and metadata included.
totalBytes: 684,
});
// Index files count toward totalBytes too (only deletion files and
// manifests are excluded).
await table.createIndex("id", { config: Index.btree() });
const statsWithIndex = await table.stats();
expect(statsWithIndex.numIndices).toBe(1);
expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
});
it("should overwrite data if asked", async () => {
@@ -3294,3 +3340,120 @@ describe("LSM merge insert", () => {
await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
});
});
describe("LSM convergence and stats", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => tmpDir.removeCallback());
async function lsmTable(conn: Connection): Promise<Table> {
const table = await conn.createEmptyTable(
"t",
new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
);
await table.setUnenforcedPrimaryKey("id");
await table.setLsmWriteSpec({ specType: "unsharded" });
return table;
}
// These four route through the server that owns the MemWAL, so a local table
// rejects them rather than answering. What is asserted here is that the
// bindings reach the core at all; the behavior against a real endpoint is
// covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
it("rejects flushLsm on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
});
it("rejects compactLsm on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
});
it("rejects getLsmStats on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
});
it("rejects checkpointLsm on a local table", async () => {
const conn = await connect(tmpDir.name);
const table = await lsmTable(conn);
// checkpointLsm seals first, so it surfaces flushLsm's rejection.
await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
});
});
describe("computed columns", () => {
let tmpDir: tmp.DirResult;
beforeEach(() => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
});
afterEach(() => tmpDir.removeCallback());
it("declares a column and fills it on refresh", async () => {
const db = await connect(tmpDir.name);
const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]);
await table.addColumns({
computed: [{ name: "doubled", valueSql: "x * 2" }],
});
let rows = await table.query().toArray();
expect(rows.map((r) => r.doubled)).toEqual([null, null]);
const result = await table.refreshColumn("doubled");
expect(result.rowsFilled).toBe(2);
rows = await table.query().toArray();
expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
});
it("returns a job handle from refreshColumnAsync", async () => {
const db = await connect(tmpDir.name);
const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]);
await table.addColumns({
computed: [{ name: "doubled", valueSql: "x * 2" }],
});
const job = await table.refreshColumnAsync("doubled");
expect(job.id).toBeNull();
await job.wait();
expect(await job.status()).toBe("finished");
const rows = await table.query().toArray();
expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
// Bad input rejects at the call, not through the job.
await expect(table.refreshColumnAsync("x")).rejects.toThrow(
"not a computed column",
);
});
it("fills rows added since the last refresh", async () => {
const db = await connect(tmpDir.name);
const table = await db.createTable("computed_append", [{ x: 1 }]);
await table.addColumns({
computed: [{ name: "doubled", valueSql: "x * 2" }],
});
await table.refreshColumn("doubled");
await table.add([{ x: 5 }]);
const result = await table.refreshColumn("doubled");
expect(result.rowsFilled).toBe(1);
const rows = await table.query().toArray();
expect(rows.map((r) => r.doubled).sort()).toEqual([10, 2]);
});
});
nodejs/lancedb/connection.ts
+12
View File
@@ -327,6 +327,14 @@ export abstract class Connection {
*/
abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
/**
* Start dropping a table and return its cleanup job.
*
* The table may become unavailable before its data files are removed. Wait
* on the returned job to know when cleanup has finished.
*/
abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
/**
* Drop all tables in the database.
* @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
@@ -705,6 +713,10 @@ export class LocalConnection extends Connection {
return this.inner.dropTable(name, namespacePath ?? []);
}
async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
return this.inner.dropTableAsync(name, namespacePath ?? []);
}
async dropAllTables(namespacePath?: string[]): Promise<void> {
return this.inner.dropAllTables(namespacePath ?? []);
}
nodejs/lancedb/index.ts
+5
View File
@@ -50,6 +50,7 @@ export {
MergeResult,
AddResult,
AddColumnsResult,
RefreshColumnResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
DeleteResult,
@@ -146,6 +147,10 @@ export {
FtsToken,
TokenizeTableOptions,
LsmWriteSpec,
LsmStats,
BucketStats,
GenerationStats,
MemtableStats,
ColumnAlteration,
FieldMetadataUpdate,
} from "./table";
nodejs/lancedb/sanitize.ts
+174 -29
View File
@@ -9,7 +9,7 @@
// comes from the exact same library instance. This is not always the case
// and so we must sanitize the input to ensure that it is compatible.
import { BufferType, Data } from "apache-arrow";
import { BufferType, Data, Vector } from "apache-arrow";
import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
import {
Binary,
@@ -74,6 +74,20 @@ import {
Utf8,
} from "./arrow";
type SanitizationContext = {
types: WeakMap<object, DataType>;
vectors: WeakMap<object, Vector>;
data: WeakMap<object, Data<DataType>>;
};
function createSanitizationContext(): SanitizationContext {
return {
types: new WeakMap(),
vectors: new WeakMap(),
data: new WeakMap(),
};
}
export function sanitizeMetadata(
metadataLike?: unknown,
): Map<string, string> | undefined {
@@ -186,6 +200,13 @@ export function sanitizeInterval(typeLike: object) {
}
export function sanitizeList(typeLike: object) {
return sanitizeListWithContext(typeLike, createSanitizationContext());
}
function sanitizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a List type to have an array-like `children` property",
@@ -194,19 +215,35 @@ export function sanitizeList(typeLike: object) {
if (typeLike.children.length !== 1) {
throw Error("Expected a List type to have exactly one child");
}
return new List(sanitizeField(typeLike.children[0]));
return new List(sanitizeFieldWithContext(typeLike.children[0], context));
}
export function sanitizeStruct(typeLike: object) {
return sanitizeStructWithContext(typeLike, createSanitizationContext());
}
function sanitizeStructWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a Struct type to have an array-like `children` property",
);
}
return new Struct(typeLike.children.map((child) => sanitizeField(child)));
return new Struct(
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
export function sanitizeUnion(typeLike: object) {
return sanitizeUnionWithContext(typeLike, createSanitizationContext());
}
function sanitizeUnionWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (
!("typeIds" in typeLike) ||
!("mode" in typeLike) ||
@@ -226,7 +263,7 @@ export function sanitizeUnion(typeLike: object) {
typeLike.mode,
// biome-ignore lint/suspicious/noExplicitAny: skip
typeLike.typeIds as any,
typeLike.children.map((child) => sanitizeField(child)),
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
@@ -234,6 +271,19 @@ export function sanitizeTypedUnion(
typeLike: object,
// eslint-disable-next-line @typescript-eslint/naming-convention
UnionType: typeof DenseUnion | typeof SparseUnion,
) {
return sanitizeTypedUnionWithContext(
typeLike,
UnionType,
createSanitizationContext(),
);
}
function sanitizeTypedUnionWithContext(
typeLike: object,
// eslint-disable-next-line @typescript-eslint/naming-convention
UnionType: typeof DenseUnion | typeof SparseUnion,
context: SanitizationContext,
) {
if (!("typeIds" in typeLike)) {
throw Error(
@@ -248,7 +298,7 @@ export function sanitizeTypedUnion(
return new UnionType(
typeLike.typeIds as Int32Array | number[],
typeLike.children.map((child) => sanitizeField(child)),
typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
);
}
@@ -262,6 +312,16 @@ export function sanitizeFixedSizeBinary(typeLike: object) {
}
export function sanitizeFixedSizeList(typeLike: object) {
return sanitizeFixedSizeListWithContext(
typeLike,
createSanitizationContext(),
);
}
function sanitizeFixedSizeListWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
throw Error("Expected a FixedSizeList type to have a `listSize` property");
}
@@ -275,11 +335,18 @@ export function sanitizeFixedSizeList(typeLike: object) {
}
return new FixedSizeList(
typeLike.listSize,
sanitizeField(typeLike.children[0]),
sanitizeFieldWithContext(typeLike.children[0], context),
);
}
export function sanitizeMap(typeLike: object) {
return sanitizeMapWithContext(typeLike, createSanitizationContext());
}
function sanitizeMapWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
throw Error(
"Expected a Map type to have an array-like `children` property",
@@ -292,7 +359,10 @@ export function sanitizeMap(typeLike: object) {
throw Error("Expected a Map type to have exactly one child");
}
return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
return new Map_(
sanitizeFieldWithContext(typeLike.children[0], context),
typeLike.keysSorted,
);
}
export function sanitizeDuration(typeLike: object) {
@@ -303,6 +373,13 @@ export function sanitizeDuration(typeLike: object) {
}
export function sanitizeDictionary(typeLike: object) {
return sanitizeDictionaryWithContext(typeLike, createSanitizationContext());
}
function sanitizeDictionaryWithContext(
typeLike: object,
context: SanitizationContext,
) {
if (!("id" in typeLike) || typeof typeLike.id !== "number") {
throw Error("Expected a Dictionary type to have an `id` property");
}
@@ -316,8 +393,8 @@ export function sanitizeDictionary(typeLike: object) {
throw Error("Expected a Dictionary type to have an `isOrdered` property");
}
return new Dictionary(
sanitizeType(typeLike.dictionary),
sanitizeType(typeLike.indices) as TKeys,
sanitizeTypeWithContext(typeLike.dictionary, context),
sanitizeTypeWithContext(typeLike.indices, context) as TKeys,
typeLike.id,
typeLike.isOrdered,
);
@@ -325,12 +402,23 @@ export function sanitizeDictionary(typeLike: object) {
// biome-ignore lint/suspicious/noExplicitAny: skip
export function sanitizeType(typeLike: unknown): DataType<any> {
return sanitizeTypeWithContext(typeLike, createSanitizationContext());
}
function sanitizeTypeWithContext(
typeLike: unknown,
context: SanitizationContext,
): DataType {
if (typeof typeLike === "string") {
return dataTypeFromName(typeLike);
}
if (typeof typeLike !== "object" || typeLike === null) {
throw Error("Expected a Type but object was null/undefined");
}
const cached = context.types.get(typeLike);
if (cached !== undefined) {
return cached;
}
if (
!("typeId" in typeLike) ||
!(
@@ -349,6 +437,16 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
throw Error("Type's typeId property was not a function or number");
}
const type = sanitizeTypeById(typeLike, typeId, context);
context.types.set(typeLike, type);
return type;
}
function sanitizeTypeById(
typeLike: object,
typeId: Type,
context: SanitizationContext,
): DataType {
switch (typeId) {
case Type.NONE:
throw Error("Received a Type with a typeId of NONE");
@@ -375,21 +473,21 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
case Type.Interval:
return sanitizeInterval(typeLike);
case Type.List:
return sanitizeList(typeLike);
return sanitizeListWithContext(typeLike, context);
case Type.Struct:
return sanitizeStruct(typeLike);
return sanitizeStructWithContext(typeLike, context);
case Type.Union:
return sanitizeUnion(typeLike);
return sanitizeUnionWithContext(typeLike, context);
case Type.FixedSizeBinary:
return sanitizeFixedSizeBinary(typeLike);
case Type.FixedSizeList:
return sanitizeFixedSizeList(typeLike);
return sanitizeFixedSizeListWithContext(typeLike, context);
case Type.Map:
return sanitizeMap(typeLike);
return sanitizeMapWithContext(typeLike, context);
case Type.Duration:
return sanitizeDuration(typeLike);
case Type.Dictionary:
return sanitizeDictionary(typeLike);
return sanitizeDictionaryWithContext(typeLike, context);
case Type.Int8:
return new Int8();
case Type.Int16:
@@ -433,9 +531,9 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
case Type.TimestampSecond:
return sanitizeTypedTimestamp(typeLike, TimestampSecond);
case Type.DenseUnion:
return sanitizeTypedUnion(typeLike, DenseUnion);
return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context);
case Type.SparseUnion:
return sanitizeTypedUnion(typeLike, SparseUnion);
return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context);
case Type.IntervalDayTime:
return new IntervalDayTime();
case Type.IntervalYearMonth:
@@ -454,6 +552,13 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
}
export function sanitizeField(fieldLike: unknown): Field {
return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
}
function sanitizeFieldWithContext(
fieldLike: unknown,
context: SanitizationContext,
): Field {
if (fieldLike instanceof Field) {
return fieldLike;
}
@@ -471,7 +576,7 @@ export function sanitizeField(fieldLike: unknown): Field {
}
let type: DataType;
try {
type = sanitizeType(fieldLike.type);
type = sanitizeTypeWithContext(fieldLike.type, context);
} catch (error: unknown) {
throw Error(
`Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
@@ -501,6 +606,13 @@ export function sanitizeField(fieldLike: unknown): Field {
* than lancedb is using.
*/
export function sanitizeSchema(schemaLike: SchemaLike): Schema {
return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
}
function sanitizeSchemaWithContext(
schemaLike: SchemaLike,
context: SanitizationContext,
): Schema {
if (schemaLike instanceof Schema) {
return schemaLike;
}
@@ -522,7 +634,7 @@ export function sanitizeSchema(schemaLike: SchemaLike): Schema {
);
}
const sanitizedFields = schemaLike.fields.map((field) =>
sanitizeField(field),
sanitizeFieldWithContext(field, context),
);
return new Schema(sanitizedFields, metadata);
}
@@ -544,13 +656,18 @@ export function sanitizeTable(tableLike: TableLike): Table {
"The table passed in does not appear to be a table (no 'columns' property)",
);
}
const schema = sanitizeSchema(tableLike.schema);
const batches = tableLike.batches.map(sanitizeRecordBatch);
const context = createSanitizationContext();
const schema = sanitizeSchemaWithContext(tableLike.schema, context);
const batches = tableLike.batches.map((batch) =>
sanitizeRecordBatch(batch, context),
);
return new Table(schema, batches);
}
function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
function sanitizeRecordBatch(
batchLike: RecordBatchLike,
context: SanitizationContext,
): RecordBatch {
if (batchLike instanceof RecordBatch) {
return batchLike;
}
@@ -567,19 +684,43 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
"The record batch passed in does not appear to be a record batch (no 'data' property)",
);
}
const schema = sanitizeSchema(batchLike.schema);
const data = sanitizeData(batchLike.data);
const schema = sanitizeSchemaWithContext(batchLike.schema, context);
const data = sanitizeData(batchLike.data, context) as Data<Struct>;
return new RecordBatch(schema, data);
}
type DictionaryVectorLike = {
data: readonly DataLike[];
};
type DictionaryDataLike = DataLike & {
dictionary?: DictionaryVectorLike;
};
function sanitizeData(
dataLike: DataLike,
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
): import("apache-arrow").Data<Struct<any>> {
context: SanitizationContext,
): Data<DataType> {
if (dataLike instanceof Data) {
return dataLike;
}
return new Data(
dataLike.type,
const cachedData = context.data.get(dataLike);
if (cachedData !== undefined) {
return cachedData;
}
const dictionaryLike = (dataLike as DictionaryDataLike).dictionary;
let dictionary: Vector | undefined;
if (dictionaryLike !== undefined) {
dictionary = context.vectors.get(dictionaryLike);
if (dictionary === undefined) {
dictionary = new Vector(
dictionaryLike.data.map((data) => sanitizeData(data, context)),
);
context.vectors.set(dictionaryLike, dictionary);
}
}
const data = new Data(
sanitizeTypeWithContext(dataLike.type, context),
dataLike.offset,
dataLike.length,
dataLike.nullCount,
@@ -589,7 +730,11 @@ function sanitizeData(
[BufferType.VALIDITY]: dataLike.nullBitmap,
[BufferType.TYPE]: dataLike.typeIds,
},
dataLike.children.map((child) => sanitizeData(child, context)),
dictionary,
);
context.data.set(dataLike, data);
return data;
}
const constructorsByTypeName = {
nodejs/lancedb/table.ts
+174 -6
View File
@@ -31,8 +31,10 @@ import {
IndexConfig,
IndexStatistics,
Job,
LsmStats,
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
TableStatistics,
Tags,
UpdateFieldMetadataResult,
@@ -49,6 +51,12 @@ import {
import { sanitizeType } from "./sanitize";
import { IntoSql, toSQL } from "./util";
export { IndexConfig } from "./native";
export {
BucketStats,
GenerationStats,
LsmStats,
MemtableStats,
} from "./native";
/**
* Progress snapshot for a write operation, delivered to the `progress`
@@ -197,7 +205,11 @@ export interface LsmWriteSpec {
column?: string;
/** Bucket variant: the number of buckets, in `[1, 1024]`. */
numBuckets?: number;
/** Names of indexes the MemWAL should keep up to date during writes. */
/**
* Indexes the MemWAL keeps up to date. Omit to maintain every supported
* index, resolved on install — a snapshot, so indexes created later are not
* maintained. Pass `[]` for none.
*/
maintainedIndexes?: string[];
/** Default `ShardWriter` configuration recorded in the MemWAL index. */
writerConfigDefaults?: Record<string, string>;
@@ -521,18 +533,75 @@ export abstract class Table {
abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
/**
* Add new columns with defined values.
*
* The `{ computed }` form stores the expression rather than evaluating it
* now: the column is committed with no values, and rows get them from
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
* large table as on an empty one.
*
* A refresh does not revisit rows it has already filled, so mutating an
* input leaves the value computed at fill time; recomputing means dropping
* the column and declaring it again. While a declaration reads a column,
* that column cannot be renamed, retyped or dropped.
*
* On LanceDB Cloud and Enterprise the expression is planned by the
* server, and the refresh runs as a server job -- see
* {@link Table#refreshColumnAsync}.
* @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
* - An array of objects with column names and SQL expressions to calculate values
* - A single Arrow Field defining one column with its data type (column will be initialized with null values)
* - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
* - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
* - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
* @returns {Promise<AddColumnsResult>} A promise that resolves to an object
* containing the new version number of the table after adding the columns.
* @example
* ```ts
* await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
* const { rowsFilled } = await table.refreshColumn("doubled");
* ```
*/
abstract addColumns(
newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
newColumnTransforms:
| AddColumnsSql[]
| Field
| Field[]
| Schema
| { computed: AddColumnsSql[] },
): Promise<AddColumnsResult>;
/**
* Fill the rows of a computed column that hold no value yet.
*
* Rows appended since the last refresh are filled by the next one; rows
* already filled are left as they are, so the call is idempotent and does
* not observe a mutated input. Local tables only: a remote refresh runs
* as a server job, through {@link Table#refreshColumnAsync}.
* @param {string} column The name of the computed column to fill.
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
* number of rows filled and the new version number of the table.
*/
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
/**
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
* job instead of blocking until it completes.
*
* The job may already be complete when returned; callers must not assume
* the column is filled until {@link Job.wait} resolves. Invalid input --
* an unknown column, or one that is not computed -- rejects here rather
* than failing the job. On local tables the job runs in-process; on
* LanceDB Cloud and Enterprise it is the server's backfill job.
* @param {string} column The name of the computed column to fill.
* @example
* ```ts
* const job = await table.refreshColumnAsync("doubled");
* await job.wait();
* console.log(await job.status()); // "finished"
* ```
*/
abstract refreshColumnAsync(column: string): Promise<Job>;
/**
* Alter the name or nullability of columns.
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -595,6 +664,11 @@ export abstract class Table {
* All variants require the table to have an unenforced primary key
* ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
* requires it to be the single column being bucketed.
*
* Omitting `maintainedIndexes` maintains every index on the table, resolved
* here, failing if one cannot be maintained — name them to install anyway.
* Naming them pins an exact set, and a still-building index is rejected
* rather than quietly omitted.
* @param {LsmWriteSpec} spec The sharding spec to install.
* @returns {Promise<void>}
* @example
@@ -622,9 +696,10 @@ export abstract class Table {
*
* Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
* spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
* The returned spec — including its `maintainedIndexes` and
* `writerConfigDefaults` — mirrors what was passed to
* {@link Table#setLsmWriteSpec}.
* The returned spec mirrors what was passed to
* {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
* reports the concrete list resolved when the spec was set — `undefined`
* never round-trips.
* @returns {Promise<LsmWriteSpec | undefined>}
*/
abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -638,6 +713,59 @@ export abstract class Table {
* @returns {Promise<void>}
*/
abstract closeLsmWriters(): Promise<void>;
/**
* Seal every bucket's active memtable into a new L0 generation.
*
* Returns once the seal is committed. Sealing an empty memtable is a no-op,
* so this is safe to call repeatedly.
* @returns {Promise<void>}
*/
abstract flushLsm(): Promise<void>;
/**
* Trigger a background L0 → base compaction pass per bucket.
*
* Returns once the passes are *dispatched*, not once they finish — watch
* {@link Table#getLsmStats} for progress, or use
* {@link Table#checkpointLsm} to wait for convergence.
* @returns {Promise<void>}
*/
abstract compactLsm(): Promise<void>;
/**
* Converge this table's LSM write path into its base table.
*
* Seals once, then triggers compaction and polls until the L0 that existed
* at the start is gone. The target set is fixed at the start, so
* generations created *during* the checkpoint are ignored — that is what
* lets it terminate under write load, and what makes it best-effort: it
* converges the fresh tier as of some instant. Idempotent, abandonable at
* any point, and safe to run on a cadence.
*
* There is no liveness bound — the compactor pool is shared across tables,
* so a checkpoint queued behind unrelated work looks exactly like one that
* is merging. The caller owns the deadline.
* @returns {Promise<void>}
* @example
* ```ts
* const before = await table.getLsmStats();
* await table.checkpointLsm();
* const after = await table.getLsmStats();
* ```
*/
abstract checkpointLsm(): Promise<void>;
/**
* Read live per-bucket LSM state.
*
* Answers "how far behind is my fresh tier", "which bucket is hot", and
* "why is my fresh-tier vector search brute-force". Mutates no table state.
*
* Resolves to `undefined` only when the LSM write path is not enabled.
* @param {boolean} includeGenerationRows Also count rows per L0 generation.
* Off by default because each count opens an uncached Lance dataset.
* @returns {Promise<LsmStats | undefined>}
*/
abstract getLsmStats(
includeGenerationRows?: boolean,
): Promise<LsmStats | undefined>;
/** Retrieve the version of the table */
abstract version(): Promise<number>;
@@ -1078,8 +1206,22 @@ export class LocalTable extends Table {
// TODO: Support BatchUDF
async addColumns(
newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
newColumnTransforms:
| AddColumnsSql[]
| Field
| Field[]
| Schema
| { computed: AddColumnsSql[] },
): Promise<AddColumnsResult> {
// Columns defined by an expression are declared, not materialized here.
if (
typeof newColumnTransforms === "object" &&
!Array.isArray(newColumnTransforms) &&
"computed" in newColumnTransforms
) {
return await this.inner.addComputedColumns(newColumnTransforms.computed);
}
// Handle single Field -> convert to array of Fields
if (newColumnTransforms instanceof Field) {
newColumnTransforms = [newColumnTransforms];
@@ -1114,6 +1256,14 @@ export class LocalTable extends Table {
throw new Error("Invalid input type for addColumns");
}
async refreshColumn(column: string): Promise<RefreshColumnResult> {
return await this.inner.refreshColumn(column);
}
async refreshColumnAsync(column: string): Promise<Job> {
return await this.inner.refreshColumnAsync(column);
}
async alterColumns(
columnAlterations: ColumnAlteration[],
): Promise<AlterColumnsResult> {
@@ -1176,6 +1326,24 @@ export class LocalTable extends Table {
return await this.inner.closeLsmWriters();
}
async flushLsm(): Promise<void> {
return await this.inner.flushLsm();
}
async compactLsm(): Promise<void> {
return await this.inner.compactLsm();
}
async checkpointLsm(): Promise<void> {
return await this.inner.checkpointLsm();
}
async getLsmStats(
includeGenerationRows: boolean = false,
): Promise<LsmStats | undefined> {
return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
}
async version(): Promise<number> {
return await this.inner.version();
}
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package-lock.json
Generated
+8 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"cpu": [
"x64",
"arm64"
@@ -55,7 +55,13 @@
"openai": "4.29.2"
},
"peerDependencies": {
"@types/node": ">=18",
"apache-arrow": ">=15.0.0 <=18.1.0"
},
"peerDependenciesMeta": {
"@types/node": {
"optional": true
}
}
},
"node_modules/@aws-crypto/crc32": {
nodejs/package.json
+7 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.37.1-beta.0",
"version": "0.38.0-beta.2",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
@@ -101,6 +101,12 @@
"openai": "4.29.2"
},
"peerDependencies": {
"@types/node": ">=18",
"apache-arrow": ">=15.0.0 <=18.1.0"
},
"peerDependenciesMeta": {
"@types/node": {
"optional": true
}
}
}
nodejs/src/connection.rs
+16
View File
@@ -334,6 +334,22 @@ impl Connection {
.default_error()
}
/// Start dropping a table and return its cleanup job.
#[napi(catch_unwind)]
pub async fn drop_table_async(
&self,
name: String,
namespace_path: Option<Vec<String>>,
) -> napi::Result<crate::job::Job> {
let ns = namespace_path.unwrap_or_default();
let job = self
.get_inner()?
.drop_table_async(&name, &ns)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
let ns = namespace_path.unwrap_or_default();
nodejs/src/table.rs
+210 -7
View File
@@ -347,6 +347,40 @@ impl Table {
Ok(res.into())
}
#[napi(catch_unwind)]
pub async fn add_computed_columns(
&self,
columns: Vec<AddColumnsSql>,
) -> napi::Result<AddColumnsResult> {
let table = self.inner_ref()?;
let mut builder = table.add_columns();
for column in columns {
builder = builder.computed(column.name, column.value_sql);
}
let res = builder.execute().await.default_error()?;
Ok(res.into())
}
#[napi(catch_unwind)]
pub async fn refresh_column(&self, column: String) -> napi::Result<RefreshColumnResult> {
let res = self
.inner_ref()?
.refresh_column(column)
.await
.default_error()?;
Ok(res.into())
}
#[napi(catch_unwind)]
pub async fn refresh_column_async(&self, column: String) -> napi::Result<crate::job::Job> {
let job = self
.inner_ref()?
.refresh_column_async(column)
.await
.default_error()?;
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn add_columns_with_schema(
&self,
@@ -463,6 +497,34 @@ impl Table {
self.inner_ref()?.close_lsm_writers().await.default_error()
}
#[napi(catch_unwind)]
pub async fn flush_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.flush_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn compact_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.compact_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
self.inner_ref()?.checkpoint_lsm().await.default_error()
}
#[napi(catch_unwind)]
pub async fn get_lsm_stats(
&self,
include_generation_rows: bool,
) -> napi::Result<Option<LsmStats>> {
let stats = self
.inner_ref()?
.get_lsm_stats(include_generation_rows)
.await
.default_error()?;
Ok(stats.map(LsmStats::from))
}
#[napi(catch_unwind)]
pub async fn version(&self) -> napi::Result<i64> {
self.inner_ref()?
@@ -772,7 +834,8 @@ pub struct LsmWriteSpec {
pub column: Option<String>,
/// Bucket variant: the number of buckets, in `[1, 1024]`.
pub num_buckets: Option<u32>,
/// Names of indexes the MemWAL should keep up to date during writes.
/// Indexes the MemWAL keeps up to date. Omitted resolves every
/// maintainable index on install; an empty array means none.
pub maintained_indexes: Option<Vec<String>>,
/// Default `ShardWriter` configuration recorded in the MemWAL index.
pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -782,7 +845,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
type Error = napi::Error;
fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
let maintained = value.maintained_indexes.unwrap_or_default();
let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
let spec = match value.spec_type.as_str() {
"bucket" => {
@@ -809,7 +871,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
}
};
Ok(spec
.with_maintained_indexes(maintained)
.with_maintained_indexes(value.maintained_indexes)
.with_writer_config_defaults(writer_config_defaults))
}
}
@@ -827,7 +889,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "bucket".to_string(),
column: Some(column),
num_buckets: Some(num_buckets),
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
Native::Identity {
@@ -838,7 +900,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "identity".to_string(),
column: Some(column),
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
Native::Unsharded {
@@ -848,13 +910,136 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
spec_type: "unsharded".to_string(),
column: None,
num_buckets: None,
maintained_indexes: Some(maintained_indexes),
maintained_indexes,
writer_config_defaults: Some(writer_config_defaults),
},
}
}
}
/// One flushed L0 generation.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct GenerationStats {
/// The generation number. Increases as memtables are sealed into L0.
pub generation: i64,
/// On-disk size of the generation.
pub bytes: i64,
/// Present only when `includeGenerationRows` was requested. Off by default
/// because each count opens an uncached Lance dataset.
pub rows: Option<i64>,
}
impl From<lancedb::table::GenerationStats> for GenerationStats {
fn from(g: lancedb::table::GenerationStats) -> Self {
Self {
generation: g.generation as i64,
bytes: g.bytes as i64,
rows: g.rows.map(|r| r as i64),
}
}
}
/// One in-memory memtable.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct MemtableStats {
/// The generation this memtable will become once sealed.
pub generation: i64,
/// Rows currently buffered.
pub rows: i64,
/// Estimated in-memory size.
pub bytes: i64,
/// Record batches currently buffered.
pub batches: i64,
/// Names of the indexes this memtable carries. An absent name is the whole
/// answer to "why is my fresh-tier search on that column brute-force".
pub indexes: Vec<String>,
}
impl From<lancedb::table::MemtableStats> for MemtableStats {
fn from(m: lancedb::table::MemtableStats) -> Self {
Self {
generation: m.generation as i64,
rows: m.rows as i64,
bytes: m.bytes as i64,
batches: m.batches as i64,
indexes: m.indexes,
}
}
}
/// Live state of one bucket. A table is N buckets on one node; flattening to a
/// single number hides the one hot bucket that is usually why someone opened
/// this endpoint.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct BucketStats {
/// The shard this bucket writes.
pub shard_id: String,
/// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
pub status: String,
/// Epoch of the writer that currently owns the shard.
pub writer_epoch: i64,
/// Version of the shard manifest these numbers were read from.
pub manifest_version: i64,
/// The generation the active memtable will become.
pub current_generation: i64,
/// WAL position replay resumes from.
pub replay_after_wal_entry_position: i64,
/// Highest WAL position the writer has seen. The difference against
/// `replayAfterWalEntryPosition` is the WAL lag.
pub wal_entry_position_last_seen: i64,
/// Flushed L0 generations not yet merged into the base table.
pub generations: Vec<GenerationStats>,
/// Whether a pass owns this bucket's compaction latch right now. Says *a*
/// driver is running, not *whose*, and the latch is held from dispatch —
/// including while the pass queues for a pod-wide compactor permit. Read it
/// as "do not pile on", never as "mine is progressing".
pub compacting: bool,
/// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
/// in-memory state is torn down.
pub memtables: Option<Vec<MemtableStats>>,
}
impl From<lancedb::table::BucketStats> for BucketStats {
fn from(b: lancedb::table::BucketStats) -> Self {
Self {
shard_id: b.shard_id,
status: b.status,
writer_epoch: b.writer_epoch as i64,
manifest_version: b.manifest_version as i64,
current_generation: b.current_generation as i64,
replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
generations: b.generations.into_iter().map(Into::into).collect(),
compacting: b.compacting,
memtables: b
.memtables
.map(|ms| ms.into_iter().map(Into::into).collect()),
}
}
}
/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
///
/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
/// the caller's to compute.
#[napi(object)]
#[derive(Clone, Debug)]
pub struct LsmStats {
/// One entry per bucket backing this table.
pub buckets: Vec<BucketStats>,
}
impl From<lancedb::table::LsmStats> for LsmStats {
fn from(stats: lancedb::table::LsmStats) -> Self {
Self {
buckets: stats.buckets.into_iter().map(Into::into).collect(),
}
}
}
/// Statistics about a compaction operation.
#[napi(object)]
#[derive(Clone, Debug)]
@@ -1043,7 +1228,10 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
#[napi(object)]
pub struct TableStatistics {
/// The total number of bytes in the table
/// The total size, in bytes, of the table's data files, index files, and
/// overlay files
///
/// Read from the manifest, so this excludes deletion files and manifests.
pub total_bytes: i64,
/// The number of rows in the table
@@ -1193,6 +1381,21 @@ pub struct AddColumnsResult {
pub version: i64,
}
#[napi(object)]
pub struct RefreshColumnResult {
pub rows_filled: i64,
pub version: i64,
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
Self {
rows_filled: value.rows_filled as i64,
version: value.version as i64,
}
}
}
impl From<lancedb::table::AddColumnsResult> for AddColumnsResult {
fn from(value: lancedb::table::AddColumnsResult) -> Self {
Self {