Merge remote-tracking branch 'refs/remotes/origin/main' into gatekeeper/fix-1557-1

# Conflicts:
#	nodejs/lancedb/query.ts
#	nodejs/lancedb/table.ts

nodejs/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.37.1-beta.0"
+version = "0.38.0-beta.10"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -16,12 +16,12 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
+arrow-buffer.workspace = true
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
 futures.workspace = true
-lancedb = { path = "../rust/lancedb", default-features = false }
+lancedb.workspace = true
 lance-namespace.workspace = true
 napi = { version = "3.8.3", default-features = false, features = [
     "napi9",
@@ -29,8 +29,8 @@ napi = { version = "3.8.3", default-features = false, features = [
     "chrono_date",
     "serde-json",
 ] }
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
-serde_json = "1"
+chrono.workspace = true
+serde_json.workspace = true
 napi-derive = "3.5.2"
 # Prevent dynamic linking of lzma, which comes from datafusion
 lzma-sys = { version = "0.1", features = ["static"] }

nodejs/__test__/arrow.test.ts

@@ -6,7 +6,9 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
+  Vector as CurrentVector,
   convertToTable,
+  tableFromIPC as currentTableFromIPC,
   fromBufferToRecordBatch,
   fromDataToBuffer,
   fromRecordBatchToBuffer,
@@ -19,6 +21,7 @@ import {
   FunctionOptions,
 } from "../lancedb/embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry";
+import { sanitizeTable } from "../lancedb/sanitize";
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 function sampleRecords(): Array<Record<string, any>> {
@@ -64,7 +67,11 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       tableFromIPC,
       DataType,
       Dictionary,
+      RecordBatch: ArrowRecordBatch,
+      Table: ArrowTable,
       Uint8: ArrowUint8,
+      makeData: arrowMakeData,
+      vectorFromArray,
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
     } = <any>arrow;
     type Schema = ApacheArrow["Schema"];
@@ -166,6 +173,36 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     }
 
     describe("The function makeArrowTable", function () {
+      it("accepts snake_case embedding metadata like camelCase", function () {
+        const spellings = [
+          // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+          { source_column: "text", vector_column: "vector" },
+          { sourceColumn: "text", vectorColumn: "vector" },
+        ];
+        for (const columns of spellings) {
+          const schema = new Schema(
+            [
+              new Field("text", new Utf8(), false),
+              new Field(
+                "vector",
+                new FixedSizeList(3, new Field("item", new Float32(), true)),
+                false,
+              ),
+            ],
+            new Map([
+              [
+                "embedding_functions",
+                JSON.stringify([{ name: "mock", model: {}, ...columns }]),
+              ],
+            ]),
+          );
+          // The vector field is non-nullable and absent from the data; only a
+          // recognized embedding config makes that acceptable.
+          const table = makeArrowTable([{ text: "hello" }], { schema });
+          expect(table.numRows).toBe(1);
+        }
+      });
+
       it("will use data types from a provided schema instead of inference", async function () {
         const schema = new Schema([
           new Field("a", new Int32(), false),
@@ -197,6 +234,35 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         expect(table.getChild("d")?.toJSON()).toEqual([9n, 10n, null]);
       });
 
+      it("will use a provided FixedSizeList schema with typed array values", function () {
+        const schema = new Schema([
+          new Field("text", new Utf8(), false),
+          new Field(
+            "vector",
+            new FixedSizeList(3, new Field("item", new Float32(), false)),
+            false,
+          ),
+        ]);
+
+        const table = makeArrowTable(
+          [
+            {
+              text: "foo",
+              vector: new Float32Array([1, 2, 3]),
+            },
+          ],
+          { schema },
+        );
+
+        expect(table.getChild("text")?.toJSON()).toEqual(["foo"]);
+        expect(
+          table
+            .getChild("vector")
+            ?.toJSON()
+            .map((value) => value.toJSON()),
+        ).toEqual([[1, 2, 3]]);
+      });
+
       it("will assume the column `vector` is FixedSizeList<Float32> by default", async function () {
         const schema = new Schema([
           new Field("a", new Float(Precision.DOUBLE), true),
@@ -449,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         );
       });
 
+      it("will allow matching inferred types across records", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: 2 }]),
+        ).not.toThrow();
+      });
+
+      it("will reject mismatched inferred types across records", function () {
+        expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.",
+        );
+      });
+
+      it("will ignore generated dictionary IDs when comparing inferred types", function () {
+        const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
+          dictionaryEncodeStrings: true,
+        });
+
+        expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]);
+      });
+
+      it("will preserve null values without treating them as type mismatches", function () {
+        for (const records of [
+          [{ vector: [1, 2, 3] }, { vector: null }],
+          [{ vector: null }, { vector: [1, 2, 3] }],
+        ]) {
+          const table = makeArrowTable(records);
+
+          expect(table.numRows).toBe(2);
+          expect(table.getChild("vector")?.nullCount).toBe(1);
+        }
+      });
+
+      it("will preserve empty variable-size lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [] }],
+          [{ items: [] }, { items: [1] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+      });
+
+      it("will propagate deferred evidence through nested lists", function () {
+        for (const records of [
+          [{ items: [1] }, { items: [null] }],
+          [{ items: [null] }, { items: [1] }],
+          [{ items: [null, 1] }, { items: [2, null] }],
+        ]) {
+          const table = makeArrowTable(records);
+          expect(
+            table
+              .getChild("items")
+              ?.toJSON()
+              .map((value) => value.toJSON()),
+          ).toEqual(records.map((record) => record.items));
+        }
+
+        const nestedRecords = [{ items: [[1]] }, { items: [[null]] }];
+        const nestedTable = makeArrowTable(nestedRecords);
+        expect(
+          nestedTable
+            .getChild("items")
+            ?.toJSON()
+            .map((value) =>
+              value
+                .toJSON()
+                .map((nestedValue: { toJSON: () => unknown[] }) =>
+                  nestedValue.toJSON(),
+                ),
+            ),
+        ).toEqual(nestedRecords.map((record) => record.items));
+      });
+
+      it("will reject incompatible deferred evidence within a list", function () {
+        for (const items of [
+          [[], 1],
+          [1, []],
+          [[null], 1],
+          [1, [null]],
+        ]) {
+          expect(() => makeArrowTable([{ items }])).toThrow(
+            "Failed to infer data type for field items at row 0.",
+          );
+        }
+      });
+
+      it("will reject empty fixed-size lists", function () {
+        expect(() =>
+          makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type FixedSizeList[3]<Float32> but found List[0] for field vector at row 1.",
+        );
+      });
+
+      it("will reject inferred leaf and branch shape changes", function () {
+        expect(() =>
+          makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.",
+        );
+        expect(() =>
+          makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]),
+        ).toThrow(
+          "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.",
+        );
+      });
+
+      it("will allow null values around inferred struct values", function () {
+        for (const { records, nullIndex } of [
+          {
+            records: [{ value: null }, { value: { nested: 2 } }],
+            nullIndex: 0,
+          },
+          {
+            records: [{ value: { nested: 1 } }, { value: null }],
+            nullIndex: 1,
+          },
+        ]) {
+          const table = makeArrowTable(records);
+          const values = table.getChild("value");
+
+          expect(values?.nullCount).toBe(1);
+          expect(values?.get(nullIndex)).toBeNull();
+        }
+      });
+
       it("will allow a schema to be provided", async function () {
         await checkTableCreation(
           async (records, _, schema) =>
@@ -1025,6 +1222,114 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     });
 
     describe("when using two versions of arrow", function () {
+      it("preserves a dictionary shared by multiple fields", async function () {
+        const values = ["alpha", "beta", "alpha"];
+        const dictionaryVector = vectorFromArray(values);
+        const batch = new ArrowRecordBatch({
+          first: dictionaryVector.data[0],
+          second: dictionaryVector.data[0],
+        });
+        const table = new ArrowTable([batch]);
+
+        const sanitized = sanitizeTable(table);
+        expect([...sanitized.getChild("first")!]).toEqual(values);
+        expect([...sanitized.getChild("second")!]).toEqual(values);
+        const firstType = sanitized.schema.fields[0].type as {
+          dictionary: unknown;
+        };
+        const secondType = sanitized.schema.fields[1].type as {
+          dictionary: unknown;
+        };
+        expect(secondType.dictionary).toBe(firstType.dictionary);
+        expect(sanitized.batches[0].data.children[1].dictionary).toBe(
+          sanitized.batches[0].data.children[0].dictionary,
+        );
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("first")!]).toEqual(values);
+        expect([...actual.getChild("second")!]).toEqual(values);
+      });
+
+      it("preserves shared dictionary data from another Arrow version", async function () {
+        const values = ["alpha", "beta", "alpha"];
+        const dictionaryVector = vectorFromArray(values);
+        const firstBatch = new ArrowRecordBatch({
+          label: dictionaryVector.slice(0, 2).data[0],
+        });
+        const secondBatch = new ArrowRecordBatch({
+          label: dictionaryVector.slice(2).data[0],
+        });
+        const table = new ArrowTable([firstBatch, secondBatch]);
+
+        const sanitized = sanitizeTable(table);
+        expect([...sanitized.getChild("label")!]).toEqual(values);
+
+        const dictionaries = sanitized.batches.map(
+          (batch) => batch.data.children[0].dictionary,
+        );
+        expect(dictionaries[0]).toBeInstanceOf(CurrentVector);
+        expect(dictionaries[1]).toBe(dictionaries[0]);
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("label")!]).toEqual(values);
+      });
+
+      it("preserves shared chunks in growing dictionaries", async function () {
+        const type = new Dictionary(new Utf8(), new Int32(), 42, false);
+        const firstDictionary = vectorFromArray(["alpha", "beta"], new Utf8());
+        const secondDictionary = firstDictionary.concat(
+          vectorFromArray(["gamma"], new Utf8()),
+        );
+        const firstData = arrowMakeData({
+          type,
+          data: Int32Array.from([0, 1]),
+          dictionary: firstDictionary,
+        });
+        const secondData = arrowMakeData({
+          type,
+          data: Int32Array.from([2]),
+          dictionary: secondDictionary,
+        });
+        const table = new ArrowTable([
+          new ArrowRecordBatch({ label: firstData }),
+          new ArrowRecordBatch({ label: secondData }),
+        ]);
+
+        const sanitized = sanitizeTable(table);
+        const expected = ["alpha", "beta", "gamma"];
+        expect([...sanitized.getChild("label")!]).toEqual(expected);
+        const firstLocalDictionary =
+          sanitized.batches[0].data.children[0].dictionary!;
+        const secondLocalDictionary =
+          sanitized.batches[1].data.children[0].dictionary!;
+        expect(secondLocalDictionary.data[0]).toBe(
+          firstLocalDictionary.data[0],
+        );
+
+        const buf = await fromTableToBuffer(sanitized);
+        const actual = currentTableFromIPC(buf);
+        expect([...actual.getChild("label")!]).toEqual(expected);
+      });
+
+      it("can serialize list data from another Arrow version", async function () {
+        const values = [["anime", "action"], [], null];
+        const vector = vectorFromArray(
+          values,
+          new List(new Field("item", new Utf8(), true)),
+        );
+        const table = new ArrowTable({ tags: vector });
+
+        const buf = await fromDataToBuffer(table);
+        const actual = currentTableFromIPC(buf);
+        const actualTags = actual.getChild("tags");
+
+        expect(actualTags?.get(0)?.toJSON()).toEqual(values[0]);
+        expect(actualTags?.get(1)?.toJSON()).toEqual(values[1]);
+        expect(actualTags?.get(2)).toBeNull();
+      });
+
       it("can still import data", async function () {
         const schema = new arrow15.Schema([
           new arrow15.Field("id", new arrow15.Int32()),

nodejs/__test__/connection.test.ts

@@ -4,7 +4,13 @@
 import { readdirSync } from "fs";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import { Connection, Table, connect, connectNamespace } from "../lancedb";
+import {
+  Connection,
+  ListTablesResponse,
+  Table,
+  connect,
+  connectNamespace,
+} from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
 describe("when connecting", () => {
@@ -47,6 +53,7 @@ describe("given a connection", () => {
     await db.close();
     expect(db.isOpen()).toBe(false);
     await expect(db.tableNames()).rejects.toThrow("Connection is closed");
+    await expect(db.listTables()).rejects.toThrow("Connection is closed");
     await expect(db.renameTable("a", "b")).rejects.toThrow(
       "Connection is closed",
     );
@@ -89,6 +96,16 @@ describe("given a connection", () => {
     await db.createTable("test4", [{ id: 1 }, { id: 2 }]);
   });
 
+  it("should return a completed job when dropping a local table", async () => {
+    await db.createTable("async-drop", [{ id: 1 }]);
+
+    const job = await db.dropTableAsync("async-drop");
+    expect(job.id).toBeNull();
+    await expect(job.status()).resolves.toBe("finished");
+    await job.wait();
+    await expect(db.tableNames()).resolves.toEqual([]);
+  });
+
   it("should fail if creating table twice, unless overwrite is true", async () => {
     let tbl = await db.createTable("test", [{ id: 1 }, { id: 2 }]);
     await expect(tbl.countRows()).resolves.toBe(2);
@@ -119,6 +136,66 @@ describe("given a connection", () => {
     expect(tables).toEqual(["b", "c"]);
   });
 
+  it("should respect limit and page token when listing tables", async () => {
+    const db = await connect(tmpDir.name);
+
+    await db.createTable("b", [{ id: 1 }]);
+    await db.createTable("a", [{ id: 1 }]);
+    await db.createTable("c", [{ id: 1 }]);
+
+    const all = await db.listTables();
+    expect(all.tables).toEqual(["a", "b", "c"]);
+    expect(all.pageToken).toBeUndefined();
+
+    const first = await db.listTables({ limit: 1 });
+    expect(first.tables).toEqual(["a"]);
+    expect(first.pageToken).toBeDefined();
+
+    const second = await db.listTables({
+      limit: 1,
+      pageToken: first.pageToken,
+    });
+    expect(second.tables).toEqual(["b"]);
+  });
+
+  it("should visit every table exactly once when walking pages", async () => {
+    const db = await connect(tmpDir.name);
+
+    const created = ["a", "b", "c", "d", "e"];
+    for (const name of created) {
+      await db.createTable(name, [{ id: 1 }]);
+    }
+
+    const seen: string[] = [];
+    let pageToken: string | undefined = undefined;
+    do {
+      const page: ListTablesResponse = await db.listTables({
+        limit: 2,
+        pageToken,
+      });
+      seen.push(...page.tables);
+      pageToken = page.pageToken;
+    } while (pageToken);
+
+    expect(seen).toEqual(created);
+  });
+
+  it("should list tables in a namespace", async () => {
+    const db = await connect(tmpDir.name, {
+      // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust
+      namespaceClientProperties: { manifest_enabled: "true" },
+    });
+    await db.createNamespace(["child"]);
+    await db.createTable("nested", [{ id: 1 }], ["child"]);
+
+    await expect(db.listTables(["child"])).resolves.toEqual(
+      expect.objectContaining({ tables: ["nested"] }),
+    );
+    await expect(db.listTables()).resolves.toEqual(
+      expect.objectContaining({ tables: [] }),
+    );
+  });
+
   it("should create tables in v2 mode", async () => {
     const db = await connect(tmpDir.name);
     const data = [...Array(10000).keys()].map((i) => ({ id: i }));

nodejs/__test__/embedding.test.ts

@@ -11,8 +11,11 @@ import {
   Float16,
   Float32,
   Float64,
+  Int32,
   Schema,
   Utf8,
+  fromDataToBuffer,
+  tableFromIPC,
 } from "../lancedb/arrow";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
@@ -184,6 +187,63 @@ describe("embedding functions", () => {
     const vector0 = JSON.parse(JSON.stringify(arr[0].vector));
     expect(vector0).toEqual([1, 2, 3]);
   });
+
+  it("should append generated vectors to a non-nullable schema", async () => {
+    @register("non_nullable_schema_test")
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float64();
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+
+    const schema = new Schema([
+      new Field("id", new Int32()),
+      new Field("text", new Utf8()),
+      new Field("type", new Utf8()),
+      new Field(
+        "vector",
+        new FixedSizeList(3, new Field("item", new Float64())),
+      ),
+    ]);
+    const func = new MockEmbeddingFunction();
+    const db = await connect(tmpDir.name);
+    const table = await db.createEmptyTable("test_non_nullable", schema, {
+      embeddingFunction: {
+        function: func,
+        sourceColumn: "text",
+      },
+    });
+
+    const data = [
+      { id: 1, text: "Carrot", type: "vegetable" },
+      { id: 2, text: "Apple", type: "fruit" },
+    ];
+    const buffer = await fromDataToBuffer(
+      data,
+      undefined,
+      await table.schema(),
+    );
+    const generatedTable = tableFromIPC(buffer);
+    const vectorField = generatedTable.schema.fields.find(
+      (field) => field.name === "vector",
+    );
+    expect(vectorField?.nullable).toBe(false);
+
+    await table.add(data);
+
+    const rows = await table.query().toArray();
+    expect(rows).toHaveLength(2);
+    for (const row of rows) {
+      expect([...row.vector]).toEqual([1, 2, 3]);
+    }
+  });
+
   it("should error when appending to a table with an unregistered embedding function", async () => {
     @register("mock")
     class MockEmbeddingFunction extends EmbeddingFunction<string> {
@@ -427,4 +487,52 @@ describe("embedding functions", () => {
       expect(stringSchema3).toEqual(stringExpectedSchema);
     },
   );
+  test("parses one function writing several vector columns", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType(): Float {
+        return new Float32();
+      }
+      async computeQueryEmbeddings(_data: string) {
+        return [1, 2, 3];
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return Array.from({ length: data.length }).fill([
+          1, 2, 3,
+        ]) as number[][];
+      }
+    }
+    const registry = getRegistry();
+    registry.register("multi_output_mock")(MockEmbeddingFunction);
+
+    // A materialized view can project one source vector column under two
+    // names, so a table's configuration names the same function twice.
+    const parsed = await registry.parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "multi_output_mock",
+              sourceColumn: "text",
+              vectorColumn: "vector_a",
+              model: {},
+            },
+            {
+              name: "multi_output_mock",
+              sourceColumn: "text",
+              vectorColumn: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+
+    expect(
+      [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
+    ).toEqual(["vector_a", "vector_b"]);
+  });
 });

nodejs/__test__/embedding_registry.test.ts

@@ -0,0 +1,95 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { execFileSync } from "node:child_process";
+import { resolve } from "node:path";
+
+import type { OpenAIEmbeddingFunction } from "../lancedb/embedding/openai";
+import type { EmbeddingFunctionRegistry } from "../lancedb/embedding/registry";
+
+type EmbeddingModule = typeof import("../lancedb/embedding");
+type OpenAIModule = typeof import("../lancedb/embedding/openai");
+type RegistryModule = typeof import("../lancedb/embedding/registry");
+
+describe("embedding function registry", () => {
+  const registries: EmbeddingFunctionRegistry[] = [];
+
+  afterEach(() => {
+    for (const registry of registries) {
+      registry.reset();
+    }
+    registries.length = 0;
+  });
+
+  it("defers built-in providers until the public registry API is used", () => {
+    jest.isolateModules(() => {
+      const embedding = require("../lancedb/embedding") as EmbeddingModule;
+      const { getRegistry: getInternalRegistry } =
+        require("../lancedb/embedding/registry") as RegistryModule;
+      const registry = getInternalRegistry();
+      registries.push(registry);
+
+      expect(registry.length()).toBe(0);
+      expect(embedding.getRegistry()).toBe(registry);
+      expect(registry.get("openai")).toBeDefined();
+      expect(registry.get("huggingface")).toBeDefined();
+    });
+  });
+
+  it("preserves automatic FTS search in a fresh process", () => {
+    execFileSync(
+      process.execPath,
+      [resolve(__dirname, "fixtures", "auto_fts_search.cjs")],
+      { stdio: "pipe" },
+    );
+  });
+
+  it("shares registrations across duplicated provider module graphs", () => {
+    let registeringRegistry: EmbeddingFunctionRegistry | undefined;
+    let latestOpenAIConstructor: typeof OpenAIEmbeddingFunction | undefined;
+
+    jest.isolateModules(() => {
+      require("../lancedb/embedding/openai");
+      const { getRegistry } =
+        require("../lancedb/embedding/registry") as RegistryModule;
+      registeringRegistry = getRegistry();
+      registries.push(registeringRegistry);
+      expect(registeringRegistry.get("openai")).toBeDefined();
+    });
+
+    expect(() => {
+      jest.isolateModules(() => {
+        const { OpenAIEmbeddingFunction } =
+          require("../lancedb/embedding/openai") as OpenAIModule;
+        latestOpenAIConstructor = OpenAIEmbeddingFunction;
+        const { getRegistry } =
+          require("../lancedb/embedding/registry") as RegistryModule;
+        registries.push(getRegistry());
+      });
+    }).not.toThrow();
+
+    const previousApiKey = process.env.OPENAI_API_KEY;
+    process.env.OPENAI_API_KEY = "test";
+    try {
+      const latestOpenAI = registeringRegistry!
+        .get<OpenAIEmbeddingFunction>("openai")!
+        .create();
+      expect(latestOpenAI).toBeInstanceOf(latestOpenAIConstructor!);
+    } finally {
+      if (previousApiKey === undefined) {
+        delete process.env.OPENAI_API_KEY;
+      } else {
+        process.env.OPENAI_API_KEY = previousApiKey;
+      }
+    }
+
+    jest.isolateModules(() => {
+      const { getRegistry } =
+        require("../lancedb/embedding") as EmbeddingModule;
+      const publicRegistry = getRegistry();
+      registries.push(publicRegistry);
+      expect(publicRegistry).toBe(registeringRegistry);
+      expect(publicRegistry.get("openai")).toBeDefined();
+    });
+  });
+});

nodejs/__test__/fixtures/auto_fts_search.cjs

@@ -0,0 +1,33 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+const assert = require("node:assert/strict");
+const tmp = require("tmp");
+const { connect, embedding, Index } = require("../../dist");
+const { getRegistry } = require("../../dist/embedding/registry");
+
+async function main() {
+  assert.equal(typeof embedding.getRegistry, "function");
+  assert.equal(getRegistry().length(), 0);
+  assert.equal(embedding.getRegistry(), getRegistry());
+  assert.equal(getRegistry().length(), 2);
+
+  const dir = tmp.dirSync({ unsafeCleanup: true });
+  let db;
+  try {
+    db = await connect(dir.name);
+    const table = await db.createTable("docs", [{ text: "hello world" }]);
+    await table.createIndex("text", { config: Index.fts() });
+
+    const rows = await table.search("hello").toArray();
+    assert.equal(rows[0].text, "hello world");
+  } finally {
+    db?.close();
+    dir.removeCallback();
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

nodejs/__test__/materialized_view.test.ts

@@ -0,0 +1,147 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import * as tmp from "tmp";
+
+import { Connection, connect } from "../lancedb";
+import {
+  DEFINITION_META_KEY,
+  definitionFromMetadata,
+} from "../lancedb/materialized_view";
+
+describe("materialized views", () => {
+  let tmpDir: tmp.DirResult;
+  let db: Connection;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    db = await connect(tmpDir.name);
+    await db.createTable(
+      "people",
+      [
+        { name: "ada", age: 36 },
+        { name: "kid", age: 7 },
+        { name: "grace", age: 85 },
+      ],
+      { storageOptions: { newTableEnableStableRowIds: "true" } },
+    );
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("rejects a stored limit a number cannot carry", () => {
+    const big = new Map([
+      [
+        DEFINITION_META_KEY,
+        '{"kind":"select","source_table":"people","limit":9007199254740993}',
+      ],
+    ]);
+    expect(() => definitionFromMetadata(big, "v")).toThrow(
+      /too large to represent exactly/,
+    );
+
+    const safe = new Map([
+      [
+        DEFINITION_META_KEY,
+        '{"kind":"select","source_table":"people","limit":42}',
+      ],
+    ]);
+    expect(definitionFromMetadata(safe, "v").limit).toBe(42);
+  });
+
+  it("creates, refreshes and queries a view", async () => {
+    const view = await db.createMaterializedView("adults", "people", {
+      select: ["name", ["shout", "upper(name)"]],
+      where: "age >= 18",
+    });
+    expect(view.name).toBe("adults");
+    expect(await view.table().countRows()).toBe(0);
+
+    const result = await view.refresh();
+    expect(result.mode).toBe("rebuild");
+    expect(Number(result.rowsWritten)).toBe(2);
+
+    const rows = await view.table().query().toArray();
+    expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
+  });
+
+  it("round-trips the definition", async () => {
+    await db.createMaterializedView("adults", "people", {
+      where: "age >= 18",
+    });
+    const view = await db.openMaterializedView("adults");
+    const definition = await view.definition();
+    expect(definition.sourceTable).toBe("people");
+    expect(definition.filter).toBe("age >= 18");
+    expect(definition.projections).toEqual([
+      ["name", "`name`"],
+      ["age", "`age`"],
+    ]);
+    expect(definition.inputs).toEqual(["age", "name"]);
+  });
+
+  it("refreshes incrementally after an append", async () => {
+    const view = await db.createMaterializedView("copy", "people");
+    await view.refresh();
+
+    const people = await db.openTable("people");
+    await people.add([{ name: "alan", age: 41 }]);
+    const result = await view.refresh();
+    expect(result.mode).toBe("incremental");
+    expect(Number(result.rowsWritten)).toBe(1);
+    expect(await view.table().countRows()).toBe(4);
+
+    expect((await view.refresh()).mode).toBe("no_op");
+  });
+
+  it("lists views and rejects non-views", async () => {
+    await db.createMaterializedView("adults", "people", {
+      where: "age >= 18",
+    });
+    expect(await db.listMaterializedViews()).toEqual(["adults"]);
+    await expect(db.openMaterializedView("people")).rejects.toThrow(
+      "not a materialized view",
+    );
+  });
+
+  it("rejects an invalid expression at create time", async () => {
+    await expect(
+      db.createMaterializedView("bad", "people", {
+        select: [["x", "missing + 1"]],
+      }),
+    ).rejects.toThrow("missing");
+  });
+
+  it("rejects invalid numeric options before creating anything", async () => {
+    for (const limit of [-5, 1.5, Infinity, NaN]) {
+      await expect(
+        db.createMaterializedView("bad", "people", { limit }),
+      ).rejects.toThrow("non-negative integer");
+    }
+    expect(await db.listMaterializedViews()).toEqual([]);
+
+    const view = await db.createMaterializedView("copy", "people");
+    for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
+      await expect(view.refresh({ sourceVersion })).rejects.toThrow(
+        "non-negative integer",
+      );
+    }
+  });
+
+  it("quotes bare select names", async () => {
+    await db.createTable("odd_names", [{ "order item": "widget" }], {
+      storageOptions: { newTableEnableStableRowIds: "true" },
+    });
+    const view = await db.createMaterializedView("quoted", "odd_names", {
+      select: ["order item"],
+    });
+    const result = await view.refresh();
+    expect(Number(result.rowsWritten)).toBe(1);
+  });
+
+  it("requires stable row ids on the source", async () => {
+    await db.createTable("plain", [{ x: 1 }]);
+    await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
+      "stable row ids",
+    );
+  });
+});

nodejs/__test__/package.test.ts

@@ -0,0 +1,14 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import packageJson = require("../package.json");
+
+describe("package metadata", () => {
+  it("requires Node.js type declarations compatible with the runtime", () => {
+    expect(packageJson.engines.node).toBe(">= 18");
+    expect(packageJson.peerDependencies["@types/node"]).toBe(">=18");
+    expect(packageJson.peerDependenciesMeta["@types/node"]).toEqual({
+      optional: true,
+    });
+  });
+});

nodejs/__test__/query.test.ts

@@ -110,6 +110,81 @@ describe("Query outputSchema", () => {
   });
 });
 
+describe("Search pagination", () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), false),
+      new Field("text", new Utf8(), false),
+      new Field(
+        "vector",
+        new FixedSizeList(2, new Field("item", new Float32())),
+        false,
+      ),
+    ]);
+    const data = makeArrowTable(
+      [
+        { id: 1n, text: "common", vector: [0, 0] },
+        { id: 2n, text: "common common", vector: [1, 1] },
+        { id: 3n, text: "common common common", vector: [2, 2] },
+        { id: 4n, text: "common common common common", vector: [3, 3] },
+      ],
+      { schema },
+    );
+    table = await db.createTable("test", data);
+  });
+
+  afterEach(() => {
+    tmpDir.removeCallback();
+  });
+
+  it("applies offset after the vector search limit", async () => {
+    const allResults = await table
+      .vectorSearch([0, 0])
+      .select(["id"])
+      .limit(4)
+      .toArray();
+    const secondPage = await table
+      .vectorSearch([0, 0])
+      .select(["id"])
+      .limit(2)
+      .offset(2)
+      .toArray();
+
+    expect(allResults).toHaveLength(4);
+    expect(secondPage).toHaveLength(2);
+    expect(secondPage.map((row) => row.id)).toEqual(
+      allResults.slice(2, 4).map((row) => row.id),
+    );
+  });
+
+  it("applies offset after the full-text search limit", async () => {
+    await table.createIndex("text", { config: Index.fts() });
+
+    const allResults = await table
+      .search("common", "fts")
+      .select(["id"])
+      .limit(4)
+      .toArray();
+    const secondPage = await table
+      .search("common", "fts")
+      .select(["id"])
+      .limit(2)
+      .offset(2)
+      .toArray();
+
+    expect(allResults).toHaveLength(4);
+    expect(secondPage).toHaveLength(2);
+    expect(secondPage.map((row) => row.id)).toEqual(
+      allResults.slice(2, 4).map((row) => row.id),
+    );
+  });
+});
+
 describe("Query orderBy", () => {
   let tmpDir: tmp.DirResult;
   let table: Table;

nodejs/__test__/registry.test.ts

@@ -106,6 +106,77 @@ describe.each([arrow15, arrow16, arrow17, arrow18])("Registry", (arrow) => {
       'Embedding function with alias "mock-embedding" already exists',
     );
   });
+  test("parseFunctions keeps entries sharing a function name", async () => {
+    class MockEmbeddingFunction extends EmbeddingFunction<string> {
+      ndims() {
+        return 3;
+      }
+      embeddingDataType() {
+        return new arrow.Float32() as apiArrow.Float;
+      }
+      async computeSourceEmbeddings(data: string[]) {
+        return data.map(() => [1, 2, 3]);
+      }
+    }
+    register("mock-embedding")(MockEmbeddingFunction);
+    const parsed = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              sourceColumn: "text",
+              vectorColumn: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...parsed.values()].map((f) => f.vectorColumn)).toEqual([
+      "vector_a",
+      "vector_b",
+    ]);
+
+    // The Python bindings write snake_case keys.
+    const snake = await getRegistry().parseFunctions(
+      new Map([
+        [
+          "embedding_functions",
+          JSON.stringify([
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_a",
+              model: {},
+            },
+            {
+              name: "mock-embedding",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              source_column: "text",
+              // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+              vector_column: "vector_b",
+              model: {},
+            },
+          ]),
+        ],
+      ]),
+    );
+    expect([...snake.keys()]).toEqual(["vector_a", "vector_b"]);
+    expect([...snake.values()].map((f) => f.sourceColumn)).toEqual([
+      "text",
+      "text",
+    ]);
+  });
   test("schema should contain correct metadata", async () => {
     class MockEmbeddingFunction extends EmbeddingFunction<string> {
       constructor(args: FunctionOptions = {}) {

nodejs/__test__/remote.test.ts

@@ -75,6 +75,25 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it("refuses materialized views before issuing any request", async () => {
+    const paths: string[] = [];
+    await withMockDatabase(
+      (req, res) => {
+        paths.push(req.url ?? "");
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
+          /only on local databases/,
+        );
+        await expect(db.listMaterializedViews()).rejects.toThrow(
+          /only on local databases/,
+        );
+        expect(paths).toEqual([]);
+      },
+    );
+  });
+
   it("should accept partial connection options", async () => {
     await connect("db://test", {
       apiKey: "fake",
@@ -170,6 +189,38 @@ describe("remote connection", () => {
     );
   });
 
+  it("surfaces JSON server errors from remote table operations", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        const path = req.url ?? "";
+        if (path.endsWith("/describe/")) {
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "broken_table",
+              version: 1,
+              schema: { fields: [] },
+            }),
+          );
+          return;
+        }
+
+        if (path.endsWith("/count_rows/")) {
+          res
+            .writeHead(400, { "Content-Type": "application/json" })
+            .end(JSON.stringify({ error: "count rows failed" }));
+          return;
+        }
+
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("broken_table");
+
+        await expect(table.countRows()).rejects.toThrow("count rows failed");
+      },
+    );
+  });
+
   it("should pass on requested extra headers", async () => {
     await withMockDatabase(
       (req, res) => {
@@ -279,7 +330,7 @@ describe("remote connection", () => {
     expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
   });
 
-  it("diffs and merges remote branches", async () => {
+  it("diffs and cherry-picks remote branches", async () => {
     const sampleDiff = {
       fromBranch: "exp",
       parentVersion: 1,
@@ -301,10 +352,9 @@ describe("remote connection", () => {
       changedColumns: [],
       addedIndexes: [],
       removedIndexes: [],
-      mergeable: true,
-      mergeBlockers: [],
+      errors: [],
     };
-    const mergeBodies: Record<string, unknown>[] = [];
+    const cherryPickBodies: Record<string, unknown>[] = [];
 
     await withMockDatabase(
       (req, res) => {
@@ -334,17 +384,16 @@ describe("remote connection", () => {
               .end(JSON.stringify(sampleDiff));
             return;
           }
-          if (path.endsWith("/branches/merge/")) {
-            mergeBodies.push(body);
+          if (path.endsWith("/branches/cherry_pick/")) {
+            cherryPickBodies.push(body);
             const dryRun = body["dry_run"] === true;
             const response = {
-              status: dryRun ? "ready" : "rejected",
+              status: dryRun ? "ready" : "failed",
               diff: dryRun
                 ? sampleDiff
                 : {
                     ...sampleDiff,
-                    mergeable: false,
-                    mergeBlockers: [
+                    errors: [
                       { code: "baseMoved", message: "main has advanced" },
                     ],
                   },
@@ -366,19 +415,19 @@ describe("remote connection", () => {
 
         await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
 
-        const rejected = await branches.merge("exp");
-        expect(rejected.status).toBe("rejected");
-        expect(rejected.diff.mergeBlockers).toEqual([
+        const failed = await branches.cherryPick("exp");
+        expect(failed.status).toBe("failed");
+        expect(failed.diff.errors).toEqual([
           { code: "baseMoved", message: "main has advanced" },
         ]);
 
-        const preview = await branches.merge("exp", true);
+        const preview = await branches.cherryPick("exp", true);
         expect(preview.status).toBe("ready");
         expect(preview.preview.promotedColumns).toEqual(["tag"]);
       },
     );
 
-    expect(mergeBodies).toEqual([
+    expect(cherryPickBodies).toEqual([
       // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
       { from_branch: "exp", dry_run: false },
       // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format

nodejs/__test__/table.test.ts

@@ -11,10 +11,13 @@ import * as arrow17 from "apache-arrow-17";
 import * as arrow18 from "apache-arrow-18";
 
 import {
+  AutoQuery,
   Connection,
   MatchQuery,
   PhraseQuery,
+  Query,
   Table,
+  VectorQuery,
   connect,
   tokenize,
 } from "../lancedb";
@@ -47,7 +50,6 @@ import {
   BooleanQuery,
   Occur,
   Operator,
-  VectorQuery,
   instanceOfFullTextQuery,
 } from "../lancedb/query";
 
@@ -87,6 +89,44 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       await expect(table.countRows()).resolves.toBe(3);
     });
 
+    it("should support a foreign Float64 vector schema end to end", async () => {
+      const conn = await connect(tmpDir.name);
+      const schema = new arrow.Schema([
+        new arrow.Field("resource_id", new arrow.Int32(), false),
+        new arrow.Field(
+          "vector",
+          new arrow.FixedSizeList(
+            3,
+            new arrow.Field("value", new arrow.Float64(), true),
+          ),
+          false,
+        ),
+      ]);
+      const data = [
+        {
+          // biome-ignore lint/style/useNamingConvention: matches the reported schema
+          resource_id: 0,
+          vector: [0.1, 0.1, 0.1],
+        },
+      ];
+
+      const resources = await conn.createTable("resources", data, { schema });
+
+      const existing = await resources
+        .query()
+        .where("resource_id = 0")
+        .limit(1)
+        .toArray();
+      expect(existing).toHaveLength(1);
+
+      const matched = await resources
+        .search(Float64Array.from(data[0].vector))
+        .limit(1)
+        .toArray();
+      expect(matched).toHaveLength(1);
+      expect(matched[0]["resource_id"]).toBe(0);
+    });
+
     it("should support branches", async () => {
       await table.add([{ id: 1 }]);
       expect(await table.countRows()).toBe(1);
@@ -240,8 +280,16 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         },
         numIndices: 0,
         numRows: 3,
-        totalBytes: 44,
+        // Full on-disk size of the two data files, footers and metadata included.
+        totalBytes: 684,
       });
+
+      // Index files count toward totalBytes too (only deletion files and
+      // manifests are excluded).
+      await table.createIndex("id", { config: Index.btree() });
+      const statsWithIndex = await table.stats();
+      expect(statsWithIndex.numIndices).toBe(1);
+      expect(statsWithIndex.totalBytes).toBeGreaterThan(684);
     });
 
     it("should overwrite data if asked", async () => {
@@ -1732,6 +1780,194 @@ describe("Read consistency interval", () => {
   });
 });
 
+describe("automatic search schema consistency", () => {
+  let tmpDir: tmp.DirResult;
+
+  class SchemaRefreshEmbedding extends EmbeddingFunction<string> {
+    ndims() {
+      return 2;
+    }
+
+    embeddingDataType() {
+      return new Float32();
+    }
+
+    async computeSourceEmbeddings(data: string[]) {
+      return data.map((value) => [value.length, 1]);
+    }
+
+    async computeQueryEmbeddings(value: string) {
+      return [value.length, 1];
+    }
+  }
+
+  function embeddingSchema() {
+    const func = new SchemaRefreshEmbedding();
+    return LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+  }
+
+  beforeEach(() => {
+    getRegistry().reset();
+    register("schema-refresh")(SchemaRefreshEmbedding);
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+
+  afterEach(() => {
+    getRegistry().reset();
+    tmpDir.removeCallback();
+  });
+
+  it("uses the schema refreshed from another connection", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const stale = await first.createTable("docs", [{ text: "before" }], {
+        schema: embeddingSchema(),
+      });
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "after hello" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+
+      const search = stale.search("hello");
+      expect(search).toBeInstanceOf(AutoQuery);
+      expect(search).not.toBeInstanceOf(Query);
+      expect(search).not.toBeInstanceOf(VectorQuery);
+      expect("nprobes" in search).toBe(false);
+
+      const rows = await search.toArray();
+      expect(rows[0].text).toBe("after hello");
+      expect((await stale.schema()).metadata.has("embedding_functions")).toBe(
+        false,
+      );
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+
+  it("tracks embedding metadata across checkout and restore", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      await first.createTable("docs", [{ text: "before" }], {
+        schema: embeddingSchema(),
+      });
+      const table = await second.createTable(
+        "docs",
+        [{ text: "after hello" }],
+        { mode: "overwrite" },
+      );
+      await table.createIndex("text", { config: Index.fts() });
+
+      await table.checkout(1);
+      expect((await table.search("before").toArray())[0].text).toBe("before");
+
+      await table.checkoutLatest();
+      expect((await table.search("hello").toArray())[0].text).toBe(
+        "after hello",
+      );
+
+      await table.checkout(1);
+      await table.restore();
+      expect((await table.search("before").toArray())[0].text).toBe("before");
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+
+  it("pins automatic search while computing an embedding", async () => {
+    let markStarted!: () => void;
+    let releaseEmbedding!: () => void;
+    const started = new Promise<void>((resolve) => {
+      markStarted = resolve;
+    });
+    const released = new Promise<void>((resolve) => {
+      releaseEmbedding = resolve;
+    });
+
+    class BlockingEmbedding extends SchemaRefreshEmbedding {
+      async computeQueryEmbeddings(value: string) {
+        markStarted();
+        await released;
+        return [value.length, 1];
+      }
+    }
+
+    register("schema-refresh-blocking")(BlockingEmbedding);
+    const func = new BlockingEmbedding();
+    const schema = LanceSchema({
+      text: func.sourceField(new Utf8()),
+      vector: func.vectorField(),
+    });
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const table = await first.createTable(
+        "docs",
+        [{ text: "hello before" }],
+        { schema },
+      );
+      const pending = table.search("hello").toArray();
+      await started;
+
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "hello after" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+      releaseEmbedding();
+
+      expect((await pending)[0].text).toBe("hello before");
+    } finally {
+      releaseEmbedding();
+      first.close();
+      second.close();
+    }
+  });
+
+  it("refreshes a reused automatic search for every execution", async () => {
+    const first = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+    const second = await connect(tmpDir.name, { readConsistencyInterval: 0 });
+
+    try {
+      const table = await first.createTable("docs", [
+        { text: "hello before", marker: "before" },
+      ]);
+      await table.createIndex("text", { config: Index.fts() });
+      const search = table.search("hello").select(["text"]);
+
+      const before = (await search.toArray())[0];
+      expect(before.text).toBe("hello before");
+      expect(before.marker).toBeUndefined();
+
+      const replacement = await second.createTable(
+        "docs",
+        [{ text: "hello after", marker: "after" }],
+        { mode: "overwrite" },
+      );
+      await replacement.createIndex("text", { config: Index.fts() });
+
+      const after = (await search.toArray())[0];
+      expect(after.text).toBe("hello after");
+      expect(after.marker).toBeUndefined();
+    } finally {
+      first.close();
+      second.close();
+    }
+  });
+});
+
 describe("schema evolution", function () {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -2393,10 +2629,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         snapshotCalls += 1;
         return await querySnapshot();
       };
-      const autoQuery = (tracked.search("greetings") as VectorQuery)
-        .nprobes(1)
-        .select(["text"])
-        .limit(1);
+      const autoQuery = tracked.search("greetings").select(["text"]).limit(1);
 
       const func = new TestEmbedding();
       const schema = LanceSchema({
@@ -2414,31 +2647,13 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(results[0].text).toBe(data[0].text);
       expect(initCalls).toBe(baselineInitCalls + 1);
       expect(queryCalls).toBe(1);
-      expect(snapshotCalls).toBe(2);
+      expect(snapshotCalls).toBe(1);
 
       const repeatedResults = await autoQuery.toArray();
       expect(repeatedResults[0].text).toBe(data[0].text);
       expect(initCalls).toBe(baselineInitCalls + 1);
       expect(queryCalls).toBe(1);
-      expect(snapshotCalls).toBe(3);
-
-      await expect(
-        (tracked.search("greetings") as VectorQuery)
-          .addQueryVector(Promise.reject(new Error("extra vector failed")))
-          .toArray(),
-      ).rejects.toThrow("extra vector failed");
-
-      const multiVectorResults = await (
-        tracked.search("greetings") as VectorQuery
-      )
-        .addQueryVector(Promise.resolve([0.2]))
-        .select(["text"])
-        .limit(1)
-        .toArray();
-      expect(multiVectorResults).toHaveLength(2);
-      expect(multiVectorResults.map((row) => row.text).sort()).toEqual(
-        data.map((row) => row.text).sort(),
-      );
+      expect(snapshotCalls).toBe(2);
 
       const pending = tracked
         .search("blocked")
@@ -2458,20 +2673,13 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       releaseEmbedding();
 
       const pendingResults = await pending;
-      expect(pendingResults[0].text).toBe(ftsData[1].text);
+      expect(pendingResults[0].text).toBe(data[1].text);
 
       expect(
         (await tracked.schema()).metadata.get("embedding_functions"),
       ).toBeUndefined();
       const ftsResults = await autoQuery.toArray();
       expect(ftsResults[0].text).toBe(ftsData[0].text);
-
-      const rejectedVector = Promise.reject(new Error("unused vector failed"));
-      const ftsWithRejectedVector = (
-        tracked.search("greetings") as VectorQuery
-      ).addQueryVector(rejectedVector);
-      await expect(ftsWithRejectedVector.toArray()).resolves.toBeDefined();
-      await new Promise<void>((resolve) => setImmediate(resolve));
     });
 
     test("auto search keeps newer preparation during a revision race", async () => {
@@ -3253,7 +3461,7 @@ describe("column name options", () => {
       .limit(10)
       .toArray();
     expect(results2.length).toBe(10);
-  });
+  }, 30_000);
 });
 
 describe("when creating an empty table", () => {
@@ -3640,3 +3848,120 @@ describe("LSM merge insert", () => {
     await expect(table.query().useLsm(true).toArray()).rejects.toThrow();
   });
 });
+
+describe("LSM convergence and stats", () => {
+  let tmpDir: tmp.DirResult;
+
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  async function lsmTable(conn: Connection): Promise<Table> {
+    const table = await conn.createEmptyTable(
+      "t",
+      new arrow.Schema([new arrow.Field("id", new arrow.Utf8(), false)]),
+    );
+    await table.setUnenforcedPrimaryKey("id");
+    await table.setLsmWriteSpec({ specType: "unsharded" });
+    return table;
+  }
+
+  // These four route through the server that owns the MemWAL, so a local table
+  // rejects them rather than answering. What is asserted here is that the
+  // bindings reach the core at all; the behavior against a real endpoint is
+  // covered by the mocked endpoint tests in rust/lancedb/src/remote/table.rs.
+  it("rejects flushLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.flushLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects compactLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.compactLsm()).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects getLsmStats on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    await expect(table.getLsmStats()).rejects.toThrow(/not supported/i);
+    await expect(table.getLsmStats(true)).rejects.toThrow(/not supported/i);
+  });
+
+  it("rejects checkpointLsm on a local table", async () => {
+    const conn = await connect(tmpDir.name);
+    const table = await lsmTable(conn);
+
+    // checkpointLsm seals first, so it surfaces flushLsm's rejection.
+    await expect(table.checkpointLsm()).rejects.toThrow(/not supported/i);
+  });
+});
+
+describe("computed columns", () => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("declares a column and fills it on refresh", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed", [{ x: 1 }, { x: 2 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+    let rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled)).toEqual([null, null]);
+
+    const result = await table.refreshColumn("doubled");
+    expect(result.rowsFilled).toBe(2);
+
+    rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
+  });
+
+  it("returns a job handle from refreshColumnAsync", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+
+    const job = await table.refreshColumnAsync("doubled");
+    expect(job.id).toBeNull();
+    await job.wait();
+    expect(await job.status()).toBe("finished");
+
+    const rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]);
+
+    // Bad input rejects at the call, not through the job.
+    await expect(table.refreshColumnAsync("x")).rejects.toThrow(
+      "not a computed column",
+    );
+  });
+
+  it("fills rows added since the last refresh", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable("computed_append", [{ x: 1 }]);
+
+    await table.addColumns({
+      computed: [{ name: "doubled", valueSql: "x * 2" }],
+    });
+    await table.refreshColumn("doubled");
+    await table.add([{ x: 5 }]);
+
+    const result = await table.refreshColumn("doubled");
+    expect(result.rowsFilled).toBe(1);
+
+    const rows = await table.query().toArray();
+    expect(rows.map((r) => r.doubled).sort()).toEqual([10, 2]);
+  });
+});

nodejs/lancedb/arrow.ts

@@ -5,7 +5,6 @@ import {
   Data as ArrowData,
   Table as ArrowTable,
   Binary,
-  Bool,
   BufferType,
   DataType,
   DateUnit,
@@ -18,12 +17,7 @@ import {
   FixedSizeList,
   Float,
   Float32,
-  Float64,
   Int,
-  Int8,
-  Int16,
-  Int32,
-  Int64,
   LargeBinary,
   List,
   Null,
@@ -36,33 +30,29 @@ import {
   Struct,
   Timestamp,
   Type,
-  Uint8,
-  Uint16,
-  Uint32,
   Utf8,
   Vector,
   makeVector as arrowMakeVector,
+  util as arrowUtil,
   vectorFromArray as badVectorFromArray,
   makeBuilder,
   makeData,
 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
+import { typedArrayToArrowType } from "./arrow_type";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import {
+  EmbeddingFunctionConfig,
+  getRegistry,
+  parseEmbeddingMetadata,
+} from "./embedding/registry";
 import {
   sanitizeField,
   sanitizeSchema,
   sanitizeTable,
   sanitizeType,
 } from "./sanitize";
-
-/**
- * Check if a field name indicates a vector column.
- */
-function nameSuggestsVectorColumn(fieldName: string): boolean {
-  const nameLower = fieldName.toLowerCase();
-  return nameLower.includes("vector") || nameLower.includes("embedding");
-}
+import { inferSchema } from "./schema";
 
 export * from "apache-arrow";
 export type SchemaLike =
@@ -455,110 +445,6 @@ export function makeArrowTable(
   return new ArrowTable(inferredSchema, finalColumns);
 }
 
-function inferSchema(
-  data: Array<Record<string, unknown>>,
-  schema: Schema | undefined,
-  opts: MakeArrowTableOptions,
-): Schema {
-  // We will collect all fields we see in the data.
-  const pathTree = new PathTree<DataType>();
-
-  for (const [rowI, row] of data.entries()) {
-    for (const [path, value] of rowPathsAndValues(row)) {
-      if (!pathTree.has(path)) {
-        // First time seeing this field.
-        if (schema !== undefined) {
-          const field = getFieldForPath(schema, path);
-          if (field === undefined) {
-            throw new Error(
-              `Found field not in schema: ${path.join(".")} at row ${rowI}`,
-            );
-          } else {
-            pathTree.set(path, field.type);
-          }
-        } else {
-          const inferredType = inferType(value, path, opts);
-          if (inferredType === undefined) {
-            throw new Error(`Failed to infer data type for field ${path.join(
-              ".",
-            )} at row ${rowI}. \
-                             Consider providing an explicit schema.`);
-          }
-          pathTree.set(path, inferredType);
-        }
-      } else if (schema === undefined) {
-        const currentType = pathTree.get(path);
-        const newType = inferType(value, path, opts);
-        if (currentType !== newType) {
-          new Error(`Failed to infer schema for data. Previously inferred type \
-                     ${currentType} but found ${newType} at row ${rowI}. Consider \
-                     providing an explicit schema.`);
-        }
-      }
-    }
-  }
-
-  if (schema === undefined) {
-    function fieldsFromPathTree(pathTree: PathTree<DataType>): Field[] {
-      const fields = [];
-      for (const [name, value] of pathTree.map.entries()) {
-        if (value instanceof PathTree) {
-          const children = fieldsFromPathTree(value);
-          fields.push(new Field(name, new Struct(children), true));
-        } else {
-          fields.push(new Field(name, value, true));
-        }
-      }
-      return fields;
-    }
-    const fields = fieldsFromPathTree(pathTree);
-    return new Schema(fields);
-  } else {
-    function takeMatchingFields(
-      fields: Field[],
-      pathTree: PathTree<DataType>,
-    ): Field[] {
-      const outFields = [];
-      for (const field of fields) {
-        if (pathTree.map.has(field.name)) {
-          const value = pathTree.get([field.name]);
-          if (value instanceof PathTree) {
-            const struct = field.type as Struct;
-            const children = takeMatchingFields(struct.children, value);
-            outFields.push(
-              new Field(field.name, new Struct(children), field.nullable),
-            );
-          } else {
-            outFields.push(
-              new Field(field.name, value as DataType, field.nullable),
-            );
-          }
-        }
-      }
-      return outFields;
-    }
-    const fields = takeMatchingFields(schema.fields, pathTree);
-    return new Schema(fields);
-  }
-}
-
-function* rowPathsAndValues(
-  row: Record<string, unknown>,
-  basePath: string[] = [],
-): Generator<[string[], unknown]> {
-  for (const [key, value] of Object.entries(row)) {
-    if (isObject(value)) {
-      yield* rowPathsAndValues(value, [...basePath, key]);
-    } else {
-      // Skip undefined values - they should be treated the same as missing fields
-      // for embedding function purposes
-      if (value !== undefined) {
-        yield [[...basePath, key], value];
-      }
-    }
-  }
-}
-
 function isObject(value: unknown): value is Record<string, unknown> {
   return (
     typeof value === "object" &&
@@ -573,146 +459,19 @@ function isObject(value: unknown): value is Record<string, unknown> {
   );
 }
 
-function getFieldForPath(schema: Schema, path: string[]): Field | undefined {
-  let current: Field | Schema = schema;
+function valueAtPath(datum: Record<string, unknown>, path: string[]): unknown {
+  let current: unknown = datum;
   for (const key of path) {
-    if (current instanceof Schema) {
-      const field: Field | undefined = current.fields.find(
-        (f) => f.name === key,
-      );
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
-    } else if (current instanceof Field && DataType.isStruct(current.type)) {
-      const struct: Struct = current.type;
-      const field = struct.children.find((f) => f.name === key);
-      if (field === undefined) {
-        return undefined;
-      }
-      current = field;
+    if (current == null) {
+      return null;
+    }
+    if (isObject(current) && (Object.hasOwn(current, key) || key in current)) {
+      current = current[key];
     } else {
       return undefined;
     }
   }
-  if (current instanceof Field) {
-    return current;
-  } else {
-    return undefined;
-  }
-}
-
-/**
- * Try to infer which Arrow type to use for a given value.
- *
- * May return undefined if the type cannot be inferred.
- */
-function inferType(
-  value: unknown,
-  path: string[],
-  opts: MakeArrowTableOptions,
-): DataType | undefined {
-  if (typeof value === "bigint") {
-    return new Int64();
-  } else if (typeof value === "number") {
-    // Even if it's an integer, it's safer to assume Float64. Users can
-    // always provide an explicit schema or use BigInt if they mean integer.
-    return new Float64();
-  } else if (typeof value === "string") {
-    if (opts.dictionaryEncodeStrings) {
-      return new Dictionary(new Utf8(), new Int32());
-    } else {
-      return new Utf8();
-    }
-  } else if (typeof value === "boolean") {
-    return new Bool();
-  } else if (value instanceof Buffer) {
-    return new Binary();
-  } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
-    const info = typedArrayToArrowType(value);
-    if (info !== undefined) {
-      const child = new Field("item", info.elementType, true);
-      return new FixedSizeList(info.length, child);
-    }
-    return undefined;
-  } else if (Array.isArray(value)) {
-    if (value.length === 0) {
-      return undefined; // Without any values we can't infer the type
-    }
-    if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
-      const floatType = sanitizeType(opts.vectorColumns[path[0]].type);
-      return new FixedSizeList(
-        value.length,
-        new Field("item", floatType, true),
-      );
-    }
-    const valueType = inferType(value[0], path, opts);
-    if (valueType === undefined) {
-      return undefined;
-    }
-    // Try to automatically detect embedding columns.
-    if (nameSuggestsVectorColumn(path[path.length - 1])) {
-      // Check if value is a Uint8Array for integer vector type determination
-      if (value instanceof Uint8Array) {
-        // For integer vectors, we default to Uint8 (matching Python implementation)
-        const child = new Field("item", new Uint8(), true);
-        return new FixedSizeList(value.length, child);
-      } else {
-        // For float vectors, we default to Float32
-        const child = new Field("item", new Float32(), true);
-        return new FixedSizeList(value.length, child);
-      }
-    } else {
-      const child = new Field("item", valueType, true);
-      return new List(child);
-    }
-  } else {
-    // TODO: timestamp
-    return undefined;
-  }
-}
-
-class PathTree<V> {
-  map: Map<string, V | PathTree<V>>;
-
-  constructor(entries?: [string[], V][]) {
-    this.map = new Map();
-    if (entries !== undefined) {
-      for (const [path, value] of entries) {
-        this.set(path, value);
-      }
-    }
-  }
-  has(path: string[]): boolean {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return false;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return true;
-  }
-  get(path: string[]): V | undefined {
-    let ref: PathTree<V> = this;
-    for (const part of path) {
-      if (!(ref instanceof PathTree) || !ref.map.has(part)) {
-        return undefined;
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    return ref as V;
-  }
-  set(path: string[], value: V): void {
-    let ref: PathTree<V> = this;
-    for (const part of path.slice(0, path.length - 1)) {
-      if (!ref.map.has(part)) {
-        ref.map.set(part, new PathTree<V>());
-      }
-      ref = ref.map.get(part) as PathTree<V>;
-    }
-    ref.map.set(path[path.length - 1], value);
-  }
+  return current;
 }
 
 function transposeData(
@@ -720,37 +479,26 @@ function transposeData(
   field: Field,
   path: string[] = [],
 ): Vector {
+  const valuesPath = [...path, field.name];
+  const values = data.map((datum) => valueAtPath(datum, valuesPath));
   if (field.type instanceof Struct) {
     const childFields = field.type.children;
-    const fullPath = [...path, field.name];
     const childVectors = childFields.map((child) => {
-      return transposeData(data, child, fullPath);
+      return transposeData(data, child, valuesPath);
     });
+    const nullCount = values.filter((value) => value === null).length;
     const structData = makeData({
       type: field.type,
+      length: values.length,
+      nullCount,
+      nullBitmap:
+        nullCount > 0
+          ? arrowUtil.packBools(values.map((value) => value !== null))
+          : undefined,
       children: childVectors as unknown as ArrowData<DataType>[],
     });
     return arrowMakeVector(structData);
   } else {
-    const valuesPath = [...path, field.name];
-    const values = data.map((datum) => {
-      let current: unknown = datum;
-      for (const key of valuesPath) {
-        if (current == null) {
-          return null;
-        }
-
-        if (
-          isObject(current) &&
-          (Object.hasOwn(current, key) || key in current)
-        ) {
-          current = current[key];
-        } else {
-          return null;
-        }
-      }
-      return current;
-    });
     return makeVector(values, field.type, undefined, field.nullable);
   }
 }
@@ -793,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector<unknown> {
   return listBuilder.finish().toVector();
 }
 
-/**
- * Map a JS TypedArray instance to the corresponding Arrow element DataType
- * and its length. Returns undefined if the value is not a recognized TypedArray.
- */
-function typedArrayToArrowType(
-  value: ArrayBufferView,
-): { elementType: DataType; length: number } | undefined {
-  if (value instanceof Float32Array)
-    return { elementType: new Float32(), length: value.length };
-  if (value instanceof Float64Array)
-    return { elementType: new Float64(), length: value.length };
-  if (value instanceof Uint8Array)
-    return { elementType: new Uint8(), length: value.length };
-  if (value instanceof Uint16Array)
-    return { elementType: new Uint16(), length: value.length };
-  if (value instanceof Uint32Array)
-    return { elementType: new Uint32(), length: value.length };
-  if (value instanceof Int8Array)
-    return { elementType: new Int8(), length: value.length };
-  if (value instanceof Int16Array)
-    return { elementType: new Int16(), length: value.length };
-  if (value instanceof Int32Array)
-    return { elementType: new Int32(), length: value.length };
-  return undefined;
-}
-
 /** Helper function to convert an Array of JS values to an Arrow Vector */
 function makeVector(
   values: unknown[],
@@ -933,7 +655,7 @@ async function applyEmbeddingsFromMetadata(
 
   for (const functionEntry of functions.values()) {
     const sourceColumn = columns[functionEntry.sourceColumn];
-    const destColumn = functionEntry.vectorColumn ?? "vector";
+    const destColumn = functionEntry.vectorColumn;
     if (sourceColumn === undefined) {
       throw new Error(
         `Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`,
@@ -1385,11 +1107,10 @@ function validateSchemaEmbeddings(
 
         // Check schema metadata for embedding functions
         if (schema.metadata.has("embedding_functions")) {
-          const embeddings = JSON.parse(
+          const entries = parseEmbeddingMetadata(
             schema.metadata.get("embedding_functions")!,
           );
-          // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
-          if (embeddings.find((f: any) => f["vectorColumn"] === field.name)) {
+          if (entries.some((f) => f.vectorColumn === field.name)) {
             hasEmbeddingFunction = true;
           }
         }
@@ -1459,8 +1180,12 @@ export function ensureNestedFieldsExist(
           completeRow[field.name] = row[field.name];
         }
       } else {
-        // Field is missing from the data - set to null
-        completeRow[field.name] = null;
+        // Keep a missing struct valid while filling each of its children with
+        // null. This is distinct from an explicitly null struct value.
+        completeRow[field.name] =
+          field.type.constructor.name === "Struct"
+            ? ensureStructFieldsExist({}, field.type as Struct)
+            : null;
       }
     }
 
@@ -1495,8 +1220,12 @@ function ensureStructFieldsExist(
         completeStruct[childField.name] = data[childField.name];
       }
     } else {
-      // Field is missing - set to null
-      completeStruct[childField.name] = null;
+      // Keep a missing struct valid while filling each of its children with
+      // null. This is distinct from an explicitly null struct value.
+      completeStruct[childField.name] =
+        childField.type.constructor.name === "Struct"
+          ? ensureStructFieldsExist({}, childField.type as Struct)
+          : null;
     }
   }
 

nodejs/lancedb/arrow_type.ts

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  type DataType,
+  Float32,
+  Float64,
+  Int8,
+  Int16,
+  Int32,
+  Uint8,
+  Uint16,
+  Uint32,
+} from "apache-arrow";
+
+/**
+ * Map a JS TypedArray instance to the corresponding Arrow element type and
+ * length. Returns undefined when the view is not a supported TypedArray.
+ */
+export function typedArrayToArrowType(
+  value: ArrayBufferView,
+): { elementType: DataType; length: number } | undefined {
+  if (value instanceof Float32Array)
+    return { elementType: new Float32(), length: value.length };
+  if (value instanceof Float64Array)
+    return { elementType: new Float64(), length: value.length };
+  if (value instanceof Uint8Array)
+    return { elementType: new Uint8(), length: value.length };
+  if (value instanceof Uint16Array)
+    return { elementType: new Uint16(), length: value.length };
+  if (value instanceof Uint32Array)
+    return { elementType: new Uint32(), length: value.length };
+  if (value instanceof Int8Array)
+    return { elementType: new Int8(), length: value.length };
+  if (value instanceof Int16Array)
+    return { elementType: new Int16(), length: value.length };
+  if (value instanceof Int32Array)
+    return { elementType: new Int32(), length: value.length };
+  return undefined;
+}

nodejs/lancedb/connection.ts

@@ -16,6 +16,12 @@ import {
   makeEmptyTable,
 } from "./arrow";
 import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
+import {
+  MaterializedView,
+  MaterializedViewSelect,
+  normalizeSelect,
+  validateNonNegativeInteger,
+} from "./materialized_view";
 import { Connection as LanceDbConnection } from "./native";
 import type {
   CreateNamespaceResponse,
@@ -25,12 +31,14 @@ import type {
   JobDescription,
   JobInfo,
   ListNamespacesResponse,
+  ListTablesResponse,
 } from "./native";
 export type {
   CreateNamespaceResponse,
   DescribeNamespaceResponse,
   DropNamespaceResponse,
   ListNamespacesResponse,
+  ListTablesResponse,
 };
 import { sanitizeTable } from "./sanitize";
 import { LocalTable, Table } from "./table";
@@ -128,6 +136,10 @@ export interface OpenTableOptions {
   indexCacheSize?: number;
 }
 
+/**
+ * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables}
+ * instead.
+ */
 export interface TableNamesOptions {
   /**
    * If present, only return names that come lexicographically after the
@@ -141,6 +153,24 @@ export interface TableNamesOptions {
   limit?: number;
 }
 
+export interface ListTablesOptions {
+  /**
+   * Token from a previous response, to resume listing where it left off.
+   *
+   * The token is opaque: it carries whatever the database needs to resume, and
+   * callers should not construct or interpret one.
+   */
+  pageToken?: string;
+  /**
+   * An upper bound on how many tables to return.
+   *
+   * A page may hold fewer than this and still not be the last one, so keep
+   * going while the response carries a page token rather than while pages are
+   * full.
+   */
+  limit?: number;
+}
+
 export interface ListNamespacesOptions {
   /** Token from a previous response for pagination. */
   pageToken?: string;
@@ -225,6 +255,7 @@ export abstract class Connection {
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point (backwards compatibility)
    *
+   * @deprecated Use {@link Connection.listTables} instead.
    */
   abstract tableNames(options?: Partial<TableNamesOptions>): Promise<string[]>;
   /**
@@ -235,18 +266,94 @@ export abstract class Connection {
    * @param {Partial<TableNamesOptions>} options - options to control the
    * paging / start point
    *
+   * @deprecated Use {@link Connection.listTables} instead.
    */
   abstract tableNames(
     namespacePath?: string[],
     options?: Partial<TableNamesOptions>,
   ): Promise<string[]>;
 
+  /**
+   * List a page of the tables in this database.
+   *
+   * To retrieve the tables after the page, pass the `pageToken` the response
+   * carries back in. A page can be shorter than `limit` without being the last
+   * one, so walk until a response carries no page token:
+   *
+   * ```ts
+   * const names = [];
+   * let pageToken = undefined;
+   * do {
+   *   const page = await conn.listTables({ pageToken, limit: 100 });
+   *   names.push(...page.tables);
+   *   pageToken = page.pageToken;
+   * } while (pageToken);
+   * ```
+   *
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+  /**
+   * List a page of the tables in this database.
+   *
+   * @param {string[]} namespacePath - The namespace path to list tables from
+   *   (defaults to root namespace)
+   * @param {Partial<ListTablesOptions>} options - Pagination options
+   *   (`pageToken`, `limit`).
+   * @returns {Promise<ListTablesResponse>} A page of table names and an
+   *   optional token for the tables after it.
+   */
+  abstract listTables(
+    namespacePath?: string[],
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse>;
+
   /**
    * Open a table in the database.
    * @param {string} name - The name of the table
    * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
    * @param {Partial<OpenTableOptions>} options - Additional options
    */
+  /**
+   * Define a materialized view named `name` over the table `source`.
+   *
+   * The view is created empty, with the query recorded in its schema
+   * metadata; `view.refresh()` computes the rows. The view is a normal
+   * table: it can be queried, indexed and searched, and it appears in
+   * `tableNames`. The source table must have stable row ids (create it with
+   * the `newTableEnableStableRowIds` storage option); they keep the view's
+   * provenance valid across source compactions and cannot be enabled after
+   * a table exists. Local databases only.
+   */
+  abstract createMaterializedView(
+    name: string,
+    source: string,
+    options?: {
+      select?: MaterializedViewSelect;
+      where?: string;
+      limit?: number;
+    },
+  ): Promise<MaterializedView>;
+
+  /**
+   * Open the materialized view named `name`.
+   *
+   * Rejects a table that exists but is not a materialized view.
+   */
+  abstract openMaterializedView(name: string): Promise<MaterializedView>;
+
+  /**
+   * The names of the materialized views in this database.
+   *
+   * Found by reading every table's schema, so this costs an open per table.
+   */
+  abstract listMaterializedViews(): Promise<string[]>;
+
   abstract openTable(
     name: string,
     namespacePath?: string[],
@@ -327,6 +434,14 @@ export abstract class Connection {
    */
   abstract dropTable(name: string, namespacePath?: string[]): Promise<void>;
 
+  /**
+   * Start dropping a table and return its cleanup job.
+   *
+   * The table may become unavailable before its data files are removed. Wait
+   * on the returned job to know when cleanup has finished.
+   */
+  abstract dropTableAsync(name: string, namespacePath?: string[]): Promise<Job>;
+
   /**
    * Drop all tables in the database.
    * @param {string[]} namespacePath The namespace path to drop tables from (defaults to root namespace).
@@ -523,6 +638,54 @@ export class LocalConnection extends Connection {
     );
   }
 
+  async createMaterializedView(
+    name: string,
+    source: string,
+    options?: {
+      select?: MaterializedViewSelect;
+      where?: string;
+      limit?: number;
+    },
+  ): Promise<MaterializedView> {
+    validateNonNegativeInteger(options?.limit, "limit");
+    const innerTable = await this.inner.createMaterializedView(
+      name,
+      source,
+      normalizeSelect(options?.select),
+      options?.where,
+      options?.limit,
+    );
+    return new MaterializedView(new LocalTable(innerTable));
+  }
+
+  async openMaterializedView(name: string): Promise<MaterializedView> {
+    const innerTable = await this.inner.openMaterializedView(name);
+    return new MaterializedView(new LocalTable(innerTable));
+  }
+
+  async listMaterializedViews(): Promise<string[]> {
+    return await this.inner.listMaterializedViews();
+  }
+
+  async listTables(
+    namespacePathOrOptions?: string[] | Partial<ListTablesOptions>,
+    options?: Partial<ListTablesOptions>,
+  ): Promise<ListTablesResponse> {
+    // Detect if first argument is namespacePath array or options object
+    const namespacePath = Array.isArray(namespacePathOrOptions)
+      ? namespacePathOrOptions
+      : undefined;
+    const listTablesOptions = Array.isArray(namespacePathOrOptions)
+      ? options
+      : namespacePathOrOptions;
+
+    return this.inner.listTables(
+      namespacePath ?? [],
+      listTablesOptions?.pageToken,
+      listTablesOptions?.limit,
+    );
+  }
+
   async openTable(
     name: string,
     namespacePath?: string[],
@@ -705,6 +868,10 @@ export class LocalConnection extends Connection {
     return this.inner.dropTable(name, namespacePath ?? []);
   }
 
+  async dropTableAsync(name: string, namespacePath?: string[]): Promise<Job> {
+    return this.inner.dropTableAsync(name, namespacePath ?? []);
+  }
+
   async dropAllTables(namespacePath?: string[]): Promise<void> {
     return this.inner.dropAllTables(namespacePath ?? []);
   }

nodejs/lancedb/embedding/index.ts

@@ -4,7 +4,15 @@
 import { Field, Schema } from "../arrow";
 import { sanitizeType } from "../sanitize";
 import { EmbeddingFunction } from "./embedding_function";
-import { EmbeddingFunctionConfig, getRegistry } from "./registry";
+import {
+  EmbeddingFunctionConfig,
+  EmbeddingFunctionRegistry,
+  getRegistry as getGlobalRegistry,
+  registerBuiltIn,
+} from "./registry";
+
+type OpenAIModule = typeof import("./openai");
+type TransformersModule = typeof import("./transformers");
 
 export {
   FieldOptions,
@@ -14,7 +22,39 @@ export {
   EmbeddingFunctionConstructor,
 } from "./embedding_function";
 
-export * from "./registry";
+export {
+  EmbeddingFunctionRegistry,
+  parseEmbeddingMetadata,
+  register,
+} from "./registry";
+export type {
+  CreateReturnType,
+  EmbeddingFunctionConfig,
+  EmbeddingFunctionCreate,
+  EmbeddingMetadataEntry,
+  ResolvedEmbeddingFunctionConfig,
+} from "./registry";
+
+function initializeBuiltInProviders() {
+  const { OpenAIEmbeddingFunction } = require("./openai") as OpenAIModule;
+  const { TransformersEmbeddingFunction } =
+    require("./transformers") as TransformersModule;
+
+  registerBuiltIn("openai", OpenAIEmbeddingFunction);
+  registerBuiltIn("huggingface", TransformersEmbeddingFunction);
+}
+
+/**
+ * Get the global embedding function registry.
+ *
+ * LanceDB built-in providers are initialized when this public API is first
+ * used, so importing the root package does not change automatic search
+ * selection for tables without embedding metadata.
+ */
+export function getRegistry(): EmbeddingFunctionRegistry {
+  initializeBuiltInProviders();
+  return getGlobalRegistry();
+}
 
 /**
  * Create a schema with embedding functions.

nodejs/lancedb/embedding/openai.ts

@@ -5,14 +5,13 @@ import type OpenAI from "openai";
 import type { EmbeddingCreateParams } from "openai/resources/index";
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 
 export type OpenAIOptions = {
   apiKey: string;
   model: EmbeddingCreateParams["model"];
 };
 
-@register("openai")
 export class OpenAIEmbeddingFunction extends EmbeddingFunction<
   string,
   Partial<OpenAIOptions>
@@ -100,3 +99,5 @@ export class OpenAIEmbeddingFunction extends EmbeddingFunction<
     return response.data[0].embedding;
   }
 }
+
+registerBuiltIn("openai", OpenAIEmbeddingFunction);

nodejs/lancedb/embedding/registry.ts

@@ -7,6 +7,10 @@ import {
 } from "./embedding_function";
 import "reflect-metadata";
 
+const builtInFunctionsKey = Symbol.for(
+  "@lancedb/lancedb::embedding-built-in-functions::v1",
+);
+
 export type CreateReturnType<T> = T extends { init: () => Promise<void> }
   ? Promise<T>
   : T;
@@ -59,6 +63,15 @@ export class EmbeddingFunctionRegistry {
     };
   }
 
+  /** @ignore */
+  setBuiltIn<
+    T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
+  >(name: string, ctor: T): T {
+    this.#functions.set(name, ctor);
+    Reflect.defineMetadata("lancedb::embedding::name", name, ctor);
+    return ctor;
+  }
+
   get<T extends EmbeddingFunction<unknown>>(
     name: string,
   ): EmbeddingFunctionCreate<T> | undefined;
@@ -96,6 +109,7 @@ export class EmbeddingFunctionRegistry {
    */
   reset(this: EmbeddingFunctionRegistry) {
     this.#functions.clear();
+    getBuiltInFunctions(this).clear();
   }
 
   /**
@@ -104,41 +118,29 @@ export class EmbeddingFunctionRegistry {
   async parseFunctions(
     this: EmbeddingFunctionRegistry,
     metadata: Map<string, string>,
-  ): Promise<Map<string, EmbeddingFunctionConfig>> {
+  ): Promise<Map<string, ResolvedEmbeddingFunctionConfig>> {
     if (!metadata.has("embedding_functions")) {
       return new Map();
-    } else {
-      type FunctionConfig = {
-        name: string;
-        sourceColumn: string;
-        vectorColumn: string;
-        model: EmbeddingFunction["TOptions"];
-      };
-
-      const functions = <FunctionConfig[]>(
-        JSON.parse(metadata.get("embedding_functions")!)
-      );
-
-      const items: [string, EmbeddingFunctionConfig][] = await Promise.all(
-        functions.map(async (f) => {
-          const fn = this.get(f.name);
-          if (!fn) {
-            throw new Error(`Function "${f.name}" not found in registry`);
-          }
-          const func = await this.get(f.name)!.create(f.model);
-          return [
-            f.name,
-            {
-              sourceColumn: f.sourceColumn,
-              vectorColumn: f.vectorColumn,
-              function: func,
-            },
-          ];
-        }),
-      );
-
-      return new Map(items);
     }
+    const entries = parseEmbeddingMetadata(
+      metadata.get("embedding_functions")!,
+    );
+    const items = await Promise.all(
+      entries.map(async (f): Promise<ResolvedEmbeddingFunctionConfig> => {
+        const fn = this.get(f.name);
+        if (!fn) {
+          throw new Error(`Function "${f.name}" not found in registry`);
+        }
+        const func = await fn.create(f.model);
+        return {
+          sourceColumn: f.sourceColumn,
+          vectorColumn: f.vectorColumn,
+          function: func,
+        };
+      }),
+    );
+    // Keyed by output column: one function may serve several columns.
+    return new Map(items.map((config) => [config.vectorColumn, config]));
   }
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any> {
@@ -195,12 +197,56 @@ export class EmbeddingFunctionRegistry {
   }
 }
 
-const _REGISTRY = new EmbeddingFunctionRegistry();
+function getBuiltInFunctions(registry: EmbeddingFunctionRegistry): Set<string> {
+  const registryWithBuiltIns = registry as EmbeddingFunctionRegistry & {
+    [key: symbol]: Set<string> | undefined;
+  };
+  let builtInFunctions = registryWithBuiltIns[builtInFunctionsKey];
+  if (builtInFunctions === undefined) {
+    builtInFunctions = new Set<string>();
+    registryWithBuiltIns[builtInFunctionsKey] = builtInFunctions;
+  }
+  return builtInFunctions;
+}
+
+// Server bundlers can load the side-effect embedding entry points and the public
+// embedding API from separate module graphs. Keep their registry shared.
+const registryKey = Symbol.for(
+  "@lancedb/lancedb::embedding-function-registry::v1",
+);
+const registryGlobal = globalThis as typeof globalThis & {
+  [key: symbol]: EmbeddingFunctionRegistry | undefined;
+};
+
+function getGlobalRegistry(): EmbeddingFunctionRegistry {
+  const existingRegistry = registryGlobal[registryKey];
+  if (existingRegistry !== undefined) {
+    return existingRegistry;
+  }
+  const registry = new EmbeddingFunctionRegistry();
+  registryGlobal[registryKey] = registry;
+  return registry;
+}
+
+const _REGISTRY = getGlobalRegistry();
 
 export function register(name?: string) {
   return _REGISTRY.register(name);
 }
 
+/** @ignore */
+export function registerBuiltIn<
+  T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor,
+>(name: string, ctor: T): T {
+  const builtInFunctions = getBuiltInFunctions(_REGISTRY);
+  if (builtInFunctions.has(name)) {
+    return _REGISTRY.setBuiltIn(name, ctor);
+  }
+  _REGISTRY.register(name)(ctor);
+  builtInFunctions.add(name);
+  return ctor;
+}
+
 /**
  * Utility function to get the global instance of the registry
  * @returns `EmbeddingFunctionRegistry` The global instance of the registry
@@ -218,3 +264,52 @@ export interface EmbeddingFunctionConfig {
   vectorColumn?: string;
   function: EmbeddingFunction;
 }
+
+/** An [EmbeddingFunctionConfig] read back from table metadata, where the
+ * vector column is always recorded. */
+export type ResolvedEmbeddingFunctionConfig = EmbeddingFunctionConfig & {
+  vectorColumn: string;
+};
+
+/** One entry of the `embedding_functions` schema metadata, with the column
+ * keys normalized across the bindings' spellings. */
+export type EmbeddingMetadataEntry = {
+  name: string;
+  sourceColumn: string;
+  vectorColumn: string;
+  model: EmbeddingFunction["TOptions"];
+};
+
+/** The single parser for `embedding_functions` schema metadata: every reader
+ * goes through here, so the wire contract cannot fork between them. */
+export function parseEmbeddingMetadata(json: string): EmbeddingMetadataEntry[] {
+  // The wire format, honestly: the Python bindings write snake_case keys.
+  type Raw = {
+    name: string;
+    sourceColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    source_column?: string;
+    vectorColumn?: string;
+    // biome-ignore lint/style/useNamingConvention: the Python wire spelling
+    vector_column?: string;
+    model: EmbeddingFunction["TOptions"];
+  };
+  const entries = <Raw[]>JSON.parse(json);
+  const seen = new Set<string>();
+  return entries.map((f) => {
+    const sourceColumn = f.sourceColumn ?? f.source_column;
+    const vectorColumn = f.vectorColumn ?? f.vector_column;
+    if (sourceColumn === undefined || vectorColumn === undefined) {
+      throw new Error(
+        `Embedding function "${f.name}" metadata names no source or vector column`,
+      );
+    }
+    if (seen.has(vectorColumn)) {
+      throw new Error(
+        `Multiple embedding configs claim vector column "${vectorColumn}"`,
+      );
+    }
+    seen.add(vectorColumn);
+    return { name: f.name, sourceColumn, vectorColumn, model: f.model };
+  });
+}

nodejs/lancedb/embedding/transformers.ts

@@ -3,7 +3,7 @@
 
 import { Float, Float32 } from "../arrow";
 import { EmbeddingFunction } from "./embedding_function";
-import { register } from "./registry";
+import { registerBuiltIn } from "./registry";
 
 export type XenovaTransformerOptions = {
   /** The wasm compatible model to use */
@@ -31,7 +31,6 @@ export type XenovaTransformerOptions = {
   };
 };
 
-@register("huggingface")
 export class TransformersEmbeddingFunction extends EmbeddingFunction<
   string,
   Partial<XenovaTransformerOptions>
@@ -158,6 +157,8 @@ export class TransformersEmbeddingFunction extends EmbeddingFunction<
   }
 }
 
+registerBuiltIn("huggingface", TransformersEmbeddingFunction);
+
 const tensorDiv = (
   src: import("@huggingface/transformers").Tensor,
   divBy: number,

nodejs/lancedb/index.ts

@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
 import type { FtsToken } from "./table";
 
 // Re-export native header provider for use with connectWithHeaderProvider
+export {
+  MaterializedView,
+  MaterializedViewDefinition,
+  MaterializedViewSelect,
+} from "./materialized_view";
 export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
 
 // OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -50,6 +55,8 @@ export {
   MergeResult,
   AddResult,
   AddColumnsResult,
+  RefreshColumnResult,
+  RefreshMaterializedViewResult,
   AlterColumnsResult,
   UpdateFieldMetadataResult,
   DeleteResult,
@@ -74,11 +81,13 @@ export {
   Connection,
   CreateTableOptions,
   TableNamesOptions,
+  ListTablesOptions,
   OpenTableOptions,
   ListNamespacesOptions,
   CreateNamespaceOptions,
   DropNamespaceOptions,
   ListNamespacesResponse,
+  ListTablesResponse,
   CreateNamespaceResponse,
   DropNamespaceResponse,
   DescribeNamespaceResponse,
@@ -94,6 +103,7 @@ export {
 } from "./native.js";
 
 export {
+  AutoQuery,
   ExecutableQuery,
   Query,
   QueryBase,
@@ -134,10 +144,10 @@ export {
   BranchColumnChange,
   BranchIndexSummary,
   BranchRowCountSummary,
-  MergeBlocker,
+  CherryPickError,
   BranchDiff,
-  MergePreview,
-  MergeBranchResult,
+  CherryPickPreview,
+  CherryPickResult,
   AddDataOptions,
   UpdateOptions,
   OptimizeOptions,
@@ -146,6 +156,10 @@ export {
   FtsToken,
   TokenizeTableOptions,
   LsmWriteSpec,
+  LsmStats,
+  BucketStats,
+  GenerationStats,
+  MemtableStats,
   ColumnAlteration,
   FieldMetadataUpdate,
 } from "./table";

nodejs/lancedb/materialized_view.ts

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { RefreshMaterializedViewResult } from "./native";
+import { Table } from "./table";
+
+/** Schema metadata key holding a materialized view's definition. */
+export const DEFINITION_META_KEY = "mv.definition";
+
+/** The query that defines a materialized view. */
+export interface MaterializedViewDefinition {
+  /** Name of the source table, in the same database as the view. */
+  sourceTable: string;
+  /** `[output column, SQL expression]` pairs, in view schema order. */
+  projections: [string, string][];
+  /** SQL predicate selecting the source rows the view holds. */
+  filter?: string;
+  /** Cap on the number of rows the view holds. */
+  limit?: number;
+  /** Source columns the projections and filter read. */
+  inputs: string[];
+}
+
+/**
+ * The view's columns: column names, `[alias, SQL expression]` pairs, or a
+ * record of the same. A bare name projects itself.
+ */
+export type MaterializedViewSelect =
+  | (string | [string, string])[]
+  | Record<string, string>;
+
+/**
+ * @internal Reject a numeric option N-API would otherwise silently coerce:
+ * `Infinity` reaches Rust as 0, `1.5` as 1.
+ */
+export function validateNonNegativeInteger(
+  value: number | undefined,
+  name: string,
+): void {
+  if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
+    throw new Error(`${name} must be a non-negative integer`);
+  }
+}
+
+/** @internal Quote a column name as a Lance SQL identifier (backticks). */
+function quoteIdentifier(name: string): string {
+  return "`" + name.replace(/`/g, "``") + "`";
+}
+
+/**
+ * @internal Normalize a select argument into `[alias, expression]` pairs.
+ * A bare name projects itself and is quoted, so any valid column name works;
+ * pair and record entries are kept verbatim because their right side is an
+ * expression.
+ */
+export function normalizeSelect(
+  select?: MaterializedViewSelect,
+): [string, string][] | undefined {
+  if (select === undefined) {
+    return undefined;
+  }
+  if (Array.isArray(select)) {
+    return select.map((item) =>
+      typeof item === "string" ? [item, quoteIdentifier(item)] : item,
+    );
+  }
+  return Object.entries(select);
+}
+
+/** @internal Parse a definition off a table's stored schema metadata. */
+export function definitionFromMetadata(
+  metadata: Map<string, string>,
+  name: string,
+): MaterializedViewDefinition {
+  const raw = metadata.get(DEFINITION_META_KEY);
+  if (raw === undefined) {
+    throw new Error(`Table '${name}' is not a materialized view`);
+  }
+  // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+  const value: any = JSON.parse(raw);
+  if (value.kind !== "select") {
+    throw new Error(
+      `materialized view '${name}' is defined by '${value.kind}', which this ` +
+        "version of lancedb cannot refresh",
+    );
+  }
+  const limit = value.limit ?? undefined;
+  // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
+  // integer and every rounded one does not, so this rejects precisely the
+  // values a number cannot carry.
+  if (limit !== undefined && !Number.isSafeInteger(limit)) {
+    throw new Error(
+      `materialized view '${name}' has a stored limit too large to represent exactly`,
+    );
+  }
+  return {
+    sourceTable: value.source_table,
+    // biome-ignore lint/suspicious/noExplicitAny: raw JSON
+    projections: (value.projections ?? []).map((p: any) => [
+      p.output,
+      p.expression,
+    ]),
+    filter: value.filter ?? undefined,
+    limit,
+    inputs: value.inputs ?? [],
+  };
+}
+
+/**
+ * A handle on a materialized view: its table plus its definition.
+ *
+ * Obtained from {@link Connection#createMaterializedView} or
+ * {@link Connection#openMaterializedView}. The view is a normal table --
+ * queries, indexes and search all apply through {@link MaterializedView#table}
+ * -- whose contents are maintained by {@link MaterializedView#refresh}.
+ */
+export class MaterializedView {
+  private readonly inner: Table;
+
+  constructor(table: Table) {
+    this.inner = table;
+  }
+
+  get name(): string {
+    return this.inner.name;
+  }
+
+  /** The view, as the table it is. */
+  table(): Table {
+    return this.inner;
+  }
+
+  /** The query that defines the view, read from its stored schema. */
+  async definition(): Promise<MaterializedViewDefinition> {
+    const schema = await this.inner.schema();
+    return definitionFromMetadata(schema.metadata, this.name);
+  }
+
+  /**
+   * Recompute the view from its source.
+   *
+   * The refresh is incremental when the source's changes can be reconciled
+   * into the view -- rows added, changed or removed since the last one --
+   * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
+   * refreshes to that source version instead of the latest.
+   *
+   * Concurrent refreshes of one view do not duplicate its rows. Two that
+   * plan the same source rows conflict on commit, and the loser throws
+   * rather than writing them a second time.
+   */
+  async refresh(options?: {
+    full?: boolean;
+    sourceVersion?: number;
+  }): Promise<RefreshMaterializedViewResult> {
+    validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
+    return await this.inner.refreshMaterializedView(
+      options?.full,
+      options?.sourceVersion,
+    );
+  }
+}

nodejs/lancedb/query.ts

@@ -100,26 +100,6 @@ export interface FullTextSearchOptions {
   columns?: string | string[];
 }
 
-type NativeQueryLike = NativeQuery | NativeVectorQuery | NativeTakeQuery;
-
-class DeferredNativeQuery<NativeQueryType extends NativeQueryLike> {
-  protected readonly calls: Array<(inner: NativeQueryType) => void> = [];
-
-  constructor(private readonly factory: () => Promise<NativeQueryType>) {}
-
-  doCall(fn: (inner: NativeQueryType) => void) {
-    this.calls.push(fn);
-  }
-
-  async resolve(): Promise<NativeQueryType> {
-    const inner = await this.factory();
-    for (const call of this.calls) {
-      call(inner);
-    }
-    return inner;
-  }
-}
-
 function nearestToNative(
   inner: NativeQuery,
   vector: Awaited<IntoVector>,
@@ -154,24 +134,15 @@ export class QueryBase<
   NativeQueryType extends NativeQuery | NativeVectorQuery | NativeTakeQuery,
 > implements AsyncIterable<RecordBatch>
 {
-  /**
-   * @hidden
-   */
-  protected inner:
-    | NativeQueryType
-    | Promise<NativeQueryType>
-    | DeferredNativeQuery<NativeQueryType>;
+  protected inner!: NativeQueryType | Promise<NativeQueryType>;
 
   /**
    * @hidden
    */
-  protected constructor(
-    inner:
-      | NativeQueryType
-      | Promise<NativeQueryType>
-      | DeferredNativeQuery<NativeQueryType>,
-  ) {
-    this.inner = inner;
+  protected constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
+    if (inner !== undefined) {
+      this.inner = inner;
+    }
   }
 
   // call a function on the inner (either a promise or the actual object)
@@ -179,9 +150,7 @@ export class QueryBase<
    * @hidden
    */
   protected doCall(fn: (inner: NativeQueryType) => void) {
-    if (this.inner instanceof DeferredNativeQuery) {
-      this.inner.doCall(fn);
-    } else if (this.inner instanceof Promise) {
+    if (this.inner instanceof Promise) {
       this.inner = this.inner.then((inner) => {
         fn(inner);
         return inner;
@@ -192,12 +161,11 @@ export class QueryBase<
   }
 
   /**
+   * Return the native query used by the next terminal operation.
+   *
    * @hidden
    */
-  protected resolveInner(): NativeQueryType | Promise<NativeQueryType> {
-    if (this.inner instanceof DeferredNativeQuery) {
-      return this.inner.resolve();
-    }
+  protected async getInner(): Promise<NativeQueryType> {
     return this.inner;
   }
 
@@ -273,17 +241,11 @@ export class QueryBase<
   /**
    * @hidden
    */
-  protected nativeExecute(
+  protected async nativeExecute(
     options?: Partial<QueryExecutionOptions>,
   ): Promise<NativeBatchIterator> {
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) =>
-        inner.execute(options?.maxBatchLength, options?.timeoutMs),
-      );
-    } else {
-      return inner.execute(options?.maxBatchLength, options?.timeoutMs);
-    }
+    const inner = await this.getInner();
+    return inner.execute(options?.maxBatchLength, options?.timeoutMs);
   }
 
   /**
@@ -312,7 +274,7 @@ export class QueryBase<
   /** Collect the results as an Arrow @see {@link ArrowTable}. */
   async toArrow(options?: Partial<QueryExecutionOptions>): Promise<ArrowTable> {
     const batches = [];
-    const inner = await this.resolveInner();
+    const inner = await this.getInner();
     for await (const batch of new RecordBatchIterable(inner, options)) {
       batches.push(batch);
     }
@@ -341,12 +303,8 @@ export class QueryBase<
    * @returns A Promise that resolves to a string containing the query execution plan explanation.
    */
   async explainPlan(verbose = false): Promise<string> {
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) => inner.explainPlan(verbose));
-    } else {
-      return inner.explainPlan(verbose);
-    }
+    const inner = await this.getInner();
+    return inner.explainPlan(verbose);
   }
 
   /**
@@ -384,12 +342,8 @@ export class QueryBase<
     distributedMetrics?: AnalyzePlanDistributedMetrics,
   ): Promise<string> {
     const distributedMetricsMode = distributedMetrics ?? "aggregate";
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      return inner.then((inner) => inner.analyzePlan(distributedMetricsMode));
-    } else {
-      return inner.analyzePlan(distributedMetricsMode);
-    }
+    const inner = await this.getInner();
+    return inner.analyzePlan(distributedMetricsMode);
   }
 
   /**
@@ -401,13 +355,8 @@ export class QueryBase<
    * @returns An Arrow Schema describing the output columns.
    */
   async outputSchema(): Promise<import("./arrow").Schema> {
-    let schemaBuffer: Buffer;
-    const inner = this.resolveInner();
-    if (inner instanceof Promise) {
-      schemaBuffer = await inner.then((inner) => inner.outputSchema());
-    } else {
-      schemaBuffer = await inner.outputSchema();
-    }
+    const inner = await this.getInner();
+    const schemaBuffer = await inner.outputSchema();
     const schema = tableFromIPC(schemaBuffer).schema;
     return schema;
   }
@@ -419,12 +368,7 @@ export class StandardQueryBase<
   extends QueryBase<NativeQueryType>
   implements ExecutableQuery
 {
-  constructor(
-    inner:
-      | NativeQueryType
-      | Promise<NativeQueryType>
-      | DeferredNativeQuery<NativeQueryType>,
-  ) {
+  constructor(inner?: NativeQueryType | Promise<NativeQueryType>) {
     super(inner);
   }
 
@@ -574,12 +518,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   /**
    * @hidden
    */
-  constructor(
-    inner:
-      | NativeVectorQuery
-      | Promise<NativeVectorQuery>
-      | DeferredNativeQuery<NativeVectorQuery>,
-  ) {
+  constructor(inner: NativeVectorQuery | Promise<NativeVectorQuery>) {
     super(inner);
   }
 
@@ -797,7 +736,7 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   addQueryVector(vector: IntoVector): VectorQuery {
     if (vector instanceof Promise) {
       const res = (async () => {
-        const inner = (await this.resolveInner()) as NativeVectorQuery;
+        const inner = await this.getInner();
         addQueryVectorToNative(inner, await vector);
         return inner;
       })();
@@ -828,70 +767,6 @@ export class VectorQuery extends StandardQueryBase<NativeVectorQuery> {
   }
 }
 
-type AutoQueryResolution = {
-  inner: NativeQuery | NativeVectorQuery;
-  route: "fts" | "vector";
-};
-
-class DeferredAutoNativeQuery extends DeferredNativeQuery<NativeVectorQuery> {
-  private readonly vectorCalls: Array<
-    (inner: NativeVectorQuery) => void | Promise<void>
-  > = [];
-
-  constructor(
-    private readonly autoFactory: () => Promise<AutoQueryResolution>,
-  ) {
-    super(async () => (await autoFactory()).inner as NativeVectorQuery);
-  }
-
-  doVectorCall(fn: (inner: NativeVectorQuery) => void | Promise<void>) {
-    this.vectorCalls.push(fn);
-  }
-
-  async resolve(): Promise<NativeVectorQuery> {
-    const resolution = await this.autoFactory();
-    for (const call of this.calls) {
-      call(resolution.inner as NativeVectorQuery);
-    }
-    if (resolution.route === "vector") {
-      for (const call of this.vectorCalls) {
-        await call(resolution.inner as NativeVectorQuery);
-      }
-    }
-    return resolution.inner as NativeVectorQuery;
-  }
-}
-
-class DeferredAutoQuery extends VectorQuery {
-  constructor(private readonly deferred: DeferredAutoNativeQuery) {
-    super(deferred);
-  }
-
-  protected doVectorCall(fn: (inner: NativeVectorQuery) => void) {
-    this.deferred.doVectorCall(fn);
-  }
-
-  addQueryVector(vector: IntoVector): VectorQuery {
-    // Observe promised vectors immediately so a rejection cannot become an
-    // unhandled rejection while auto routing is still resolving (or when the
-    // eventual route is FTS and vector-only calls are intentionally skipped).
-    // The settled outcome remains fulfilled and is rethrown only if a vector
-    // execution actually consumes it.
-    const settledVector = Promise.resolve(vector).then(
-      (value) => ({ status: "fulfilled" as const, value }),
-      (reason) => ({ status: "rejected" as const, reason }),
-    );
-    this.deferred.doVectorCall(async (inner) => {
-      const outcome = await settledVector;
-      if (outcome.status === "rejected") {
-        throw outcome.reason;
-      }
-      addQueryVectorToNative(inner, outcome.value);
-    });
-    return this;
-  }
-}
-
 /**
  * Create a string query whose vector/FTS routing is resolved against the active
  * table schema when the query executes.
@@ -903,7 +778,7 @@ export function createAutoQuery(
   query: string,
   columns: string[] | null,
   getVector: (metadata: string) => Promise<Awaited<IntoVector>>,
-): VectorQuery {
+): AutoQuery {
   type RouteSnapshot = {
     table: NativeTable;
     embeddingMetadata: string | undefined;
@@ -911,7 +786,6 @@ export function createAutoQuery(
   type CachedPreparation = {
     metadata: string;
     vector: Promise<Awaited<IntoVector>>;
-    settled: boolean;
   };
 
   let cachedPreparation: CachedPreparation | undefined;
@@ -925,75 +799,37 @@ export function createAutoQuery(
     };
   };
 
-  const deferred = new DeferredAutoNativeQuery(async () => {
-    while (true) {
-      const initial = await snapshotRoute();
-      if (initial.embeddingMetadata === undefined) {
-        const inner = initial.table.query();
-        inner.fullTextSearch({ query, columns });
-        return { inner, route: "fts" };
-      }
+  const createInner = async (): Promise<NativeQuery | NativeVectorQuery> => {
+    const route = await snapshotRoute();
+    if (route.embeddingMetadata === undefined) {
+      const inner = route.table.query();
+      inner.fullTextSearch({ query, columns });
+      return inner;
+    }
 
-      const metadata = initial.embeddingMetadata;
-      if (cachedPreparation?.metadata !== metadata) {
-        const vector = Promise.resolve().then(() => getVector(metadata));
-        const preparation: CachedPreparation = {
-          metadata,
-          settled: false,
-          vector,
-        };
-        void vector.then(
-          () => {
-            preparation.settled = true;
-          },
-          () => {
-            preparation.settled = true;
-          },
-        );
-        cachedPreparation = preparation;
-      }
-
-      const preparation = cachedPreparation;
-      const preparationWasWarm = preparation.settled;
-      let vector: Awaited<IntoVector>;
-      try {
-        vector = await preparation.vector;
-      } catch (error) {
-        if (cachedPreparation === preparation) {
-          cachedPreparation = undefined;
-        }
-        throw error;
-      }
-
-      // Newly started (or still in-flight) provider preparation can perform
-      // arbitrary asynchronous work. Revalidate after that work, but reuse the
-      // initial pinned snapshot once preparation was already warm. This avoids
-      // a second remote describe request on every repeated execution.
-      if (!preparationWasWarm) {
-        const current = await snapshotRoute();
-        if (current.embeddingMetadata !== metadata) {
-          // A stale execution must not erase another revision's newer in-flight
-          // preparation.
-          if (cachedPreparation === preparation) {
-            cachedPreparation = undefined;
-          }
-          continue;
-        }
-
-        return {
-          inner: nearestToNative(current.table.query(), vector),
-          route: "vector",
-        };
-      }
-
-      return {
-        inner: nearestToNative(initial.table.query(), vector),
-        route: "vector",
+    const metadata = route.embeddingMetadata;
+    if (cachedPreparation?.metadata !== metadata) {
+      cachedPreparation = {
+        metadata,
+        vector: Promise.resolve().then(() => getVector(metadata)),
       };
     }
-  });
 
-  return new DeferredAutoQuery(deferred);
+    const preparation = cachedPreparation;
+    let vector: Awaited<IntoVector>;
+    try {
+      vector = await preparation.vector;
+    } catch (error) {
+      if (cachedPreparation === preparation) {
+        cachedPreparation = undefined;
+      }
+      throw error;
+    }
+
+    return nearestToNative(route.table.query(), vector);
+  };
+
+  return new AutoQuery(createInner);
 }
 
 /**
@@ -1021,6 +857,51 @@ export class TakeQuery extends QueryBase<NativeTakeQuery> {
   }
 }
 
+/**
+ * A builder for automatic string searches.
+ *
+ * Automatic search determines whether to use full-text or vector search from
+ * the table revision selected for each execution. This builder exposes the
+ * common operations supported by both query families.
+ *
+ * @hideconstructor
+ */
+export class AutoQuery extends StandardQueryBase<
+  NativeQuery | NativeVectorQuery
+> {
+  private readonly calls: Array<
+    (inner: NativeQuery | NativeVectorQuery) => void
+  > = [];
+
+  /** @hidden */
+  constructor(
+    private readonly createInner: () => Promise<
+      NativeQuery | NativeVectorQuery
+    >,
+  ) {
+    super();
+  }
+
+  /** @hidden */
+  protected override doCall(
+    fn: (inner: NativeQuery | NativeVectorQuery) => void,
+  ) {
+    this.calls.push(fn);
+  }
+
+  /** @hidden */
+  protected override async getInner(): Promise<
+    NativeQuery | NativeVectorQuery
+  > {
+    const calls = [...this.calls];
+    const inner = await this.createInner();
+    for (const call of calls) {
+      call(inner);
+    }
+    return inner;
+  }
+}
+
 /** A builder for LanceDB queries.
  *
  * @see {@link Table#query}, {@link Table#search}
@@ -1073,33 +954,19 @@ export class Query extends StandardQueryBase<NativeQuery> {
    * a default `limit` of 10 will be used.  @see {@link Query#limit}
    */
   nearestTo(vector: IntoVector): VectorQuery {
-    const inner = this.resolveInner();
+    const inner = this.inner;
     if (inner instanceof Promise) {
-      const nativeQuery = inner.then(async (inner) => {
-        const resolved = vector instanceof Promise ? await vector : vector;
-        return nearestToNative(inner, resolved);
-      });
+      const nativeQuery = inner.then(async (resolvedInner) =>
+        nearestToNative(resolvedInner, await vector),
+      );
       return new VectorQuery(nativeQuery);
     }
     if (vector instanceof Promise) {
-      const res = (async () => {
-        try {
-          const v = await vector;
-          // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-          const value: any = this.nearestTo(v);
-          const inner = value.inner as
-            | NativeVectorQuery
-            | Promise<NativeVectorQuery>;
-          return inner;
-        } catch (e) {
-          return Promise.reject(e);
-        }
-      })();
-      return new VectorQuery(res);
-    } else {
-      const vectorQuery = nearestToNative(inner, vector);
-      return new VectorQuery(vectorQuery);
+      return new VectorQuery(
+        vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)),
+      );
     }
+    return new VectorQuery(nearestToNative(inner, vector));
   }
 
   nearestToText(query: string | FullTextQuery, columns?: string[]): Query {

nodejs/lancedb/sanitize.ts

@@ -9,7 +9,7 @@
 // comes from the exact same library instance.  This is not always the case
 // and so we must sanitize the input to ensure that it is compatible.
 
-import { BufferType, Data } from "apache-arrow";
+import { BufferType, Data, Vector } from "apache-arrow";
 import type { IntBitWidth, TKeys, TimeBitWidth } from "apache-arrow/type";
 import {
   Binary,
@@ -74,6 +74,20 @@ import {
   Utf8,
 } from "./arrow";
 
+type SanitizationContext = {
+  types: WeakMap<object, DataType>;
+  vectors: WeakMap<object, Vector>;
+  data: WeakMap<object, Data<DataType>>;
+};
+
+function createSanitizationContext(): SanitizationContext {
+  return {
+    types: new WeakMap(),
+    vectors: new WeakMap(),
+    data: new WeakMap(),
+  };
+}
+
 export function sanitizeMetadata(
   metadataLike?: unknown,
 ): Map<string, string> | undefined {
@@ -186,6 +200,13 @@ export function sanitizeInterval(typeLike: object) {
 }
 
 export function sanitizeList(typeLike: object) {
+  return sanitizeListWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeListWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a List type to have an array-like `children` property",
@@ -194,19 +215,35 @@ export function sanitizeList(typeLike: object) {
   if (typeLike.children.length !== 1) {
     throw Error("Expected a List type to have exactly one child");
   }
-  return new List(sanitizeField(typeLike.children[0]));
+  return new List(sanitizeFieldWithContext(typeLike.children[0], context));
 }
 
 export function sanitizeStruct(typeLike: object) {
+  return sanitizeStructWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeStructWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Struct type to have an array-like `children` property",
     );
   }
-  return new Struct(typeLike.children.map((child) => sanitizeField(child)));
+  return new Struct(
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
+  );
 }
 
 export function sanitizeUnion(typeLike: object) {
+  return sanitizeUnionWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeUnionWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (
     !("typeIds" in typeLike) ||
     !("mode" in typeLike) ||
@@ -226,7 +263,7 @@ export function sanitizeUnion(typeLike: object) {
     typeLike.mode,
     // biome-ignore lint/suspicious/noExplicitAny: skip
     typeLike.typeIds as any,
-    typeLike.children.map((child) => sanitizeField(child)),
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
   );
 }
 
@@ -234,6 +271,19 @@ export function sanitizeTypedUnion(
   typeLike: object,
   // eslint-disable-next-line @typescript-eslint/naming-convention
   UnionType: typeof DenseUnion | typeof SparseUnion,
+) {
+  return sanitizeTypedUnionWithContext(
+    typeLike,
+    UnionType,
+    createSanitizationContext(),
+  );
+}
+
+function sanitizeTypedUnionWithContext(
+  typeLike: object,
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  UnionType: typeof DenseUnion | typeof SparseUnion,
+  context: SanitizationContext,
 ) {
   if (!("typeIds" in typeLike)) {
     throw Error(
@@ -248,7 +298,7 @@ export function sanitizeTypedUnion(
 
   return new UnionType(
     typeLike.typeIds as Int32Array | number[],
-    typeLike.children.map((child) => sanitizeField(child)),
+    typeLike.children.map((child) => sanitizeFieldWithContext(child, context)),
   );
 }
 
@@ -262,6 +312,16 @@ export function sanitizeFixedSizeBinary(typeLike: object) {
 }
 
 export function sanitizeFixedSizeList(typeLike: object) {
+  return sanitizeFixedSizeListWithContext(
+    typeLike,
+    createSanitizationContext(),
+  );
+}
+
+function sanitizeFixedSizeListWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("listSize" in typeLike) || typeof typeLike.listSize !== "number") {
     throw Error("Expected a FixedSizeList type to have a `listSize` property");
   }
@@ -275,11 +335,18 @@ export function sanitizeFixedSizeList(typeLike: object) {
   }
   return new FixedSizeList(
     typeLike.listSize,
-    sanitizeField(typeLike.children[0]),
+    sanitizeFieldWithContext(typeLike.children[0], context),
   );
 }
 
 export function sanitizeMap(typeLike: object) {
+  return sanitizeMapWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeMapWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("children" in typeLike) || !Array.isArray(typeLike.children)) {
     throw Error(
       "Expected a Map type to have an array-like `children` property",
@@ -292,7 +359,10 @@ export function sanitizeMap(typeLike: object) {
     throw Error("Expected a Map type to have exactly one child");
   }
 
-  return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
+  return new Map_(
+    sanitizeFieldWithContext(typeLike.children[0], context),
+    typeLike.keysSorted,
+  );
 }
 
 export function sanitizeDuration(typeLike: object) {
@@ -303,6 +373,13 @@ export function sanitizeDuration(typeLike: object) {
 }
 
 export function sanitizeDictionary(typeLike: object) {
+  return sanitizeDictionaryWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeDictionaryWithContext(
+  typeLike: object,
+  context: SanitizationContext,
+) {
   if (!("id" in typeLike) || typeof typeLike.id !== "number") {
     throw Error("Expected a Dictionary type to have an `id` property");
   }
@@ -316,8 +393,8 @@ export function sanitizeDictionary(typeLike: object) {
     throw Error("Expected a Dictionary type to have an `isOrdered` property");
   }
   return new Dictionary(
-    sanitizeType(typeLike.dictionary),
-    sanitizeType(typeLike.indices) as TKeys,
+    sanitizeTypeWithContext(typeLike.dictionary, context),
+    sanitizeTypeWithContext(typeLike.indices, context) as TKeys,
     typeLike.id,
     typeLike.isOrdered,
   );
@@ -325,12 +402,23 @@ export function sanitizeDictionary(typeLike: object) {
 
 // biome-ignore lint/suspicious/noExplicitAny: skip
 export function sanitizeType(typeLike: unknown): DataType<any> {
+  return sanitizeTypeWithContext(typeLike, createSanitizationContext());
+}
+
+function sanitizeTypeWithContext(
+  typeLike: unknown,
+  context: SanitizationContext,
+): DataType {
   if (typeof typeLike === "string") {
     return dataTypeFromName(typeLike);
   }
   if (typeof typeLike !== "object" || typeLike === null) {
     throw Error("Expected a Type but object was null/undefined");
   }
+  const cached = context.types.get(typeLike);
+  if (cached !== undefined) {
+    return cached;
+  }
   if (
     !("typeId" in typeLike) ||
     !(
@@ -349,6 +437,16 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     throw Error("Type's typeId property was not a function or number");
   }
 
+  const type = sanitizeTypeById(typeLike, typeId, context);
+  context.types.set(typeLike, type);
+  return type;
+}
+
+function sanitizeTypeById(
+  typeLike: object,
+  typeId: Type,
+  context: SanitizationContext,
+): DataType {
   switch (typeId) {
     case Type.NONE:
       throw Error("Received a Type with a typeId of NONE");
@@ -375,21 +473,21 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     case Type.Interval:
       return sanitizeInterval(typeLike);
     case Type.List:
-      return sanitizeList(typeLike);
+      return sanitizeListWithContext(typeLike, context);
     case Type.Struct:
-      return sanitizeStruct(typeLike);
+      return sanitizeStructWithContext(typeLike, context);
     case Type.Union:
-      return sanitizeUnion(typeLike);
+      return sanitizeUnionWithContext(typeLike, context);
     case Type.FixedSizeBinary:
       return sanitizeFixedSizeBinary(typeLike);
     case Type.FixedSizeList:
-      return sanitizeFixedSizeList(typeLike);
+      return sanitizeFixedSizeListWithContext(typeLike, context);
     case Type.Map:
-      return sanitizeMap(typeLike);
+      return sanitizeMapWithContext(typeLike, context);
     case Type.Duration:
       return sanitizeDuration(typeLike);
     case Type.Dictionary:
-      return sanitizeDictionary(typeLike);
+      return sanitizeDictionaryWithContext(typeLike, context);
     case Type.Int8:
       return new Int8();
     case Type.Int16:
@@ -433,9 +531,9 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
     case Type.TimestampSecond:
       return sanitizeTypedTimestamp(typeLike, TimestampSecond);
     case Type.DenseUnion:
-      return sanitizeTypedUnion(typeLike, DenseUnion);
+      return sanitizeTypedUnionWithContext(typeLike, DenseUnion, context);
     case Type.SparseUnion:
-      return sanitizeTypedUnion(typeLike, SparseUnion);
+      return sanitizeTypedUnionWithContext(typeLike, SparseUnion, context);
     case Type.IntervalDayTime:
       return new IntervalDayTime();
     case Type.IntervalYearMonth:
@@ -454,6 +552,13 @@ export function sanitizeType(typeLike: unknown): DataType<any> {
 }
 
 export function sanitizeField(fieldLike: unknown): Field {
+  return sanitizeFieldWithContext(fieldLike, createSanitizationContext());
+}
+
+function sanitizeFieldWithContext(
+  fieldLike: unknown,
+  context: SanitizationContext,
+): Field {
   if (fieldLike instanceof Field) {
     return fieldLike;
   }
@@ -471,7 +576,7 @@ export function sanitizeField(fieldLike: unknown): Field {
   }
   let type: DataType;
   try {
-    type = sanitizeType(fieldLike.type);
+    type = sanitizeTypeWithContext(fieldLike.type, context);
   } catch (error: unknown) {
     throw Error(
       `Unable to sanitize type for field: ${fieldLike.name} due to error: ${error}`,
@@ -501,6 +606,13 @@ export function sanitizeField(fieldLike: unknown): Field {
  * than lancedb is using.
  */
 export function sanitizeSchema(schemaLike: SchemaLike): Schema {
+  return sanitizeSchemaWithContext(schemaLike, createSanitizationContext());
+}
+
+function sanitizeSchemaWithContext(
+  schemaLike: SchemaLike,
+  context: SanitizationContext,
+): Schema {
   if (schemaLike instanceof Schema) {
     return schemaLike;
   }
@@ -522,7 +634,7 @@ export function sanitizeSchema(schemaLike: SchemaLike): Schema {
     );
   }
   const sanitizedFields = schemaLike.fields.map((field) =>
-    sanitizeField(field),
+    sanitizeFieldWithContext(field, context),
   );
   return new Schema(sanitizedFields, metadata);
 }
@@ -544,13 +656,18 @@ export function sanitizeTable(tableLike: TableLike): Table {
       "The table passed in does not appear to be a table (no 'columns' property)",
     );
   }
-  const schema = sanitizeSchema(tableLike.schema);
-
-  const batches = tableLike.batches.map(sanitizeRecordBatch);
+  const context = createSanitizationContext();
+  const schema = sanitizeSchemaWithContext(tableLike.schema, context);
+  const batches = tableLike.batches.map((batch) =>
+    sanitizeRecordBatch(batch, context),
+  );
   return new Table(schema, batches);
 }
 
-function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
+function sanitizeRecordBatch(
+  batchLike: RecordBatchLike,
+  context: SanitizationContext,
+): RecordBatch {
   if (batchLike instanceof RecordBatch) {
     return batchLike;
   }
@@ -567,19 +684,43 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
       "The record batch passed in does not appear to be a record batch (no 'data' property)",
     );
   }
-  const schema = sanitizeSchema(batchLike.schema);
-  const data = sanitizeData(batchLike.data);
+  const schema = sanitizeSchemaWithContext(batchLike.schema, context);
+  const data = sanitizeData(batchLike.data, context) as Data<Struct>;
   return new RecordBatch(schema, data);
 }
+
+type DictionaryVectorLike = {
+  data: readonly DataLike[];
+};
+
+type DictionaryDataLike = DataLike & {
+  dictionary?: DictionaryVectorLike;
+};
+
 function sanitizeData(
   dataLike: DataLike,
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-): import("apache-arrow").Data<Struct<any>> {
+  context: SanitizationContext,
+): Data<DataType> {
   if (dataLike instanceof Data) {
     return dataLike;
   }
-  return new Data(
-    dataLike.type,
+  const cachedData = context.data.get(dataLike);
+  if (cachedData !== undefined) {
+    return cachedData;
+  }
+  const dictionaryLike = (dataLike as DictionaryDataLike).dictionary;
+  let dictionary: Vector | undefined;
+  if (dictionaryLike !== undefined) {
+    dictionary = context.vectors.get(dictionaryLike);
+    if (dictionary === undefined) {
+      dictionary = new Vector(
+        dictionaryLike.data.map((data) => sanitizeData(data, context)),
+      );
+      context.vectors.set(dictionaryLike, dictionary);
+    }
+  }
+  const data = new Data(
+    sanitizeTypeWithContext(dataLike.type, context),
     dataLike.offset,
     dataLike.length,
     dataLike.nullCount,
@@ -589,7 +730,11 @@ function sanitizeData(
       [BufferType.VALIDITY]: dataLike.nullBitmap,
       [BufferType.TYPE]: dataLike.typeIds,
     },
+    dataLike.children.map((child) => sanitizeData(child, context)),
+    dictionary,
   );
+  context.data.set(dataLike, data);
+  return data;
 }
 
 const constructorsByTypeName = {

nodejs/lancedb/schema.ts

@@ -0,0 +1,566 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import {
+  Binary,
+  Bool,
+  DataType,
+  Dictionary,
+  Field,
+  FixedSizeList,
+  Float32,
+  Float64,
+  Int32,
+  Int64,
+  List,
+  Schema,
+  Struct,
+  Utf8,
+  util as arrowUtil,
+} from "apache-arrow";
+import { typedArrayToArrowType } from "./arrow_type";
+import { sanitizeType } from "./sanitize";
+
+type InferenceOptions = {
+  dictionaryEncodeStrings: boolean;
+  vectorColumns: Record<string, { type: unknown }>;
+};
+
+/**
+ * Infer the Arrow schema represented by a set of records.
+ *
+ * This is the intentionally small interface to schema inference. The stateful
+ * details of combining partial type evidence are encapsulated below so callers
+ * only need to provide records, an optional schema, and inference options.
+ */
+export function inferSchema(
+  data: Array<Record<string, unknown>>,
+  schema: Schema | undefined,
+  options: InferenceOptions,
+): Schema {
+  return new SchemaInferrer(schema, options).infer(data);
+}
+
+class SchemaInferrer {
+  private readonly fields = new FieldTree();
+
+  constructor(
+    private readonly providedSchema: Schema | undefined,
+    private readonly options: InferenceOptions,
+  ) {}
+
+  infer(data: Array<Record<string, unknown>>): Schema {
+    for (const [row, record] of data.entries()) {
+      for (const [path, value] of recordPathsAndValues(record)) {
+        this.observe(path, value, row);
+      }
+    }
+
+    return this.providedSchema === undefined
+      ? new Schema(fieldsFromTree(this.fields))
+      : new Schema(matchingFields(this.providedSchema.fields, this.fields));
+  }
+
+  private observe(path: string[], value: unknown, row: number): void {
+    const current = this.fields.get(path);
+    if (current === undefined) {
+      this.addField(path, value, row);
+    } else if (this.providedSchema === undefined) {
+      this.updateInferredField(path, value, row, current);
+    }
+  }
+
+  private addField(path: string[], value: unknown, row: number): void {
+    if (this.providedSchema !== undefined) {
+      this.addSchemaField(this.providedSchema, path, row);
+      return;
+    }
+
+    const evidence =
+      this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
+    if (evidence === undefined) {
+      throw typeInferenceError(path, row);
+    }
+
+    const conflict = this.fields.set(
+      path,
+      evidence,
+      (existing) =>
+        existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(),
+    );
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private addSchemaField(schema: Schema, path: string[], row: number): void {
+    const field = fieldAtPath(schema, path);
+    if (field === undefined) {
+      throw new Error(
+        `Found field not in schema: ${path.join(".")} at row ${row}`,
+      );
+    }
+
+    const conflict = this.fields.set(path, field.type);
+    if (conflict !== undefined) {
+      throw branchConflictError(conflict, row, "Struct");
+    }
+  }
+
+  private updateInferredField(
+    path: string[],
+    value: unknown,
+    row: number,
+    current: FieldNode,
+  ): void {
+    const newType = this.inferType(value, path);
+    const deferred = DeferredTypeEvidence.from(value, row);
+
+    if (current instanceof FieldTree) {
+      if (deferred?.isOnlyNulls()) {
+        return;
+      }
+      throw schemaInferenceError(
+        path,
+        row,
+        "Struct",
+        describeEvidence(newType ?? deferred),
+      );
+    }
+
+    if (current instanceof DeferredTypeEvidence) {
+      this.resolveDeferredField(path, row, current, newType, deferred);
+      return;
+    }
+
+    if (newType !== undefined) {
+      if (!inferredTypesEqual(current, newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          describeEvidence(current),
+          describeEvidence(newType),
+        );
+      }
+      return;
+    }
+
+    if (deferred === undefined || !deferred.matches(current)) {
+      throw schemaInferenceError(
+        path,
+        row,
+        describeEvidence(current),
+        describeEvidence(deferred),
+      );
+    }
+  }
+
+  private resolveDeferredField(
+    path: string[],
+    row: number,
+    current: DeferredTypeEvidence,
+    newType: DataType | undefined,
+    deferred: DeferredTypeEvidence | undefined,
+  ): void {
+    if (newType !== undefined) {
+      if (!current.matches(newType)) {
+        throw schemaInferenceError(
+          path,
+          row,
+          current.describe(),
+          describeEvidence(newType),
+        );
+      }
+      this.fields.set(path, newType);
+      return;
+    }
+
+    if (deferred !== undefined) {
+      this.fields.set(path, current.merge(deferred));
+      return;
+    }
+
+    throw schemaInferenceError(
+      path,
+      row,
+      current.describe(),
+      describeEvidence(newType),
+    );
+  }
+
+  private inferType(value: unknown, path: string[]): DataType | undefined {
+    if (typeof value === "bigint") {
+      return new Int64();
+    }
+    if (typeof value === "number") {
+      return new Float64();
+    }
+    if (typeof value === "string") {
+      return this.options.dictionaryEncodeStrings
+        ? new Dictionary(new Utf8(), new Int32())
+        : new Utf8();
+    }
+    if (typeof value === "boolean") {
+      return new Bool();
+    }
+    if (value instanceof Buffer) {
+      return new Binary();
+    }
+    if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
+      const typedArray = typedArrayToArrowType(value);
+      return typedArray === undefined
+        ? undefined
+        : new FixedSizeList(
+            typedArray.length,
+            new Field("item", typedArray.elementType, true),
+          );
+    }
+    if (!Array.isArray(value) || value.length === 0) {
+      return undefined;
+    }
+
+    const configuredVector =
+      path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
+    if (configuredVector !== undefined) {
+      return new FixedSizeList(
+        value.length,
+        new Field("item", sanitizeType(configuredVector.type), true),
+      );
+    }
+
+    const itemType = this.inferArrayItemType(value, path);
+    if (itemType === undefined) {
+      return undefined;
+    }
+
+    return nameSuggestsVectorColumn(path[path.length - 1])
+      ? new FixedSizeList(value.length, new Field("item", new Float32(), true))
+      : new List(new Field("item", itemType, true));
+  }
+
+  private inferArrayItemType(
+    values: unknown[],
+    path: string[],
+  ): DataType | undefined {
+    let itemType: DataType | undefined;
+    const deferredItems: unknown[] = [];
+
+    for (const value of values) {
+      const candidate = this.inferType(value, path);
+      if (candidate === undefined) {
+        if (!isDeferredValue(value)) {
+          return undefined;
+        }
+        deferredItems.push(value);
+      } else if (itemType === undefined) {
+        itemType = candidate;
+      } else if (!inferredTypesEqual(itemType, candidate)) {
+        return undefined;
+      }
+    }
+
+    if (itemType === undefined) {
+      return undefined;
+    }
+    return deferredItems.every((value) =>
+      deferredValueMatchesType(value, itemType),
+    )
+      ? itemType
+      : undefined;
+  }
+}
+
+/** Nulls and empty/all-null lists that do not determine a type by themselves. */
+class DeferredTypeEvidence {
+  private constructor(
+    private readonly values: Array<{ value: unknown; row: number }>,
+  ) {}
+
+  static from(value: unknown, row: number): DeferredTypeEvidence | undefined {
+    return isDeferredValue(value)
+      ? new DeferredTypeEvidence([{ value, row }])
+      : undefined;
+  }
+
+  isOnlyNulls(): boolean {
+    return this.values.every(({ value }) => value == null);
+  }
+
+  matches(type: DataType): boolean {
+    return this.values.every(({ value }) =>
+      deferredValueMatchesType(value, type),
+    );
+  }
+
+  merge(other: DeferredTypeEvidence): DeferredTypeEvidence {
+    return new DeferredTypeEvidence([...this.values, ...other.values]);
+  }
+
+  describe(): string {
+    const list = this.values.find(({ value }) => Array.isArray(value));
+    return list === undefined
+      ? "null"
+      : `List[${(list.value as unknown[]).length}]`;
+  }
+
+  firstRow(): number {
+    return this.values[0].row;
+  }
+}
+
+type FieldNode = DataType | DeferredTypeEvidence | FieldTree;
+type LeafNode = Exclude<FieldNode, FieldTree>;
+type FieldConflict = { path: string[]; value: FieldNode };
+
+/** Nested field state, kept separate from Arrow's eventual Struct types. */
+class FieldTree {
+  private readonly children = new Map<string, FieldNode>();
+
+  get(path: string[]): FieldNode | undefined {
+    let current: FieldNode = this;
+    for (const part of path) {
+      if (!(current instanceof FieldTree)) {
+        return undefined;
+      }
+      const child = current.children.get(part);
+      if (child === undefined) {
+        return undefined;
+      }
+      current = child;
+    }
+    return current;
+  }
+
+  set(
+    path: string[],
+    value: LeafNode,
+    canReplaceLeaf: (value: LeafNode) => boolean = () => false,
+  ): FieldConflict | undefined {
+    let branch: FieldTree = this;
+    for (const [index, part] of path.slice(0, -1).entries()) {
+      const child = branch.children.get(part);
+      if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
+        const nextBranch = new FieldTree();
+        branch.children.set(part, nextBranch);
+        branch = nextBranch;
+      } else if (child instanceof FieldTree) {
+        branch = child;
+      } else {
+        return { path: path.slice(0, index + 1), value: child };
+      }
+    }
+
+    const name = path[path.length - 1];
+    const current = branch.children.get(name);
+    if (current instanceof FieldTree) {
+      return { path, value: current };
+    }
+    branch.children.set(name, value);
+    return undefined;
+  }
+
+  entries(): IterableIterator<[string, FieldNode]> {
+    return this.children.entries();
+  }
+
+  has(name: string): boolean {
+    return this.children.has(name);
+  }
+}
+
+function isLeaf(value: FieldNode): value is LeafNode {
+  return !(value instanceof FieldTree);
+}
+
+function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] {
+  const fields: Field[] = [];
+  for (const [name, value] of tree.entries()) {
+    if (value instanceof FieldTree) {
+      fields.push(
+        new Field(
+          name,
+          new Struct(fieldsFromTree(value, [...path, name])),
+          true,
+        ),
+      );
+    } else if (value instanceof DeferredTypeEvidence) {
+      throw typeInferenceError([...path, name], value.firstRow());
+    } else {
+      fields.push(new Field(name, value, true));
+    }
+  }
+  return fields;
+}
+
+function matchingFields(fields: Field[], tree: FieldTree): Field[] {
+  const matches: Field[] = [];
+  for (const field of fields) {
+    if (!tree.has(field.name)) {
+      continue;
+    }
+    const value = tree.get([field.name]);
+    if (value instanceof FieldTree) {
+      const struct = field.type as Struct;
+      matches.push(
+        new Field(
+          field.name,
+          new Struct(matchingFields(struct.children, value)),
+          field.nullable,
+        ),
+      );
+    } else {
+      matches.push(new Field(field.name, value as DataType, field.nullable));
+    }
+  }
+  return matches;
+}
+
+function* recordPathsAndValues(
+  record: Record<string, unknown>,
+  path: string[] = [],
+): Generator<[string[], unknown]> {
+  for (const [name, value] of Object.entries(record)) {
+    if (isRecord(value)) {
+      yield* recordPathsAndValues(value, [...path, name]);
+    } else if (value !== undefined) {
+      yield [[...path, name], value];
+    }
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    !Array.isArray(value) &&
+    !(value instanceof RegExp) &&
+    !(value instanceof Date) &&
+    !(value instanceof Set) &&
+    !(value instanceof Map) &&
+    !(value instanceof Buffer) &&
+    !ArrayBuffer.isView(value)
+  );
+}
+
+function fieldAtPath(schema: Schema, path: string[]): Field | undefined {
+  let fields = schema.fields;
+  let field: Field | undefined;
+  for (const [index, name] of path.entries()) {
+    field = fields.find((candidate) => candidate.name === name);
+    if (field === undefined || index === path.length - 1) {
+      return field;
+    }
+    if (!DataType.isStruct(field.type)) {
+      return undefined;
+    }
+    fields = field.type.children;
+  }
+  return field;
+}
+
+function isDeferredValue(value: unknown): boolean {
+  return (
+    value == null || (Array.isArray(value) && value.every(isDeferredValue))
+  );
+}
+
+function deferredValueMatchesType(value: unknown, type: DataType): boolean {
+  if (value == null) {
+    return true;
+  }
+  if (!Array.isArray(value)) {
+    return false;
+  }
+  if (DataType.isList(type)) {
+    return value.every((item) =>
+      deferredValueMatchesType(item, type.valueType),
+    );
+  }
+  if (DataType.isFixedSizeList(type)) {
+    return (
+      value.length === type.listSize &&
+      value.every((item) => deferredValueMatchesType(item, type.valueType))
+    );
+  }
+  return false;
+}
+
+function inferredTypesEqual(current: DataType, candidate: DataType): boolean {
+  if (DataType.isDictionary(current)) {
+    return (
+      DataType.isDictionary(candidate) &&
+      current.isOrdered === candidate.isOrdered &&
+      inferredTypesEqual(current.indices, candidate.indices) &&
+      inferredTypesEqual(current.dictionary, candidate.dictionary)
+    );
+  }
+  if (DataType.isList(current)) {
+    return (
+      DataType.isList(candidate) &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  if (DataType.isFixedSizeList(current)) {
+    return (
+      DataType.isFixedSizeList(candidate) &&
+      current.listSize === candidate.listSize &&
+      current.valueField.name === candidate.valueField.name &&
+      current.valueField.nullable === candidate.valueField.nullable &&
+      inferredTypesEqual(current.valueType, candidate.valueType)
+    );
+  }
+  return arrowUtil.compareTypes(current, candidate);
+}
+
+function describeEvidence(
+  evidence: DataType | DeferredTypeEvidence | undefined,
+): string {
+  if (evidence === undefined) {
+    return "an unsupported value";
+  }
+  return evidence instanceof DeferredTypeEvidence
+    ? evidence.describe()
+    : evidence.toString();
+}
+
+function branchConflictError(
+  conflict: FieldConflict,
+  row: number,
+  candidate: string,
+): Error {
+  return schemaInferenceError(
+    conflict.path,
+    row,
+    conflict.value instanceof FieldTree
+      ? "Struct"
+      : describeEvidence(conflict.value),
+    candidate,
+  );
+}
+
+function schemaInferenceError(
+  path: string[],
+  row: number,
+  currentType: string,
+  newType: string,
+): Error {
+  return new Error(
+    `Failed to infer schema for data. Previously inferred type ${currentType} ` +
+      `but found ${newType} for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function typeInferenceError(path: string[], row: number): Error {
+  return new Error(
+    `Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
+      "Consider providing an explicit schema.",
+  );
+}
+
+function nameSuggestsVectorColumn(name: string): boolean {
+  const normalized = name.toLowerCase();
+  return normalized.includes("vector") || normalized.includes("embedding");
+}

nodejs/lancedb/table.ts

@@ -31,8 +31,11 @@ import {
   IndexConfig,
   IndexStatistics,
   Job,
+  LsmStats,
   Branches as NativeBranches,
   OptimizeStats,
+  RefreshColumnResult,
+  RefreshMaterializedViewResult,
   TableStatistics,
   Tags,
   UpdateFieldMetadataResult,
@@ -40,6 +43,7 @@ import {
   Table as _NativeTable,
 } from "./native";
 import {
+  AutoQuery,
   FullTextQuery,
   Query,
   TakeQuery,
@@ -50,6 +54,12 @@ import {
 import { sanitizeType } from "./sanitize";
 import { IntoSql, toSQL } from "./util";
 export { IndexConfig } from "./native";
+export {
+  BucketStats,
+  GenerationStats,
+  LsmStats,
+  MemtableStats,
+} from "./native";
 
 /**
  * Progress snapshot for a write operation, delivered to the `progress`
@@ -198,7 +208,11 @@ export interface LsmWriteSpec {
   column?: string;
   /** Bucket variant: the number of buckets, in `[1, 1024]`. */
   numBuckets?: number;
-  /** Names of indexes the MemWAL should keep up to date during writes. */
+  /**
+   * Indexes the MemWAL keeps up to date. Omit to maintain every supported
+   * index, resolved on install — a snapshot, so indexes created later are not
+   * maintained. Pass `[]` for none.
+   */
   maintainedIndexes?: string[];
   /** Default `ShardWriter` configuration recorded in the MemWAL index. */
   writerConfigDefaults?: Record<string, string>;
@@ -511,7 +525,7 @@ export abstract class Table {
     query: string | IntoVector | MultiVector | FullTextQuery,
     queryType?: string,
     ftsColumns?: string | string[],
-  ): VectorQuery | Query;
+  ): VectorQuery | Query | AutoQuery;
   /**
    * Search the table with a given query vector.
    *
@@ -522,18 +536,87 @@ export abstract class Table {
   abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
   /**
    * Add new columns with defined values.
+   *
+   * The `{ computed }` form stores the expression rather than evaluating it
+   * now: the column is committed with no values, and rows get them from
+   * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
+   * large table as on an empty one.
+   *
+   * A refresh does not revisit rows it has already filled, so mutating an
+   * input leaves the value computed at fill time; recomputing means dropping
+   * the column and declaring it again. While a declaration reads a column,
+   * that column cannot be renamed, retyped or dropped.
+   *
+   * On LanceDB Cloud and Enterprise the expression is planned by the
+   * server, and the refresh runs as a server job -- see
+   * {@link Table#refreshColumnAsync}.
    * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
    *   - An array of objects with column names and SQL expressions to calculate values
    *   - A single Arrow Field defining one column with its data type (column will be initialized with null values)
    *   - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
    *   - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
+   *   - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
    * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
    * containing the new version number of the table after adding the columns.
+   * @example
+   * ```ts
+   * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
+   * const { rowsFilled } = await table.refreshColumn("doubled");
+   * ```
    */
   abstract addColumns(
-    newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
+    newColumnTransforms:
+      | AddColumnsSql[]
+      | Field
+      | Field[]
+      | Schema
+      | { computed: AddColumnsSql[] },
   ): Promise<AddColumnsResult>;
 
+  /**
+   * Fill the rows of a computed column that hold no value yet.
+   *
+   * Rows appended since the last refresh are filled by the next one; rows
+   * already filled are left as they are, so the call is idempotent and does
+   * not observe a mutated input. Local tables only: a remote refresh runs
+   * as a server job, through {@link Table#refreshColumnAsync}.
+   * @param {string} column The name of the computed column to fill.
+   * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
+   * number of rows filled and the new version number of the table.
+   */
+  abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
+
+  /**
+   * Like {@link Table#refreshColumn}, but returns a handle to the refresh
+   * job instead of blocking until it completes.
+   *
+   * The job may already be complete when returned; callers must not assume
+   * the column is filled until {@link Job.wait} resolves. Invalid input --
+   * an unknown column, or one that is not computed -- rejects here rather
+   * than failing the job. On local tables the job runs in-process; on
+   * LanceDB Cloud and Enterprise it is the server's backfill job.
+   * @param {string} column The name of the computed column to fill.
+   * @example
+   * ```ts
+   * const job = await table.refreshColumnAsync("doubled");
+   * await job.wait();
+   * console.log(await job.status()); // "finished"
+   * ```
+   */
+  abstract refreshColumnAsync(column: string): Promise<Job>;
+
+  /**
+   * Recompute this table's contents from its materialized-view definition.
+   *
+   * Plumbing for {@link MaterializedView.refresh}, which is the way to call
+   * it: rejects tables that carry no view definition. Local tables only.
+   * @ignore
+   */
+  abstract refreshMaterializedView(
+    full?: boolean,
+    sourceVersion?: number,
+  ): Promise<RefreshMaterializedViewResult>;
+
   /**
    * Alter the name or nullability of columns.
    * @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -596,6 +679,11 @@ export abstract class Table {
    * All variants require the table to have an unenforced primary key
    * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
    * requires it to be the single column being bucketed.
+   *
+   * Omitting `maintainedIndexes` maintains every index on the table, resolved
+   * here, failing if one cannot be maintained — name them to install anyway.
+   * Naming them pins an exact set, and a still-building index is rejected
+   * rather than quietly omitted.
    * @param {LsmWriteSpec} spec The sharding spec to install.
    * @returns {Promise<void>}
    * @example
@@ -623,9 +711,10 @@ export abstract class Table {
    *
    * Resolves to `undefined` when the MemWAL LSM write path is not enabled (no
    * spec has been set, or it was removed with {@link Table#unsetLsmWriteSpec}).
-   * The returned spec — including its `maintainedIndexes` and
-   * `writerConfigDefaults` — mirrors what was passed to
-   * {@link Table#setLsmWriteSpec}.
+   * The returned spec mirrors what was passed to
+   * {@link Table#setLsmWriteSpec}, except that `maintainedIndexes` always
+   * reports the concrete list resolved when the spec was set — `undefined`
+   * never round-trips.
    * @returns {Promise<LsmWriteSpec | undefined>}
    */
   abstract getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
@@ -639,6 +728,59 @@ export abstract class Table {
    * @returns {Promise<void>}
    */
   abstract closeLsmWriters(): Promise<void>;
+  /**
+   * Seal every bucket's active memtable into a new L0 generation.
+   *
+   * Returns once the seal is committed. Sealing an empty memtable is a no-op,
+   * so this is safe to call repeatedly.
+   * @returns {Promise<void>}
+   */
+  abstract flushLsm(): Promise<void>;
+  /**
+   * Trigger a background L0 → base compaction pass per bucket.
+   *
+   * Returns once the passes are *dispatched*, not once they finish — watch
+   * {@link Table#getLsmStats} for progress, or use
+   * {@link Table#checkpointLsm} to wait for convergence.
+   * @returns {Promise<void>}
+   */
+  abstract compactLsm(): Promise<void>;
+  /**
+   * Converge this table's LSM write path into its base table.
+   *
+   * Seals once, then triggers compaction and polls until the L0 that existed
+   * at the start is gone. The target set is fixed at the start, so
+   * generations created *during* the checkpoint are ignored — that is what
+   * lets it terminate under write load, and what makes it best-effort: it
+   * converges the fresh tier as of some instant. Idempotent, abandonable at
+   * any point, and safe to run on a cadence.
+   *
+   * There is no liveness bound — the compactor pool is shared across tables,
+   * so a checkpoint queued behind unrelated work looks exactly like one that
+   * is merging. The caller owns the deadline.
+   * @returns {Promise<void>}
+   * @example
+   * ```ts
+   * const before = await table.getLsmStats();
+   * await table.checkpointLsm();
+   * const after = await table.getLsmStats();
+   * ```
+   */
+  abstract checkpointLsm(): Promise<void>;
+  /**
+   * Read live per-bucket LSM state.
+   *
+   * Answers "how far behind is my fresh tier", "which bucket is hot", and
+   * "why is my fresh-tier vector search brute-force". Mutates no table state.
+   *
+   * Resolves to `undefined` only when the LSM write path is not enabled.
+   * @param {boolean} includeGenerationRows Also count rows per L0 generation.
+   *   Off by default because each count opens an uncached Lance dataset.
+   * @returns {Promise<LsmStats | undefined>}
+   */
+  abstract getLsmStats(
+    includeGenerationRows?: boolean,
+  ): Promise<LsmStats | undefined>;
   /** Retrieve the version of the table */
 
   abstract version(): Promise<number>;
@@ -835,10 +977,11 @@ export class LocalTable extends Table {
     return this.inner.display();
   }
 
-  private async getEmbeddingFunctions(): Promise<
-    Map<string, EmbeddingFunctionConfig>
-  > {
-    const schema = await this.schema();
+  private async getEmbeddingFunctions(
+    inner: _NativeTable = this.inner,
+  ): Promise<Map<string, EmbeddingFunctionConfig>> {
+    const schemaBuf = await inner.schema();
+    const schema = tableFromIPC(schemaBuf).schema;
     const registry = getRegistry();
     return registry.parseFunctions(schema.metadata);
   }
@@ -1020,7 +1163,7 @@ export class LocalTable extends Table {
     query: string | IntoVector | MultiVector | FullTextQuery,
     queryType: string = "auto",
     ftsColumns?: string | string[],
-  ): VectorQuery | Query {
+  ): VectorQuery | Query | AutoQuery {
     if (typeof query !== "string" && !instanceOfFullTextQuery(query)) {
       if (queryType === "fts") {
         throw new Error("Cannot perform full text search on a vector query");
@@ -1093,8 +1236,22 @@ export class LocalTable extends Table {
   // TODO: Support BatchUDF
 
   async addColumns(
-    newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema,
+    newColumnTransforms:
+      | AddColumnsSql[]
+      | Field
+      | Field[]
+      | Schema
+      | { computed: AddColumnsSql[] },
   ): Promise<AddColumnsResult> {
+    // Columns defined by an expression are declared, not materialized here.
+    if (
+      typeof newColumnTransforms === "object" &&
+      !Array.isArray(newColumnTransforms) &&
+      "computed" in newColumnTransforms
+    ) {
+      return await this.inner.addComputedColumns(newColumnTransforms.computed);
+    }
+
     // Handle single Field -> convert to array of Fields
     if (newColumnTransforms instanceof Field) {
       newColumnTransforms = [newColumnTransforms];
@@ -1129,6 +1286,21 @@ export class LocalTable extends Table {
     throw new Error("Invalid input type for addColumns");
   }
 
+  async refreshColumn(column: string): Promise<RefreshColumnResult> {
+    return await this.inner.refreshColumn(column);
+  }
+
+  async refreshColumnAsync(column: string): Promise<Job> {
+    return await this.inner.refreshColumnAsync(column);
+  }
+
+  async refreshMaterializedView(
+    full?: boolean,
+    sourceVersion?: number,
+  ): Promise<RefreshMaterializedViewResult> {
+    return await this.inner.refreshMaterializedView(full, sourceVersion);
+  }
+
   async alterColumns(
     columnAlterations: ColumnAlteration[],
   ): Promise<AlterColumnsResult> {
@@ -1191,6 +1363,24 @@ export class LocalTable extends Table {
     return await this.inner.closeLsmWriters();
   }
 
+  async flushLsm(): Promise<void> {
+    return await this.inner.flushLsm();
+  }
+
+  async compactLsm(): Promise<void> {
+    return await this.inner.compactLsm();
+  }
+
+  async checkpointLsm(): Promise<void> {
+    return await this.inner.checkpointLsm();
+  }
+
+  async getLsmStats(
+    includeGenerationRows: boolean = false,
+  ): Promise<LsmStats | undefined> {
+    return (await this.inner.getLsmStats(includeGenerationRows)) ?? undefined;
+  }
+
   async version(): Promise<number> {
     return await this.inner.version();
   }
@@ -1404,8 +1594,8 @@ export interface BranchRowCountSummary {
   deltaAvailable: boolean;
 }
 
-/** A reason why a branch cannot currently be merged. */
-export interface MergeBlocker {
+/** A reason why a cherry-pick cannot currently land. */
+export interface CherryPickError {
   code: string;
   message: string;
 }
@@ -1425,20 +1615,19 @@ export interface BranchDiff {
   changedColumns: BranchColumnChange[];
   addedIndexes: BranchIndexSummary[];
   removedIndexes: BranchIndexSummary[];
-  mergeable: boolean;
-  mergeBlockers: MergeBlocker[];
+  errors: CherryPickError[];
 }
 
-/** Changes that would be, or were, promoted by a branch merge. */
-export interface MergePreview {
+/** Changes that would be, or were, promoted by a cherry-pick. */
+export interface CherryPickPreview {
   promotedColumns: string[];
 }
 
-/** Result of previewing or attempting a branch merge. */
-export interface MergeBranchResult {
-  status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
+/** Result of previewing or attempting a cherry-pick. */
+export interface CherryPickResult {
+  status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
   diff: BranchDiff;
-  preview: MergePreview;
+  preview: CherryPickPreview;
   mainVersionAfter?: number;
 }
 
@@ -1501,21 +1690,21 @@ export class Branches {
   }
 
   /**
-   * Merge a branch into main.
+   * Cherry-pick a branch onto main.
    *
-   * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
-   * with `status: "rejected"` instead of throwing.
+   * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
+   * with `status: "failed"` instead of throwing.
    *
-   * @param fromBranch Branch to merge from.
-   * @param dryRun When true, only preview the merge. Defaults to false.
+   * @param fromBranch Branch to cherry-pick from.
+   * @param dryRun When true, only preview. Defaults to false.
    */
-  async merge(
+  async cherryPick(
     fromBranch: string,
     dryRun: boolean = false,
-  ): Promise<MergeBranchResult> {
-    return (await this.#inner.merge(
+  ): Promise<CherryPickResult> {
+    return (await this.#inner.cherryPick(
       fromBranch,
       dryRun,
-    )) as unknown as MergeBranchResult;
+    )) as unknown as CherryPickResult;
   }
 }

nodejs/npm/darwin-arm64/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",

nodejs/npm/linux-arm64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",

nodejs/npm/linux-arm64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",

nodejs/npm/linux-x64-gnu/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",

nodejs/npm/linux-x64-musl/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",

nodejs/npm/win32-arm64-msvc/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "os": [
     "win32"
   ],

nodejs/npm/win32-x64-msvc/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.37.1-beta.0",
+	"version": "0.38.0-beta.10",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",

nodejs/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@lancedb/lancedb",
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@lancedb/lancedb",
-      "version": "0.37.1-beta.0",
+      "version": "0.38.0-beta.10",
       "cpu": [
         "x64",
         "arm64"
@@ -55,7 +55,13 @@
         "openai": "4.29.2"
       },
       "peerDependencies": {
+        "@types/node": ">=18",
         "apache-arrow": ">=15.0.0 <=18.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/node": {
+          "optional": true
+        }
       }
     },
     "node_modules/@aws-crypto/crc32": {

nodejs/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.37.1-beta.0",
+  "version": "0.38.0-beta.10",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -101,6 +101,12 @@
     "openai": "4.29.2"
   },
   "peerDependencies": {
+    "@types/node": ">=18",
     "apache-arrow": ">=15.0.0 <=18.1.0"
+  },
+  "peerDependenciesMeta": {
+    "@types/node": {
+      "optional": true
+    }
   }
 }

nodejs/src/connection.rs

@@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne
 
 use lance_namespace::models::{
     CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest,
+    ListTablesRequest,
 };
 use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema};
 
@@ -36,6 +37,12 @@ pub struct ListNamespacesResponse {
     pub page_token: Option<String>,
 }
 
+#[napi(object)]
+pub struct ListTablesResponse {
+    pub tables: Vec<String>,
+    pub page_token: Option<String>,
+}
+
 #[napi(object)]
 pub struct CreateNamespaceResponse {
     pub properties: Option<HashMap<String, String>>,
@@ -206,6 +213,33 @@ impl Connection {
         op.execute().await.default_error()
     }
 
+    /// List a page of tables in the database.
+    #[napi(catch_unwind)]
+    pub async fn list_tables(
+        &self,
+        namespace_path: Option<Vec<String>>,
+        page_token: Option<String>,
+        limit: Option<u32>,
+    ) -> napi::Result<ListTablesResponse> {
+        let request = ListTablesRequest {
+            // The root namespace is an empty path, not an absent one: a namespace-backed
+            // database rejects a request that names no namespace.
+            id: Some(namespace_path.unwrap_or_default()),
+            page_token,
+            limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)),
+            ..Default::default()
+        };
+        let response = self
+            .get_inner()?
+            .list_tables(request)
+            .await
+            .default_error()?;
+        Ok(ListTablesResponse {
+            tables: response.tables,
+            page_token: response.page_token,
+        })
+    }
+
     /// Create table from a Apache Arrow IPC (file) buffer.
     ///
     /// Parameters:
@@ -266,6 +300,58 @@ impl Connection {
         Ok(Table::new(tbl))
     }
 
+    #[napi(catch_unwind)]
+    pub async fn create_materialized_view(
+        &self,
+        name: String,
+        source: String,
+        projections: Option<Vec<Vec<String>>>,
+        filter: Option<String>,
+        limit: Option<i64>,
+    ) -> napi::Result<Table> {
+        let mut builder = self.get_inner()?.create_materialized_view(name, source);
+        if let Some(projections) = projections {
+            let mut pairs = Vec::with_capacity(projections.len());
+            for pair in projections {
+                let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
+                    napi::Error::from_reason("each projection must be an [output, expression] pair")
+                })?;
+                pairs.push((output, expression));
+            }
+            builder = builder.select(pairs);
+        }
+        if let Some(filter) = filter {
+            builder = builder.only_if(filter);
+        }
+        if let Some(limit) = limit {
+            let limit = u64::try_from(limit)
+                .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
+            builder = builder.limit(limit);
+        }
+        let view = builder.execute().await.default_error()?;
+        Ok(Table::new(view.table().clone()))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
+        let view = self
+            .get_inner()?
+            .open_materialized_view(&name)
+            .await
+            .default_error()?;
+        Ok(Table::new(view.table().clone()))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
+        let views = self
+            .get_inner()?
+            .list_materialized_views()
+            .await
+            .default_error()?;
+        Ok(views.into_iter().map(|v| v.name).collect())
+    }
+
     #[napi(catch_unwind)]
     pub async fn open_table(
         &self,
@@ -334,6 +420,22 @@ impl Connection {
             .default_error()
     }
 
+    /// Start dropping a table and return its cleanup job.
+    #[napi(catch_unwind)]
+    pub async fn drop_table_async(
+        &self,
+        name: String,
+        namespace_path: Option<Vec<String>>,
+    ) -> napi::Result<crate::job::Job> {
+        let ns = namespace_path.unwrap_or_default();
+        let job = self
+            .get_inner()?
+            .drop_table_async(&name, &ns)
+            .await
+            .default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
     #[napi(catch_unwind)]
     pub async fn drop_all_tables(&self, namespace_path: Option<Vec<String>>) -> napi::Result<()> {
         let ns = namespace_path.unwrap_or_default();

nodejs/src/job.rs

@@ -14,9 +14,12 @@ pub struct Job {
 }
 
 impl Job {
-    pub(crate) fn new(inner: lancedb::Job) -> Self {
+    pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
+    where
+        T: Clone + Send + Sync + 'static,
+    {
         Self {
-            inner: Arc::new(inner),
+            inner: Arc::new(inner.map(|_| ())),
         }
     }
 }

nodejs/src/lib.rs

@@ -1,6 +1,10 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
+// The materialized-view refresh future deepens the type graph past the
+// default trait-recursion depth; same raise as the core crate applies.
+#![recursion_limit = "256"]
+
 use std::collections::HashMap;
 
 use env_logger::Env;

nodejs/src/table.rs

@@ -354,6 +354,60 @@ impl Table {
         Ok(res.into())
     }
 
+    #[napi(catch_unwind)]
+    pub async fn add_computed_columns(
+        &self,
+        columns: Vec<AddColumnsSql>,
+    ) -> napi::Result<AddColumnsResult> {
+        let table = self.inner_ref()?;
+        let mut builder = table.add_columns();
+        for column in columns {
+            builder = builder.computed(column.name, column.value_sql);
+        }
+        let res = builder.execute().await.default_error()?;
+        Ok(res.into())
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_column(&self, column: String) -> napi::Result<RefreshColumnResult> {
+        let res = self
+            .inner_ref()?
+            .refresh_column(column)
+            .await
+            .default_error()?;
+        Ok(res.into())
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_column_async(&self, column: String) -> napi::Result<crate::job::Job> {
+        let job = self
+            .inner_ref()?
+            .refresh_column_async(column)
+            .await
+            .default_error()?;
+        Ok(crate::job::Job::new(job))
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn refresh_materialized_view(
+        &self,
+        full: Option<bool>,
+        source_version: Option<i64>,
+    ) -> napi::Result<RefreshMaterializedViewResult> {
+        let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
+            .await
+            .default_error()?;
+        let mut builder = view.refresh().full(full.unwrap_or(false));
+        if let Some(version) = source_version {
+            let version = u64::try_from(version).map_err(|_| {
+                napi::Error::from_reason("sourceVersion must be a non-negative integer")
+            })?;
+            builder = builder.source_version(version);
+        }
+        let result = builder.execute().await.default_error()?;
+        Ok(result.into())
+    }
+
     #[napi(catch_unwind)]
     pub async fn add_columns_with_schema(
         &self,
@@ -470,6 +524,34 @@ impl Table {
         self.inner_ref()?.close_lsm_writers().await.default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn flush_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.flush_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn compact_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.compact_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn checkpoint_lsm(&self) -> napi::Result<()> {
+        self.inner_ref()?.checkpoint_lsm().await.default_error()
+    }
+
+    #[napi(catch_unwind)]
+    pub async fn get_lsm_stats(
+        &self,
+        include_generation_rows: bool,
+    ) -> napi::Result<Option<LsmStats>> {
+        let stats = self
+            .inner_ref()?
+            .get_lsm_stats(include_generation_rows)
+            .await
+            .default_error()?;
+        Ok(stats.map(LsmStats::from))
+    }
+
     #[napi(catch_unwind)]
     pub async fn version(&self) -> napi::Result<i64> {
         self.inner_ref()?
@@ -479,6 +561,12 @@ impl Table {
             .default_error()
     }
 
+    #[napi(catch_unwind)]
+    pub async fn checkout_current(&self) -> napi::Result<Self> {
+        let table = self.inner_ref()?.checkout_current().await.default_error()?;
+        Ok(Self::new(table))
+    }
+
     #[napi(catch_unwind)]
     pub async fn checkout(&self, version: i64) -> napi::Result<()> {
         self.inner_ref()?
@@ -779,7 +867,8 @@ pub struct LsmWriteSpec {
     pub column: Option<String>,
     /// Bucket variant: the number of buckets, in `[1, 1024]`.
     pub num_buckets: Option<u32>,
-    /// Names of indexes the MemWAL should keep up to date during writes.
+    /// Indexes the MemWAL keeps up to date. Omitted resolves every
+    /// maintainable index on install; an empty array means none.
     pub maintained_indexes: Option<Vec<String>>,
     /// Default `ShardWriter` configuration recorded in the MemWAL index.
     pub writer_config_defaults: Option<HashMap<String, String>>,
@@ -789,7 +878,6 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
     type Error = napi::Error;
 
     fn try_from(value: LsmWriteSpec) -> napi::Result<Self> {
-        let maintained = value.maintained_indexes.unwrap_or_default();
         let writer_config_defaults = value.writer_config_defaults.unwrap_or_default();
         let spec = match value.spec_type.as_str() {
             "bucket" => {
@@ -816,7 +904,7 @@ impl TryFrom<LsmWriteSpec> for lancedb::table::LsmWriteSpec {
             }
         };
         Ok(spec
-            .with_maintained_indexes(maintained)
+            .with_maintained_indexes(value.maintained_indexes)
             .with_writer_config_defaults(writer_config_defaults))
     }
 }
@@ -834,7 +922,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "bucket".to_string(),
                 column: Some(column),
                 num_buckets: Some(num_buckets),
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
             Native::Identity {
@@ -845,7 +933,7 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "identity".to_string(),
                 column: Some(column),
                 num_buckets: None,
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
             Native::Unsharded {
@@ -855,13 +943,136 @@ impl From<lancedb::table::LsmWriteSpec> for LsmWriteSpec {
                 spec_type: "unsharded".to_string(),
                 column: None,
                 num_buckets: None,
-                maintained_indexes: Some(maintained_indexes),
+                maintained_indexes,
                 writer_config_defaults: Some(writer_config_defaults),
             },
         }
     }
 }
 
+/// One flushed L0 generation.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct GenerationStats {
+    /// The generation number. Increases as memtables are sealed into L0.
+    pub generation: i64,
+    /// On-disk size of the generation.
+    pub bytes: i64,
+    /// Present only when `includeGenerationRows` was requested. Off by default
+    /// because each count opens an uncached Lance dataset.
+    pub rows: Option<i64>,
+}
+
+impl From<lancedb::table::GenerationStats> for GenerationStats {
+    fn from(g: lancedb::table::GenerationStats) -> Self {
+        Self {
+            generation: g.generation as i64,
+            bytes: g.bytes as i64,
+            rows: g.rows.map(|r| r as i64),
+        }
+    }
+}
+
+/// One in-memory memtable.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct MemtableStats {
+    /// The generation this memtable will become once sealed.
+    pub generation: i64,
+    /// Rows currently buffered.
+    pub rows: i64,
+    /// Estimated in-memory size.
+    pub bytes: i64,
+    /// Record batches currently buffered.
+    pub batches: i64,
+    /// Names of the indexes this memtable carries. An absent name is the whole
+    /// answer to "why is my fresh-tier search on that column brute-force".
+    pub indexes: Vec<String>,
+}
+
+impl From<lancedb::table::MemtableStats> for MemtableStats {
+    fn from(m: lancedb::table::MemtableStats) -> Self {
+        Self {
+            generation: m.generation as i64,
+            rows: m.rows as i64,
+            bytes: m.bytes as i64,
+            batches: m.batches as i64,
+            indexes: m.indexes,
+        }
+    }
+}
+
+/// Live state of one bucket. A table is N buckets on one node; flattening to a
+/// single number hides the one hot bucket that is usually why someone opened
+/// this endpoint.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct BucketStats {
+    /// The shard this bucket writes.
+    pub shard_id: String,
+    /// `"Active"` or `"Sealed"` (drop-table 2PC in flight).
+    pub status: String,
+    /// Epoch of the writer that currently owns the shard.
+    pub writer_epoch: i64,
+    /// Version of the shard manifest these numbers were read from.
+    pub manifest_version: i64,
+    /// The generation the active memtable will become.
+    pub current_generation: i64,
+    /// WAL position replay resumes from.
+    pub replay_after_wal_entry_position: i64,
+    /// Highest WAL position the writer has seen. The difference against
+    /// `replayAfterWalEntryPosition` is the WAL lag.
+    pub wal_entry_position_last_seen: i64,
+    /// Flushed L0 generations not yet merged into the base table.
+    pub generations: Vec<GenerationStats>,
+    /// Whether a pass owns this bucket's compaction latch right now. Says *a*
+    /// driver is running, not *whose*, and the latch is held from dispatch —
+    /// including while the pass queues for a pod-wide compactor permit. Read it
+    /// as "do not pile on", never as "mine is progressing".
+    pub compacting: bool,
+    /// Oldest first, active last. Absent for a `"Sealed"` bucket, whose
+    /// in-memory state is torn down.
+    pub memtables: Option<Vec<MemtableStats>>,
+}
+
+impl From<lancedb::table::BucketStats> for BucketStats {
+    fn from(b: lancedb::table::BucketStats) -> Self {
+        Self {
+            shard_id: b.shard_id,
+            status: b.status,
+            writer_epoch: b.writer_epoch as i64,
+            manifest_version: b.manifest_version as i64,
+            current_generation: b.current_generation as i64,
+            replay_after_wal_entry_position: b.replay_after_wal_entry_position as i64,
+            wal_entry_position_last_seen: b.wal_entry_position_last_seen as i64,
+            generations: b.generations.into_iter().map(Into::into).collect(),
+            compacting: b.compacting,
+            memtables: b
+                .memtables
+                .map(|ms| ms.into_iter().map(Into::into).collect()),
+        }
+    }
+}
+
+/// Live per-bucket LSM state, as returned by `Table#getLsmStats`.
+///
+/// Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
+/// the caller's to compute.
+#[napi(object)]
+#[derive(Clone, Debug)]
+pub struct LsmStats {
+    /// One entry per bucket backing this table.
+    pub buckets: Vec<BucketStats>,
+}
+
+impl From<lancedb::table::LsmStats> for LsmStats {
+    fn from(stats: lancedb::table::LsmStats) -> Self {
+        Self {
+            buckets: stats.buckets.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
 /// Statistics about a compaction operation.
 #[napi(object)]
 #[derive(Clone, Debug)]
@@ -1050,7 +1261,10 @@ impl From<lancedb::index::IndexStatistics> for IndexStatistics {
 
 #[napi(object)]
 pub struct TableStatistics {
-    /// The total number of bytes in the table
+    /// The total size, in bytes, of the table's data files, index files, and
+    /// overlay files
+    ///
+    /// Read from the manifest, so this excludes deletion files and manifests.
     pub total_bytes: i64,
 
     /// The number of rows in the table
@@ -1200,6 +1414,46 @@ pub struct AddColumnsResult {
     pub version: i64,
 }
 
+#[napi(object)]
+pub struct RefreshColumnResult {
+    pub rows_filled: i64,
+    pub version: i64,
+}
+
+#[napi(object)]
+pub struct RefreshMaterializedViewResult {
+    /// How the view was brought up to date: "rebuild", "incremental" or "no_op".
+    pub mode: String,
+    pub rows_written: i64,
+    pub source_version: i64,
+    pub version: i64,
+}
+
+impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
+    fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
+        let mode = match value.mode {
+            lancedb::RefreshMode::Rebuild => "rebuild",
+            lancedb::RefreshMode::Incremental => "incremental",
+            lancedb::RefreshMode::NoOp => "no_op",
+        };
+        Self {
+            mode: mode.to_string(),
+            rows_written: value.rows_written as i64,
+            source_version: value.source_version as i64,
+            version: value.version as i64,
+        }
+    }
+}
+
+impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
+    fn from(value: lancedb::table::RefreshColumnResult) -> Self {
+        Self {
+            rows_filled: value.rows_filled as i64,
+            version: value.version as i64,
+        }
+    }
+}
+
 impl From<lancedb::table::AddColumnsResult> for AddColumnsResult {
     fn from(value: lancedb::table::AddColumnsResult) -> Self {
         Self {
@@ -1409,18 +1663,18 @@ impl Branches {
     }
 
     #[napi(ts_return_type = "Promise<Record<string, unknown>>")]
-    pub async fn merge(
+    pub async fn cherry_pick(
         &self,
         from_branch: String,
         dry_run: Option<bool>,
     ) -> napi::Result<serde_json::Value> {
         let result = self
             .inner
-            .merge_branch(&from_branch, dry_run.unwrap_or(false))
+            .cherry_pick(&from_branch, dry_run.unwrap_or(false))
             .await
             .default_error()?;
         serde_json::to_value(result).map_err(|err| {
-            napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
+            napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
         })
     }
 }