chore: merge main to pick up Tokio advisory fix

nodejs/__test__/connection.test.ts

@@ -5,15 +5,15 @@ import { mkdirSync, readdirSync } from "fs";
 import { join } from "path";
 import { Field, Float64, Schema } from "apache-arrow";
 import * as tmp from "tmp";
-import {
-  Connection,
-  ListTablesResponse,
-  Table,
-  connect,
-  connectNamespace,
-} from "../lancedb";
+import { Connection, Table, connect, connectNamespace } from "../lancedb";
 import { LocalTable } from "../lancedb/table";
 
+async function collectNames(items: AsyncIterable<string>): Promise<string[]> {
+  const names: string[] = [];
+  for await (const name of items) names.push(name);
+  return names;
+}
+
 describe("when connecting", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -36,6 +36,87 @@ describe("when connecting", () => {
   });
 });
 
+describe.each([
+  [
+    "URI overload",
+    (uri: string, options: { storageOptions?: Record<string, string> }) =>
+      connect(uri, options),
+  ],
+  [
+    "object overload",
+    async (
+      uri: string,
+      options: { storageOptions?: Record<string, string> },
+    ) => {
+      const input = { uri, ...options };
+      if (Object.isFrozen(options)) {
+        Object.freeze(input);
+      }
+      const db = await connect(input);
+      expect(input).toEqual({ uri, ...options });
+      expect(input.storageOptions).toBe(options.storageOptions);
+      return db;
+    },
+  ],
+  [
+    "namespace",
+    (uri: string, options: { storageOptions?: Record<string, string> }) =>
+      connectNamespace("dir", { root: uri }, options),
+  ],
+])("%s connection options", (_name, open) => {
+  let tmpDir: tmp.DirResult;
+  beforeEach(() => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+  });
+  afterEach(() => tmpDir.removeCallback());
+
+  it("should accept frozen options without storage options", async () => {
+    const options = Object.freeze({});
+    const db = await open(tmpDir.name, options);
+    try {
+      await db.createTable("frozen", [{ id: 1 }]);
+      await expect(db.tableNames()).resolves.toContain("frozen");
+      expect(options).toEqual({});
+    } finally {
+      await db.close();
+    }
+  });
+
+  it("should accept frozen options with storage options", async () => {
+    const storageOptions = Object.freeze({
+      newTableDataStorageVersion: "stable",
+    });
+    const options = Object.freeze({ storageOptions });
+    const db = await open(tmpDir.name, options);
+    try {
+      const table = await db.createTable("frozen", [{ id: 1 }]);
+      await expect(table.countRows()).resolves.toBe(1);
+      expect(options.storageOptions).toBe(storageOptions);
+      expect(storageOptions).toEqual({ newTableDataStorageVersion: "stable" });
+    } finally {
+      await db.close();
+    }
+  });
+
+  it("should preserve reused options and their storage options", async () => {
+    const storageOptions = { newTableDataStorageVersion: "stable" };
+    const options = { storageOptions };
+    for (let i = 0; i < 2; i++) {
+      const db = await open(tmpDir.name, options);
+      try {
+        const table = await db.createTable(`reused_${i}`, [{ id: i }]);
+        await expect(table.countRows()).resolves.toBe(1);
+        expect(options.storageOptions).toBe(storageOptions);
+        expect(storageOptions).toEqual({
+          newTableDataStorageVersion: "stable",
+        });
+      } finally {
+        await db.close();
+      }
+    }
+  });
+});
+
 describe("given a connection", () => {
   let tmpDir: tmp.DirResult;
   let db: Connection;
@@ -54,7 +135,7 @@ describe("given a connection", () => {
     await db.close();
     expect(db.isOpen()).toBe(false);
     await expect(db.tableNames()).rejects.toThrow("Connection is closed");
-    await expect(db.listTables()).rejects.toThrow("Connection is closed");
+    expect(() => db.listTables()).toThrow("Connection is closed");
     await expect(db.renameTable("a", "b")).rejects.toThrow(
       "Connection is closed",
     );
@@ -117,6 +198,80 @@ describe("given a connection", () => {
     await expect(tbl.countRows()).resolves.toBe(1);
   });
 
+  describe.each([undefined, []])("root namespace %p", (namespacePath) => {
+    it("should preserve createTable options after the namespace", async () => {
+      await db.createTable("test", [{ id: 1 }, { id: 2 }]);
+
+      const table = await db.createTable("test", [{ id: 3 }], namespacePath, {
+        mode: "overwrite",
+      });
+
+      const rows = await table.query().toArray();
+      expect(rows.map((row) => row.id)).toEqual([3]);
+    });
+
+    it("should preserve createEmptyTable overwrite after the namespace", async () => {
+      await db.createTable("test", [{ id: 1 }]);
+      const schema = new Schema([new Field("id", new Float64(), true)]);
+
+      const table = await db.createEmptyTable("test", schema, namespacePath, {
+        mode: "overwrite",
+      });
+
+      await expect(table.countRows()).resolves.toBe(0);
+    });
+
+    it("should preserve createEmptyTable existOk after the namespace", async () => {
+      await db.createTable("test", [{ id: 1 }]);
+      const schema = new Schema([new Field("id", new Float64(), true)]);
+
+      const table = await db.createEmptyTable("test", schema, namespacePath, {
+        existOk: true,
+      });
+
+      await expect(table.countRows()).resolves.toBe(1);
+    });
+
+    it("should preserve tableNames options after the namespace", async () => {
+      for (const name of ["a", "b", "c"]) {
+        await db.createTable(name, [{ id: 1 }]);
+      }
+
+      await expect(
+        db.tableNames(namespacePath, { limit: 1, startAfter: "a" }),
+      ).resolves.toEqual(["b"]);
+    });
+
+    it("should preserve listTables options after the namespace", async () => {
+      for (const name of ["a", "b", "c"]) {
+        await db.createTable(name, [{ id: 1 }]);
+      }
+
+      const first = db.listTables(namespacePath, { pageLimit: 1 });
+      expect(await first.next()).toEqual({ done: false, value: "a" });
+      expect(first.pageToken()).toBeDefined();
+
+      const second = db.listTables(namespacePath, {
+        pageLimit: 1,
+        pageToken: first.pageToken(),
+      });
+      expect(await second.next()).toEqual({ done: false, value: "b" });
+    });
+  });
+  it("should keep the existing table when mode is exist_ok", async () => {
+    const created = await db.createTable("test", [{ id: 1 }, { id: 2 }], {
+      mode: "exist_ok",
+    });
+    await expect(created.countRows()).resolves.toBe(2);
+
+    const reopened = await db.createTable("test", [{ id: 9 }], {
+      mode: "exist_ok",
+    });
+    await expect(reopened.countRows()).resolves.toBe(2);
+    const rows = (await reopened.query().toArray()) as { id: number }[];
+    expect(rows.map((row) => row.id).sort()).toEqual([1, 2]);
+  });
+
   it("should respect limit and page token when listing tables", async () => {
     const db = await connect(tmpDir.name);
 
@@ -144,19 +299,18 @@ describe("given a connection", () => {
     await db.createTable("a", [{ id: 1 }]);
     await db.createTable("c", [{ id: 1 }]);
 
-    const all = await db.listTables();
-    expect(all.tables).toEqual(["a", "b", "c"]);
-    expect(all.pageToken).toBeUndefined();
+    expect(await collectNames(db.listTables())).toEqual(["a", "b", "c"]);
 
-    const first = await db.listTables({ limit: 1 });
-    expect(first.tables).toEqual(["a"]);
-    expect(first.pageToken).toBeDefined();
+    const first = db.listTables([], { pageLimit: 1 });
+    expect(first.numPageResults()).toBe(0);
+    expect(await first.next()).toEqual({ done: false, value: "a" });
+    expect(first.pageToken()).toBeDefined();
 
-    const second = await db.listTables({
-      limit: 1,
-      pageToken: first.pageToken,
+    const second = db.listTables([], {
+      pageLimit: 1,
+      pageToken: first.pageToken(),
     });
-    expect(second.tables).toEqual(["b"]);
+    expect(await second.next()).toEqual({ done: false, value: "b" });
   });
 
   it("should visit every table exactly once when walking pages", async () => {
@@ -167,18 +321,71 @@ describe("given a connection", () => {
       await db.createTable(name, [{ id: 1 }]);
     }
 
-    const seen: string[] = [];
-    let pageToken: string | undefined = undefined;
-    do {
-      const page: ListTablesResponse = await db.listTables({
-        limit: 2,
-        pageToken,
-      });
-      seen.push(...page.tables);
-      pageToken = page.pageToken;
-    } while (pageToken);
+    expect(await collectNames(db.listTables([], { pageLimit: 2 }))).toEqual(
+      created,
+    );
+  });
 
-    expect(seen).toEqual(created);
+  it.each([2 ** 32, 2 ** 32 + 1, Number.MAX_SAFE_INTEGER])(
+    "should not wrap a large table listing limit %s",
+    async (limit) => {
+      await db.createTable("a", [{ id: 1 }]);
+      await db.createTable("b", [{ id: 2 }]);
+      for (const names of [
+        db.tableNames({ limit }),
+        db.tableNames([], { limit }),
+      ]) {
+        await expect(names).resolves.toEqual(["a", "b"]);
+      }
+      for (const namespacePath of [undefined, []]) {
+        expect(() =>
+          db.listTables(namespacePath, { pageLimit: limit }),
+        ).toThrow(/limit/);
+      }
+    },
+  );
+
+  it.each([
+    -1,
+    1.5,
+    Number.NaN,
+    Number.POSITIVE_INFINITY,
+    Number.NEGATIVE_INFINITY,
+    Number.MAX_SAFE_INTEGER + 1,
+  ])("should reject an invalid table listing limit %s", async (limit) => {
+    await expect(db.tableNames({ limit })).rejects.toThrow(/limit/);
+    await expect(db.tableNames([], { limit })).rejects.toThrow(/limit/);
+    for (const namespacePath of [undefined, []]) {
+      expect(() => db.listTables(namespacePath, { pageLimit: limit })).toThrow(
+        /limit/,
+      );
+    }
+  });
+
+  it("should preserve zero, small and unsigned-32-bit table listing limits", async () => {
+    await db.createTable("a", [{ id: 1 }]);
+    await db.createTable("b", [{ id: 2 }]);
+    for (const [limit, names] of [
+      [0, []],
+      [1, ["a"]],
+      [2 ** 32 - 1, ["a", "b"]],
+    ] as const) {
+      await expect(db.tableNames({ limit })).resolves.toEqual(names);
+    }
+    await expect(db.tableNames()).resolves.toEqual(["a", "b"]);
+    for (const namespacePath of [undefined, []]) {
+      for (const pageLimit of [1, 2 ** 31 - 1]) {
+        expect(
+          await collectNames(db.listTables(namespacePath, { pageLimit })),
+        ).toEqual(["a", "b"]);
+      }
+      for (const pageLimit of [0, 2 ** 31, 2 ** 32 - 1]) {
+        expect(() => db.listTables(namespacePath, { pageLimit })).toThrow(
+          /limit/,
+        );
+      }
+    }
+    expect(await collectNames(db.listTables())).toEqual(["a", "b"]);
   });
 
   it("should list all names and use a default page size of 100", async () => {
@@ -191,20 +398,17 @@ describe("given a connection", () => {
     }
     await expect(db.tableNames()).resolves.toEqual(names);
     await expect(db.tableNames({ limit: 0 })).resolves.toEqual([]);
-    const first = await db.listTables();
-    expect(first.tables).toEqual(names.slice(0, 100));
-    expect(first.pageToken).toBeDefined();
-    const second = await db.listTables({ pageToken: first.pageToken });
-    expect(second.tables).toEqual(names.slice(100));
-    expect(second.pageToken).toBeUndefined();
-    await expect(db.listTables({ limit: 200 })).resolves.toEqual({
-      tables: names,
-      pageToken: undefined,
-    });
-    await expect(db.listTables({ limit: 0 })).resolves.toEqual({
-      tables: [],
-      pageToken: undefined,
-    });
+    const first = db.listTables();
+    const firstPage: string[] = [];
+    for (let i = 0; i < 100; i++) firstPage.push((await first.next()).value!);
+    expect(firstPage).toEqual(names.slice(0, 100));
+    expect(first.pageToken()).toBeDefined();
+    expect(first.numPageResults()).toBe(0);
+    expect(await collectNames(first)).toEqual(names.slice(100));
+    expect(await collectNames(db.listTables([], { pageLimit: 200 }))).toEqual(
+      names,
+    );
+    expect(() => db.listTables([], { pageLimit: 0 })).toThrow(/limit/);
   });
 
   it("should list tables in a namespace", async () => {
@@ -215,12 +419,8 @@ describe("given a connection", () => {
     await db.createNamespace(["child"]);
     await db.createTable("nested", [{ id: 1 }], ["child"]);
 
-    await expect(db.listTables(["child"])).resolves.toEqual(
-      expect.objectContaining({ tables: ["nested"] }),
-    );
-    await expect(db.listTables()).resolves.toEqual(
-      expect.objectContaining({ tables: [] }),
-    );
+    expect(await collectNames(db.listTables(["child"]))).toEqual(["nested"]);
+    expect(await collectNames(db.listTables())).toEqual([]);
   });
 
   it("should create tables in v2 mode", async () => {
@@ -293,6 +493,69 @@ describe("given a connection", () => {
       });
   });
 
+  describe.each(["createTable", "createEmptyTable"] as const)(
+    "%s storage options",
+    (method) => {
+      const schema = new Schema([new Field("id", new Float64(), true)]);
+
+      it("should accept frozen creation options without storage options", async () => {
+        const options = Object.freeze({
+          dataStorageVersion: "stable",
+          enableV2ManifestPaths: true,
+        });
+        const table =
+          method === "createTable"
+            ? await db.createTable("frozen_options", [{ id: 1 }], options)
+            : await db.createEmptyTable("frozen_options", schema, options);
+        expect(await (table as LocalTable).usesV2ManifestPaths()).toBe(true);
+        expect(options).not.toHaveProperty("storageOptions");
+      });
+
+      it("should accept frozen storage options and preserve override precedence", async () => {
+        const storageOptions = Object.freeze({
+          newTableDataStorageVersion: "legacy",
+          newTableEnableV2ManifestPaths: "false",
+        });
+        const options = Object.freeze({
+          storageOptions,
+          dataStorageVersion: "stable",
+          enableV2ManifestPaths: true,
+        });
+        const table =
+          method === "createTable"
+            ? await db.createTable("frozen_storage", [{ id: 1 }], options)
+            : await db.createEmptyTable("frozen_storage", schema, options);
+        expect(await (table as LocalTable).usesV2ManifestPaths()).toBe(true);
+        expect(options.storageOptions).toBe(storageOptions);
+        expect(storageOptions).toEqual({
+          newTableDataStorageVersion: "legacy",
+          newTableEnableV2ManifestPaths: "false",
+        });
+      });
+
+      it("should not retain a previous override when reusing storage options", async () => {
+        const storageOptions = { newTableEnableV2ManifestPaths: "false" };
+        const create = async (name: string, enableV2ManifestPaths?: boolean) =>
+          method === "createTable"
+            ? db.createTable(name, [{ id: 1 }], {
+                storageOptions,
+                enableV2ManifestPaths,
+              })
+            : db.createEmptyTable(name, schema, {
+                storageOptions,
+                enableV2ManifestPaths,
+              });
+        const first = await create("override", true);
+        expect(await (first as LocalTable).usesV2ManifestPaths()).toBe(true);
+        const second = await create("without_override");
+        expect(await (second as LocalTable).usesV2ManifestPaths()).toBe(false);
+        expect(storageOptions).toEqual({
+          newTableEnableV2ManifestPaths: "false",
+        });
+      });
+    },
+  );
+
   it("should be able to migrate tables to the V2 manifest paths", async () => {
     const db = await connect(tmpDir.name);
     const table = (await db.createEmptyTable(
@@ -451,22 +714,35 @@ describe("namespaces", () => {
   it("should list namespaces created at the root", async () => {
     await db.createNamespace(["alpha"]);
     await db.createNamespace(["beta"]);
-    const list = await db.listNamespaces();
-    expect(list.namespaces).toEqual(expect.arrayContaining(["alpha", "beta"]));
+    const list = await collectNames(db.listNamespaces());
+    expect(list).toEqual(expect.arrayContaining(["alpha", "beta"]));
   });
 
   it("should list child namespaces under a parent", async () => {
     await db.createNamespace(["parent"]);
     await db.createNamespace(["parent", "child"]);
-    const list = await db.listNamespaces(["parent"]);
-    expect(list.namespaces).toContain("child");
+    const list = await collectNames(db.listNamespaces(["parent"]));
+    expect(list).toContain("child");
+  });
+
+  it("should resume a namespace listing from its page token", async () => {
+    await db.createNamespace(["alpha"]);
+    await db.createNamespace(["beta"]);
+    const first = db.listNamespaces([], { pageLimit: 1 });
+    expect(first.numPageResults()).toBe(0);
+    expect((await first.next()).value).toBe("alpha");
+    const second = db.listNamespaces([], {
+      pageLimit: 1,
+      pageToken: first.pageToken(),
+    });
+    expect(await collectNames(second)).toEqual(["beta"]);
   });
 
   it("should drop a namespace", async () => {
     await db.createNamespace(["ephemeral"]);
     await db.dropNamespace(["ephemeral"]);
-    const list = await db.listNamespaces();
-    expect(list.namespaces).not.toContain("ephemeral");
+    const list = await collectNames(db.listNamespaces());
+    expect(list).not.toContain("ephemeral");
   });
 
   it("should raise an error on any namespace op after close", async () => {
@@ -474,7 +750,7 @@ describe("namespaces", () => {
     await expect(db.describeNamespace(["foo"])).rejects.toThrow(
       "Connection is closed",
     );
-    await expect(db.listNamespaces()).rejects.toThrow("Connection is closed");
+    expect(() => db.listNamespaces()).toThrow("Connection is closed");
     await expect(db.createNamespace(["foo"])).rejects.toThrow(
       "Connection is closed",
     );

nodejs/__test__/fixtures/transformers_tokenizer.cjs

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+const assert = require("node:assert/strict");
+const {
+  TransformersEmbeddingFunction,
+} = require("../../dist/embedding/transformers");
+const { getRegistry } = require("../../dist/embedding/registry");
+
+async function main() {
+  const transformers = await import("@huggingface/transformers");
+  // Never fetch a real model, including if a loader mock stops intercepting.
+  transformers.env.allowRemoteModels = false;
+  transformers.env.allowLocalModels = false;
+  const modelCalls = [];
+  const tokenizerCalls = [];
+  const testCase = Number(process.argv[2]);
+  const defaults = "Xenova/all-MiniLM-L6-v2";
+  const cases = [
+    [{}, defaults, defaults],
+    [{ model: "test/model" }, "test/model", "test/model"],
+    [
+      { model: "test/model", tokenizer: "test/tokenizer" },
+      "test/model",
+      "test/tokenizer",
+    ],
+    [
+      { model: "test/model", tokenizer: "$var:tokenizer" },
+      "test/model",
+      "test/tokenizer",
+    ],
+    [
+      { model: "test/model", tokenizer: "test/tokenizer" },
+      "test/model",
+      "test/tokenizer",
+    ],
+  ];
+  assert.ok(
+    Number.isInteger(testCase) && testCase >= 0 && testCase < cases.length,
+  );
+  transformers.AutoModel.from_pretrained = async (...args) => {
+    modelCalls.push(args);
+    return {};
+  };
+  transformers.AutoTokenizer.from_pretrained = async (...args) => {
+    tokenizerCalls.push(args);
+    if (testCase === 4) throw new Error("missing tokenizer");
+    return {};
+  };
+  getRegistry().setVar("tokenizer", "test/tokenizer");
+  const [options, model, tokenizer] = cases[testCase];
+  const fn = new TransformersEmbeddingFunction(options);
+  if (testCase === 4) {
+    await assert.rejects(fn.init(), /error loading tokenizer test\/tokenizer/);
+  } else {
+    await fn.init();
+  }
+  assert.deepEqual(modelCalls, [[model, { dtype: "fp32" }]]);
+  assert.deepEqual(tokenizerCalls, [[tokenizer]]);
+  assert.deepEqual(fn.toJSON(), options);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

nodejs/__test__/native_symbols.test.ts

@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+const {
+  checkNativeBinary,
+  findForbiddenUndefinedSymbols,
+} = require("../scripts/check-native-symbols.js");
+
+test("detects the unresolved AVX-512 symbol from broken musl binaries", () => {
+  const output = [
+    "                 U napi_create_function",
+    "                 U sum_4bit_dist_table_32bytes_batch_avx512",
+    "                 U strlen",
+  ].join("\n");
+
+  expect(findForbiddenUndefinedSymbols(output)).toEqual([
+    "sum_4bit_dist_table_32bytes_batch_avx512",
+  ]);
+});
+
+test("accepts native binaries without unresolved internal Lance symbols", () => {
+  const runNm = jest.fn(() => ({
+    status: 0,
+    stdout:
+      "                 U napi_create_function\n                 U strlen\n",
+    stderr: "",
+  }));
+
+  expect(() => checkNativeBinary("lancedb.node", runNm)).not.toThrow();
+  expect(runNm).toHaveBeenCalledWith(
+    "nm",
+    ["-D", "--undefined-only", "lancedb.node"],
+    { encoding: "utf8" },
+  );
+});
+
+test("rejects native binaries with the unresolved AVX-512 symbol", () => {
+  const runNm = jest.fn(() => ({
+    status: 0,
+    stdout: "                 U sum_4bit_dist_table_32bytes_batch_avx512\n",
+    stderr: "",
+  }));
+
+  expect(() => checkNativeBinary("lancedb.node", runNm)).toThrow(
+    "lancedb.node contains unresolved internal Lance symbols: sum_4bit_dist_table_32bytes_batch_avx512",
+  );
+});

nodejs/__test__/nested_blob.test.ts

@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import * as tmp from "tmp";
+import { type Table, blob, connect, isBlobField } from "../lancedb";
+import {
+  DataType,
+  Field,
+  Int64,
+  List,
+  Schema,
+  Struct,
+  Utf8,
+} from "../lancedb/arrow";
+
+describe('nested blobs with blobMode "bytes"', () => {
+  let tmpDir: tmp.DirResult;
+  let table: Table;
+
+  const caption = new Field(
+    "caption",
+    new Utf8(),
+    false,
+    new Map([["label", "caption"]]),
+  );
+  const info = new Field(
+    "info",
+    new Struct([
+      blob("image"),
+      caption,
+      new Field("preview", new Struct([blob("image")]), true),
+    ]),
+    true,
+    new Map([["label", "info"]]),
+  );
+  const schema = new Schema(
+    [new Field("id", new Int64(), false), info, blob("cover")],
+    new Map([["source", "nested-blobs"]]),
+  );
+
+  beforeEach(async () => {
+    tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
+    table = await db.createTable(
+      "nested_blobs",
+      [
+        {
+          id: 1n,
+          info: {
+            image: Buffer.from("hello"),
+            caption: "first",
+            preview: { image: Buffer.from("preview") },
+          },
+          cover: Buffer.from("cover"),
+        },
+        { id: 2n, info: null, cover: null },
+        {
+          id: 3n,
+          info: { image: null, caption: "third", preview: null },
+          cover: null,
+        },
+        {
+          id: 4n,
+          info: {
+            image: Buffer.alloc(0),
+            caption: "fourth",
+            preview: { image: null },
+          },
+          cover: Buffer.alloc(0),
+        },
+      ],
+      { schema },
+    );
+  });
+
+  afterEach(() => tmpDir.removeCallback());
+
+  it.each([false, true])(
+    "reads struct leaves across batches (select parent: %s)",
+    async (selectParent) => {
+      const query = table
+        .query()
+        .orderBy({ columnName: "id", ascending: true });
+      if (selectParent) {
+        query.select(["info"]);
+      }
+      const result = await query.toArrow({
+        blobMode: "bytes",
+        maxBatchLength: 1,
+      });
+      expect(result.batches.length).toBeGreaterThan(1);
+      expect(result.schema.fields.map((field) => field.name)).toEqual(
+        selectParent ? ["info"] : ["id", "info", "cover"],
+      );
+      expect(result.schema.metadata).toEqual(schema.metadata);
+      const infoField = result.schema.fields.find(
+        (field) => field.name === "info",
+      )!;
+      expect(infoField.nullable).toBe(true);
+      expect(infoField.metadata).toEqual(info.metadata);
+      const fields = (infoField.type as Struct).children;
+      expect(DataType.isLargeBinary(fields[0].type)).toBe(true);
+      expect(isBlobField(fields[0])).toBe(false);
+      expect(fields[0].metadata.size).toBe(0);
+      expect(fields[1]).toEqual(caption);
+      expect(
+        DataType.isLargeBinary((fields[2].type as Struct).children[0].type),
+      ).toBe(true);
+
+      const rows = result.toArray();
+      expect(Buffer.from(rows[0].info.image).toString()).toBe("hello");
+      expect(rows[0].info.caption).toBe("first");
+      expect(Buffer.from(rows[0].info.preview.image).toString()).toBe(
+        "preview",
+      );
+      expect(rows[1].info).toBeNull();
+      expect(rows[2].info.image).toBeNull();
+      expect(rows[2].info.caption).toBe("third");
+      expect(rows[2].info.preview).toBeNull();
+      expect(Buffer.from(rows[3].info.image)).toEqual(Buffer.alloc(0));
+      expect(rows[3].info.preview.image).toBeNull();
+      if (!selectParent) {
+        expect(Buffer.from(rows[0].cover).toString()).toBe("cover");
+        expect(rows[1].cover).toBeNull();
+      }
+    },
+  );
+
+  it("reads direct leaf and struct aliases", async () => {
+    const rows = await table
+      .query()
+      .orderBy({ columnName: "id", ascending: true })
+      .select({
+        image: "info.image",
+        details: "info",
+        preview: "info.preview",
+        caption: "info.caption",
+      })
+      .toArray({ blobMode: "bytes", maxBatchLength: 1 });
+    expect(Object.keys(rows[0])).toEqual([
+      "image",
+      "details",
+      "preview",
+      "caption",
+    ]);
+    expect(Buffer.from(rows[0].image).toString()).toBe("hello");
+    expect(Buffer.from(rows[0].details.image).toString()).toBe("hello");
+    expect(Buffer.from(rows[0].details.preview.image).toString()).toBe(
+      "preview",
+    );
+    expect(Buffer.from(rows[0].preview.image).toString()).toBe("preview");
+    expect(rows[0].caption).toBe("first");
+    expect(rows[1].details).toBeNull();
+    expect(rows[1].image).toBeNull();
+    expect(rows[2].image).toBeNull();
+    expect(Buffer.from(rows[3].image)).toEqual(Buffer.alloc(0));
+  });
+
+  it.each([false, true])(
+    "preserves a literal scalar alias matching a nested blob path (reverse: %s)",
+    async (reverse) => {
+      const projection: [string, string][] = [
+        ["info", "info"],
+        ["info.image", "id"],
+      ];
+      if (reverse) {
+        projection.reverse();
+      }
+      const [row] = await table
+        .query()
+        .where("id = 1")
+        .select(new Map(projection))
+        .toArray({ blobMode: "bytes" });
+      expect(row["info.image"]).toBe(1n);
+      expect(Buffer.from(row.info.image).toString()).toBe("hello");
+    },
+  );
+
+  it.each([false, true])(
+    "materializes every overlapping blob projection (reverse: %s)",
+    async (reverse) => {
+      const columns = ["info.image", "info"];
+      if (reverse) {
+        columns.reverse();
+      }
+      const result = await table
+        .query()
+        .orderBy({ columnName: "id", ascending: true })
+        .select(columns)
+        .toArrow({ blobMode: "bytes", maxBatchLength: 1 });
+      expect(result.schema.fields.map((field) => field.name)).toEqual(columns);
+      expect(DataType.isLargeBinary(result.getChild("info.image")!.type)).toBe(
+        true,
+      );
+      expect(
+        DataType.isLargeBinary(
+          result.getChild("info")!.getChild("image")!.type,
+        ),
+      ).toBe(true);
+      const rows = result.toArray();
+      expect(Buffer.from(rows[0]["info.image"]).toString()).toBe("hello");
+      expect(Buffer.from(rows[0].info.image).toString()).toBe("hello");
+      expect(rows[0].info.caption).toBe("first");
+      expect(rows[1]["info.image"]).toBeNull();
+      expect(rows[1].info).toBeNull();
+      expect(rows[2]["info.image"]).toBeNull();
+      expect(rows[2].info.image).toBeNull();
+      expect(Buffer.from(rows[3]["info.image"])).toEqual(Buffer.alloc(0));
+      expect(Buffer.from(rows[3].info.image)).toEqual(Buffer.alloc(0));
+    },
+  );
+
+  it("materializes dotted aliases independently of nested fields", async () => {
+    const [row] = await table
+      .query()
+      .where("id = 1")
+      .select({
+        details: "info",
+        "details.image": "info.image",
+        "details.preview": "info.preview",
+        "details.preview.image": "info.preview.image",
+      })
+      .toArray({ blobMode: "bytes" });
+    expect(Buffer.from(row.details.image).toString()).toBe("hello");
+    expect(Buffer.from(row["details.image"]).toString()).toBe("hello");
+    expect(Buffer.from(row.details.preview.image).toString()).toBe("preview");
+    expect(Buffer.from(row["details.preview"].image).toString()).toBe(
+      "preview",
+    );
+    expect(Buffer.from(row["details.preview.image"]).toString()).toBe(
+      "preview",
+    );
+  });
+
+  it.each([false, true])(
+    "converts the empty result schema (alias: %s)",
+    async (alias) => {
+      const query = table.query().where("id < 0");
+      if (alias) {
+        query.select({ image: "info.image", details: "info" });
+      }
+      const result = await query.toArrow({ blobMode: "bytes" });
+      expect(result.numRows).toBe(0);
+      expect(result.schema.fields.map((field) => field.name)).toEqual(
+        alias ? ["image", "details"] : ["id", "info", "cover"],
+      );
+      const fields = (
+        result.schema.fields.find(
+          (field) => field.name === (alias ? "details" : "info"),
+        )!.type as Struct
+      ).children;
+      expect(DataType.isLargeBinary(fields[0].type)).toBe(true);
+      expect(isBlobField(fields[0])).toBe(false);
+      expect(fields[1]).toEqual(caption);
+      expect(
+        DataType.isLargeBinary((fields[2].type as Struct).children[0].type),
+      ).toBe(true);
+      if (alias) {
+        expect(DataType.isLargeBinary(result.schema.fields[0].type)).toBe(true);
+      }
+    },
+  );
+
+  it("preserves requested row IDs and leaves subsequent descriptors intact", async () => {
+    const query = table.query().where("id = 1").select(["info"]);
+    const [bytes] = await query.toArray({ blobMode: "bytes" });
+    expect(Object.keys(bytes)).toEqual(["info"]);
+    const [description] = await query.toArray();
+    expect(Object.keys(description)).toEqual(["info"]);
+    expect(Number(description.info.image.size)).toBe(5);
+    const [withRowId] = await query.withRowId().toArray({ blobMode: "bytes" });
+    expect(typeof withRowId._rowid).toBe("bigint");
+    expect(await table.fetchBlobs("info.image", [withRowId._rowid])).toEqual([
+      Buffer.from(withRowId.info.image),
+    ]);
+  });
+
+  it("keeps descriptors in description mode", async () => {
+    const [description] = await table
+      .query()
+      .where("id = 1")
+      .toArray({ blobMode: "descriptions" });
+    expect(Number(description.info.image.size)).toBe(5);
+  });
+
+  it("keeps list elements as descriptors alongside struct blob bytes", async () => {
+    const db = await connect(tmpDir.name);
+    const table = await db.createTable(
+      "list_blobs",
+      [
+        {
+          info: { image: Buffer.from("hello"), images: [Buffer.from("list")] },
+        },
+      ],
+      {
+        schema: new Schema([
+          new Field(
+            "info",
+            new Struct([
+              blob("image"),
+              new Field("images", new List(blob("item")), true),
+            ]),
+            true,
+          ),
+        ]),
+      },
+    );
+    const [bytes] = await table.query().toArray({ blobMode: "bytes" });
+    expect(Buffer.from(bytes.info.image).toString()).toBe("hello");
+    expect(Number(bytes.info.images.get(0).size)).toBe(4);
+    const [row] = await table
+      .query()
+      .select(["info.images"])
+      .toArray({ blobMode: "bytes" });
+    expect(Number(row["info.images"].get(0).size)).toBe(4);
+    expect(Object.keys(row)).toEqual(["info.images"]);
+    const result = await table.query().toArrow({ blobMode: "bytes" });
+    const fields = (result.schema.fields[0].type as Struct).children;
+    const descriptions = await table.query().toArrow();
+    expect(fields[1]).toEqual(
+      (descriptions.schema.fields[0].type as Struct).children[1],
+    );
+    expect(DataType.isStruct((fields[1].type as List).children[0].type)).toBe(
+      true,
+    );
+  });
+});

nodejs/__test__/table.test.ts

@@ -3314,51 +3314,64 @@ describe("when dealing with blob columns", () => {
     ).rejects.toThrow(/blobMode must be/);
   });
 
-  it("reruns the query when the table changes before the version check", async () => {
-    const { table, native, update } = await openVersionedBlobTable();
-    const liveVersion = native.version.bind(native);
-    native.version = async () => {
-      native.version = liveVersion;
-      await update(2n, "new");
-      return liveVersion();
-    };
-    const [row] = await table.query().toArray({ blobMode: "bytes" });
-    expect(row.generation).toBe(2n);
-    expect(Buffer.from(row.image).toString()).toBe("new");
-  });
+  it.each([false, true])(
+    "reruns the query when the table changes before the version check (nested: %s)",
+    async (nested) => {
+      const { table, native, update } = await openVersionedBlobTable(nested);
+      const liveVersion = native.version.bind(native);
+      native.version = async () => {
+        native.version = liveVersion;
+        await update(2n, "new");
+        return liveVersion();
+      };
+      const [row] = await table.query().toArray({ blobMode: "bytes" });
+      expect(row.generation).toBe(2n);
+      expect(Buffer.from(nested ? row.info.image : row.image).toString()).toBe(
+        "new",
+      );
+    },
+  );
 
-  it("reads bytes from the query's version when a write lands before the fetch", async () => {
-    const { table, native, update } = await openVersionedBlobTable();
-    const liveVersion = native.version.bind(native);
-    native.version = async () => {
-      native.version = liveVersion;
-      const version = await liveVersion();
-      await update(2n, "new");
-      return version;
-    };
-    const [row] = await table.query().toArray({ blobMode: "bytes" });
-    expect(row.generation).toBe(1n);
-    expect(Buffer.from(row.image).toString()).toBe("old");
-  });
+  it.each([false, true])(
+    "reads bytes from the query's version when a write lands before the fetch (nested: %s)",
+    async (nested) => {
+      const { table, native, update } = await openVersionedBlobTable(nested);
+      const liveVersion = native.version.bind(native);
+      native.version = async () => {
+        native.version = liveVersion;
+        const version = await liveVersion();
+        await update(2n, "new");
+        return version;
+      };
+      const [row] = await table.query().toArray({ blobMode: "bytes" });
+      expect(row.generation).toBe(1n);
+      expect(Buffer.from(nested ? row.info.image : row.image).toString()).toBe(
+        "old",
+      );
+    },
+  );
 
-  it("fails when the table changes during every attempt", async () => {
-    const { table, native, update } = await openVersionedBlobTable();
-    const liveVersion = native.version.bind(native);
-    let generation = 1n;
-    native.version = async () => {
-      generation += 1n;
-      await update(generation, `generation ${generation}`);
-      return liveVersion();
-    };
-    try {
-      await expect(
-        table.query().toArray({ blobMode: "bytes" }),
-      ).rejects.toThrow(/changed during each of 3 attempts/);
-      expect(generation).toBe(4n);
-    } finally {
-      native.version = liveVersion;
-    }
-  });
+  it.each([false, true])(
+    "fails when the table changes during every attempt (nested: %s)",
+    async (nested) => {
+      const { table, native, update } = await openVersionedBlobTable(nested);
+      const liveVersion = native.version.bind(native);
+      let generation = 1n;
+      native.version = async () => {
+        generation += 1n;
+        await update(generation, `generation ${generation}`);
+        return liveVersion();
+      };
+      try {
+        await expect(
+          table.query().toArray({ blobMode: "bytes" }),
+        ).rejects.toThrow(/changed during each of 3 attempts/);
+        expect(generation).toBe(4n);
+      } finally {
+        native.version = liveVersion;
+      }
+    },
+  );
 
   it("rejects fetchBlobs on a non-blob column", async () => {
     const { table, rowIds } = await openBlobTable();
@@ -3385,6 +3398,106 @@ describe("when dealing with blob columns", () => {
     expect(bytes[0]!.equals(payload)).toBe(true);
   });
 
+  it("returns bytes for blobs nested in structs", async () => {
+    const { table, alpha, empty } = await openNestedBlobTable();
+    const rows = await table.query().toArray({ blobMode: "bytes" });
+    const byId = new Map(rows.map((row) => [Number(row.id), row]));
+    expect(Buffer.from(byId.get(1)!.info.image)).toEqual(alpha);
+    expect(byId.get(1)!.info.caption).toBe("one");
+    expect(byId.get(2)!.info.image).toBeNull();
+    expect(byId.get(2)!.info.caption).toBe("two");
+    expect(byId.get(3)!.info).toBeNull();
+    expect(Buffer.from(byId.get(4)!.info.image)).toEqual(empty);
+  });
+
+  it("lines up nested rows and bytes across single-row batches", async () => {
+    const { table, alpha, empty } = await openNestedBlobTable();
+    const result = await table
+      .query()
+      .toArrow({ blobMode: "bytes", maxBatchLength: 1 });
+    expect(result.batches.length).toBe(4);
+    const byId = new Map(result.toArray().map((row) => [Number(row.id), row]));
+    expect(Buffer.from(byId.get(1)!.info.image)).toEqual(alpha);
+    expect(byId.get(1)!.info.caption).toBe("one");
+    expect(byId.get(2)!.info.image).toBeNull();
+    expect(byId.get(2)!.info.caption).toBe("two");
+    expect(byId.get(3)!.info).toBeNull();
+    expect(Buffer.from(byId.get(4)!.info.image)).toEqual(empty);
+  });
+
+  it("handles struct fields named like Object prototypes", async () => {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      new Field(
+        "info",
+        new Struct([
+          new Field("constructor", new Struct([blob("image")]), true),
+        ]),
+        true,
+      ),
+    ]);
+    const payload = Buffer.from("proto");
+    const table = await db.createTable(
+      "prototype_named_blobs",
+      [{ id: 1n, info: { constructor: { image: payload } } }],
+      { schema },
+    );
+    expect(await table.blobColumns()).toEqual(["info.constructor.image"]);
+    const [row] = await table.query().toArray({ blobMode: "bytes" });
+    // Arrow's StructRow shadows the `constructor` property, so read the
+    // field through the JSON form instead of property access.
+    const info = JSON.parse(JSON.stringify(row.info));
+    expect(Buffer.from(Object.values<number>(info.constructor.image))).toEqual(
+      payload,
+    );
+  });
+
+  it("returns nested blob bytes for a struct selection", async () => {
+    const { table, alpha } = await openNestedBlobTable();
+    const [row] = await table
+      .query()
+      .where("id = 1")
+      .select(["info"])
+      .toArray({ blobMode: "bytes" });
+    expect(Object.keys(row)).toEqual(["info"]);
+    expect(Buffer.from(row.info.image)).toEqual(alpha);
+    expect(row.info.caption).toBe("one");
+  });
+
+  it("returns nested blob bytes through aliases", async () => {
+    const { table, alpha } = await openNestedBlobTable();
+    const [row] = await table
+      .query()
+      .where("id = 1")
+      .select(
+        new Map([
+          ["picture", "info.image"],
+          ["details", "info"],
+        ]),
+      )
+      .toArray({ blobMode: "bytes" });
+    expect(Buffer.from(row.picture)).toEqual(alpha);
+    expect(Buffer.from(row.details.image)).toEqual(alpha);
+    expect(row.details.caption).toBe("one");
+  });
+
+  it("returns a nested bytes schema for an empty result", async () => {
+    const { table } = await openNestedBlobTable();
+    const result = await table
+      .query()
+      .where("id > 100")
+      .toArrow({ blobMode: "bytes" });
+    expect(result.numRows).toBe(0);
+    const field = result.schema.fields.find((f) => f.name === "info")!;
+    expect(arrow.DataType.isStruct(field.type)).toBe(true);
+    const image = (field.type as Struct).children.find(
+      (f) => f.name === "image",
+    )!;
+    expect(arrow.DataType.isLargeBinary(image.type)).toBe(true);
+    expect(isBlobField(image)).toBe(false);
+  });
+
   it("creates list blobs without advertising unfetchable elements", async () => {
     const db = await connect(tmpDir.name);
     const schema = new Schema([
@@ -3542,18 +3655,28 @@ describe("when dealing with blob columns", () => {
     ).map((value) => (value == null ? null : Number(value.size)));
   }
 
-  async function openVersionedBlobTable() {
+  async function openVersionedBlobTable(nested = false) {
     const db = await connect(tmpDir.name, {
       storageOptions: { newTableEnableStableRowIds: "true" },
     });
     const schema = new Schema([
       new Field("id", new Int64(), false),
       new Field("generation", new Int64(), true),
-      blob("image"),
+      nested
+        ? new Field("info", new Struct([blob("image")]), true)
+        : blob("image"),
     ]);
     const table = await db.createTable(
       "versions",
-      [{ id: 1n, generation: 1n, image: Buffer.from("old") }],
+      [
+        {
+          id: 1n,
+          generation: 1n,
+          ...(nested
+            ? { info: { image: Buffer.from("old") } }
+            : { image: Buffer.from("old") }),
+        },
+      ],
       { schema },
     );
     const native = (
@@ -3563,7 +3686,15 @@ describe("when dealing with blob columns", () => {
       await table
         .mergeInsert("id")
         .whenMatchedUpdateAll()
-        .execute([{ id: 1n, generation, image: Buffer.from(image) }]);
+        .execute([
+          {
+            id: 1n,
+            generation,
+            ...(nested
+              ? { info: { image: Buffer.from(image) } }
+              : { image: Buffer.from(image) }),
+          },
+        ]);
     };
     return { table, native, update };
   }
@@ -3593,6 +3724,31 @@ describe("when dealing with blob columns", () => {
     return { table, rowIds, alpha, beta };
   }
 
+  async function openNestedBlobTable() {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      new Field(
+        "info",
+        new Struct([blob("image"), new Field("caption", new Utf8(), true)]),
+        true,
+      ),
+    ]);
+    const alpha = Buffer.from("alpha");
+    const empty = Buffer.alloc(0);
+    const table = await db.createTable(
+      "nested_blobs",
+      [{ id: 1n, info: { image: alpha, caption: "one" } }],
+      { schema },
+    );
+    await table.add([
+      { id: 2n, info: { image: null, caption: "two" } },
+      { id: 3n, info: null },
+      { id: 4n, info: { image: empty, caption: "four" } },
+    ]);
+    return { table, alpha, empty };
+  }
+
   async function openExternalBlobTable() {
     const db = await connect(path.join(tmpDir.name, "db"));
     const payload = Buffer.from("external-payload");
@@ -3742,6 +3898,21 @@ describe("when optimizing a dataset", () => {
     expect(stats.prune.oldVersionsRemoved).toBe(2);
   });
 
+  it("retains tagged versions older than the cleanup cutoff", async () => {
+    await (await table.tags()).create("keep", 1);
+    await table.add([{ id: 3 }]);
+
+    const cutoff = await nextMillisecond();
+    const stats = await table.optimize({ cleanupOlderThan: cutoff });
+    // Versions 2 and 3 are pruned, tagged version 1 is retained
+    expect(stats.prune.oldVersionsRemoved).toBe(2);
+
+    await table.checkout("keep");
+    expect(await table.countRows()).toBe(1);
+    await table.checkoutLatest();
+    expect(await table.countRows()).toBe(3);
+  });
+
   it("delete unverified", async () => {
     const version = await table.version();
     const versionFile = `${tmpDir.name}/${table.name}.lance/_versions/${String(

nodejs/__test__/transformers.test.ts

@@ -0,0 +1,26 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+import { execFileSync } from "node:child_process";
+import { resolve } from "node:path";
+
+describe("Transformers embedding tokenizer selection", () => {
+  // Run init's native ESM import outside Jest's CommonJS sandbox, like a user.
+  // The fixture mocks model loading and forbids local/remote model access.
+  test.each([
+    ["default model", 0],
+    ["default tokenizer", 1],
+    ["custom tokenizer", 2],
+    ["tokenizer registry variable", 3],
+    ["tokenizer loading error", 4],
+  ])("handles %s", (_name, testCase) => {
+    execFileSync(
+      process.execPath,
+      [
+        resolve(__dirname, "fixtures", "transformers_tokenizer.cjs"),
+        String(testCase),
+      ],
+      { stdio: "pipe", timeout: 15_000 },
+    );
+  });
+});