Merge main into gatekeeper/fix-4299-1

nodejs/__test__/arrow.test.ts

@@ -601,6 +601,103 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
         );
       });
 
+      it.each([
+        ["ArrayBuffer", new Uint8Array([104]).buffer],
+        ["URL", new URL("https://example.com/")],
+        ["empty object", {}],
+        [
+          "class instance with enumerable fields",
+          Object.assign(new Blob(["x"]), { position: 0 }),
+        ],
+      ])("rejects %s values without a schema", (_name, value) => {
+        expect(() => makeArrowTable([{ id: 1, value }])).toThrow(
+          /field value at row 0/,
+        );
+        expect(() => makeArrowTable([{ value }])).toThrow(
+          /field value at row 0/,
+        );
+      });
+
+      it.each([
+        ["Binary", new Binary(), new Uint8Array([104]).buffer],
+        ["Utf8", new Utf8(), new URL("https://example.com/")],
+      ])("rejects %s values with a schema", (_name, type, value) => {
+        const schema = new Schema([new Field("value", type, true)]);
+        expect(() => makeArrowTable([{ value }], { schema })).toThrow(
+          /field value at row 0/,
+        );
+      });
+
+      it("rejects unsupported values in nested struct fields", () => {
+        const schema = new Schema([
+          new Field(
+            "metadata",
+            new Struct([new Field("bytes", new Binary(), true)]),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable(
+            [{ metadata: { bytes: new Uint8Array([104]).buffer } }],
+            { schema },
+          ),
+        ).toThrow(/field metadata\.bytes at row 0/);
+      });
+
+      it("rejects unsupported objects in list elements", () => {
+        const schema = new Schema([
+          new Field(
+            "images",
+            new List(new Field("item", new Binary(), true)),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable([{ images: [new Blob(["abc"])] }], { schema }),
+        ).toThrow(/field images\[0\] at row 0/);
+      });
+
+      it("rejects unsupported objects in nested lists", () => {
+        const schema = new Schema([
+          new Field(
+            "images",
+            new List(
+              new Field(
+                "item",
+                new List(new Field("item", new Binary(), true)),
+                true,
+              ),
+            ),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable([{ images: [[new Blob(["abc"])]] }], { schema }),
+        ).toThrow(/field images\[0\]\[0\] at row 0/);
+      });
+
+      it("rejects unsupported nested struct fields in lists", () => {
+        const schema = new Schema([
+          new Field(
+            "items",
+            new List(
+              new Field(
+                "item",
+                new Struct([new Field("bytes", new Binary(), true)]),
+                true,
+              ),
+            ),
+            true,
+          ),
+        ]);
+        expect(() =>
+          makeArrowTable(
+            [{ items: [{ bytes: new Uint8Array([104]).buffer }] }],
+            { schema },
+          ),
+        ).toThrow(/field items\[0\]\.bytes at row 0/);
+      });
+
       it("will ignore generated dictionary IDs when comparing inferred types", function () {
         const table = makeArrowTable([{ str: "a" }, { str: "b" }], {
           dictionaryEncodeStrings: true,

nodejs/__test__/blob.test.ts

@@ -102,6 +102,27 @@ describe("BlobFile", () => {
 });
 
 describe("makeArrowTable blob columns", () => {
+  it.each([
+    ["ArrayBuffer", new Uint8Array([104]).buffer],
+    ["Blob", new Blob(["hello"])],
+    ["File", new File(["hello"], "hello.txt")],
+    ["URL", new URL("https://example.com/blob")],
+    ["ReadableStream", new ReadableStream()],
+    ["data struct with ArrayBuffer", { data: new Uint8Array([104]).buffer }],
+    ["uri struct with URL", { uri: new URL("https://example.com/blob") }],
+  ])("rejects %s input rather than writing a null blob", (_name, image) => {
+    const schema = new Schema([blob("image")]);
+    expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
+  });
+
+  it("rejects Blob instances with enumerable fields as one invalid value", () => {
+    const schema = new Schema([blob("image")]);
+    const image = Object.assign(new Blob(["hello"]), { position: 0 });
+    expect(() => makeArrowTable([{ image }], { schema })).toThrow(
+      "Unsupported object value for field image at row 0.",
+    );
+  });
+
   it("coerces Buffer input onto a blob field", () => {
     const schema = new Schema([
       new Field("id", new Int64(), true),

nodejs/__test__/materialized_view.test.ts

@@ -52,9 +52,19 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
+    for (const [format, fn] of [
+      [3, "vector_duplicate_pairs"],
+      [4, "vector_dedup"],
+    ]) {
+      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
+      expect(read(JSON.stringify({ format, query: native })).query).toBe(
+        native,
+      );
+    }
+
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":3,"query":${JSON.stringify(query)}}`,
+      `{"format":5,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);

nodejs/__test__/remote.test.ts

@@ -85,6 +85,69 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it("rejects the external blob opt-in without blocking regular adds", async () => {
+    let insertRequests = 0;
+    const describeRequests: string[] = [];
+
+    await withMockDatabase(
+      (req, res) => {
+        const requestPath = req.url ?? "";
+        if (requestPath.endsWith("/describe/")) {
+          describeRequests.push(requestPath);
+          res.writeHead(200, { "Content-Type": "application/json" }).end(
+            JSON.stringify({
+              name: "items",
+              version: 1,
+              schema: {
+                fields: [
+                  {
+                    name: "id",
+                    type: { type: "int64" },
+                    nullable: true,
+                  },
+                ],
+              },
+            }),
+          );
+          return;
+        }
+        if (requestPath.endsWith("/insert/")) {
+          insertRequests++;
+          req.resume();
+          req.on("end", () => {
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(JSON.stringify({ version: insertRequests + 1 }));
+          });
+          return;
+        }
+        res.writeHead(404).end();
+      },
+      async (db) => {
+        const table = await db.openTable("items");
+
+        await expect(
+          table.add([{ id: 1n }], {
+            allowExternalBlobOutsideBases: true,
+          }),
+        ).rejects.toThrow("only supported on local tables");
+        expect(insertRequests).toBe(0);
+
+        await expect(table.add([{ id: 2n }])).resolves.toMatchObject({
+          version: 2,
+        });
+        await expect(
+          table.add([{ id: 3n }], {
+            allowExternalBlobOutsideBases: false,
+          }),
+        ).resolves.toMatchObject({ version: 3 });
+      },
+    );
+
+    expect(describeRequests.length).toBeGreaterThan(0);
+    expect(insertRequests).toBe(2);
+  });
+
   it("lists materialized views through the namespace route", async () => {
     await withMockDatabase(
       (req, res) => {
@@ -161,6 +224,37 @@ describe("remote connection", () => {
     );
   });
 
+  it("reports the cleanup job when a view drop is accepted", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.method).toBe("POST");
+        expect(req.url).toBe("/v1/view/adults/drop");
+        res
+          .writeHead(202, { "content-type": "application/json" })
+          .end('{"job_id": "j1-do-abc"}');
+      },
+      async (db) => {
+        const job = await db.dropViewAsync("adults");
+        expect(job.id).toBe("j1-do-abc");
+      },
+    );
+  });
+
+  it("reports a finished job when a view drop had nothing to delete", async () => {
+    await withMockDatabase(
+      (req, res) => {
+        expect(req.url).toBe("/v1/view/adults/drop");
+        res.writeHead(200, { "content-type": "application/json" }).end("{}");
+      },
+      async (db) => {
+        // A 200 means the name was not bound, so there is no cleanup to wait on.
+        const job = await db.dropViewAsync("adults");
+        expect(job.id).toBeNull();
+        await job.wait();
+      },
+    );
+  });
+
   it("should accept partial connection options", async () => {
     await connect("db://test", {
       apiKey: "fake",
@@ -1152,6 +1246,24 @@ describe("remote connection jobs surface", () => {
             res
               .writeHead(200, { "Content-Type": "application/json" })
               .end('{"job_id": "job-1"}');
+          } else if (req.url === "/v1/jobs/pause") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end('{"job_id": "job-1", "paused": true}');
+          } else if (req.url === "/v1/jobs/resume") {
+            if (payload["job_id"] !== "job-1") {
+              res.writeHead(404).end("no such job");
+              return;
+            }
+            res
+              .writeHead(200, { "Content-Type": "application/json" })
+              .end(
+                '{"job_id": "job-1", "resumed": false, "still_pausing": true}',
+              );
           } else if (req.url === "/v1/jobs/query_events") {
             queryEventsPayloads.push(payload);
             res
@@ -1173,6 +1285,9 @@ describe("remote connection jobs surface", () => {
         expect(await db.cancelJob("job-1")).toBe(true);
         expect(await db.cancelJob("missing")).toBe(false);
 
+        expect(await db.pauseJob("job-1")).toEqual("pausing");
+        expect(await db.resumeJob("job-1")).toEqual("still_pausing");
+
         // Opening a job hands back a populated handle; a missing one rejects.
         await expect(db.openJob("missing")).rejects.toThrow("not found");
         const finished = await db.openJob("job-2");

nodejs/__test__/table.test.ts

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import * as fs from "fs";
+import { pathToFileURL } from "node:url";
 import * as path from "path";
 import * as tmp from "tmp";
 
@@ -416,6 +417,26 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       expect(await table.countRows("id == 10")).toBe(1);
     });
 
+    it("should let me update float values to NaN and Infinity", async () => {
+      const db = await connect(tmpDir.name);
+      const floats = await db.createTable("floats", [
+        { id: 1, price: 1.5 },
+        { id: 2, price: 2.5 },
+        { id: 3, price: 3.5 },
+        { id: 4, price: 4.5 },
+      ]);
+      await floats.update({ values: { price: NaN }, where: "id = 1" });
+      await floats.update({ values: { price: Infinity }, where: "id = 2" });
+      await floats.update({ values: { price: -Infinity }, where: "id = 3" });
+
+      const rows = await floats.query().toArray();
+      const price = (id: number) => rows.find((row) => row.id === id).price;
+      expect(price(1)).toBeNaN();
+      expect(price(2)).toBe(Infinity);
+      expect(price(3)).toBe(-Infinity);
+      expect(price(4)).toBe(4.5);
+    });
+
     // https://github.com/lancedb/lancedb/issues/1293
     test.each([new arrow.Float16(), new arrow.Float32(), new arrow.Float64()])(
       "can create empty table with non default float type: %s",
@@ -2412,6 +2433,135 @@ describe("when dealing with blob columns", () => {
     tmpDir.removeCallback();
   });
 
+  it("rejects unsupported blob values in createTable and add", async () => {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("image"),
+    ]);
+    const row = { id: 1n, image: new Uint8Array([104]).buffer };
+
+    await expect(db.createTable("invalid", [row], { schema })).rejects.toThrow(
+      /field image at row 0/,
+    );
+
+    const table = await db.createEmptyTable("empty", schema);
+    await expect(table.add([row])).rejects.toThrow(/field image at row 0/);
+    await expect(table.countRows()).resolves.toBe(0);
+  });
+
+  it("requires an explicit opt-in for blob URIs outside registered bases", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await expect(table.add([{ id: 1n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(0);
+
+    await expect(
+      table.add([{ id: 2n, payload: uri }], {
+        allowExternalBlobOutsideBases: false,
+      }),
+    ).rejects.toThrow("allow_external_blob_outside_bases");
+    await expect(table.countRows()).resolves.toBe(0);
+  });
+
+  it.each([
+    ["URI string", (uri: string) => uri],
+    ["URI struct", (uri: string) => ({ uri })],
+  ])(
+    "round-trips an external blob from a %s after reopening",
+    async (_label, blobValue) => {
+      const { db, table, uri, payload } = await openExternalBlobTable();
+
+      const result = await table.add([{ id: 1n, payload: blobValue(uri) }], {
+        allowExternalBlobOutsideBases: true,
+      });
+      expect(result.version).toBeGreaterThan(0);
+      await expect(table.countRows()).resolves.toBe(1);
+
+      table.close();
+      const reopened = await db.openTable("external_blobs");
+      const rows = await reopened.query().withRowId().toArray();
+      expect(rows).toHaveLength(1);
+      expect(rows[0].payload.blob_uri).toBe(uri);
+      const bytes = await reopened.fetchBlobs("payload", [
+        rows[0]._rowid as bigint,
+      ]);
+      expect(bytes[0]).toEqual(payload);
+    },
+  );
+
+  it("applies the external blob opt-in to Arrow table input", async () => {
+    const { table, schema, uri, payload } = await openExternalBlobTable();
+    const input = makeArrowTable([{ id: 1n, payload: uri }], { schema });
+
+    const result = await table.add(input, {
+      allowExternalBlobOutsideBases: true,
+    });
+
+    expect(result.version).toBeGreaterThan(0);
+    const rows = await table.query().withRowId().toArray();
+    expect(rows[0].payload.blob_uri).toBe(uri);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
+  it("applies the external blob opt-in only to the current add", async () => {
+    const { table, uri } = await openExternalBlobTable();
+
+    await table.add([{ id: 1n, payload: uri }], {
+      allowExternalBlobOutsideBases: true,
+    });
+    await expect(table.countRows()).resolves.toBe(1);
+
+    await expect(table.add([{ id: 2n, payload: uri }])).rejects.toThrow(
+      "allow_external_blob_outside_bases",
+    );
+    await expect(table.countRows()).resolves.toBe(1);
+  });
+
+  it("supports progress and overwrite with the external blob opt-in", async () => {
+    const { table, uri, payload } = await openExternalBlobTable();
+    await table.add([{ id: 1n, payload: Buffer.from("old") }]);
+
+    let resolveFinal!: (progress: import("../lancedb").WriteProgress) => void;
+    let rejectFinal!: (error: Error) => void;
+    const finalProgress = new Promise<import("../lancedb").WriteProgress>(
+      (resolve, reject) => {
+        resolveFinal = resolve;
+        rejectFinal = reject;
+      },
+    );
+    const timeout = setTimeout(
+      () => rejectFinal(new Error("timed out waiting for final progress")),
+      5_000,
+    );
+    try {
+      const result = await table.add([{ id: 2n, payload: uri }], {
+        mode: "overwrite",
+        allowExternalBlobOutsideBases: true,
+        progress: (progress) => {
+          if (progress.done) {
+            resolveFinal(progress);
+          }
+        },
+      });
+      expect(result.version).toBeGreaterThan(0);
+      const progress = await finalProgress;
+      expect(progress.outputRows).toBe(1);
+      expect(progress.totalRows).toBe(1);
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    const rows = await table.query().withRowId().toArray();
+    expect(rows).toHaveLength(1);
+    expect(rows[0].id).toBe(2n);
+    const bytes = await table.fetchBlobs("payload", [rows[0]._rowid as bigint]);
+    expect(bytes[0]).toEqual(payload);
+  });
+
   it("discovers blob columns", async () => {
     const { table } = await openBlobTable();
     expect(await table.blobColumns()).toEqual(["image"]);
@@ -2681,6 +2831,20 @@ describe("when dealing with blob columns", () => {
     const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
     return { table, rowIds, alpha, beta };
   }
+
+  async function openExternalBlobTable() {
+    const db = await connect(path.join(tmpDir.name, "db"));
+    const payload = Buffer.from("external-payload");
+    const payloadPath = path.join(tmpDir.name, "payload.bin");
+    fs.writeFileSync(payloadPath, payload);
+    const uri = pathToFileURL(payloadPath).href;
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("payload"),
+    ]);
+    const table = await db.createEmptyTable("external_blobs", schema);
+    return { db, table, schema, uri, payload };
+  }
 });
 
 describe("when dealing with tags", () => {

nodejs/__test__/util.test.ts

@@ -14,6 +14,9 @@ test.each([
   [new ArrayBuffer(8), "X'0000000000000000'"],
   [Buffer.from("hello"), "X'68656c6c6f'"],
   ["Hello 'world'", "'Hello ''world'''"],
+  [NaN, "CAST('NaN' AS DOUBLE)"],
+  [Infinity, "CAST('Infinity' AS DOUBLE)"],
+  [-Infinity, "CAST('-Infinity' AS DOUBLE)"],
 ])("toSQL(%p) === %p", (value, expected) => {
   expect(toSQL(value)).toBe(expected);
 });