Merge main into gatekeeper/fix-4393-1

nodejs/__test__/arrow.test.ts

@@ -790,6 +790,43 @@ describe.each(arrowVersions)("Arrow %#", (arrow: ApacheArrow) => {
       }
     });
 
+    it("will allow null values around struct values with a schema", function () {
+      const schema = new Schema([
+        new Field("value", new Struct([new Field("nested", new Int32())])),
+      ]);
+      for (const { records, nullIndex } of [
+        {
+          records: [{ value: null }, { value: { nested: 2 } }],
+          nullIndex: 0,
+        },
+        {
+          records: [{ value: { nested: 1 } }, { value: null }],
+          nullIndex: 1,
+        },
+      ]) {
+        const table = makeArrowTable(records, { schema });
+        const values = table.getChild("value");
+
+        expect(values?.nullCount).toBe(1);
+        expect(values?.get(nullIndex)).toBeNull();
+      }
+    });
+
+    it("will reject non-object values for struct fields in a schema", function () {
+      const schema = new Schema([
+        new Field("value", new Struct([new Field("nested", new Int32())])),
+      ]);
+      for (const records of [
+        [{ value: 5 }],
+        [{ value: 5 }, { value: { nested: 1 } }],
+        [{ value: { nested: 1 } }, { value: "oops" }],
+      ]) {
+        expect(() => makeArrowTable(records, { schema })).toThrow(
+          /Expected a struct value for field value at row \d, got/,
+        );
+      }
+    });
+
     it("will allow a schema to be provided", async function () {
       await checkTableCreation(
         async (records, _, schema) =>

nodejs/__test__/blob.test.ts

@@ -1,9 +1,25 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-import { Field, Int64, List, Schema, Struct, Utf8 } from "apache-arrow";
-import { makeArrowTable } from "../lancedb/arrow";
+import {
+  Field,
+  Float,
+  Float32,
+  Int32,
+  Int64,
+  List,
+  Schema,
+  Struct,
+  Utf8,
+} from "apache-arrow";
+import {
+  convertToTable,
+  makeArrowTable,
+  resolveBlobInputs,
+} from "../lancedb/arrow";
 import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
+import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
+import { getRegistry } from "../lancedb/embedding/registry";
 
 describe("blob()", () => {
   it("marks the field as lance.blob.v2", () => {
@@ -46,6 +62,13 @@ describe("blob()", () => {
   });
 });
 
+class BlobStruct {
+  constructor(
+    public data?: unknown,
+    public uri?: unknown,
+  ) {}
+}
+
 describe("coerceBlobValue", () => {
   it.each([
     ["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }],
@@ -65,6 +88,41 @@ describe("coerceBlobValue", () => {
       { uri: "s3://bucket/key" },
       { data: null, uri: "s3://bucket/key" },
     ],
+    [
+      "ArrayBuffer",
+      new Uint8Array([120]).buffer,
+      { data: new Uint8Array([120]), uri: null },
+    ],
+    [
+      "URL",
+      new URL("https://example.com/blob"),
+      { data: null, uri: "https://example.com/blob" },
+    ],
+    [
+      "data struct with ArrayBuffer",
+      { data: new Uint8Array([121]).buffer },
+      { data: new Uint8Array([121]), uri: null },
+    ],
+    [
+      "uri struct with URL",
+      { uri: new URL("file:///tmp/a.png") },
+      { data: null, uri: "file:///tmp/a.png" },
+    ],
+    [
+      "class instance with Uint8Array data",
+      new BlobStruct(new Uint8Array([122])),
+      { data: new Uint8Array([122]), uri: null },
+    ],
+    [
+      "class instance with ArrayBuffer data",
+      new BlobStruct(new Uint8Array([122]).buffer),
+      { data: new Uint8Array([122]), uri: null },
+    ],
+    [
+      "class instance with URL uri",
+      new BlobStruct(undefined, new URL("s3://bucket/key")),
+      { data: null, uri: "s3://bucket/key" },
+    ],
     ["null", null, null],
   ])("accepts %s", (_name, input, expected) => {
     expect(coerceBlobValue(input)).toEqual(expected);
@@ -76,7 +134,7 @@ describe("coerceBlobValue", () => {
     [
       "Int16Array",
       new Int16Array([1]),
-      /Blob data must be Buffer or Uint8Array/,
+      /Blob data must be Buffer, Uint8Array, ArrayBuffer, or Blob/,
     ],
     [
       "both data and uri",
@@ -88,11 +146,160 @@ describe("coerceBlobValue", () => {
       { data: null, uri: null },
       /exactly one of 'data' or 'uri'/,
     ],
+    ["Blob", new Blob(["x"]), /must be read asynchronously/],
+    ["data struct with Blob", { data: new Blob(["x"]) }, /asynchronously/],
+    [
+      "class instance with Blob data",
+      new BlobStruct(new Blob(["x"])),
+      /asynchronously/,
+    ],
+    [
+      "unknown struct field",
+      { data: Buffer.from("y"), mime: "image/png" },
+      /Blob struct values only support 'data' and 'uri', got 'mime'/,
+    ],
+    [
+      "misspelled struct field",
+      { uri: "s3://bucket/key", dat: Buffer.from("y") },
+      /got 'dat'/,
+    ],
+    [
+      "SharedArrayBuffer",
+      new SharedArrayBuffer(2),
+      /Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob/,
+    ],
+    ["Date", new Date(0), /Blob column values must be/],
   ])("rejects %s", (_name, input, message) => {
     expect(() => coerceBlobValue(input)).toThrow(message);
   });
 });
 
+describe("resolveBlobInputs", () => {
+  it("reads Blob and File values in blob columns only", async () => {
+    const schema = new Schema([
+      new Field("name", new Utf8(), true),
+      blob("image"),
+      new Field("images", new List(blob("image")), true),
+    ]);
+    const shared = new Blob(["same"]);
+    const notABlobColumn = new Blob(["other"]);
+    const data = [
+      {
+        name: notABlobColumn,
+        image: new File(["file"], "a.txt"),
+        images: [shared, { data: shared }, null],
+      },
+      { name: "x", image: Buffer.from("buf"), images: null },
+    ];
+    const resolved = await resolveBlobInputs(data, schema);
+
+    expect(resolved[0].name).toBe(notABlobColumn);
+    expect(Buffer.from(resolved[0].image as Uint8Array).toString()).toBe(
+      "file",
+    );
+    const images = resolved[0].images as [Uint8Array, { data: Uint8Array }];
+    expect(Buffer.from(images[0]).toString()).toBe("same");
+    expect(Buffer.from(images[1].data).toString()).toBe("same");
+    // Rows without Blob values and the caller's records are left alone.
+    expect(resolved[1]).toBe(data[1]);
+    expect(data[0].images?.[0]).toBe(shared);
+  });
+
+  it("reads each distinct Blob once, bare or inside data structs", async () => {
+    class CountingBlob extends Blob {
+      calls = 0;
+      async arrayBuffer(): Promise<ArrayBuffer> {
+        this.calls++;
+        return super.arrayBuffer();
+      }
+    }
+    const shared = new CountingBlob(["same"]);
+    const schema = new Schema([blob("image")]);
+    const resolved = await resolveBlobInputs(
+      [
+        { image: { data: shared } },
+        { image: { data: shared } },
+        { image: shared },
+      ],
+      schema,
+    );
+    expect(shared.calls).toBe(1);
+    const first = (resolved[0].image as { data: Uint8Array }).data;
+    expect(Buffer.from(first).toString()).toBe("same");
+    expect((resolved[1].image as { data: Uint8Array }).data).toBe(first);
+    expect(resolved[2].image).toBe(first);
+  });
+
+  it("reads a Blob held by a class instance", async () => {
+    const schema = new Schema([blob("image")]);
+    const [row] = await resolveBlobInputs(
+      [{ image: new BlobStruct(new Blob(["hi"])) }],
+      schema,
+    );
+    expect(
+      Buffer.from((row.image as { data: Uint8Array }).data).toString(),
+    ).toBe("hi");
+  });
+
+  it("returns the input when there is nothing to read", async () => {
+    const data = [{ image: Buffer.from("x") }];
+    const schema = new Schema([blob("image")]);
+    await expect(resolveBlobInputs(data, schema)).resolves.toBe(data);
+    await expect(resolveBlobInputs(data)).resolves.toBe(data);
+  });
+});
+
+describe("convertToTable blob columns with embedding functions", () => {
+  // Embedding metadata routes rows through ensureNestedFieldsExist, which must
+  // leave blob values for the blob coercion instead of treating them as structs.
+  class LengthEmbedding extends EmbeddingFunction<string> {
+    ndims() {
+      return 2;
+    }
+    embeddingDataType(): Float {
+      return new Float32();
+    }
+    async computeSourceEmbeddings(data: string[]) {
+      return data.map((text) => [text.length, 1]);
+    }
+  }
+  getRegistry().register("blob-test-length")(LengthEmbedding);
+  const func = getRegistry().get<LengthEmbedding>("blob-test-length")!.create();
+  const embeddingSchema = LanceSchema({
+    text: func.sourceField(new Utf8()),
+    vector: func.vectorField(),
+  });
+  const schema = new Schema(
+    [...embeddingSchema.fields, blob("image")],
+    embeddingSchema.metadata,
+  );
+
+  it.each([
+    ["Uint8Array", new Uint8Array([104, 105]), "hi", null],
+    ["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
+    ["Blob", new Blob(["hi"]), "hi", null],
+    ["URI string", "s3://b/k", null, "s3://b/k"],
+    ["URL", new URL("s3://b/k"), null, "s3://b/k"],
+    ["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
+  ])("accepts %s input", async (_name, image, data, uri) => {
+    const table = await convertToTable([{ text: "a", image }], undefined, {
+      schema,
+    });
+    const column = table.getChild("image")!;
+    expect(column.nullCount).toBe(0);
+    const bytes = column.getChild("data")!.get(0);
+    expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
+    expect(column.getChild("uri")!.get(0)).toBe(uri);
+  });
+
+  it("writes a missing blob as null", async () => {
+    const table = await convertToTable([{ text: "a" }], undefined, { schema });
+    const column = table.getChild("image")!;
+    expect(column.nullCount).toBe(1);
+    expect(column.get(0)).toBeNull();
+  });
+});
+
 describe("BlobFile", () => {
   it("rejects constructing BlobFile without a native handle", () => {
     expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow(
@@ -103,26 +310,122 @@ describe("BlobFile", () => {
 
 describe("makeArrowTable blob columns", () => {
   it.each([
-    ["ArrayBuffer", new Uint8Array([104]).buffer],
-    ["Blob", new Blob(["hello"])],
-    ["File", new File(["hello"], "hello.txt")],
-    ["URL", new URL("https://example.com/blob")],
     ["ReadableStream", new ReadableStream()],
-    ["data struct with ArrayBuffer", { data: new Uint8Array([104]).buffer }],
-    ["uri struct with URL", { uri: new URL("https://example.com/blob") }],
+    ["Int16Array", new Int16Array([1])],
   ])("rejects %s input rather than writing a null blob", (_name, image) => {
     const schema = new Schema([blob("image")]);
     expect(() => makeArrowTable([{ image }], { schema })).toThrow(/image/);
   });
 
-  it("rejects Blob instances with enumerable fields as one invalid value", () => {
+  it.each([
+    ["Blob", new Blob(["hello"])],
+    ["File", new File(["hello"], "hello.txt")],
+    ["data struct with Blob", { data: new Blob(["hello"]) }],
+    [
+      "Blob with enumerable fields",
+      Object.assign(new Blob(["hello"]), { position: 0 }),
+    ],
+  ])("points %s input at the async APIs", (_name, image) => {
     const schema = new Schema([blob("image")]);
-    const image = Object.assign(new Blob(["hello"]), { position: 0 });
-    expect(() => makeArrowTable([{ image }], { schema })).toThrow(
-      "Unsupported object value for field image at row 0.",
+    expect(() => makeArrowTable([{ id: 1, image }], { schema })).toThrow(
+      /blob field image at row 0: Blob and File values must be read asynchronously/,
     );
   });
 
+  it.each([
+    ["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
+    [
+      "data struct with ArrayBuffer",
+      { data: new Uint8Array([104, 105]).buffer },
+      "hi",
+      null,
+    ],
+    ["URL", new URL("https://example.com/a"), null, "https://example.com/a"],
+    [
+      "uri struct with URL",
+      { uri: new URL("file:///tmp/a.png") },
+      null,
+      "file:///tmp/a.png",
+    ],
+  ])("accepts %s input", (_name, image, data, uri) => {
+    const schema = new Schema([blob("image")]);
+    const input = { image };
+    const table = makeArrowTable([input], { schema });
+    const column = table.getChild("image")!;
+    expect(column.nullCount).toBe(0);
+    const bytes = column.getChild("data")!.get(0);
+    expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
+    expect(column.getChild("uri")!.get(0)).toBe(uri);
+    // The caller's record is left untouched.
+    expect(input.image).toBe(image);
+  });
+
+  it.each([
+    ["Uint8Array", new Uint8Array([104, 105]), "hi", null],
+    ["URI string", "s3://b/k", null, "s3://b/k"],
+    ["data struct", { data: new Uint8Array([104, 105]) }, "hi", null],
+  ])("accepts %s input after a null row", (_name, image, data, uri) => {
+    const schema = new Schema([blob("image")]);
+    const table = makeArrowTable([{ image: null }, { image }], { schema });
+    const column = table.getChild("image")!;
+    expect(column.nullCount).toBe(1);
+    expect(column.get(0)).toBeNull();
+    const bytes = column.getChild("data")!.get(1);
+    expect(bytes == null ? null : Buffer.from(bytes).toString()).toBe(data);
+    expect(column.getChild("uri")!.get(1)).toBe(uri);
+  });
+
+  it("accepts a null struct before one holding a blob", () => {
+    const schema = new Schema([
+      new Field("info", new Struct([blob("image")]), true),
+    ]);
+    const table = makeArrowTable(
+      [{ info: null }, { info: { image: new Uint8Array([97]) } }],
+      { schema },
+    );
+    const info = table.getChild("info")!;
+    expect(info.get(0)).toBeNull();
+    const image = (info.get(1) as { image: { data: Uint8Array } }).image;
+    expect(Buffer.from(image.data).toString()).toBe("a");
+  });
+
+  it("skips a missing blob column named like an Object.prototype member", () => {
+    const schema = new Schema([
+      new Field("id", new Int32(), true),
+      blob("constructor"),
+    ]);
+    const table = makeArrowTable([{ id: 1 }], { schema });
+    expect(table.schema.fields.map((f) => f.name)).toEqual(["id"]);
+  });
+
+  it("accepts ArrayBuffer and URL inside lists and structs", () => {
+    const schema = new Schema([
+      new Field("images", new List(blob("image")), true),
+      new Field("info", new Struct([blob("image")]), true),
+    ]);
+    const table = makeArrowTable(
+      [
+        {
+          images: [new Uint8Array([97]).buffer, new URL("s3://b/k")],
+          info: { image: new Uint8Array([98]).buffer },
+        },
+      ],
+      { schema },
+    );
+    const images = Array.from(
+      table.getChild("images")!.get(0) as Iterable<{
+        data: Uint8Array | null;
+        uri: string | null;
+      }>,
+    );
+    expect(Buffer.from(images[0].data!).toString()).toBe("a");
+    expect(images[1].uri).toBe("s3://b/k");
+    const info = table.getChild("info")!.get(0) as {
+      image: { data: Uint8Array };
+    };
+    expect(Buffer.from(info.image.data).toString()).toBe("b");
+  });
+
   it("coerces Buffer input onto a blob field", () => {
     const schema = new Schema([
       new Field("id", new Int64(), true),

nodejs/__test__/catalog.test.ts

@@ -80,9 +80,12 @@ describe("remote catalog", () => {
         const second = await catalog.connectDatabase("other");
         expect(await second.tableNames()).toEqual([]);
         expect(await first.tableNames()).toEqual([]);
-        expect(
-          await catalog.listDatabases({ limit: 1, pageToken: "a/b" }),
-        ).toEqual({ databases: ["team/search"], pageToken: "next" });
+        const names = catalog.listDatabases({ pageLimit: 1, pageToken: "a/b" });
+        expect(await names.next()).toEqual({
+          done: false,
+          value: "team/search",
+        });
+        expect(names.pageToken()).toBe("next");
         await catalog.dropDatabase("team/search", { ignoreMissing: true });
         expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create");
         expect(requests[0].body).toEqual({ mode: "ExistOk" });
@@ -129,11 +132,139 @@ describe("remote catalog", () => {
     );
   });
 
+  it("lazily traverses pages, including empty pages, and exposes cached state", async () => {
+    await withCatalog(
+      [
+        // biome-ignore lint/style/useNamingConvention: server wire format
+        [200, { namespaces: ["a", "b"], page_token: "empty" }],
+        // biome-ignore lint/style/useNamingConvention: server wire format
+        [200, { namespaces: [], page_token: "last" }],
+        // biome-ignore lint/style/useNamingConvention: server wire format
+        [200, { namespaces: ["c", "d"], page_token: "" }],
+      ],
+      async (catalog, requests) => {
+        const names = catalog.listDatabases({ pageLimit: 2 });
+        expect(names[Symbol.asyncIterator]()).toBe(names);
+        expect(requests).toHaveLength(0);
+        expect(names.numPageResults()).toBe(0);
+        expect(names.pageToken()).toBeUndefined();
+        expect(await names.next()).toEqual({ done: false, value: "a" });
+        expect(names.numPageResults()).toBe(1);
+        expect(names.pageToken()).toBe("empty");
+        expect(await names.next()).toEqual({ done: false, value: "b" });
+        expect(requests).toHaveLength(1);
+        expect(await names.next()).toEqual({ done: false, value: "c" });
+        expect(requests).toHaveLength(3);
+        expect(names.pageToken()).toBeUndefined();
+        expect(names.numPageResults()).toBe(1);
+        const remaining = [];
+        for await (const name of names) remaining.push(name);
+        expect(remaining).toEqual(["d"]);
+        expect(names.numPageResults()).toBe(0);
+        expect(await names.next()).toEqual({ done: true, value: undefined });
+        expect(requests.map((request) => request.url)).toEqual([
+          "/v1/namespace/$/list?limit=2",
+          "/v1/namespace/$/list?limit=2&page_token=empty",
+          "/v1/namespace/$/list?limit=2&page_token=last",
+        ]);
+      },
+    );
+  });
+
+  it("resumes from a saved token after draining the cache", async () => {
+    await withCatalog(
+      [
+        // biome-ignore lint/style/useNamingConvention: server wire format
+        [200, { namespaces: ["a", "b"], page_token: "next/token" }],
+        [200, { namespaces: ["c"] }],
+      ],
+      async (catalog, requests) => {
+        const names = catalog.listDatabases({
+          pageToken: "start/token",
+          pageLimit: 2,
+        });
+        expect(names.pageToken()).toBe("start/token");
+        expect(await names.next()).toEqual({ done: false, value: "a" });
+        while (names.numPageResults() > 0) await names.next();
+        const resumed = catalog.listDatabases({
+          pageToken: names.pageToken(),
+          pageLimit: 2,
+        });
+        const results = [];
+        for await (const name of resumed) results.push(name);
+        expect(results).toEqual(["c"]);
+        expect(requests.map((request) => request.url)).toEqual([
+          "/v1/namespace/$/list?limit=2&page_token=start%2Ftoken",
+          "/v1/namespace/$/list?limit=2&page_token=next%2Ftoken",
+        ]);
+      },
+    );
+  });
+
+  it("terminates on errors and retains the failed request token", async () => {
+    await withCatalog(
+      [
+        // biome-ignore lint/style/useNamingConvention: server wire format
+        [200, { namespaces: ["a"], page_token: "retry" }],
+        [400, {}],
+        [200, { namespaces: ["b"] }],
+      ],
+      async (catalog, requests) => {
+        const names = catalog.listDatabases();
+        await names.next();
+        await expect(names.next()).rejects.toThrow();
+        expect(names.pageToken()).toBe("retry");
+        expect(names.numPageResults()).toBe(0);
+        expect(await names.next()).toEqual({ done: true, value: undefined });
+        expect(requests).toHaveLength(2);
+        const resumed = catalog.listDatabases({ pageToken: names.pageToken() });
+        expect(await resumed.next()).toEqual({ done: false, value: "b" });
+        expect(await resumed.next()).toEqual({ done: true, value: undefined });
+      },
+    );
+  });
+
+  it("handles an empty listing and concurrent advances", async () => {
+    await withCatalog(
+      [
+        [200, { namespaces: [] }],
+        [200, { namespaces: ["a", "b"] }],
+      ],
+      async (catalog, requests) => {
+        const empty = catalog.listDatabases();
+        expect(await empty.next()).toEqual({ done: true, value: undefined });
+        expect(await empty.next()).toEqual({ done: true, value: undefined });
+        const names = catalog.listDatabases();
+        const pending = [names.next(), names.next(), names.next()];
+        expect(names.numPageResults()).toBe(0);
+        expect(names.pageToken()).toBeUndefined();
+        const results = await Promise.all(pending);
+        expect(
+          results
+            .filter((result) => !result.done)
+            .map((result) => result.value)
+            .sort(),
+        ).toEqual(["a", "b"]);
+        expect(results.filter((result) => result.done)).toHaveLength(1);
+        expect(requests).toHaveLength(2);
+      },
+    );
+  });
+
   it("validates endpoints and pagination", async () => {
     await expect(connectCatalog("/tmp/catalog")).rejects.toThrow();
     const catalog = await connectCatalog("http://127.0.0.1:1");
-    for (const limit of [0, -1, 1.5, 2147483648]) {
-      await expect(catalog.listDatabases({ limit })).rejects.toThrow("limit");
+    for (const limit of [
+      0,
+      -1,
+      1.5,
+      2147483648,
+      Number.NaN,
+      Number.POSITIVE_INFINITY,
+    ]) {
+      expect(() => catalog.listDatabases({ pageLimit: limit })).toThrow(
+        "limit",
+      );
     }
     await expect(catalog.connectDatabase("a$b")).rejects.toThrow(
       "Invalid database name",

nodejs/__test__/materialized_view.test.ts

@@ -16,15 +16,11 @@ describe("materialized views", () => {
   beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
     db = await connect(tmpDir.name);
-    await db.createTable(
-      "people",
-      [
-        { name: "ada", age: 36 },
-        { name: "kid", age: 7 },
-        { name: "grace", age: 85 },
-      ],
-      { storageOptions: { newTableEnableStableRowIds: "true" } },
-    );
+    await db.createTable("people", [
+      { name: "ada", age: 36 },
+      { name: "kid", age: 7 },
+      { name: "grace", age: 85 },
+    ]);
   });
   afterEach(() => tmpDir.removeCallback());
 
@@ -52,19 +48,9 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
-    for (const [format, fn] of [
-      [3, "vector_duplicate_pairs"],
-      [4, "vector_dedup"],
-    ]) {
-      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
-      expect(read(JSON.stringify({ format, query: native })).query).toBe(
-        native,
-      );
-    }
-
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":5,"query":${JSON.stringify(query)}}`,
+      `{"format":3,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);
@@ -106,7 +92,7 @@ describe("materialized views", () => {
     );
   });
 
-  it("refreshes incrementally after an append", async () => {
+  it("fully refreshes after an append", async () => {
     const view = await db.createMaterializedView("copy", "people", {
       withNoData: true,
     });
@@ -115,11 +101,11 @@ describe("materialized views", () => {
     const people = await db.openTable("people");
     await people.add([{ name: "alan", age: 41 }]);
     const result = await view.refresh();
-    expect(result.mode).toBe("incremental");
-    expect(Number(result.rowsWritten)).toBe(1);
+    expect(result.mode).toBe("rebuild");
+    expect(Number(result.rowsWritten)).toBe(4);
     expect(await view.table().countRows()).toBe(4);
 
-    expect((await view.refresh()).mode).toBe("no_op");
+    expect((await view.refresh()).mode).toBe("rebuild");
   });
 
   it("lists views and rejects non-views", async () => {
@@ -172,9 +158,7 @@ describe("materialized views", () => {
   });
 
   it("quotes bare select names", async () => {
-    await db.createTable("odd_names", [{ "order item": "widget" }], {
-      storageOptions: { newTableEnableStableRowIds: "true" },
-    });
+    await db.createTable("odd_names", [{ "order item": "widget" }]);
     const view = await db.createMaterializedView("quoted", "odd_names", {
       select: ["order item"],
       withNoData: true,
@@ -183,10 +167,9 @@ describe("materialized views", () => {
     expect(Number(result.rowsWritten)).toBe(1);
   });
 
-  it("requires stable row ids on the source", async () => {
+  it("does not require stable row ids on the source", async () => {
     await db.createTable("plain", [{ x: 1 }]);
-    await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
-      "stable row ids",
-    );
+    const view = await db.createMaterializedView("v", "plain");
+    expect(await view.table().countRows()).toBe(1);
   });
 });

nodejs/__test__/table.test.ts

@@ -9,6 +9,7 @@ import * as tmp from "tmp";
 
 import {
   AutoQuery,
+  type BlobInput,
   Connection,
   MatchQuery,
   PhraseQuery,
@@ -2583,7 +2584,7 @@ describe("when dealing with blob columns", () => {
       new Field("id", new Int64(), true),
       blob("image"),
     ]);
-    const row = { id: 1n, image: new Uint8Array([104]).buffer };
+    const row = { id: 1n, image: new ReadableStream() };
 
     await expect(db.createTable("invalid", [row], { schema })).rejects.toThrow(
       /field image at row 0/,
@@ -2613,6 +2614,8 @@ describe("when dealing with blob columns", () => {
   it.each([
     ["URI string", (uri: string) => uri],
     ["URI struct", (uri: string) => ({ uri })],
+    ["URL", (uri: string) => new URL(uri)],
+    ["URL struct", (uri: string) => ({ uri: new URL(uri) })],
   ])(
     "round-trips an external blob from a %s after reopening",
     async (_label, blobValue) => {
@@ -2706,6 +2709,63 @@ describe("when dealing with blob columns", () => {
     expect(bytes[0]).toEqual(payload);
   });
 
+  it("accepts ArrayBuffer, Blob, and File in createTable and add", async () => {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("image"),
+    ]);
+    const rows: { id: bigint; image: BlobInput }[] = [
+      { id: 1n, image: new TextEncoder().encode("array-buffer").buffer },
+      { id: 2n, image: new Blob(["blob"]) },
+      { id: 3n, image: { data: new File(["file"], "f.txt") } },
+    ];
+    const table = await db.createTable("widened", rows.slice(0, 2), {
+      schema,
+    });
+    await table.add(rows.slice(2));
+
+    const results = await table.query().select(["id"]).withRowId().toArray();
+    results.sort((a, b) => Number(a.id - b.id));
+    const bytes = await table.fetchBlobs(
+      "image",
+      results.map((row) => row._rowid as bigint),
+    );
+    expect(bytes.map((b) => b?.toString())).toEqual([
+      "array-buffer",
+      "blob",
+      "file",
+    ]);
+  });
+
+  it("accepts Blob values in mergeInsert", async () => {
+    const db = await connect(tmpDir.name);
+    const schema = new Schema([
+      new Field("id", new Int64(), true),
+      blob("image"),
+    ]);
+    const table = await db.createTable(
+      "merge_blobs",
+      [{ id: 1n, image: Buffer.from("old") }],
+      { schema },
+    );
+    await table
+      .mergeInsert("id")
+      .whenMatchedUpdateAll()
+      .whenNotMatchedInsertAll()
+      .execute([
+        { id: 1n, image: new Blob(["new"]) },
+        { id: 2n, image: new Uint8Array([104, 105]).buffer },
+      ]);
+    const results = await table.query().select(["id"]).withRowId().toArray();
+    results.sort((a, b) => Number(a.id - b.id));
+    const bytes = await table.fetchBlobs(
+      "image",
+      results.map((row) => row._rowid as bigint),
+    );
+    expect(bytes.map((b) => b?.toString())).toEqual(["new", "hi"]);
+  });
+
   it("discovers blob columns", async () => {
     const { table } = await openBlobTable();
     expect(await table.blobColumns()).toEqual(["image"]);