fix!: normalize blob layouts and support external byte ranges (#4589)

nodejs/__test__/blob.test.ts

@@ -21,7 +21,41 @@ import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
 import { EmbeddingFunction, LanceSchema } from "../lancedb/embedding";
 import { getRegistry } from "../lancedb/embedding/registry";
 
+function legacyBlob(name: string): Field {
+  const field = blob(name);
+  return new Field(
+    name,
+    new Struct((field.type as Struct).children.slice(0, 2)),
+    field.nullable,
+    field.metadata,
+  );
+}
+
 describe("blob()", () => {
+  it("declares the four-field range layout", () => {
+    const field = blob("image", {
+      nullable: false,
+      inlineSizeThreshold: 12,
+    });
+    expect(field.nullable).toBe(false);
+    expect(isBlobField(field)).toBe(true);
+    expect(
+      field.metadata.get("lance-encoding:blob-inline-size-threshold"),
+    ).toBe("12");
+    expect(
+      (field.type as Struct).children.map((f) => [
+        f.name,
+        f.type.toString(),
+        f.nullable,
+      ]),
+    ).toEqual([
+      ["data", "LargeBinary", true],
+      ["uri", "Utf8", true],
+      ["position", "Uint64", true],
+      ["size", "Uint64", true],
+    ]);
+  });
+
   it("marks the field as lance.blob.v2", () => {
     const field = blob("image", { nullable: false });
     expect(field.nullable).toBe(false);
@@ -156,7 +190,7 @@ describe("coerceBlobValue", () => {
     [
       "unknown struct field",
       { data: Buffer.from("y"), mime: "image/png" },
-      /Blob struct values only support 'data' and 'uri', got 'mime'/,
+      /Blob struct values only support .*got 'mime'/,
     ],
     [
       "misspelled struct field",
@@ -174,6 +208,158 @@ describe("coerceBlobValue", () => {
   });
 });
 
+describe("external blob ranges", () => {
+  const uri = "s3://bucket/key";
+
+  it.each([0, 4, 0n, 4n, 9007199254740993n])(
+    "preserves position %s",
+    (position) => {
+      expect(coerceBlobValue({ uri, position, size: 6 })).toEqual({
+        data: null,
+        uri,
+        position: BigInt(position),
+        size: 6n,
+      });
+    },
+  );
+
+  it.each([
+    [{ uri, position: 0 }, /both position and size/],
+    [{ uri, size: 6 }, /both position and size/],
+    [{ uri, position: null, size: 6 }, /both position and size/],
+    [{ uri, position: 0, size: 0 }, /size must be positive/],
+    [{ uri, position: -1, size: 6 }, /position must be non-negative/],
+    [{ uri, position: 0, size: -1n }, /size must be positive/],
+    [{ uri, position: 0.5, size: 6 }, /safe integer or bigint/],
+    [
+      { uri, position: Number.MAX_SAFE_INTEGER + 1, size: 6 },
+      /safe integer or bigint/,
+    ],
+    [{ uri, position: 0, size: Number.NaN }, /safe integer or bigint/],
+    [{ uri, position: "4", size: 6 }, /safe integer or bigint/],
+    [{ uri, position: false, size: 6 }, /safe integer or bigint/],
+    [{ uri, position: 1n << 64n, size: 6 }, /fit in uint64/],
+    [{ uri, position: 0xffffffffffffffffn, size: 1 }, /overflows uint64/],
+    [{ data: Buffer.from("x"), position: 0, size: 6 }, /require a uri/],
+  ])("rejects malformed input %p", (value, message) => {
+    expect(() => coerceBlobValue(value)).toThrow(message);
+  });
+
+  it("preserves ranges while normalizing a URL held by getters", () => {
+    class Range {
+      get uri() {
+        return new URL(uri);
+      }
+      get position() {
+        return 4;
+      }
+      get size() {
+        return 6;
+      }
+    }
+    expect(coerceBlobValue(new Range())).toEqual({
+      data: null,
+      uri,
+      position: 4n,
+      size: 6n,
+    });
+  });
+
+  it("writes ranges and mixed whole-file, inline, and null values to Arrow", () => {
+    const schema = new Schema([blob("image")]);
+    const rows = [
+      null,
+      Buffer.from("x"),
+      uri,
+      { uri, position: 4, size: 6 },
+    ].map((image) => ({ image }));
+    const column = makeArrowTable(rows, { schema }).getChild("image")!;
+    expect(column.get(0)).toBeNull();
+    expect(Buffer.from(column.getChild("data")!.get(1))).toEqual(
+      Buffer.from("x"),
+    );
+    expect(column.getChild("uri")!.get(2)).toBe(uri);
+    expect(column.getChild("position")!.toArray()).toEqual(
+      new BigUint64Array([0n, 0n, 0n, 4n]),
+    );
+    expect(column.getChild("position")!.get(2)).toBeNull();
+    expect(column.getChild("size")!.get(3)).toBe(6n);
+  });
+
+  it.each([
+    Buffer.from("inline"),
+    uri,
+    { uri, position: null, size: null },
+    null,
+  ])("retains the full declared range layout when seeded with %p", (image) => {
+    const schema = new Schema([blob("image")]);
+    const table = makeArrowTable([{ image }], { schema });
+    expect(table.schema).toEqual(schema);
+    expect(table.getChild("image")!.getChild("position")!.get(0)).toBeNull();
+  });
+
+  it.each([0, 4])(
+    "rejects position %s on a minimal schema before dropping keys",
+    (position) => {
+      expect(() =>
+        makeArrowTable(
+          [
+            { image: Buffer.from("inline") },
+            { image: { uri, position, size: 6 } },
+          ],
+          { schema: new Schema([legacyBlob("image")]) },
+        ),
+      ).toThrow(
+        /field image at row 1: Blob storage schema cannot store byte ranges/,
+      );
+    },
+  );
+
+  it("accepts omitted or null range fields on the minimal schema", () => {
+    const column = makeArrowTable(
+      [{ image: { uri, position: null, size: null } }],
+      {
+        schema: new Schema([legacyBlob("image")]),
+      },
+    ).getChild("image")!;
+    expect(column.getChild("uri")!.get(0)).toBe(uri);
+  });
+
+  it("validates ranges nested in structs and lists against their storage fields", () => {
+    const value = { uri, position: 4, size: 6 };
+    for (const field of [legacyBlob("image"), blob("image")]) {
+      const schema = new Schema([
+        new Field("info", new Struct([field]), true),
+        new Field("images", new List(field), true),
+      ]);
+      if ((field.type as Struct).children.length === 2) {
+        expect(() =>
+          makeArrowTable([{ info: { image: value } }], { schema }),
+        ).toThrow(/field info.image.*storage schema/);
+        expect(() => makeArrowTable([{ images: [value] }], { schema })).toThrow(
+          /field images\[0\].*storage schema/,
+        );
+      } else {
+        const table = makeArrowTable(
+          [{ info: { image: value }, images: [value] }],
+          { schema },
+        );
+        const info = table.getChild("info")!.get(0) as {
+          image: { position: bigint; size: bigint };
+        };
+        expect(info.image.position).toBe(4n);
+        const images = Array.from(
+          table.getChild("images")!.get(0) as Iterable<{
+            position: bigint;
+            size: bigint;
+          }>,
+        );
+        expect(images[0].size).toBe(6n);
+      }
+    }
+  });
+});
+
 describe("resolveBlobInputs", () => {
   it("reads Blob and File values in blob columns only", async () => {
     const schema = new Schema([

nodejs/__test__/remote.test.ts

@@ -12,6 +12,7 @@ import {
   OAuthConfig,
   OAuthFlowType,
   TlsConfig,
+  blob,
   connect,
 } from "../lancedb";
 import {
@@ -19,7 +20,10 @@ import {
   Field,
   Int64,
   RecordBatch,
+  RecordBatchReader,
   Schema,
+  Struct,
+  makeArrowTable,
   tableToIPC,
 } from "../lancedb/arrow";
 import {
@@ -94,6 +98,137 @@ async function withMockDatabase(
 }
 
 describe("remote connection", () => {
+  it.each([false, true])(
+    "normalizes remote blob writes (range target: %s)",
+    async (wide) => {
+      const field = (full: boolean) => {
+        const f = blob("image");
+        return full
+          ? f
+          : new Field(
+              f.name,
+              new Struct((f.type as Struct).children.slice(0, 2)),
+              f.nullable,
+              f.metadata,
+            );
+      };
+      const schema = (full: boolean) =>
+        new Schema([new Field("id", new Int64(), true), field(full)]);
+      const writes: { path: string; schema: Schema }[] = [];
+      let storedWide = wide;
+      const describe = () => ({
+        version: writes.length + 1,
+        schema: {
+          fields: [
+            { name: "id", type: { type: "int64" }, nullable: true },
+            {
+              name: "image",
+              nullable: true,
+              metadata: { "ARROW:extension:name": "lance.blob.v2" },
+              type: {
+                type: "struct",
+                fields: [
+                  {
+                    name: "data",
+                    nullable: true,
+                    type: { type: "large_binary" },
+                  },
+                  { name: "uri", nullable: true, type: { type: "string" } },
+                  ...(storedWide
+                    ? ["position", "size"].map((name) => ({
+                        name,
+                        nullable: true,
+                        type: { type: "uint64" },
+                      }))
+                    : []),
+                ],
+              },
+            },
+          ],
+        },
+      });
+      await withMockDatabase(
+        (req, res) => {
+          if (req.url?.includes("/describe/")) {
+            res
+              .writeHead(200, { "content-type": "application/json" })
+              .end(JSON.stringify(describe()));
+            return;
+          }
+          const chunks: Buffer[] = [];
+          // Failed streaming input aborts before end; only complete bodies commit.
+          req.on("error", () => {
+            /* Aborted inputs must not commit. */
+          });
+          req.on("data", (chunk) => chunks.push(chunk));
+          req.on("end", () => {
+            const reader = RecordBatchReader.from(Buffer.concat(chunks)).open();
+            // Only read the schema: Rust separately checks compressed wire values.
+            writes.push({ path: req.url ?? "", schema: reader.schema });
+            storedWide =
+              (reader.schema.fields[1].type as Struct).children.length === 4;
+            res.writeHead(200, { "content-type": "application/json" }).end(
+              JSON.stringify({
+                version: writes.length + 1,
+                // biome-ignore lint/style/useNamingConvention: server wire format
+                num_inserted_rows: 1,
+              }),
+            );
+          });
+        },
+        async (db) => {
+          const table = await db.openTable("test");
+          const input = makeArrowTable(
+            [{ id: 1n, image: Buffer.from("payload") }],
+            { schema: schema(!wide) },
+          );
+          await table.add(input);
+          await table
+            .mergeInsert("id")
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute(input);
+          expect(writes).toHaveLength(2);
+          for (const write of writes)
+            expect(write.schema).toEqual(schema(wide));
+          expect(writes[0].path).toContain("/insert/");
+          expect(writes[1].path).toContain("/merge_insert/");
+          if (!wide) {
+            const invalid = makeArrowTable(
+              [
+                {
+                  id: 1n,
+                  image: {
+                    uri: "s3://registered/payload",
+                    position: 4,
+                    size: 6,
+                  },
+                },
+              ],
+              { schema: schema(true) },
+            );
+            await expect(table.add(invalid)).rejects.toThrow(
+              /cannot store byte ranges/,
+            );
+            await expect(
+              table
+                .mergeInsert("id")
+                .whenMatchedUpdateAll()
+                .whenNotMatchedInsertAll()
+                .execute(invalid),
+            ).rejects.toThrow(/cannot store byte ranges/);
+            expect(writes).toHaveLength(2);
+          }
+          await table.add(input, { mode: "overwrite" });
+          expect(writes).toHaveLength(3);
+          expect(writes[2].path).toContain("mode=overwrite");
+          expect(writes[2].schema).toEqual(schema(!wide));
+          expect(await table.schema()).toEqual(schema(!wide));
+        },
+      );
+    },
+  );
+
   it("reports unsupported operations with backend-neutral errors", async () => {
     await withMockDatabase(
       (req, res) => {

nodejs/__test__/table.test.ts

@@ -2583,6 +2583,16 @@ describe("when dealing with versioning", () => {
   });
 });
 
+function legacyBlob(name: string): Field {
+  const field = blob(name);
+  return new Field(
+    name,
+    new Struct((field.type as Struct).children.slice(0, 2)),
+    field.nullable,
+    field.metadata,
+  );
+}
+
 describe("when dealing with blob columns", () => {
   let tmpDir: tmp.DirResult;
   beforeEach(() => {
@@ -2592,6 +2602,89 @@ describe("when dealing with blob columns", () => {
     tmpDir.removeCallback();
   });
 
+  it.each([false, true])(
+    "normalizes Arrow blob layouts for append and merge (range target: %s)",
+    async (wide) => {
+      const db = await connect(tmpDir.name);
+      const schema = (full: boolean) =>
+        new Schema([
+          new Field("id", new Int64(), true),
+          full ? blob("image") : legacyBlob("image"),
+        ]);
+      const table = await db.createTable(
+        "layouts",
+        [{ id: 1n, image: Buffer.from("old") }],
+        { schema: schema(wide) },
+      );
+      await table.add(
+        makeArrowTable([{ id: 2n, image: Buffer.from("appended") }], {
+          schema: schema(!wide),
+        }),
+      );
+      await table
+        .mergeInsert("id")
+        .whenMatchedUpdateAll()
+        .whenNotMatchedInsertAll()
+        .execute(
+          makeArrowTable(
+            [
+              { id: 1n, image: Buffer.from("updated") },
+              { id: 3n, image: Buffer.from("inserted") },
+            ],
+            { schema: schema(!wide) },
+          ),
+        );
+      expect(await table.schema()).toEqual(schema(wide));
+      const rows = await table.query().withRowId().toArray();
+      rows.sort((a, b) => Number(a.id - b.id));
+      expect(
+        await table.fetchBlobs(
+          "image",
+          rows.map((row) => row._rowid as bigint),
+        ),
+      ).toEqual([
+        Buffer.from("updated"),
+        Buffer.from("appended"),
+        Buffer.from("inserted"),
+      ]);
+      if (!wide) {
+        const bad = makeArrowTable(
+          [
+            {
+              id: 1n,
+              image: { uri: "s3://registered/file", position: 0, size: 6 },
+            },
+          ],
+          { schema: schema(true) },
+        );
+        const version = await table.version();
+        await expect(table.add(bad)).rejects.toThrow(
+          /cannot store byte ranges/,
+        );
+        await expect(
+          table
+            .mergeInsert("id")
+            .whenMatchedUpdateAll()
+            .whenNotMatchedInsertAll()
+            .execute(bad),
+        ).rejects.toThrow(/cannot store byte ranges/);
+        expect(await table.version()).toBe(version);
+        expect(await table.countRows()).toBe(3);
+      }
+      const replacement = makeArrowTable(
+        [{ id: 4n, image: Buffer.from("replacement") }],
+        { schema: schema(!wide) },
+      );
+      await table.add(replacement, { mode: "overwrite" });
+      expect(await table.schema()).toEqual(schema(!wide));
+      expect(await table.countRows()).toBe(1);
+      const [row] = await table.query().withRowId().toArray();
+      expect(await table.fetchBlobs("image", [row._rowid as bigint])).toEqual([
+        Buffer.from("replacement"),
+      ]);
+    },
+  );
+
   it("rejects unsupported blob values in createTable and add", async () => {
     const db = await connect(tmpDir.name);
     const schema = new Schema([
@@ -2668,6 +2761,100 @@ describe("when dealing with blob columns", () => {
     expect(bytes[0]).toEqual(payload);
   });
 
+  it.each([0, 4])(
+    "fetches the exact external range at position %s after reopening",
+    async (position) => {
+      const { db, uri, payload } = await openExternalBlobTable();
+      const schema = new Schema([
+        new Field("id", new Int64(), true),
+        blob("payload"),
+      ]);
+      const rows: { id: bigint; payload: BlobInput }[] = [
+        { id: 1n, payload: Buffer.from("inline") },
+        { id: 2n, payload: { uri: new URL(uri), position, size: 6 } },
+        { id: 3n, payload: uri },
+        { id: 4n, payload: null },
+      ];
+      const table = await db.createTable("ranges", rows.slice(0, 1), {
+        schema,
+      });
+      await table.add(rows.slice(1), { allowExternalBlobOutsideBases: true });
+      table.close();
+      const reopened = await db.openTable("ranges");
+      const results = await reopened.query().withRowId().toArray();
+      results.sort((a, b) => Number(a.id - b.id));
+      expect(results[1].payload.position).toBe(BigInt(position));
+      expect(results[1].payload.size).toBe(6n);
+      expect(
+        await reopened.fetchBlobs(
+          "payload",
+          results.map((r) => r._rowid as bigint),
+        ),
+      ).toEqual([
+        Buffer.from("inline"),
+        payload.subarray(position, position + 6),
+        payload,
+        null,
+      ]);
+    },
+  );
+
+  it.each([0, 4])(
+    "rejects ranges at position %s on reopened minimal tables without writing",
+    async (position) => {
+      const { db, uri, payload } = await openExternalBlobTable();
+      const table = await db.createEmptyTable(
+        "legacy_blobs",
+        new Schema([new Field("id", new Int64(), true), legacyBlob("payload")]),
+      );
+      table.close();
+      const reopened = await db.openTable("legacy_blobs");
+      await expect(
+        reopened.add(
+          [
+            { id: 1n, payload: Buffer.from("inline") },
+            { id: 2n, payload: { uri, position, size: 6 } },
+          ],
+          { allowExternalBlobOutsideBases: true },
+        ),
+      ).rejects.toThrow(/storage schema cannot store byte ranges/);
+      expect(await reopened.countRows()).toBe(0);
+      await reopened.add(
+        [
+          { id: 3n, payload: uri },
+          { id: 4n, payload: Buffer.from("inline") },
+        ],
+        { allowExternalBlobOutsideBases: true },
+      );
+      const rows = await reopened.query().withRowId().toArray();
+      rows.sort((a, b) => Number(a.id - b.id));
+      expect(
+        await reopened.fetchBlobs(
+          "payload",
+          rows.map((r) => r._rowid as bigint),
+        ),
+      ).toEqual([payload, Buffer.from("inline")]);
+    },
+  );
+
+  it.each([{ position: 0 }, { size: 6 }, { position: 0, size: 0 }])(
+    "rejects partial or empty ranges without writing: %p",
+    async (range) => {
+      const { db, uri } = await openExternalBlobTable();
+      const table = await db.createEmptyTable(
+        "invalid_ranges",
+        new Schema([blob("payload")]),
+      );
+      await expect(
+        table.add(
+          [{ payload: Buffer.from("inline") }, { payload: { uri, ...range } }],
+          { allowExternalBlobOutsideBases: true },
+        ),
+      ).rejects.toThrow(/both position and size|size must be positive/);
+      expect(await table.countRows()).toBe(0);
+    },
+  );
+
   it("applies the external blob opt-in only to the current add", async () => {
     const { table, uri } = await openExternalBlobTable();