mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-24 04:32:27 +00:00
fix(node): support nested blobs with blobMode 'bytes' (#4610)
Closes #4609. `blobMode: "bytes"` only resolved top-level blob columns, so a blob nested in a struct came back as its descriptor. Now struct projections that contain blob columns are resolved too: the bytes are fetched per blob column and the struct is rebuilt with `LargeBinary` leaves, preserving struct validity so a null struct stays null. Sibling fields pass through untouched, aliases work for both `info.image AS image` and `info AS details`, and blobs nested in lists still keep their descriptors. Tested with `npx tsc`, `biome check`, and the full `table.test.ts` suite (580 tests, including new regressions for select-all, struct selection, aliases, null structs, null and empty blobs, sibling fields, empty results, and single-row batches). Not run against remote tables.
This commit is contained in:
@@ -3385,6 +3385,106 @@ describe("when dealing with blob columns", () => {
|
||||
expect(bytes[0]!.equals(payload)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns bytes for blobs nested in structs", async () => {
|
||||
const { table, alpha, empty } = await openNestedBlobTable();
|
||||
const rows = await table.query().toArray({ blobMode: "bytes" });
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(Buffer.from(byId.get(1)!.info.image)).toEqual(alpha);
|
||||
expect(byId.get(1)!.info.caption).toBe("one");
|
||||
expect(byId.get(2)!.info.image).toBeNull();
|
||||
expect(byId.get(2)!.info.caption).toBe("two");
|
||||
expect(byId.get(3)!.info).toBeNull();
|
||||
expect(Buffer.from(byId.get(4)!.info.image)).toEqual(empty);
|
||||
});
|
||||
|
||||
it("lines up nested rows and bytes across single-row batches", async () => {
|
||||
const { table, alpha, empty } = await openNestedBlobTable();
|
||||
const result = await table
|
||||
.query()
|
||||
.toArrow({ blobMode: "bytes", maxBatchLength: 1 });
|
||||
expect(result.batches.length).toBe(4);
|
||||
const byId = new Map(result.toArray().map((row) => [Number(row.id), row]));
|
||||
expect(Buffer.from(byId.get(1)!.info.image)).toEqual(alpha);
|
||||
expect(byId.get(1)!.info.caption).toBe("one");
|
||||
expect(byId.get(2)!.info.image).toBeNull();
|
||||
expect(byId.get(2)!.info.caption).toBe("two");
|
||||
expect(byId.get(3)!.info).toBeNull();
|
||||
expect(Buffer.from(byId.get(4)!.info.image)).toEqual(empty);
|
||||
});
|
||||
|
||||
it("handles struct fields named like Object prototypes", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"info",
|
||||
new Struct([
|
||||
new Field("constructor", new Struct([blob("image")]), true),
|
||||
]),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const payload = Buffer.from("proto");
|
||||
const table = await db.createTable(
|
||||
"prototype_named_blobs",
|
||||
[{ id: 1n, info: { constructor: { image: payload } } }],
|
||||
{ schema },
|
||||
);
|
||||
expect(await table.blobColumns()).toEqual(["info.constructor.image"]);
|
||||
const [row] = await table.query().toArray({ blobMode: "bytes" });
|
||||
// Arrow's StructRow shadows the `constructor` property, so read the
|
||||
// field through the JSON form instead of property access.
|
||||
const info = JSON.parse(JSON.stringify(row.info));
|
||||
expect(Buffer.from(Object.values<number>(info.constructor.image))).toEqual(
|
||||
payload,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns nested blob bytes for a struct selection", async () => {
|
||||
const { table, alpha } = await openNestedBlobTable();
|
||||
const [row] = await table
|
||||
.query()
|
||||
.where("id = 1")
|
||||
.select(["info"])
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(Object.keys(row)).toEqual(["info"]);
|
||||
expect(Buffer.from(row.info.image)).toEqual(alpha);
|
||||
expect(row.info.caption).toBe("one");
|
||||
});
|
||||
|
||||
it("returns nested blob bytes through aliases", async () => {
|
||||
const { table, alpha } = await openNestedBlobTable();
|
||||
const [row] = await table
|
||||
.query()
|
||||
.where("id = 1")
|
||||
.select(
|
||||
new Map([
|
||||
["picture", "info.image"],
|
||||
["details", "info"],
|
||||
]),
|
||||
)
|
||||
.toArray({ blobMode: "bytes" });
|
||||
expect(Buffer.from(row.picture)).toEqual(alpha);
|
||||
expect(Buffer.from(row.details.image)).toEqual(alpha);
|
||||
expect(row.details.caption).toBe("one");
|
||||
});
|
||||
|
||||
it("returns a nested bytes schema for an empty result", async () => {
|
||||
const { table } = await openNestedBlobTable();
|
||||
const result = await table
|
||||
.query()
|
||||
.where("id > 100")
|
||||
.toArrow({ blobMode: "bytes" });
|
||||
expect(result.numRows).toBe(0);
|
||||
const field = result.schema.fields.find((f) => f.name === "info")!;
|
||||
expect(arrow.DataType.isStruct(field.type)).toBe(true);
|
||||
const image = (field.type as Struct).children.find(
|
||||
(f) => f.name === "image",
|
||||
)!;
|
||||
expect(arrow.DataType.isLargeBinary(image.type)).toBe(true);
|
||||
expect(isBlobField(image)).toBe(false);
|
||||
});
|
||||
|
||||
it("creates list blobs without advertising unfetchable elements", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
@@ -3593,6 +3693,31 @@ describe("when dealing with blob columns", () => {
|
||||
return { table, rowIds, alpha, beta };
|
||||
}
|
||||
|
||||
async function openNestedBlobTable() {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"info",
|
||||
new Struct([blob("image"), new Field("caption", new Utf8(), true)]),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const alpha = Buffer.from("alpha");
|
||||
const empty = Buffer.alloc(0);
|
||||
const table = await db.createTable(
|
||||
"nested_blobs",
|
||||
[{ id: 1n, info: { image: alpha, caption: "one" } }],
|
||||
{ schema },
|
||||
);
|
||||
await table.add([
|
||||
{ id: 2n, info: { image: null, caption: "two" } },
|
||||
{ id: 3n, info: null },
|
||||
{ id: 4n, info: { image: empty, caption: "four" } },
|
||||
]);
|
||||
return { table, alpha, empty };
|
||||
}
|
||||
|
||||
async function openExternalBlobTable() {
|
||||
const db = await connect(path.join(tmpDir.name, "db"));
|
||||
const payload = Buffer.from("external-payload");
|
||||
|
||||
Reference in New Issue
Block a user