mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-22 04:55:39 +00:00
feat(node): add blob v2 fetch and field helpers (#4155)
this PR blob v2 field helpers and reads to the Node SDK.
`blob()` marks a field as blob v2 and lets you set the storage
thresholds. Inputs can be bytes, a URI, or a data/uri struct.
Queries return descriptors. `fetchBlobs()` reads the bytes by row ID,
and `fetchBlobFiles()` gives you lazy handles for full or range reads.
`blobColumns()` lists the blob fields, including nested ones.
Fetch uses the table’s current checkout. It preserves order, duplicates,
and nulls. Holding row IDs across compaction still requires stable row
IDs.
```javascript
const db = await connect("./data");
const video = await readFile("clip.mp4");
const table = await db.createTable(
"videos",
[{ id: 1n, video }],
{
schema: new Schema([
new Field("id", new Int64()),
blob("video"),
]),
},
);
const rows = await table.query().select(["id"]).withRowId().toArray();
const rowIds = rows.map((row) => row._rowid as bigint);
const bytes = await table.fetchBlobs("video", rowIds);
const [handle] = await table.fetchBlobFiles("video", rowIds);
const header = await handle!.readRange(0n, 65536n);
```
### Testing
- cover input validation, thresholds, nested fields, fetch ordering,
nulls, and range reads.
This commit is contained in:
Drew
@@ -0,0 +1,185 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import { Field, Int64, List, Schema, Struct, Utf8 } from "apache-arrow";
|
||||
import { makeArrowTable } from "../lancedb/arrow";
|
||||
import { BlobFile, blob, coerceBlobValue, isBlobField } from "../lancedb/blob";
|
||||
|
||||
describe("blob()", () => {
|
||||
it("marks the field as lance.blob.v2", () => {
|
||||
const field = blob("image", { nullable: false });
|
||||
expect(field.nullable).toBe(false);
|
||||
expect(isBlobField(field)).toBe(true);
|
||||
expect(field.metadata.get("ARROW:extension:name")).toBe("lance.blob.v2");
|
||||
});
|
||||
|
||||
it("writes encoding thresholds as field metadata", () => {
|
||||
const field = blob("video", {
|
||||
inlineSizeThreshold: 1024,
|
||||
dedicatedSizeThreshold: 2 * 1024 * 1024,
|
||||
packFileSizeThreshold: 64 * 1024 * 1024,
|
||||
});
|
||||
expect(
|
||||
field.metadata.get("lance-encoding:blob-inline-size-threshold"),
|
||||
).toBe("1024");
|
||||
expect(
|
||||
field.metadata.get("lance-encoding:blob-dedicated-size-threshold"),
|
||||
).toBe(String(2 * 1024 * 1024));
|
||||
expect(
|
||||
field.metadata.get("lance-encoding:blob-pack-file-size-threshold"),
|
||||
).toBe(String(64 * 1024 * 1024));
|
||||
});
|
||||
|
||||
it("rejects invalid thresholds", () => {
|
||||
expect(() => blob("image", { inlineSizeThreshold: -1 })).toThrow(
|
||||
/inlineSizeThreshold must be non-negative/,
|
||||
);
|
||||
expect(() => blob("image", { dedicatedSizeThreshold: 0 })).toThrow(
|
||||
/dedicatedSizeThreshold must be positive/,
|
||||
);
|
||||
expect(() => blob("image", { packFileSizeThreshold: 1.5 })).toThrow(
|
||||
/packFileSizeThreshold must be a safe integer/,
|
||||
);
|
||||
expect(() =>
|
||||
blob("image", { dedicatedSizeThreshold: Number.MAX_SAFE_INTEGER + 1 }),
|
||||
).toThrow(/dedicatedSizeThreshold must be a safe integer/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("coerceBlobValue", () => {
|
||||
it.each([
|
||||
["Buffer", Buffer.from("x"), { data: Buffer.from("x"), uri: null }],
|
||||
[
|
||||
"Uint8Array",
|
||||
new Uint8Array([120]),
|
||||
{ data: new Uint8Array([120]), uri: null },
|
||||
],
|
||||
["URI string", "s3://bucket/key", { data: null, uri: "s3://bucket/key" }],
|
||||
[
|
||||
"data struct",
|
||||
{ data: Buffer.from("y") },
|
||||
{ data: Buffer.from("y"), uri: null },
|
||||
],
|
||||
[
|
||||
"uri struct",
|
||||
{ uri: "s3://bucket/key" },
|
||||
{ data: null, uri: "s3://bucket/key" },
|
||||
],
|
||||
["null", null, null],
|
||||
])("accepts %s", (_name, input, expected) => {
|
||||
expect(coerceBlobValue(input)).toEqual(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["empty URI", "", /uri cannot be empty/],
|
||||
["object without data or uri", { position: 0 }, /data' or 'uri/],
|
||||
[
|
||||
"Int16Array",
|
||||
new Int16Array([1]),
|
||||
/Blob data must be Buffer or Uint8Array/,
|
||||
],
|
||||
[
|
||||
"both data and uri",
|
||||
{ data: Buffer.from("y"), uri: "s3://bucket/key" },
|
||||
/exactly one of 'data' or 'uri'/,
|
||||
],
|
||||
[
|
||||
"neither data nor uri",
|
||||
{ data: null, uri: null },
|
||||
/exactly one of 'data' or 'uri'/,
|
||||
],
|
||||
])("rejects %s", (_name, input, message) => {
|
||||
expect(() => coerceBlobValue(input)).toThrow(message);
|
||||
});
|
||||
});
|
||||
|
||||
describe("BlobFile", () => {
|
||||
it("rejects constructing BlobFile without a native handle", () => {
|
||||
expect(() => new (BlobFile as unknown as { new (): BlobFile })()).toThrow(
|
||||
/fetchBlobFiles/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("makeArrowTable blob columns", () => {
|
||||
it("coerces Buffer input onto a blob field", () => {
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
blob("image"),
|
||||
]);
|
||||
const table = makeArrowTable([{ id: 1n, image: Buffer.from("hello") }], {
|
||||
schema,
|
||||
});
|
||||
expect(isBlobField(table.schema.fields[1])).toBe(true);
|
||||
const image = table.getChild("image")!;
|
||||
expect(image.nullCount).toBe(0);
|
||||
expect(image.getChild("uri")!.get(0)).toBeNull();
|
||||
expect(image.getChild("data")!.nullCount).toBe(0);
|
||||
expect(Buffer.from(image.getChild("data")!.get(0)!).toString()).toBe(
|
||||
"hello",
|
||||
);
|
||||
});
|
||||
|
||||
it("coerces Buffer elements inside a list and keeps null slots", () => {
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field("images", new List(blob("image")), true),
|
||||
]);
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{ id: 1n, images: [Buffer.from("a"), Buffer.from("bb")] },
|
||||
{ id: 2n, images: null },
|
||||
{ id: 3n, images: [Buffer.from("c"), null] },
|
||||
{ id: 4n, images: [] },
|
||||
],
|
||||
{ schema },
|
||||
);
|
||||
const images = table.getChild("images")!;
|
||||
expect(images.nullCount).toBe(1);
|
||||
const rows = images.toArray();
|
||||
expect(rows[1]).toBeNull();
|
||||
expect(Array.from(rows[3] as Iterable<unknown>)).toHaveLength(0);
|
||||
const first = Array.from(rows[0] as Iterable<{ data: Uint8Array | null }>);
|
||||
expect(Buffer.from(first[0].data!).toString()).toBe("a");
|
||||
expect(Buffer.from(first[1].data!).toString()).toBe("bb");
|
||||
const third = Array.from(
|
||||
rows[2] as Iterable<{ data: Uint8Array | null } | null>,
|
||||
);
|
||||
expect(Buffer.from(third[0]!.data!).toString()).toBe("c");
|
||||
expect(third[1]).toBeNull();
|
||||
});
|
||||
|
||||
it("coerces Buffer fields inside list structs", () => {
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"items",
|
||||
new List(
|
||||
new Field(
|
||||
"item",
|
||||
new Struct([new Field("name", new Utf8(), true), blob("image")]),
|
||||
true,
|
||||
),
|
||||
),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const table = makeArrowTable(
|
||||
[
|
||||
{
|
||||
id: 1n,
|
||||
items: [{ name: "one", image: Buffer.from("alpha") }],
|
||||
},
|
||||
],
|
||||
{ schema },
|
||||
);
|
||||
const items = Array.from(
|
||||
table.getChild("items")!.toArray()[0] as Iterable<{
|
||||
name: string;
|
||||
image: { data: Uint8Array | null };
|
||||
}>,
|
||||
);
|
||||
expect(items[0].name).toBe("one");
|
||||
expect(Buffer.from(items[0].image.data!).toString()).toBe("alpha");
|
||||
});
|
||||
});
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
Query,
|
||||
Table,
|
||||
VectorQuery,
|
||||
blob,
|
||||
connect,
|
||||
tokenize,
|
||||
} from "../lancedb";
|
||||
@@ -2401,6 +2402,276 @@ describe("when dealing with versioning", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("when dealing with blob columns", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
});
|
||||
afterEach(() => {
|
||||
tmpDir.removeCallback();
|
||||
});
|
||||
|
||||
it("discovers blob columns", async () => {
|
||||
const { table } = await openBlobTable();
|
||||
expect(await table.blobColumns()).toEqual(["image"]);
|
||||
});
|
||||
|
||||
it("preserves order, duplicates, and nulls", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const [alphaId, betaId, nullId] = rowIds;
|
||||
const bytes = await table.fetchBlobs("image", [
|
||||
betaId,
|
||||
alphaId,
|
||||
betaId,
|
||||
nullId,
|
||||
]);
|
||||
expect(bytes.map((b) => (b == null ? null : b.toString()))).toEqual([
|
||||
"beta",
|
||||
"alpha",
|
||||
"beta",
|
||||
null,
|
||||
]);
|
||||
const files = await table.fetchBlobFiles("image", [
|
||||
betaId,
|
||||
nullId,
|
||||
alphaId,
|
||||
]);
|
||||
expect(files.map((f) => f == null)).toEqual([false, true, false]);
|
||||
});
|
||||
|
||||
it("reads full blob contents", async () => {
|
||||
const { table, rowIds, alpha, beta } = await openBlobTable();
|
||||
const bytes = await table.fetchBlobs("image", rowIds);
|
||||
expect(bytes[0]!.equals(alpha)).toBe(true);
|
||||
expect(bytes[1]!.equals(beta)).toBe(true);
|
||||
const files = await table.fetchBlobFiles("image", rowIds);
|
||||
expect(files[0]!.size()).toBe(BigInt(alpha.length));
|
||||
expect(Buffer.from(await files[0]!.read()).toString()).toBe("alpha");
|
||||
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
|
||||
});
|
||||
|
||||
it("reads a half-open range", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const files = await table.fetchBlobFiles("image", rowIds);
|
||||
expect(Buffer.from(await files[0]!.readRange(0n, 2n)).toString()).toBe(
|
||||
"al",
|
||||
);
|
||||
});
|
||||
|
||||
it("readRange does not move the cursor", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const [handle] = await table.fetchBlobFiles("image", rowIds);
|
||||
expect((await handle!.readRange(1n, 3n)).toString()).toBe("lp");
|
||||
expect(await handle!.read()).toEqual(alpha);
|
||||
expect(await handle!.read()).toEqual(Buffer.alloc(0));
|
||||
});
|
||||
|
||||
it("fails when readRange end is past the blob size", async () => {
|
||||
const { table, rowIds, alpha } = await openBlobTable();
|
||||
const files = await table.fetchBlobFiles("image", rowIds);
|
||||
await expect(
|
||||
files[0]!.readRange(0n, BigInt(alpha.length + 1)),
|
||||
).rejects.toThrow(/exceeds blob size/);
|
||||
});
|
||||
|
||||
it("rejects fetchBlobs on a non-blob column", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
await expect(table.fetchBlobs("id", rowIds)).rejects.toThrow(/blob/i);
|
||||
});
|
||||
|
||||
it("discovers and fetches nested blob columns", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field("info", new Struct([blob("image")]), true),
|
||||
]);
|
||||
const payload = Buffer.from("nested");
|
||||
const table = await db.createTable(
|
||||
"nested_blobs",
|
||||
[{ id: 1n, info: { image: payload } }],
|
||||
{ schema },
|
||||
);
|
||||
expect(await table.blobColumns()).toEqual(["info.image"]);
|
||||
const rows = await table.query().withRowId().toArray();
|
||||
const bytes = await table.fetchBlobs("info.image", [
|
||||
rows[0]._rowid as bigint,
|
||||
]);
|
||||
expect(bytes[0]!.equals(payload)).toBe(true);
|
||||
});
|
||||
|
||||
it("creates and adds list blob columns", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field("images", new List(blob("image")), true),
|
||||
]);
|
||||
const alpha = Buffer.from("alpha");
|
||||
const beta = Buffer.from("beta");
|
||||
const gamma = Buffer.from("gamma");
|
||||
const table = await db.createTable(
|
||||
"list_blobs",
|
||||
[{ id: 1n, images: [alpha, beta] }],
|
||||
{ schema },
|
||||
);
|
||||
await table.add([
|
||||
{ id: 2n, images: null },
|
||||
{ id: 3n, images: [gamma, null] },
|
||||
{ id: 4n, images: [] },
|
||||
]);
|
||||
expect(await table.blobColumns()).toEqual(["images.image"]);
|
||||
const rows = await table.query().toArray();
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(descriptorSizes(byId.get(1)!.images)).toEqual([
|
||||
alpha.length,
|
||||
beta.length,
|
||||
]);
|
||||
expect(byId.get(2)!.images).toBeNull();
|
||||
expect(descriptorSizes(byId.get(3)!.images)).toEqual([gamma.length, null]);
|
||||
expect(Array.from(byId.get(4)!.images as Iterable<unknown>)).toHaveLength(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it("creates and adds list struct blob columns", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"items",
|
||||
new List(
|
||||
new Field(
|
||||
"item",
|
||||
new Struct([new Field("name", new Utf8(), true), blob("image")]),
|
||||
true,
|
||||
),
|
||||
),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
const alpha = Buffer.from("nested-alpha");
|
||||
const beta = Buffer.from("nested-beta");
|
||||
const table = await db.createTable(
|
||||
"list_struct_blobs",
|
||||
[{ id: 1n, items: [{ name: "one", image: alpha }] }],
|
||||
{ schema },
|
||||
);
|
||||
await table.add([
|
||||
{
|
||||
id: 2n,
|
||||
items: [
|
||||
{ name: "two", image: beta },
|
||||
{ name: "three", image: null },
|
||||
],
|
||||
},
|
||||
]);
|
||||
const rows = await table.query().toArray();
|
||||
const byId = new Map(rows.map((row) => [Number(row.id), row]));
|
||||
expect(
|
||||
descriptorSizes(
|
||||
Array.from(byId.get(1)!.items as Iterable<{ image: unknown }>).map(
|
||||
(item) => item.image,
|
||||
),
|
||||
),
|
||||
).toEqual([alpha.length]);
|
||||
expect(
|
||||
descriptorSizes(
|
||||
Array.from(byId.get(2)!.items as Iterable<{ image: unknown }>).map(
|
||||
(item) => item.image,
|
||||
),
|
||||
),
|
||||
).toEqual([beta.length, null]);
|
||||
});
|
||||
|
||||
it("rejects blob fields inside a fixed-size list", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field("frames", new FixedSizeList(2, blob("frame")), true),
|
||||
]);
|
||||
await expect(
|
||||
db.createTable(
|
||||
"fsl_blobs",
|
||||
[{ id: 1n, frames: [Buffer.from("a"), Buffer.from("b")] }],
|
||||
{ schema },
|
||||
),
|
||||
).rejects.toThrow(
|
||||
"Blob fields inside FixedSizeList are not supported. Use List instead.",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects blob fields inside a nested fixed-size list", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field(
|
||||
"clip",
|
||||
new Struct([
|
||||
new Field("frames", new FixedSizeList(2, blob("frame")), true),
|
||||
]),
|
||||
true,
|
||||
),
|
||||
]);
|
||||
await expect(
|
||||
db.createTable(
|
||||
"nested_fsl_blobs",
|
||||
[
|
||||
{
|
||||
id: 1n,
|
||||
clip: { frames: [Buffer.from("a"), Buffer.from("b")] },
|
||||
},
|
||||
],
|
||||
{ schema },
|
||||
),
|
||||
).rejects.toThrow(
|
||||
"Blob fields inside FixedSizeList are not supported. Use List instead.",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an Arrow table with blob fields inside a fixed-size list", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
new Field("frames", new FixedSizeList(2, blob("frame")), true),
|
||||
]);
|
||||
await expect(
|
||||
db.createTable("fsl_blobs_ipc", new ArrowTable(schema)),
|
||||
).rejects.toThrow(
|
||||
"Blob fields inside FixedSizeList are not supported. Use List instead.",
|
||||
);
|
||||
});
|
||||
|
||||
function descriptorSizes(values: unknown): (number | null)[] {
|
||||
return Array.from(
|
||||
values as Iterable<{ size?: bigint | number } | null>,
|
||||
).map((value) => (value == null ? null : Number(value.size)));
|
||||
}
|
||||
|
||||
async function openBlobTable() {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
blob("image"),
|
||||
]);
|
||||
const alpha = Buffer.from("alpha");
|
||||
const beta = Buffer.from("beta");
|
||||
const table = await db.createTable(
|
||||
"blobs",
|
||||
[
|
||||
{ id: 1n, image: alpha },
|
||||
{ id: 2n, image: beta },
|
||||
{ id: 3n, image: null },
|
||||
],
|
||||
{ schema },
|
||||
);
|
||||
const rows = await table.query().withRowId().toArray();
|
||||
const rowIdById = new Map(
|
||||
rows.map((r) => [Number(r.id), r._rowid as bigint]),
|
||||
);
|
||||
const rowIds = [1, 2, 3].map((id) => rowIdById.get(id)!);
|
||||
return { table, rowIds, alpha, beta };
|
||||
}
|
||||
});
|
||||
|
||||
describe("when dealing with tags", () => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
beforeEach(() => {
|
||||
|
||||
Reference in New Issue
Block a user