rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-26 13:36:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix!: normalize blob layouts and support external byte ranges (#4589)

Browse Source
This commit is contained in:
lancedb-gatefixer[bot]
2026-10-19 16:44:00 -07:00
committed by GitHub
parent 6507615a6f
commit 4a03859b2a
20 changed files with 2837 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nodejs/lancedb/arrow.ts
+27 -7
View File
@@ -456,14 +456,29 @@ export function makeArrowTable(
if (schema !== undefined) {
// Validate blob values up front and give every one the full
// `{ data, uri }` shape, so inference sees the same struct whether a row
// storage shape, so inference sees the same struct whether a row
// passed bytes, a URI, or a partial struct.
data = mapBlobInputs(data, schema, (value, field, row) => {
data = mapBlobInputs(data, schema, (value, field, row, storageField) => {
if (value === undefined) {
return value;
}
try {
return coerceBlobValue(value);
const normalized = coerceBlobValue(value, storageField);
if (
normalized === null ||
!(storageField.type instanceof Struct) ||
storageField.type.children.length === 2
) {
return normalized;
}
// Keep even unused range children during schema inference. Otherwise
// creating a range-capable table with inline data would lose them.
return Object.fromEntries(
storageField.type.children.map((child) => [
child.name,
normalized[child.name as keyof typeof normalized] ?? null,
]),
);
} catch (e) {
throw new Error(
`Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,
@@ -533,7 +548,12 @@ function containsBlobField(field: Field): boolean {
);
}
type BlobInputVisitor = (value: unknown, field: string, row: number) => unknown;
type BlobInputVisitor = (
value: unknown,
field: string,
row: number,
storageField: Field,
) => unknown;
/**
* Calls `visit` on every value that lands in a blob column of `schema`,
@@ -592,7 +612,7 @@ function mapBlobField(
visit: BlobInputVisitor,
): unknown {
if (isBlobField(field)) {
return visit(value, label, row);
return visit(value, label, row, field);
}
if (field.type instanceof Struct && isObject(value)) {
return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
@@ -697,11 +717,11 @@ function transposeData(
const valuesPath = [...path, field.name];
if (isBlobField(field) && field.type instanceof Struct) {
const blobRows = data.map((datum) =>
coerceBlobValue(valueAtPath(datum, valuesPath)),
coerceBlobValue(valueAtPath(datum, valuesPath), field),
);
const childVectors = field.type.children.map((child) => {
const values = blobRows.map((row) =>
row == null ? null : (row[child.name as "data" | "uri"] ?? null),
row == null ? null : (row[child.name as keyof typeof row] ?? null),
);
return makeVector(values, child.type, undefined, child.nullable);
});
nodejs/lancedb/blob.ts
+91 -8
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { Field, LargeBinary, Struct, Utf8 } from "apache-arrow";
import { Field, LargeBinary, Struct, Uint64, Utf8 } from "apache-arrow";
import { BlobFile as NativeBlobFile } from "./native";
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
@@ -28,6 +28,9 @@ export type BlobUri = string | URL;
*
* Either inline bytes, a URI pointing at external bytes, or a struct that sets
* exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
* External byte ranges require both
* `position` (non-negative) and `size` (positive). Use safe integer numbers or
* `bigint` values for these unsigned 64-bit byte counts.
*
* @example
* ```ts
@@ -45,8 +48,14 @@ export type BlobUri = string | URL;
export type BlobInput =
| BlobData
| BlobUri
| { data: BlobData; uri?: null }
| { data?: null; uri: BlobUri }
| { data: BlobData; uri?: null; position?: null; size?: null }
| { data?: null; uri: BlobUri; position?: null; size?: null }
| {
data?: null;
uri: BlobUri;
position: number | bigint;
size: number | bigint;
}
| null
| undefined;
@@ -54,6 +63,8 @@ export type BlobInput =
export type BlobValue = {
data: Buffer | Uint8Array | null;
uri: string | null;
position?: bigint;
size?: bigint;
};
export type BlobOptions = {
@@ -78,6 +89,15 @@ export type BlobOptions = {
/**
* Declares a `lance.blob.v2` column.
* Uses nullable `data`, `uri`, `position`, and `size` fields for inline bytes,
* whole external files, and external byte ranges. Set both `position` and
* positive `size` for a range, or omit both for a whole file.
*
* Existing two-field columns remain writable. Appends and merge inserts fill
* missing range fields with nulls for four-field targets and drop all-null
* range fields for two-field targets. Non-null ranges require a four-field
* target. Overwrite with Arrow input replaces the table's schema; row objects
* are converted using the current table schema.
*
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
* or {@link Table.fetchBlobFiles} to read bytes.
@@ -142,6 +162,8 @@ export function blob(name: string, options: BlobOptions = {}): Field {
new Struct([
new Field("data", new LargeBinary(), true),
new Field("uri", new Utf8(), true),
new Field("position", new Uint64(), true),
new Field("size", new Uint64(), true),
]),
options.nullable ?? true,
metadata,
@@ -317,6 +339,8 @@ export function normalizeBlobInput(value: unknown): unknown {
...value,
data: value.data,
uri: value.uri,
position: value.position,
size: value.size,
};
let changed = false;
if (value.data instanceof ArrayBuffer) {
@@ -355,10 +379,19 @@ export function withBlobBytes(value: unknown, bytes: Uint8Array): unknown {
return bytes;
}
const record = value as Record<string, unknown>;
return { ...record, uri: record.uri, data: bytes };
return {
...record,
uri: record.uri,
position: record.position,
size: record.size,
data: bytes,
};
}
export function coerceBlobValue(input: unknown): BlobValue | null {
export function coerceBlobValue(
input: unknown,
field?: Field,
): BlobValue | null {
const value = normalizeBlobInput(input);
if (value == null) {
return null;
@@ -385,11 +418,11 @@ export function coerceBlobValue(input: unknown): BlobValue | null {
);
}
const unknownKeys = Object.keys(record).filter(
(key) => key !== "data" && key !== "uri",
(key) => !["data", "uri", "position", "size"].includes(key),
);
if (unknownKeys.length > 0) {
throw new Error(
`Blob struct values only support 'data' and 'uri', got ${unknownKeys
`Blob struct values only support 'data', 'uri', 'position', and 'size', got ${unknownKeys
.map((key) => `'${key}'`)
.join(", ")}`,
);
@@ -416,13 +449,63 @@ export function coerceBlobValue(input: unknown): BlobValue | null {
"Blob struct values must set exactly one of 'data' or 'uri'",
);
}
const hasPosition = record.position != null;
const hasSize = record.size != null;
if (hasPosition !== hasSize) {
throw new Error(
"External blob must set both position and size, or neither",
);
}
if (hasPosition) {
if (uriValue == null) {
throw new Error("External blob byte ranges require a uri");
}
const position = blobRangeInteger(record.position, "position", 0n);
const size = blobRangeInteger(record.size, "size", 1n);
if (position + size > 0xffffffffffffffffn) {
throw new Error("External blob range overflows uint64");
}
if (
field !== undefined &&
(!(field.type instanceof Struct) ||
["position", "size"].some(
(name) =>
!field.type.children.some((child: Field) => child.name === name),
))
) {
throw new Error(
"Blob storage schema cannot store byte ranges. Use a four-field blob() schema to store position and size",
);
}
return { data: null, uri: uriValue, position, size };
}
return { data: bytes, uri: uriValue };
}
throw new Error(
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri? }",
"Blob column values must be Buffer, Uint8Array, ArrayBuffer, Blob, a URI string or URL, null, or { data?, uri?, position?, size? }",
);
}
function blobRangeInteger(
value: unknown,
name: string,
minimum: bigint,
): bigint {
if (
typeof value !== "bigint" &&
(typeof value !== "number" || !Number.isSafeInteger(value))
) {
throw new Error(`Blob ${name} must be a safe integer or bigint`);
}
const integer = BigInt(value);
if (integer < minimum || integer > 0xffffffffffffffffn) {
throw new Error(
`Blob ${name} must be ${minimum === 0n ? "non-negative" : "positive"} and fit in uint64`,
);
}
return integer;
}
function isBlobLike(value: unknown): value is Blob {
return typeof Blob !== "undefined" && value instanceof Blob;
}