From b50a67c3703385b00482182174979ccd041d6dfb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 23 Sep 2026 22:37:59 +0000 Subject: [PATCH] fix(node): normalize blob values before schema inference Give every blob value the full { data, uri } shape before inference so rows mixing bytes, URIs, and partial structs infer one struct type. Also wrap blob validation errors with the field and row, regenerate the TypeScript docs, and cover URL handling end to end. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012U4KdHofT5edv3LcXHXbLv --- docs/src/js/globals.md | 5 ++-- docs/src/js/type-aliases/BlobData.md | 16 ++++++++++++ docs/src/js/type-aliases/BlobInput.md | 36 +++++++++++++++++++++++++++ docs/src/js/type-aliases/BlobUri.md | 13 ++++++++++ nodejs/__test__/blob.test.ts | 2 +- nodejs/__test__/table.test.ts | 27 ++++++++------------ nodejs/lancedb/arrow.ts | 9 +++++-- 7 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 docs/src/js/type-aliases/BlobData.md create mode 100644 docs/src/js/type-aliases/BlobInput.md create mode 100644 docs/src/js/type-aliases/BlobUri.md diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index a82d9e334..18fa8ed37 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -157,9 +157,10 @@ - [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md) - [BaseTokenizer](type-aliases/BaseTokenizer.md) +- [BlobData](type-aliases/BlobData.md) +- [BlobInput](type-aliases/BlobInput.md) - [BlobOptions](type-aliases/BlobOptions.md) -- [BlobRange](type-aliases/BlobRange.md) -- [BlobReadOptions](type-aliases/BlobReadOptions.md) +- [BlobUri](type-aliases/BlobUri.md) - [Data](type-aliases/Data.md) - [DataLike](type-aliases/DataLike.md) - [FieldLike](type-aliases/FieldLike.md) diff --git a/docs/src/js/type-aliases/BlobData.md b/docs/src/js/type-aliases/BlobData.md new file mode 100644 index 000000000..2af0a3659 --- /dev/null +++ b/docs/src/js/type-aliases/BlobData.md @@ -0,0 +1,16 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobData + +# Type Alias: BlobData + +```ts +type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob; +``` + +Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying. +`Blob` and `File` are read with `arrayBuffer()`, which only the async write +paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add), +[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do. diff --git a/docs/src/js/type-aliases/BlobInput.md b/docs/src/js/type-aliases/BlobInput.md new file mode 100644 index 000000000..d1738a760 --- /dev/null +++ b/docs/src/js/type-aliases/BlobInput.md @@ -0,0 +1,36 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobInput + +# Type Alias: BlobInput + +```ts +type BlobInput: + | BlobData + | BlobUri + | object + | object + | null + | undefined; +``` + +A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)). + +Either inline bytes, a URI pointing at external bytes, or a struct that sets +exactly one of `data` or `uri`. `null` and `undefined` write a null blob. + +## Example + +```ts +import { pathToFileURL } from "node:url"; +import type { BlobInput } from "@lancedb/lancedb"; + +const rows: { id: bigint; image: BlobInput }[] = [ + { id: 1n, image: await (await fetch(url)).arrayBuffer() }, + { id: 2n, image: pathToFileURL("/data/cat.png") }, + { id: 3n, image: { data: new Blob(["hello"]) } }, +]; +await table.add(rows); +``` diff --git a/docs/src/js/type-aliases/BlobUri.md b/docs/src/js/type-aliases/BlobUri.md new file mode 100644 index 000000000..a90b395fc --- /dev/null +++ b/docs/src/js/type-aliases/BlobUri.md @@ -0,0 +1,13 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobUri + +# Type Alias: BlobUri + +```ts +type BlobUri: string | URL; +``` + +A URI accepted for a blob value. A `URL` is stored as its `href`. diff --git a/nodejs/__test__/blob.test.ts b/nodejs/__test__/blob.test.ts index 920edfd99..ce7de494c 100644 --- a/nodejs/__test__/blob.test.ts +++ b/nodejs/__test__/blob.test.ts @@ -190,7 +190,7 @@ describe("makeArrowTable blob columns", () => { ["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null], [ "data struct with ArrayBuffer", - { data: Buffer.from("hi").buffer }, + { data: new Uint8Array([104, 105]).buffer }, "hi", null, ], diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d102a6964..93e87324d 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2503,6 +2503,8 @@ describe("when dealing with blob columns", () => { it.each([ ["URI string", (uri: string) => uri], ["URI struct", (uri: string) => ({ uri })], + ["URL", (uri: string) => new URL(uri)], + ["URL struct", (uri: string) => ({ uri: new URL(uri) })], ])( "round-trips an external blob from a %s after reopening", async (_label, blobValue) => { @@ -2596,39 +2598,32 @@ describe("when dealing with blob columns", () => { expect(bytes[0]).toEqual(payload); }); - it("accepts ArrayBuffer, Blob, File, and URL in createTable and add", async () => { + it("accepts ArrayBuffer, Blob, and File in createTable and add", async () => { const db = await connect(tmpDir.name); const schema = new Schema([ new Field("id", new Int64(), true), blob("image"), ]); const rows: { id: bigint; image: BlobInput }[] = [ - { id: 1n, image: Buffer.from("array-buffer").buffer }, + { id: 1n, image: new TextEncoder().encode("array-buffer").buffer }, { id: 2n, image: new Blob(["blob"]) }, { id: 3n, image: { data: new File(["file"], "f.txt") } }, - { id: 4n, image: new URL("https://example.com/remote.png") }, - { id: 5n, image: { uri: new URL("file:///tmp/local.png") } }, ]; - const table = await db.createTable("widened", rows.slice(0, 3), { + const table = await db.createTable("widened", rows.slice(0, 2), { schema, }); - await table.add(rows.slice(3)); - await table.add([{ id: 6n, image: new Blob(["added"]) }]); + await table.add(rows.slice(2)); const results = await table.query().select(["id"]).withRowId().toArray(); results.sort((a, b) => Number(a.id - b.id)); - const rowIds = results.map((row) => row._rowid as bigint); - const inline = await table.fetchBlobs("image", [ - rowIds[0], - rowIds[1], - rowIds[2], - rowIds[5], - ]); - expect(inline.map((b) => b?.toString())).toEqual([ + const bytes = await table.fetchBlobs( + "image", + results.map((row) => row._rowid as bigint), + ); + expect(bytes.map((b) => b?.toString())).toEqual([ "array-buffer", "blob", "file", - "added", ]); }); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 580c89c50..f009a089a 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -44,7 +44,6 @@ import { coerceBlobValue, isBlobField, needsBlobResolution, - normalizeBlobInput, resolveBlobInput, } from "./blob"; import { type EmbeddingFunction } from "./embedding/embedding_function"; @@ -447,9 +446,15 @@ export function makeArrowTable( } if (schema !== undefined) { + // Validate blob values up front and give every one the full + // `{ data, uri }` shape, so inference sees the same struct whether a row + // passed bytes, a URI, or a partial struct. data = mapBlobInputs(data, schema, (value, field, row) => { + if (value === undefined) { + return value; + } try { - return normalizeBlobInput(value); + return coerceBlobValue(value); } catch (e) { throw new Error( `Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,