diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index a82d9e334..18fa8ed37 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -157,9 +157,10 @@ - [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md) - [BaseTokenizer](type-aliases/BaseTokenizer.md) +- [BlobData](type-aliases/BlobData.md) +- [BlobInput](type-aliases/BlobInput.md) - [BlobOptions](type-aliases/BlobOptions.md) -- [BlobRange](type-aliases/BlobRange.md) -- [BlobReadOptions](type-aliases/BlobReadOptions.md) +- [BlobUri](type-aliases/BlobUri.md) - [Data](type-aliases/Data.md) - [DataLike](type-aliases/DataLike.md) - [FieldLike](type-aliases/FieldLike.md) diff --git a/docs/src/js/type-aliases/BlobData.md b/docs/src/js/type-aliases/BlobData.md new file mode 100644 index 000000000..2af0a3659 --- /dev/null +++ b/docs/src/js/type-aliases/BlobData.md @@ -0,0 +1,16 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobData + +# Type Alias: BlobData + +```ts +type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob; +``` + +Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying. +`Blob` and `File` are read with `arrayBuffer()`, which only the async write +paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add), +[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do. diff --git a/docs/src/js/type-aliases/BlobInput.md b/docs/src/js/type-aliases/BlobInput.md new file mode 100644 index 000000000..d1738a760 --- /dev/null +++ b/docs/src/js/type-aliases/BlobInput.md @@ -0,0 +1,36 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobInput + +# Type Alias: BlobInput + +```ts +type BlobInput: + | BlobData + | BlobUri + | object + | object + | null + | undefined; +``` + +A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)). + +Either inline bytes, a URI pointing at external bytes, or a struct that sets +exactly one of `data` or `uri`. `null` and `undefined` write a null blob. + +## Example + +```ts +import { pathToFileURL } from "node:url"; +import type { BlobInput } from "@lancedb/lancedb"; + +const rows: { id: bigint; image: BlobInput }[] = [ + { id: 1n, image: await (await fetch(url)).arrayBuffer() }, + { id: 2n, image: pathToFileURL("/data/cat.png") }, + { id: 3n, image: { data: new Blob(["hello"]) } }, +]; +await table.add(rows); +``` diff --git a/docs/src/js/type-aliases/BlobUri.md b/docs/src/js/type-aliases/BlobUri.md new file mode 100644 index 000000000..a90b395fc --- /dev/null +++ b/docs/src/js/type-aliases/BlobUri.md @@ -0,0 +1,13 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / BlobUri + +# Type Alias: BlobUri + +```ts +type BlobUri: string | URL; +``` + +A URI accepted for a blob value. A `URL` is stored as its `href`. diff --git a/nodejs/__test__/blob.test.ts b/nodejs/__test__/blob.test.ts index 920edfd99..ce7de494c 100644 --- a/nodejs/__test__/blob.test.ts +++ b/nodejs/__test__/blob.test.ts @@ -190,7 +190,7 @@ describe("makeArrowTable blob columns", () => { ["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null], [ "data struct with ArrayBuffer", - { data: Buffer.from("hi").buffer }, + { data: new Uint8Array([104, 105]).buffer }, "hi", null, ], diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index d102a6964..93e87324d 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2503,6 +2503,8 @@ describe("when dealing with blob columns", () => { it.each([ ["URI string", (uri: string) => uri], ["URI struct", (uri: string) => ({ uri })], + ["URL", (uri: string) => new URL(uri)], + ["URL struct", (uri: string) => ({ uri: new URL(uri) })], ])( "round-trips an external blob from a %s after reopening", async (_label, blobValue) => { @@ -2596,39 +2598,32 @@ describe("when dealing with blob columns", () => { expect(bytes[0]).toEqual(payload); }); - it("accepts ArrayBuffer, Blob, File, and URL in createTable and add", async () => { + it("accepts ArrayBuffer, Blob, and File in createTable and add", async () => { const db = await connect(tmpDir.name); const schema = new Schema([ new Field("id", new Int64(), true), blob("image"), ]); const rows: { id: bigint; image: BlobInput }[] = [ - { id: 1n, image: Buffer.from("array-buffer").buffer }, + { id: 1n, image: new TextEncoder().encode("array-buffer").buffer }, { id: 2n, image: new Blob(["blob"]) }, { id: 3n, image: { data: new File(["file"], "f.txt") } }, - { id: 4n, image: new URL("https://example.com/remote.png") }, - { id: 5n, image: { uri: new URL("file:///tmp/local.png") } }, ]; - const table = await db.createTable("widened", rows.slice(0, 3), { + const table = await db.createTable("widened", rows.slice(0, 2), { schema, }); - await table.add(rows.slice(3)); - await table.add([{ id: 6n, image: new Blob(["added"]) }]); + await table.add(rows.slice(2)); const results = await table.query().select(["id"]).withRowId().toArray(); results.sort((a, b) => Number(a.id - b.id)); - const rowIds = results.map((row) => row._rowid as bigint); - const inline = await table.fetchBlobs("image", [ - rowIds[0], - rowIds[1], - rowIds[2], - rowIds[5], - ]); - expect(inline.map((b) => b?.toString())).toEqual([ + const bytes = await table.fetchBlobs( + "image", + results.map((row) => row._rowid as bigint), + ); + expect(bytes.map((b) => b?.toString())).toEqual([ "array-buffer", "blob", "file", - "added", ]); }); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 580c89c50..f009a089a 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -44,7 +44,6 @@ import { coerceBlobValue, isBlobField, needsBlobResolution, - normalizeBlobInput, resolveBlobInput, } from "./blob"; import { type EmbeddingFunction } from "./embedding/embedding_function"; @@ -447,9 +446,15 @@ export function makeArrowTable( } if (schema !== undefined) { + // Validate blob values up front and give every one the full + // `{ data, uri }` shape, so inference sees the same struct whether a row + // passed bytes, a URI, or a partial struct. data = mapBlobInputs(data, schema, (value, field, row) => { + if (value === undefined) { + return value; + } try { - return normalizeBlobInput(value); + return coerceBlobValue(value); } catch (e) { throw new Error( `Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,