mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-25 21:22:29 +00:00
fix(node): normalize blob values before schema inference
Give every blob value the full { data, uri } shape before inference so rows
mixing bytes, URIs, and partial structs infer one struct type. Also wrap
blob validation errors with the field and row, regenerate the TypeScript
docs, and cover URL handling end to end.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012U4KdHofT5edv3LcXHXbLv
This commit is contained in:
@@ -157,9 +157,10 @@
|
||||
|
||||
- [AnalyzePlanDistributedMetrics](type-aliases/AnalyzePlanDistributedMetrics.md)
|
||||
- [BaseTokenizer](type-aliases/BaseTokenizer.md)
|
||||
- [BlobData](type-aliases/BlobData.md)
|
||||
- [BlobInput](type-aliases/BlobInput.md)
|
||||
- [BlobOptions](type-aliases/BlobOptions.md)
|
||||
- [BlobRange](type-aliases/BlobRange.md)
|
||||
- [BlobReadOptions](type-aliases/BlobReadOptions.md)
|
||||
- [BlobUri](type-aliases/BlobUri.md)
|
||||
- [Data](type-aliases/Data.md)
|
||||
- [DataLike](type-aliases/DataLike.md)
|
||||
- [FieldLike](type-aliases/FieldLike.md)
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BlobData
|
||||
|
||||
# Type Alias: BlobData
|
||||
|
||||
```ts
|
||||
type BlobData: Buffer | Uint8Array | ArrayBuffer | Blob;
|
||||
```
|
||||
|
||||
Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
|
||||
`Blob` and `File` are read with `arrayBuffer()`, which only the async write
|
||||
paths ([Connection.createTable](../classes/Connection.md#createtable), [Table.add](../classes/Table.md#add),
|
||||
[Table.mergeInsert](../classes/Table.md#mergeinsert)) can do.
|
||||
@@ -0,0 +1,36 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BlobInput
|
||||
|
||||
# Type Alias: BlobInput
|
||||
|
||||
```ts
|
||||
type BlobInput:
|
||||
| BlobData
|
||||
| BlobUri
|
||||
| object
|
||||
| object
|
||||
| null
|
||||
| undefined;
|
||||
```
|
||||
|
||||
A value accepted for a `lance.blob.v2` column (see [blob](../functions/blob.md)).
|
||||
|
||||
Either inline bytes, a URI pointing at external bytes, or a struct that sets
|
||||
exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
|
||||
|
||||
## Example
|
||||
|
||||
```ts
|
||||
import { pathToFileURL } from "node:url";
|
||||
import type { BlobInput } from "@lancedb/lancedb";
|
||||
|
||||
const rows: { id: bigint; image: BlobInput }[] = [
|
||||
{ id: 1n, image: await (await fetch(url)).arrayBuffer() },
|
||||
{ id: 2n, image: pathToFileURL("/data/cat.png") },
|
||||
{ id: 3n, image: { data: new Blob(["hello"]) } },
|
||||
];
|
||||
await table.add(rows);
|
||||
```
|
||||
@@ -0,0 +1,13 @@
|
||||
[**@lancedb/lancedb**](../README.md) • **Docs**
|
||||
|
||||
***
|
||||
|
||||
[@lancedb/lancedb](../globals.md) / BlobUri
|
||||
|
||||
# Type Alias: BlobUri
|
||||
|
||||
```ts
|
||||
type BlobUri: string | URL;
|
||||
```
|
||||
|
||||
A URI accepted for a blob value. A `URL` is stored as its `href`.
|
||||
@@ -190,7 +190,7 @@ describe("makeArrowTable blob columns", () => {
|
||||
["ArrayBuffer", new Uint8Array([104, 105]).buffer, "hi", null],
|
||||
[
|
||||
"data struct with ArrayBuffer",
|
||||
{ data: Buffer.from("hi").buffer },
|
||||
{ data: new Uint8Array([104, 105]).buffer },
|
||||
"hi",
|
||||
null,
|
||||
],
|
||||
|
||||
@@ -2503,6 +2503,8 @@ describe("when dealing with blob columns", () => {
|
||||
it.each([
|
||||
["URI string", (uri: string) => uri],
|
||||
["URI struct", (uri: string) => ({ uri })],
|
||||
["URL", (uri: string) => new URL(uri)],
|
||||
["URL struct", (uri: string) => ({ uri: new URL(uri) })],
|
||||
])(
|
||||
"round-trips an external blob from a %s after reopening",
|
||||
async (_label, blobValue) => {
|
||||
@@ -2596,39 +2598,32 @@ describe("when dealing with blob columns", () => {
|
||||
expect(bytes[0]).toEqual(payload);
|
||||
});
|
||||
|
||||
it("accepts ArrayBuffer, Blob, File, and URL in createTable and add", async () => {
|
||||
it("accepts ArrayBuffer, Blob, and File in createTable and add", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([
|
||||
new Field("id", new Int64(), true),
|
||||
blob("image"),
|
||||
]);
|
||||
const rows: { id: bigint; image: BlobInput }[] = [
|
||||
{ id: 1n, image: Buffer.from("array-buffer").buffer },
|
||||
{ id: 1n, image: new TextEncoder().encode("array-buffer").buffer },
|
||||
{ id: 2n, image: new Blob(["blob"]) },
|
||||
{ id: 3n, image: { data: new File(["file"], "f.txt") } },
|
||||
{ id: 4n, image: new URL("https://example.com/remote.png") },
|
||||
{ id: 5n, image: { uri: new URL("file:///tmp/local.png") } },
|
||||
];
|
||||
const table = await db.createTable("widened", rows.slice(0, 3), {
|
||||
const table = await db.createTable("widened", rows.slice(0, 2), {
|
||||
schema,
|
||||
});
|
||||
await table.add(rows.slice(3));
|
||||
await table.add([{ id: 6n, image: new Blob(["added"]) }]);
|
||||
await table.add(rows.slice(2));
|
||||
|
||||
const results = await table.query().select(["id"]).withRowId().toArray();
|
||||
results.sort((a, b) => Number(a.id - b.id));
|
||||
const rowIds = results.map((row) => row._rowid as bigint);
|
||||
const inline = await table.fetchBlobs("image", [
|
||||
rowIds[0],
|
||||
rowIds[1],
|
||||
rowIds[2],
|
||||
rowIds[5],
|
||||
]);
|
||||
expect(inline.map((b) => b?.toString())).toEqual([
|
||||
const bytes = await table.fetchBlobs(
|
||||
"image",
|
||||
results.map((row) => row._rowid as bigint),
|
||||
);
|
||||
expect(bytes.map((b) => b?.toString())).toEqual([
|
||||
"array-buffer",
|
||||
"blob",
|
||||
"file",
|
||||
"added",
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -44,7 +44,6 @@ import {
|
||||
coerceBlobValue,
|
||||
isBlobField,
|
||||
needsBlobResolution,
|
||||
normalizeBlobInput,
|
||||
resolveBlobInput,
|
||||
} from "./blob";
|
||||
import { type EmbeddingFunction } from "./embedding/embedding_function";
|
||||
@@ -447,9 +446,15 @@ export function makeArrowTable(
|
||||
}
|
||||
|
||||
if (schema !== undefined) {
|
||||
// Validate blob values up front and give every one the full
|
||||
// `{ data, uri }` shape, so inference sees the same struct whether a row
|
||||
// passed bytes, a URI, or a partial struct.
|
||||
data = mapBlobInputs(data, schema, (value, field, row) => {
|
||||
if (value === undefined) {
|
||||
return value;
|
||||
}
|
||||
try {
|
||||
return normalizeBlobInput(value);
|
||||
return coerceBlobValue(value);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Invalid value for blob field ${field} at row ${row}: ${(e as Error).message}`,
|
||||
|
||||
Reference in New Issue
Block a user