From 590f7a4ecace4fe9525a98bce94f87a6bbe451dc Mon Sep 17 00:00:00 2001 From: Dan Tasse Date: Thu, 1 Oct 2026 12:28:35 -0500 Subject: [PATCH] fix(node): rebuild view columns as Utf8/Binary before IPC serialization Arrow JS 21's IPC writer emits a truncated views buffer for a sliced Utf8View or BinaryView array, which the Rust reader rejects ("Need at least N bytes in buffers[0]"), so createTable and add failed for any view column with a nonzero data offset. Lance stores the view types as Utf8 and Binary anyway, so rebuild top-level view columns that way on the JS side before writing the IPC bytes. Covers the table and record batch paths. Also take the Type enum from the Arrow 21 copy in the end-to-end test so the file still type-checks when the library resolves to Arrow 19 or 20. Co-Authored-By: Claude Fable 5.1 --- nodejs/__test__/table.test.ts | 27 +++++++++++++++--- nodejs/lancedb/arrow.ts | 54 +++++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 907f185dd..9958ad25d 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -771,12 +771,14 @@ const hostArrowSupportsViewTypes = await table.add(makeData()); const schema = await table.schema(); + // Take the enum from the Arrow 21 copy: the host Arrow may predate + // `Type.LargeList`, and this file still has to type-check there. expect(schema.fields.map((f) => f.type.typeId)).toEqual([ - arrow.Type.Int, - arrow.Type.LargeList, + latestArrow.Type.Int, + latestArrow.Type.LargeList, // Lance stores the view types as their offset-based equivalents. - arrow.Type.Utf8, - arrow.Type.Binary, + latestArrow.Type.Utf8, + latestArrow.Type.Binary, ]); const rows = await table.query().toArrow(); expect(rows.numRows).toBe(4); @@ -787,6 +789,23 @@ const hostArrowSupportsViewTypes = "a string that is longer than the twelve inline bytes", ); expect(Array.from(row.bytes)).toEqual(Array(20).fill(7)); + + // Arrow JS 21 serializes a *sliced* view column with a truncated views + // buffer; the library rebuilds view columns as Utf8/Binary so this works. + const sliced = makeData().slice(1, 2); + expect(sliced.getChild("text")!.data[0].offset).toBe(1); + const slicedTable = await db.createTable("views_sliced", sliced); + await slicedTable.add(sliced); + const slicedRows = await slicedTable.query().toArrow(); + expect(slicedRows.numRows).toBe(2); + for (const i of [0, 1]) { + const r = slicedRows.get(i)!.toJSON(); + expect(r.id).toBe(2); + expect(r.text).toBe( + "a string that is longer than the twelve inline bytes", + ); + expect(Array.from(r.bytes)).toEqual(Array(20).fill(7)); + } tmpDir.removeCallback(); }, ); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 71e3abaae..4fae2fdf6 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -1068,6 +1068,49 @@ export async function fromRecordsToStreamBuffer( return Buffer.from(await writer.toUint8Array()); } +// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist +// in Arrow 21+, and this module compiles against every supported release. +const UTF8_VIEW_TYPE_ID = 24; +const BINARY_VIEW_TYPE_ID = 23; + +/** + * Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary. + * + * Lance stores the view types as their offset-based equivalents anyway, so + * nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC + * writer emits a truncated views buffer for a *sliced* view array, which the + * Rust reader rejects with "Need at least N bytes in buffers[0]". + * + * Tables without view columns are returned as-is. + */ +function materializeViewColumns(table: ArrowTable): ArrowTable { + const hasViewColumn = table.schema.fields.some( + (field) => + field.type.typeId === UTF8_VIEW_TYPE_ID || + field.type.typeId === BINARY_VIEW_TYPE_ID, + ); + if (!hasViewColumn) { + return table; + } + const columns: Record = {}; + const fields = table.schema.fields.map((field) => { + const column = table.getChild(field.name)!; + let type: DataType | undefined; + if (field.type.typeId === UTF8_VIEW_TYPE_ID) { + type = new Utf8(); + } else if (field.type.typeId === BINARY_VIEW_TYPE_ID) { + type = new Binary(); + } + if (type === undefined) { + columns[field.name] = column; + return field; + } + columns[field.name] = badVectorFromArray(column.toArray(), type); + return new Field(field.name, type, field.nullable, field.metadata); + }); + return new ArrowTable(new Schema(fields, table.schema.metadata), columns); +} + /** * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization * @@ -1084,7 +1127,9 @@ export async function fromTableToBuffer( if (schema !== undefined && schema !== null) { schema = sanitizeSchema(schema); } - const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema); + const tableWithEmbeddings = materializeViewColumns( + await applyEmbeddings(table, embeddings, schema), + ); validateBlobSchema(tableWithEmbeddings.schema); const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings); return Buffer.from(await writer.toUint8Array()); @@ -1155,7 +1200,8 @@ export async function fromRecordBatchToBuffer( export async function fromRecordBatchToStreamBuffer( batch: RecordBatch, ): Promise { - const writer = RecordBatchStreamWriter.writeAll([batch]); + const table = materializeViewColumns(new ArrowTable([batch])); + const writer = RecordBatchStreamWriter.writeAll(table); return Buffer.from(await writer.toUint8Array()); } @@ -1172,7 +1218,9 @@ export async function fromTableToStreamBuffer( embeddings?: EmbeddingFunctionConfig, schema?: SchemaLike, ): Promise { - const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema); + const tableWithEmbeddings = materializeViewColumns( + await applyEmbeddings(table, embeddings, schema), + ); const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings); return Buffer.from(await writer.toUint8Array()); }