From c155ee2787875f5b28901ce9fc3cfc70aa3e817d Mon Sep 17 00:00:00 2001 From: Dan Tasse Date: Thu, 1 Oct 2026 13:28:16 -0500 Subject: [PATCH] fix(node): rebuild view columns positionally, not by column name The view-column materialization built the new table from a Record. JavaScript enumerates integer-like keys first, so a schema with a field named e.g. "1" paired that column with the wrong values while the schema kept its declared order; compatible types let the swap through createTable and add. Rebuild each record batch from its ordered child Data instead, replacing only the view children, and add a regression test with an integer-like field name. Co-Authored-By: Claude Fable 5.1 --- nodejs/__test__/table.test.ts | 37 ++++++++++++++++++++ nodejs/lancedb/arrow.ts | 66 +++++++++++++++++++++++------------ 2 files changed, 81 insertions(+), 22 deletions(-) diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 9958ad25d..eaf59211c 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -806,6 +806,43 @@ const hostArrowSupportsViewTypes = ); expect(Array.from(r.bytes)).toEqual(Array(20).fill(7)); } + + // Regression: a field with an integer-like name. JavaScript enumerates + // such keys first, so a key-ordered rebuild would pair "1" with the text + // column's values. The batches are built explicitly to pin field order. + const numericSchema = new latestArrow.Schema([ + new latestArrow.Field("text", new latestArrow.Utf8View(), true), + new latestArrow.Field("1", new latestArrow.Utf8View(), true), + new latestArrow.Field("n", new latestArrow.Int32(), true), + ]); + const column = ( + values: unknown[], + type: InstanceType, + ) => latestArrow.vectorFromArray(values, type).data[0]; + const numericBatch = new latestArrow.RecordBatch( + numericSchema, + latestArrow.makeData({ + type: new latestArrow.Struct(numericSchema.fields), + length: 3, + nullCount: 0, + children: [ + column(["t0", "t1", "t2"], new latestArrow.Utf8View()), + column(["one0", "one1", "one2"], new latestArrow.Utf8View()), + column([0, 1, 2], new latestArrow.Int32()), + ], + }), + ); + const numericData = new latestArrow.Table(numericSchema, [numericBatch]); + const numericTable = await db.createTable("views_numeric", numericData); + await numericTable.add(numericData.slice(1)); + const numericRows = (await numericTable.query().toArrow()).toArray(); + expect(numericRows.map((r) => r.toJSON())).toEqual([ + { text: "t0", "1": "one0", n: 0 }, + { text: "t1", "1": "one1", n: 1 }, + { text: "t2", "1": "one2", n: 2 }, + { text: "t1", "1": "one1", n: 1 }, + { text: "t2", "1": "one2", n: 2 }, + ]); tmpDir.removeCallback(); }, ); diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 4fae2fdf6..3daee7d65 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -1073,6 +1073,17 @@ export async function fromRecordsToStreamBuffer( const UTF8_VIEW_TYPE_ID = 24; const BINARY_VIEW_TYPE_ID = 23; +/** + * Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`. + */ +function materializeViewData(child: ArrowData, type: DataType): ArrowData { + const builder = makeBuilder({ type, nullValues: [null] }); + for (const value of new Vector([child])) { + builder.append(value); + } + return builder.finish().flush(); +} + /** * Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary. * @@ -1081,34 +1092,45 @@ const BINARY_VIEW_TYPE_ID = 23; * writer emits a truncated views buffer for a *sliced* view array, which the * Rust reader rejects with "Need at least N bytes in buffers[0]". * + * The record batches are rebuilt positionally rather than through a + * `Record`: JavaScript enumerates integer-like keys first, so + * a field named e.g. `"1"` would otherwise be paired with the wrong column. + * * Tables without view columns are returned as-is. */ function materializeViewColumns(table: ArrowTable): ArrowTable { - const hasViewColumn = table.schema.fields.some( - (field) => - field.type.typeId === UTF8_VIEW_TYPE_ID || - field.type.typeId === BINARY_VIEW_TYPE_ID, - ); - if (!hasViewColumn) { + const replacements = new Map(); + table.schema.fields.forEach((field, i) => { + if (field.type.typeId === UTF8_VIEW_TYPE_ID) { + replacements.set(i, new Utf8()); + } else if (field.type.typeId === BINARY_VIEW_TYPE_ID) { + replacements.set(i, new Binary()); + } + }); + if (replacements.size === 0) { return table; } - const columns: Record = {}; - const fields = table.schema.fields.map((field) => { - const column = table.getChild(field.name)!; - let type: DataType | undefined; - if (field.type.typeId === UTF8_VIEW_TYPE_ID) { - type = new Utf8(); - } else if (field.type.typeId === BINARY_VIEW_TYPE_ID) { - type = new Binary(); - } - if (type === undefined) { - columns[field.name] = column; - return field; - } - columns[field.name] = badVectorFromArray(column.toArray(), type); - return new Field(field.name, type, field.nullable, field.metadata); + const fields = table.schema.fields.map((field, i) => { + const type = replacements.get(i); + return type === undefined + ? field + : new Field(field.name, type, field.nullable, field.metadata); }); - return new ArrowTable(new Schema(fields, table.schema.metadata), columns); + const schema = new Schema(fields, table.schema.metadata); + const batches = table.batches.map((batch) => { + const children = batch.data.children.map((child, i) => { + const type = replacements.get(i); + return type === undefined ? child : materializeViewData(child, type); + }); + const data = makeData({ + type: new Struct(fields), + length: batch.numRows, + nullCount: 0, + children, + }); + return new RecordBatch(schema, data); + }); + return new ArrowTable(schema, batches); } /**