mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-24 12:42:26 +00:00
fix(node): rebuild view columns positionally, not by column name
The view-column materialization built the new table from a Record<string, Vector>. JavaScript enumerates integer-like keys first, so a schema with a field named e.g. "1" paired that column with the wrong values while the schema kept its declared order; compatible types let the swap through createTable and add. Rebuild each record batch from its ordered child Data instead, replacing only the view children, and add a regression test with an integer-like field name. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5.1
parent
590f7a4eca
commit
c155ee2787
@@ -806,6 +806,43 @@ const hostArrowSupportsViewTypes =
|
||||
);
|
||||
expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
|
||||
}
|
||||
|
||||
// Regression: a field with an integer-like name. JavaScript enumerates
|
||||
// such keys first, so a key-ordered rebuild would pair "1" with the text
|
||||
// column's values. The batches are built explicitly to pin field order.
|
||||
const numericSchema = new latestArrow.Schema([
|
||||
new latestArrow.Field("text", new latestArrow.Utf8View(), true),
|
||||
new latestArrow.Field("1", new latestArrow.Utf8View(), true),
|
||||
new latestArrow.Field("n", new latestArrow.Int32(), true),
|
||||
]);
|
||||
const column = (
|
||||
values: unknown[],
|
||||
type: InstanceType<typeof latestArrow.DataType>,
|
||||
) => latestArrow.vectorFromArray(values, type).data[0];
|
||||
const numericBatch = new latestArrow.RecordBatch(
|
||||
numericSchema,
|
||||
latestArrow.makeData({
|
||||
type: new latestArrow.Struct(numericSchema.fields),
|
||||
length: 3,
|
||||
nullCount: 0,
|
||||
children: [
|
||||
column(["t0", "t1", "t2"], new latestArrow.Utf8View()),
|
||||
column(["one0", "one1", "one2"], new latestArrow.Utf8View()),
|
||||
column([0, 1, 2], new latestArrow.Int32()),
|
||||
],
|
||||
}),
|
||||
);
|
||||
const numericData = new latestArrow.Table(numericSchema, [numericBatch]);
|
||||
const numericTable = await db.createTable("views_numeric", numericData);
|
||||
await numericTable.add(numericData.slice(1));
|
||||
const numericRows = (await numericTable.query().toArrow()).toArray();
|
||||
expect(numericRows.map((r) => r.toJSON())).toEqual([
|
||||
{ text: "t0", "1": "one0", n: 0 },
|
||||
{ text: "t1", "1": "one1", n: 1 },
|
||||
{ text: "t2", "1": "one2", n: 2 },
|
||||
{ text: "t1", "1": "one1", n: 1 },
|
||||
{ text: "t2", "1": "one2", n: 2 },
|
||||
]);
|
||||
tmpDir.removeCallback();
|
||||
},
|
||||
);
|
||||
|
||||
+44
-22
@@ -1073,6 +1073,17 @@ export async function fromRecordsToStreamBuffer(
|
||||
const UTF8_VIEW_TYPE_ID = 24;
|
||||
const BINARY_VIEW_TYPE_ID = 23;
|
||||
|
||||
/**
|
||||
* Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
|
||||
*/
|
||||
function materializeViewData(child: ArrowData, type: DataType): ArrowData {
|
||||
const builder = makeBuilder({ type, nullValues: [null] });
|
||||
for (const value of new Vector([child])) {
|
||||
builder.append(value);
|
||||
}
|
||||
return builder.finish().flush();
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
|
||||
*
|
||||
@@ -1081,34 +1092,45 @@ const BINARY_VIEW_TYPE_ID = 23;
|
||||
* writer emits a truncated views buffer for a *sliced* view array, which the
|
||||
* Rust reader rejects with "Need at least N bytes in buffers[0]".
|
||||
*
|
||||
* The record batches are rebuilt positionally rather than through a
|
||||
* `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
|
||||
* a field named e.g. `"1"` would otherwise be paired with the wrong column.
|
||||
*
|
||||
* Tables without view columns are returned as-is.
|
||||
*/
|
||||
function materializeViewColumns(table: ArrowTable): ArrowTable {
|
||||
const hasViewColumn = table.schema.fields.some(
|
||||
(field) =>
|
||||
field.type.typeId === UTF8_VIEW_TYPE_ID ||
|
||||
field.type.typeId === BINARY_VIEW_TYPE_ID,
|
||||
);
|
||||
if (!hasViewColumn) {
|
||||
const replacements = new Map<number, DataType>();
|
||||
table.schema.fields.forEach((field, i) => {
|
||||
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
|
||||
replacements.set(i, new Utf8());
|
||||
} else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
|
||||
replacements.set(i, new Binary());
|
||||
}
|
||||
});
|
||||
if (replacements.size === 0) {
|
||||
return table;
|
||||
}
|
||||
const columns: Record<string, Vector> = {};
|
||||
const fields = table.schema.fields.map((field) => {
|
||||
const column = table.getChild(field.name)!;
|
||||
let type: DataType | undefined;
|
||||
if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
|
||||
type = new Utf8();
|
||||
} else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
|
||||
type = new Binary();
|
||||
}
|
||||
if (type === undefined) {
|
||||
columns[field.name] = column;
|
||||
return field;
|
||||
}
|
||||
columns[field.name] = badVectorFromArray(column.toArray(), type);
|
||||
return new Field(field.name, type, field.nullable, field.metadata);
|
||||
const fields = table.schema.fields.map((field, i) => {
|
||||
const type = replacements.get(i);
|
||||
return type === undefined
|
||||
? field
|
||||
: new Field(field.name, type, field.nullable, field.metadata);
|
||||
});
|
||||
return new ArrowTable(new Schema(fields, table.schema.metadata), columns);
|
||||
const schema = new Schema(fields, table.schema.metadata);
|
||||
const batches = table.batches.map((batch) => {
|
||||
const children = batch.data.children.map((child, i) => {
|
||||
const type = replacements.get(i);
|
||||
return type === undefined ? child : materializeViewData(child, type);
|
||||
});
|
||||
const data = makeData({
|
||||
type: new Struct(fields),
|
||||
length: batch.numRows,
|
||||
nullCount: 0,
|
||||
children,
|
||||
});
|
||||
return new RecordBatch(schema, data);
|
||||
});
|
||||
return new ArrowTable(schema, batches);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user