fix(node): rebuild view columns as Utf8/Binary before IPC serialization

Arrow JS 21's IPC writer emits a truncated views buffer for a sliced
Utf8View or BinaryView array, which the Rust reader rejects ("Need at
least N bytes in buffers[0]"), so createTable and add failed for any view
column with a nonzero data offset. Lance stores the view types as Utf8 and
Binary anyway, so rebuild top-level view columns that way on the JS side
before writing the IPC bytes. Covers the table and record batch paths.

Also take the Type enum from the Arrow 21 copy in the end-to-end test so
the file still type-checks when the library resolves to Arrow 19 or 20.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

nodejs/__test__/table.test.ts

@@ -771,12 +771,14 @@ const hostArrowSupportsViewTypes =
     await table.add(makeData());
 
     const schema = await table.schema();
+    // Take the enum from the Arrow 21 copy: the host Arrow may predate
+    // `Type.LargeList`, and this file still has to type-check there.
     expect(schema.fields.map((f) => f.type.typeId)).toEqual([
-      arrow.Type.Int,
-      arrow.Type.LargeList,
+      latestArrow.Type.Int,
+      latestArrow.Type.LargeList,
       // Lance stores the view types as their offset-based equivalents.
-      arrow.Type.Utf8,
-      arrow.Type.Binary,
+      latestArrow.Type.Utf8,
+      latestArrow.Type.Binary,
     ]);
     const rows = await table.query().toArrow();
     expect(rows.numRows).toBe(4);
@@ -787,6 +789,23 @@ const hostArrowSupportsViewTypes =
       "a string that is longer than the twelve inline bytes",
     );
     expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
+
+    // Arrow JS 21 serializes a *sliced* view column with a truncated views
+    // buffer; the library rebuilds view columns as Utf8/Binary so this works.
+    const sliced = makeData().slice(1, 2);
+    expect(sliced.getChild("text")!.data[0].offset).toBe(1);
+    const slicedTable = await db.createTable("views_sliced", sliced);
+    await slicedTable.add(sliced);
+    const slicedRows = await slicedTable.query().toArrow();
+    expect(slicedRows.numRows).toBe(2);
+    for (const i of [0, 1]) {
+      const r = slicedRows.get(i)!.toJSON();
+      expect(r.id).toBe(2);
+      expect(r.text).toBe(
+        "a string that is longer than the twelve inline bytes",
+      );
+      expect(Array.from(r.bytes)).toEqual(Array(20).fill(7));
+    }
     tmpDir.removeCallback();
   },
 );

nodejs/lancedb/arrow.ts

@@ -1068,6 +1068,49 @@ export async function fromRecordsToStreamBuffer(
   return Buffer.from(await writer.toUint8Array());
 }
 
+// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
+// in Arrow 21+, and this module compiles against every supported release.
+const UTF8_VIEW_TYPE_ID = 24;
+const BINARY_VIEW_TYPE_ID = 23;
+
+/**
+ * Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
+ *
+ * Lance stores the view types as their offset-based equivalents anyway, so
+ * nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
+ * writer emits a truncated views buffer for a *sliced* view array, which the
+ * Rust reader rejects with "Need at least N bytes in buffers[0]".
+ *
+ * Tables without view columns are returned as-is.
+ */
+function materializeViewColumns(table: ArrowTable): ArrowTable {
+  const hasViewColumn = table.schema.fields.some(
+    (field) =>
+      field.type.typeId === UTF8_VIEW_TYPE_ID ||
+      field.type.typeId === BINARY_VIEW_TYPE_ID,
+  );
+  if (!hasViewColumn) {
+    return table;
+  }
+  const columns: Record<string, Vector> = {};
+  const fields = table.schema.fields.map((field) => {
+    const column = table.getChild(field.name)!;
+    let type: DataType | undefined;
+    if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
+      type = new Utf8();
+    } else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
+      type = new Binary();
+    }
+    if (type === undefined) {
+      columns[field.name] = column;
+      return field;
+    }
+    columns[field.name] = badVectorFromArray(column.toArray(), type);
+    return new Field(field.name, type, field.nullable, field.metadata);
+  });
+  return new ArrowTable(new Schema(fields, table.schema.metadata), columns);
+}
+
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -1084,7 +1127,9 @@ export async function fromTableToBuffer(
   if (schema !== undefined && schema !== null) {
     schema = sanitizeSchema(schema);
   }
-  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+  const tableWithEmbeddings = materializeViewColumns(
+    await applyEmbeddings(table, embeddings, schema),
+  );
   validateBlobSchema(tableWithEmbeddings.schema);
   const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings);
   return Buffer.from(await writer.toUint8Array());
@@ -1155,7 +1200,8 @@ export async function fromRecordBatchToBuffer(
 export async function fromRecordBatchToStreamBuffer(
   batch: RecordBatch,
 ): Promise<Buffer> {
-  const writer = RecordBatchStreamWriter.writeAll([batch]);
+  const table = materializeViewColumns(new ArrowTable([batch]));
+  const writer = RecordBatchStreamWriter.writeAll(table);
   return Buffer.from(await writer.toUint8Array());
 }
 
@@ -1172,7 +1218,9 @@ export async function fromTableToStreamBuffer(
   embeddings?: EmbeddingFunctionConfig,
   schema?: SchemaLike,
 ): Promise<Buffer> {
-  const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+  const tableWithEmbeddings = materializeViewColumns(
+    await applyEmbeddings(table, embeddings, schema),
+  );
   const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
   return Buffer.from(await writer.toUint8Array());
 }