fix(node): preserve foreign Arrow table rows

nodejs/__test__/arrow.test.ts

@@ -61,6 +61,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
       Float32,
       FixedSizeList,
       Precision,
+      tableFromArrays,
       tableFromIPC,
       DataType,
       Dictionary,
@@ -1054,6 +1055,21 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
     });
 
     describe("when using two versions of arrow", function () {
+      it("preserves rows from a foreign Arrow Table", async function () {
+        const foreignTable = tableFromArrays({
+          id: new Int32Array([1, 2]),
+          text: ["foo", "bar"],
+        });
+
+        // biome-ignore lint/suspicious/noExplicitAny: Arrow Tables from supported versions are structurally compatible
+        const buf = await fromDataToBuffer(foreignTable as any);
+        const actual = tableFromIPC(buf);
+
+        expect(actual.numRows).toBe(2);
+        expect(actual.getChild("id")?.toJSON()).toEqual([1, 2]);
+        expect(actual.getChild("text")?.toJSON()).toEqual(["foo", "bar"]);
+      });
+
       it("can still import data", async function () {
         const schema = new arrow15.Schema([
           new arrow15.Field("id", new arrow15.Int32()),

nodejs/lancedb/arrow.ts

@@ -82,8 +82,7 @@ export type FieldLike =
     };
 
 export type DataLike =
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  | import("apache-arrow").Data<Struct<any>>
+  | import("apache-arrow").Data
   | {
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       type: any;
@@ -92,6 +91,7 @@ export type DataLike =
       stride: number;
       nullable: boolean;
       children: DataLike[];
+      dictionary?: { data: readonly DataLike[] };
       get nullCount(): number;
       // biome-ignore lint/suspicious/noExplicitAny: <explanation>
       values: Buffers<any>[BufferType.DATA];

nodejs/lancedb/sanitize.ts

@@ -72,6 +72,7 @@ import {
   Uint64,
   Union,
   Utf8,
+  Vector,
 } from "./arrow";
 
 export function sanitizeMetadata(
@@ -568,18 +569,18 @@ function sanitizeRecordBatch(batchLike: RecordBatchLike): RecordBatch {
     );
   }
   const schema = sanitizeSchema(batchLike.schema);
-  const data = sanitizeData(batchLike.data);
+  const data = sanitizeData(batchLike.data) as Data<Struct>;
   return new RecordBatch(schema, data);
 }
-function sanitizeData(
-  dataLike: DataLike,
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-): import("apache-arrow").Data<Struct<any>> {
+function sanitizeData(dataLike: DataLike): Data {
   if (dataLike instanceof Data) {
     return dataLike;
   }
+  const dictionary = dataLike.dictionary
+    ? new Vector(dataLike.dictionary.data.map(sanitizeData))
+    : undefined;
   return new Data(
-    dataLike.type,
+    sanitizeType(dataLike.type),
     dataLike.offset,
     dataLike.length,
     dataLike.nullCount,
@@ -589,6 +590,8 @@ function sanitizeData(
       [BufferType.VALIDITY]: dataLike.nullBitmap,
       [BufferType.TYPE]: dataLike.typeIds,
     },
+    dataLike.children.map(sanitizeData),
+    dictionary,
   );
 }