From 9ba1920e0e2c3ec2f49f2cf69f01d08ccf836aa9 Mon Sep 17 00:00:00 2001
From: Yang Cen <bubble-cal@outlook.com>
Date: Thu, 5 Nov 2026 19:50:50 +0800
Subject: [PATCH] fix(node): normalize record batches before IPC serialization
 (#4871)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Passing a record batch created by an ESM `apache-arrow` import to
`RRFReranker.rerankHybrid` throws an IPC footer parse error. The SDK's
CommonJS Arrow writer receives data types from another module instance.

Normalize single-record-batch input with the existing table sanitizer
before writing IPC. Extend the Arrow 15–21 round-trip test with
caller-created batches and nulls, and add a native ESM reranker
regression covering 64-bit row IDs, scores, and an empty FTS side.

Validation:

- Reproduced with published `0.42.0-beta.3`; CommonJS and explicitly
normalized ESM controls pass.
- Same native module and inputs: all 8 targeted cases fail with the
parent serializer and pass with this change.
- Local native build, TypeScript build, lint, and formatting pass.
- Current-head CI passes on Linux Node 22/24/26, macOS, and Arrow 19/20.
The macOS run passes 1,643 tests; Linux Node 24 also passes docs and
example checks.
---
 nodejs/__test__/arrow.test.ts     | 10 ++++++++--
 nodejs/__test__/rerankers.test.ts | 31 ++++++++++++++++++++++++++++++-
 nodejs/lancedb/arrow.ts           |  4 +++-
 3 files changed, 41 insertions(+), 4 deletions(-)

diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts
index 376533961..27dd14954 100644
--- a/nodejs/__test__/arrow.test.ts
+++ b/nodejs/__test__/arrow.test.ts
@@ -1567,11 +1567,17 @@ describe.each(arrowVersions)("Arrow %#", (arrow: ApacheArrow) => {
       const records = [
         { text: "dog", vector: [0.1, 0.2] },
         { text: "cat", vector: [0.3, 0.4] },
+        { text: null, vector: null },
       ];
-      const table = await convertToTable(records);
+      const table = arrow.tableFromArrays({
+        text: records.map((row) => row.text),
+        vector: records.map((row) => row.vector),
+      });
       const batch = table.batches[0];
 
-      const buffer = await fromRecordBatchToBuffer(batch);
+      const buffer = await fromRecordBatchToBuffer(
+        batch as Parameters<typeof fromRecordBatchToBuffer>[0],
+      );
       const result = await fromBufferToRecordBatch(buffer);
 
       expect(JSON.stringify(batch.toArray())).toEqual(
diff --git a/nodejs/__test__/rerankers.test.ts b/nodejs/__test__/rerankers.test.ts
index 256ef686c..8e3a36f98 100644
--- a/nodejs/__test__/rerankers.test.ts
+++ b/nodejs/__test__/rerankers.test.ts
@@ -1,13 +1,42 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-import { spawn } from "node:child_process";
+import { execFileSync, spawn } from "node:child_process";
 import * as path from "node:path";
 import { RecordBatch } from "apache-arrow";
 import * as tmp from "tmp";
 import { Connection, Index, Table, connect, makeArrowTable } from "../lancedb";
 import { RRFReranker } from "../lancedb/rerankers";
 
+it("reranks record batches created by an ESM Arrow import", () => {
+  const script = `
+import assert from "node:assert/strict";
+import { tableFromArrays } from "apache-arrow";
+import { rerankers } from "./dist/index.js";
+
+const ids = [1n, 9007199254740993n];
+const batch = tableFromArrays({ _rowid: new BigUint64Array(ids) }).batches[0];
+const reranker = await rerankers.RRFReranker.create();
+const result = await reranker.rerankHybrid("", batch, batch);
+
+assert.equal(result.numRows, 2);
+assert.deepEqual(Array.from(result.getChild("_rowid").toArray()), ids);
+const scores = result.getChild("_relevance_score").toArray();
+assert.ok(Math.abs(scores[0] - 2 / 61) < 1e-8);
+assert.ok(Math.abs(scores[1] - 2 / 62) < 1e-8);
+
+const oneSide = await reranker.rerankHybrid("", batch, batch.slice(0, 0));
+assert.deepEqual(Array.from(oneSide.getChild("_rowid").toArray()), ids);
+const oneSideScores = oneSide.getChild("_relevance_score").toArray();
+assert.ok(Math.abs(oneSideScores[0] - 1 / 61) < 1e-8);
+assert.ok(Math.abs(oneSideScores[1] - 1 / 62) < 1e-8);
+`;
+  execFileSync(process.execPath, ["--input-type=module", "-e", script], {
+    cwd: path.resolve(__dirname, ".."),
+    timeout: 10_000,
+  });
+});
+
 describe("rerankers", function () {
   let tmpDir: tmp.DirResult;
   let conn: Connection;
diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts
index 56c8e1897..f920eecb8 100644
--- a/nodejs/lancedb/arrow.ts
+++ b/nodejs/lancedb/arrow.ts
@@ -1386,7 +1386,9 @@ export async function fromBufferToRecordBatch(
 export async function fromRecordBatchToBuffer(
   batch: RecordBatch,
 ): Promise<Buffer> {
-  const writer = new RecordBatchFileWriter().writeAll([batch]);
+  // Arrow writers require data types from their own library instance.
+  const table = sanitizeTable({ schema: batch.schema, batches: [batch] });
+  const writer = new RecordBatchFileWriter().writeAll(table);
   return Buffer.from(await writer.toUint8Array());
 }