fix(node): support Apache Arrow 19 through 21 and run the library on each

Builds on #4359 (which added the Arrow 19-21 test aliases) and makes the
package genuinely work across the whole peer range:

* Resolve the library's own `apache-arrow` to 21.2.0 (devDependency) so
  the main CI jobs build, type-check and test on the newest release, and
  widen the peer range to `>=15.0.0 <22.0.0` instead of an exact pin.
* Add a CI job that re-installs with `apache-arrow` resolved to 19.0.1 and
  20.0.0 and runs the suite, covering what a consumer on those versions
  gets.
* Arrow 21 overrides `Symbol.hasInstance` on Schema/Field/Table/RecordBatch/
  Data so `instanceof` matches objects from another copy of the library,
  but not on DataType. The sanitizer's short-circuits therefore returned
  foreign schemas unchanged and Arrow's builders then failed with
  "Unrecognized type 'NONE'". Check the real prototype chain instead.
* Sanitize the types added in Arrow 20/21 (LargeList, Utf8View, BinaryView,
  IntervalMonthDayNano), looked up dynamically so the library still compiles
  and runs on older Arrow releases, with a clear error when the installed
  Arrow lacks the type. Carry `variadicBuffers` through `Data`.
* Loosen `FieldLike` so a branded DataType from another Arrow copy still
  type-checks; export the new `DataTypeLike`.
* Share the Arrow version list between test files, exercise embedding
  function types per version in the registry tests, and add sanitize and
  end-to-end coverage for the new column types.
* Enable `skipLibCheck`: flatbuffers 25 (via Arrow 21) ships declarations
  that need TypeScript 5.7+.
* Record the new dev dependencies in the third-party license inventory.

Co-Authored-By: Vincent Koc <vincentkoc@ieee.org>
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

nodejs/__test__/arrow_versions.ts

@@ -0,0 +1,39 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+
+// Every Apache Arrow release the package accepts as a peer dependency, as
+// separately installed copies so the tests can feed objects built by one
+// Arrow version into a LanceDB running on another.  When a new Arrow major
+// is released, add the alias to `package.json` and extend this list; the
+// test files pick the change up from here.
+import * as arrow15 from "apache-arrow-15";
+import * as arrow16 from "apache-arrow-16";
+import * as arrow17 from "apache-arrow-17";
+import * as arrow18 from "apache-arrow-18";
+import * as arrow19 from "apache-arrow-19";
+import * as arrow20 from "apache-arrow-20";
+import * as arrow21 from "apache-arrow-21";
+
+export { arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21 };
+
+export type ApacheArrow =
+  | typeof arrow15
+  | typeof arrow16
+  | typeof arrow17
+  | typeof arrow18
+  | typeof arrow19
+  | typeof arrow20
+  | typeof arrow21;
+
+export const arrowVersions: ApacheArrow[] = [
+  arrow15,
+  arrow16,
+  arrow17,
+  arrow18,
+  arrow19,
+  arrow20,
+  arrow21,
+];
+
+/** The newest supported Arrow release, for tests that only need one. */
+export const latestArrow = arrow21;

nodejs/__test__/registry.test.ts

@@ -3,16 +3,10 @@
 
 import * as apiArrow from "apache-arrow";
 
-import * as arrow15 from "apache-arrow-15";
-import * as arrow16 from "apache-arrow-16";
-import * as arrow17 from "apache-arrow-17";
-import * as arrow18 from "apache-arrow-18";
-import * as arrow19 from "apache-arrow-19";
-import * as arrow20 from "apache-arrow-20";
-import * as arrow21 from "apache-arrow-21";
-
 import * as tmp from "tmp";
 
+import { arrowVersions } from "./arrow_versions";
+
 import { connect } from "../lancedb";
 import {
   EmbeddingFunction,
@@ -21,16 +15,6 @@ import {
 } from "../lancedb/embedding";
 import { getRegistry, register } from "../lancedb/embedding/registry";
 
-const arrowVersions = [
-  arrow15,
-  arrow16,
-  arrow17,
-  arrow18,
-  arrow19,
-  arrow20,
-  arrow21,
-];
-
 describe.each(arrowVersions)("LanceSchema", (arrow) => {
   test("should preserve input order", async () => {
     const schema = LanceSchema({
@@ -253,7 +237,7 @@ describe("Registry.setVar", () => {
         return data.map(() => [1, 2, 3]);
       }
       embeddingDataType() {
-        return new arrow18.Float32() as apiArrow.Float;
+        return new apiArrow.Float32();
       }
       protected getSensitiveKeys() {
         return ["secretKey"];

nodejs/__test__/sanitize.test.ts

@@ -2,7 +2,13 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 import * as arrow from "../lancedb/arrow";
-import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
+import {
+  sanitizeField,
+  sanitizeMap,
+  sanitizeSchema,
+  sanitizeType,
+} from "../lancedb/sanitize";
+import { arrow21 } from "./arrow_versions";
 
 describe("sanitize", function () {
   describe("sanitizeType function", function () {
@@ -59,6 +65,8 @@ describe("sanitize", function () {
       ["durationmicrosecond", new arrow.DurationMicrosecond()],
       ["durationmillisecond", new arrow.DurationMillisecond()],
       ["durationsecond", new arrow.DurationSecond()],
+      ["largebinary", new arrow.LargeBinary()],
+      ["largeutf8", new arrow.LargeUtf8()],
     ] as const;
 
     it.each(allTypeNameTestCases)(
@@ -89,6 +97,110 @@ describe("sanitize", function () {
         "Unrecognized type name in schema: invalid_type",
       );
     });
+
+    // These types only exist in Arrow 20 (IntervalMonthDayNano) and 21 (the
+    // rest), so the objects come from a second copy of the newest release.  The
+    // library itself may be running on an older Arrow (see the CI matrix), in
+    // which case sanitizing them has to fail with a clear message instead.
+    const hostArrow = arrow as unknown as Record<string, unknown>;
+    const hostSupportsNewerTypes = typeof hostArrow.Utf8View === "function";
+    const whenSupported = hostSupportsNewerTypes ? describe : describe.skip;
+    const whenUnsupported = hostSupportsNewerTypes ? describe.skip : describe;
+
+    const newerTypeTestCases = [
+      ["Utf8View", () => new arrow21.Utf8View()],
+      ["BinaryView", () => new arrow21.BinaryView()],
+      ["IntervalMonthDayNano", () => new arrow21.IntervalMonthDayNano()],
+      [
+        "LargeList",
+        () =>
+          new arrow21.LargeList(new arrow21.Field("item", new arrow21.Utf8())),
+      ],
+    ] as const;
+
+    whenSupported("types added in Arrow 20 and 21", function () {
+      it.each(newerTypeTestCases)(
+        "should sanitize a %s type from another copy of Arrow",
+        function (name, makeForeign) {
+          const foreign = makeForeign();
+          const result = sanitizeType(foreign);
+          // Interval subclasses share `Type.Interval`, so the sanitized type
+          // is the generic `Interval` carrying the same unit, as for the
+          // other units.
+          const expected = name === "IntervalMonthDayNano" ? "Interval" : name;
+          expect(result).toBeInstanceOf(hostArrow[expected] as () => unknown);
+          expect(result.typeId).toBe(foreign.typeId);
+          expect(result.toString()).toBe(foreign.toString());
+        },
+      );
+
+      it.each([
+        ["binaryview", "BinaryView"],
+        ["utf8view", "Utf8View"],
+        ["intervalmonthdaynano", "Interval"],
+      ])('should map type name "%s" to %s', function (name, expected) {
+        expect(sanitizeType(name)).toBeInstanceOf(
+          hostArrow[expected] as () => unknown,
+        );
+      });
+
+      it("should sanitize the child field of a LargeList", function () {
+        const foreign = new arrow21.LargeList(
+          new arrow21.Field("item", new arrow21.Int32(), false),
+        );
+        const result = sanitizeType(foreign) as arrow.List;
+        expect(result.children[0]).toBeInstanceOf(arrow.Field);
+        expect(result.children[0].name).toBe("item");
+        expect(result.children[0].nullable).toBe(false);
+        const child = result.children[0].type;
+        expect(arrow.DataType.isInt(child)).toBe(true);
+        expect((child as arrow.Int).bitWidth).toBe(32);
+        expect((child as arrow.Int).isSigned).toBe(true);
+      });
+    });
+
+    whenUnsupported("types newer than the installed Arrow", function () {
+      it.each(
+        newerTypeTestCases.filter(([name]) => name !== "IntervalMonthDayNano"),
+      )(
+        "should explain that %s needs a newer apache-arrow",
+        function (name, makeForeign) {
+          expect(() => sanitizeType(makeForeign())).toThrow(
+            `The Arrow type ${name} requires apache-arrow`,
+          );
+        },
+      );
+
+      it("should sanitize IntervalMonthDayNano to a generic Interval", () => {
+        // The subclass shares `Type.Interval`, which every supported Arrow
+        // release has; only the unit is new.
+        const result = sanitizeType(new arrow21.IntervalMonthDayNano());
+        expect(result).toBeInstanceOf(arrow.Interval);
+        expect((result as arrow.Interval).unit).toBe(
+          arrow21.IntervalUnit.MONTH_DAY_NANO,
+        );
+      });
+    });
+
+    it("should rebuild a schema from a second copy of the same Arrow version", function () {
+      // Arrow 21 makes `instanceof Schema` succeed across copies of the library
+      // but not `instanceof <DataType>`; the schema must still be rebuilt.
+      const foreign = new arrow21.Schema([
+        new arrow21.Field("x", new arrow21.Float32(), true),
+      ]);
+      // Only Arrow 21+ overrides `Symbol.hasInstance` this way.
+      const crossCopyInstanceof = Object.getOwnPropertySymbols(
+        arrow.Schema,
+      ).includes(Symbol.hasInstance);
+      expect(foreign instanceof arrow.Schema).toBe(crossCopyInstanceof);
+      const result = sanitizeSchema(foreign);
+      expect(result).not.toBe(foreign);
+      const type = result.fields[0].type;
+      expect(type).toBeInstanceOf(arrow.DataType);
+      expect(arrow.DataType.isFloat(type)).toBe(true);
+      expect((type as arrow.Float).precision).toBe(arrow.Precision.SINGLE);
+      expect(() => arrow.makeBuilder({ type })).not.toThrow();
+    });
   });
 
   describe("sanitizeField function", function () {

nodejs/__test__/table.test.ts

@@ -7,14 +7,6 @@ import { pathToFileURL } from "node:url";
 import * as path from "path";
 import * as tmp from "tmp";
 
-import * as arrow15 from "apache-arrow-15";
-import * as arrow16 from "apache-arrow-16";
-import * as arrow17 from "apache-arrow-17";
-import * as arrow18 from "apache-arrow-18";
-import * as arrow19 from "apache-arrow-19";
-import * as arrow20 from "apache-arrow-20";
-import * as arrow21 from "apache-arrow-21";
-
 import {
   AutoQuery,
   Connection,
@@ -59,22 +51,21 @@ import {
   instanceOfFullTextQuery,
 } from "../lancedb/query";
 import { LocalTable } from "../lancedb/table";
+import {
+  type ApacheArrow,
+  arrow18,
+  arrowVersions,
+  latestArrow,
+} from "./arrow_versions";
 
-describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
+describe.each(arrowVersions)(
   "Given a table",
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   (arrow: any) => {
     let tmpDir: tmp.DirResult;
     let table: Table;
 
-    const schema:
-      | import("apache-arrow-15").Schema
-      | import("apache-arrow-16").Schema
-      | import("apache-arrow-17").Schema
-      | import("apache-arrow-18").Schema
-      | import("apache-arrow-19").Schema
-      | import("apache-arrow-20").Schema
-      | import("apache-arrow-21").Schema = new arrow.Schema([
+    const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
       new arrow.Field("id", new arrow.Float64(), true),
     ]);
 
@@ -748,6 +739,58 @@ describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
 );
 
 // https://github.com/lancedb/lancedb/issues/1963
+// These types were added in Arrow 21, so they are built with a second copy of
+// that release rather than the per-version matrix above, and reading them back
+// needs the library's own Arrow to know them too.
+const hostArrowSupportsViewTypes =
+  typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
+(hostArrowSupportsViewTypes ? it : it.skip)(
+  "creates and adds tables with LargeList, Utf8View and BinaryView columns",
+  async () => {
+    const tmpDir = tmp.dirSync({ unsafeCleanup: true });
+    const db = await connect(tmpDir.name);
+    const makeData = () =>
+      new latestArrow.Table({
+        id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
+        tags: latestArrow.vectorFromArray(
+          [["a", "b"], ["c"]],
+          new latestArrow.LargeList(
+            new latestArrow.Field("item", new latestArrow.Utf8(), true),
+          ),
+        ),
+        text: latestArrow.vectorFromArray(
+          ["short", "a string that is longer than the twelve inline bytes"],
+          new latestArrow.Utf8View(),
+        ),
+        bytes: latestArrow.vectorFromArray(
+          [new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
+          new latestArrow.BinaryView(),
+        ),
+      });
+    const table = await db.createTable("views", makeData());
+    await table.add(makeData());
+
+    const schema = await table.schema();
+    expect(schema.fields.map((f) => f.type.typeId)).toEqual([
+      arrow.Type.Int,
+      arrow.Type.LargeList,
+      // Lance stores the view types as their offset-based equivalents.
+      arrow.Type.Utf8,
+      arrow.Type.Binary,
+    ]);
+    const rows = await table.query().toArrow();
+    expect(rows.numRows).toBe(4);
+    const row = rows.get(1)!.toJSON();
+    expect(row.id).toBe(2);
+    expect(row.tags.toJSON()).toEqual(["c"]);
+    expect(row.text).toBe(
+      "a string that is longer than the twelve inline bytes",
+    );
+    expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
+    tmpDir.removeCallback();
+  },
+);
+
 it("should query documents with LangChain PDF metadata", async () => {
   const tmpDir = tmp.dirSync({ unsafeCleanup: true });
   try {
@@ -3209,7 +3252,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
   expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
 });
 
-describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
+describe.each(arrowVersions)(
   "when optimizing a dataset",
   // biome-ignore lint/suspicious/noExplicitAny: <explanation>
   (arrow: any) => {