fix(node): support Apache Arrow 19 through 21 and run the library on each

Builds on #4359 (which added the Arrow 19-21 test aliases) and makes the
package genuinely work across the whole peer range:

* Resolve the library's own `apache-arrow` to 21.2.0 (devDependency) so
  the main CI jobs build, type-check and test on the newest release, and
  widen the peer range to `>=15.0.0 <22.0.0` instead of an exact pin.
* Add a CI job that re-installs with `apache-arrow` resolved to 19.0.1 and
  20.0.0 and runs the suite, covering what a consumer on those versions
  gets.
* Arrow 21 overrides `Symbol.hasInstance` on Schema/Field/Table/RecordBatch/
  Data so `instanceof` matches objects from another copy of the library,
  but not on DataType. The sanitizer's short-circuits therefore returned
  foreign schemas unchanged and Arrow's builders then failed with
  "Unrecognized type 'NONE'". Check the real prototype chain instead.
* Sanitize the types added in Arrow 20/21 (LargeList, Utf8View, BinaryView,
  IntervalMonthDayNano), looked up dynamically so the library still compiles
  and runs on older Arrow releases, with a clear error when the installed
  Arrow lacks the type. Carry `variadicBuffers` through `Data`.
* Loosen `FieldLike` so a branded DataType from another Arrow copy still
  type-checks; export the new `DataTypeLike`.
* Share the Arrow version list between test files, exercise embedding
  function types per version in the registry tests, and add sanitize and
  end-to-end coverage for the new column types.
* Enable `skipLibCheck`: flatbuffers 25 (via Arrow 21) ships declarations
  that need TypeScript 5.7+.
* Record the new dev dependencies in the third-party license inventory.

Co-Authored-By: Vincent Koc <vincentkoc@ieee.org>
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Dan Tasse
2026-10-01 11:21:59 -05:00
co-authored by Vincent Koc Claude Fable 5.1
parent 8dfd371031
commit 003b87bfa0
25 changed files with 1876 additions and 1558 deletions
+61 -18
View File
@@ -7,14 +7,6 @@ import { pathToFileURL } from "node:url";
import * as path from "path";
import * as tmp from "tmp";
import * as arrow15 from "apache-arrow-15";
import * as arrow16 from "apache-arrow-16";
import * as arrow17 from "apache-arrow-17";
import * as arrow18 from "apache-arrow-18";
import * as arrow19 from "apache-arrow-19";
import * as arrow20 from "apache-arrow-20";
import * as arrow21 from "apache-arrow-21";
import {
AutoQuery,
Connection,
@@ -59,22 +51,21 @@ import {
instanceOfFullTextQuery,
} from "../lancedb/query";
import { LocalTable } from "../lancedb/table";
import {
type ApacheArrow,
arrow18,
arrowVersions,
latestArrow,
} from "./arrow_versions";
describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
describe.each(arrowVersions)(
"Given a table",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {
let tmpDir: tmp.DirResult;
let table: Table;
const schema:
| import("apache-arrow-15").Schema
| import("apache-arrow-16").Schema
| import("apache-arrow-17").Schema
| import("apache-arrow-18").Schema
| import("apache-arrow-19").Schema
| import("apache-arrow-20").Schema
| import("apache-arrow-21").Schema = new arrow.Schema([
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
new arrow.Field("id", new arrow.Float64(), true),
]);
@@ -748,6 +739,58 @@ describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
);
// https://github.com/lancedb/lancedb/issues/1963
// These types were added in Arrow 21, so they are built with a second copy of
// that release rather than the per-version matrix above, and reading them back
// needs the library's own Arrow to know them too.
const hostArrowSupportsViewTypes =
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
(hostArrowSupportsViewTypes ? it : it.skip)(
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
const db = await connect(tmpDir.name);
const makeData = () =>
new latestArrow.Table({
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
tags: latestArrow.vectorFromArray(
[["a", "b"], ["c"]],
new latestArrow.LargeList(
new latestArrow.Field("item", new latestArrow.Utf8(), true),
),
),
text: latestArrow.vectorFromArray(
["short", "a string that is longer than the twelve inline bytes"],
new latestArrow.Utf8View(),
),
bytes: latestArrow.vectorFromArray(
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
new latestArrow.BinaryView(),
),
});
const table = await db.createTable("views", makeData());
await table.add(makeData());
const schema = await table.schema();
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
arrow.Type.Int,
arrow.Type.LargeList,
// Lance stores the view types as their offset-based equivalents.
arrow.Type.Utf8,
arrow.Type.Binary,
]);
const rows = await table.query().toArrow();
expect(rows.numRows).toBe(4);
const row = rows.get(1)!.toJSON();
expect(row.id).toBe(2);
expect(row.tags.toJSON()).toEqual(["c"]);
expect(row.text).toBe(
"a string that is longer than the twelve inline bytes",
);
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
tmpDir.removeCallback();
},
);
it("should query documents with LangChain PDF metadata", async () => {
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
try {
@@ -3209,7 +3252,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
});
describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
describe.each(arrowVersions)(
"when optimizing a dataset",
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
(arrow: any) => {