mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-24 20:52:29 +00:00
fix(node): support Apache Arrow 19 through 21 and run the library on each
Builds on #4359 (which added the Arrow 19-21 test aliases) and makes the package genuinely work across the whole peer range: * Resolve the library's own `apache-arrow` to 21.2.0 (devDependency) so the main CI jobs build, type-check and test on the newest release, and widen the peer range to `>=15.0.0 <22.0.0` instead of an exact pin. * Add a CI job that re-installs with `apache-arrow` resolved to 19.0.1 and 20.0.0 and runs the suite, covering what a consumer on those versions gets. * Arrow 21 overrides `Symbol.hasInstance` on Schema/Field/Table/RecordBatch/ Data so `instanceof` matches objects from another copy of the library, but not on DataType. The sanitizer's short-circuits therefore returned foreign schemas unchanged and Arrow's builders then failed with "Unrecognized type 'NONE'". Check the real prototype chain instead. * Sanitize the types added in Arrow 20/21 (LargeList, Utf8View, BinaryView, IntervalMonthDayNano), looked up dynamically so the library still compiles and runs on older Arrow releases, with a clear error when the installed Arrow lacks the type. Carry `variadicBuffers` through `Data`. * Loosen `FieldLike` so a branded DataType from another Arrow copy still type-checks; export the new `DataTypeLike`. * Share the Arrow version list between test files, exercise embedding function types per version in the registry tests, and add sanitize and end-to-end coverage for the new column types. * Enable `skipLibCheck`: flatbuffers 25 (via Arrow 21) ships declarations that need TypeScript 5.7+. * Record the new dev dependencies in the third-party license inventory. Co-Authored-By: Vincent Koc <vincentkoc@ieee.org> Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Vincent Koc
Claude Fable 5.1
parent
8dfd371031
commit
003b87bfa0
@@ -7,14 +7,6 @@ import { pathToFileURL } from "node:url";
|
||||
import * as path from "path";
|
||||
import * as tmp from "tmp";
|
||||
|
||||
import * as arrow15 from "apache-arrow-15";
|
||||
import * as arrow16 from "apache-arrow-16";
|
||||
import * as arrow17 from "apache-arrow-17";
|
||||
import * as arrow18 from "apache-arrow-18";
|
||||
import * as arrow19 from "apache-arrow-19";
|
||||
import * as arrow20 from "apache-arrow-20";
|
||||
import * as arrow21 from "apache-arrow-21";
|
||||
|
||||
import {
|
||||
AutoQuery,
|
||||
Connection,
|
||||
@@ -59,22 +51,21 @@ import {
|
||||
instanceOfFullTextQuery,
|
||||
} from "../lancedb/query";
|
||||
import { LocalTable } from "../lancedb/table";
|
||||
import {
|
||||
type ApacheArrow,
|
||||
arrow18,
|
||||
arrowVersions,
|
||||
latestArrow,
|
||||
} from "./arrow_versions";
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
|
||||
describe.each(arrowVersions)(
|
||||
"Given a table",
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
(arrow: any) => {
|
||||
let tmpDir: tmp.DirResult;
|
||||
let table: Table;
|
||||
|
||||
const schema:
|
||||
| import("apache-arrow-15").Schema
|
||||
| import("apache-arrow-16").Schema
|
||||
| import("apache-arrow-17").Schema
|
||||
| import("apache-arrow-18").Schema
|
||||
| import("apache-arrow-19").Schema
|
||||
| import("apache-arrow-20").Schema
|
||||
| import("apache-arrow-21").Schema = new arrow.Schema([
|
||||
const schema: InstanceType<ApacheArrow["Schema"]> = new arrow.Schema([
|
||||
new arrow.Field("id", new arrow.Float64(), true),
|
||||
]);
|
||||
|
||||
@@ -748,6 +739,58 @@ describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
|
||||
);
|
||||
|
||||
// https://github.com/lancedb/lancedb/issues/1963
|
||||
// These types were added in Arrow 21, so they are built with a second copy of
|
||||
// that release rather than the per-version matrix above, and reading them back
|
||||
// needs the library's own Arrow to know them too.
|
||||
const hostArrowSupportsViewTypes =
|
||||
typeof (arrow as unknown as Record<string, unknown>).Utf8View === "function";
|
||||
(hostArrowSupportsViewTypes ? it : it.skip)(
|
||||
"creates and adds tables with LargeList, Utf8View and BinaryView columns",
|
||||
async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
const db = await connect(tmpDir.name);
|
||||
const makeData = () =>
|
||||
new latestArrow.Table({
|
||||
id: latestArrow.vectorFromArray([1, 2], new latestArrow.Int32()),
|
||||
tags: latestArrow.vectorFromArray(
|
||||
[["a", "b"], ["c"]],
|
||||
new latestArrow.LargeList(
|
||||
new latestArrow.Field("item", new latestArrow.Utf8(), true),
|
||||
),
|
||||
),
|
||||
text: latestArrow.vectorFromArray(
|
||||
["short", "a string that is longer than the twelve inline bytes"],
|
||||
new latestArrow.Utf8View(),
|
||||
),
|
||||
bytes: latestArrow.vectorFromArray(
|
||||
[new Uint8Array([1, 2, 3]), new Uint8Array(20).fill(7)],
|
||||
new latestArrow.BinaryView(),
|
||||
),
|
||||
});
|
||||
const table = await db.createTable("views", makeData());
|
||||
await table.add(makeData());
|
||||
|
||||
const schema = await table.schema();
|
||||
expect(schema.fields.map((f) => f.type.typeId)).toEqual([
|
||||
arrow.Type.Int,
|
||||
arrow.Type.LargeList,
|
||||
// Lance stores the view types as their offset-based equivalents.
|
||||
arrow.Type.Utf8,
|
||||
arrow.Type.Binary,
|
||||
]);
|
||||
const rows = await table.query().toArrow();
|
||||
expect(rows.numRows).toBe(4);
|
||||
const row = rows.get(1)!.toJSON();
|
||||
expect(row.id).toBe(2);
|
||||
expect(row.tags.toJSON()).toEqual(["c"]);
|
||||
expect(row.text).toBe(
|
||||
"a string that is longer than the twelve inline bytes",
|
||||
);
|
||||
expect(Array.from(row.bytes)).toEqual(Array(20).fill(7));
|
||||
tmpDir.removeCallback();
|
||||
},
|
||||
);
|
||||
|
||||
it("should query documents with LangChain PDF metadata", async () => {
|
||||
const tmpDir = tmp.dirSync({ unsafeCleanup: true });
|
||||
try {
|
||||
@@ -3209,7 +3252,7 @@ it("passes cleanupOlderThan to the native binding as an absolute timestamp", asy
|
||||
expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true);
|
||||
});
|
||||
|
||||
describe.each([arrow15, arrow16, arrow17, arrow18, arrow19, arrow20, arrow21])(
|
||||
describe.each(arrowVersions)(
|
||||
"when optimizing a dataset",
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
(arrow: any) => {
|
||||
|
||||
Reference in New Issue
Block a user