fix(node): support LargeUtf8 and LargeBinary Arrow columns (#4287)

## Cause

The Arrow type sanitizer handled Utf8 and Binary but omitted LargeUtf8
and LargeBinary, so Arrow tables from another Arrow installation failed
schema sanitization before reaching LanceDB.

## Fix

Reconstruct both large Arrow types during sanitization. Regression tests
cover createTable and add with Arrow 15–18 and adding a LargeBinary
Arrow column to a blob table.

## Validation

- Confirmed the new table test failed before the fix with `Unrecognized
type id in schema: 20` on all four Arrow versions.
- The table, sanitizer, and Arrow Jest suites passed: 604 tests.
- `pnpm lint`, `pnpm build`, and `pnpm run docs` passed.

Fixes #4266

<!-- lance-gatekeeper-fix:v1 agent=1090de3c0ddeca587956632b79f0193c
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
lancedb-gatefixer[bot]
2026-09-29 12:25:12 -05:00
committed by GitHub
parent 85b0eba176
commit ad79112b8e
2 changed files with 59 additions and 0 deletions
+53
View File
@@ -92,6 +92,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
await expect(table.countRows()).resolves.toBe(3);
});
it("creates and adds Arrow tables with large string and binary columns", async () => {
const data = new arrow.Table({
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
bytes: arrow.vectorFromArray(
[Buffer.from("one"), Buffer.from("two")],
new arrow.LargeBinary(),
),
});
const conn = await connect(tmpDir.name);
const largeTable = await conn.createTable("large_columns", data);
await largeTable.add(data);
const rows = await largeTable.query().toArray();
expect(rows.map((row) => row.text)).toEqual([
"alpha",
"beta",
"alpha",
"beta",
]);
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
"one",
"two",
"one",
"two",
]);
const fields = (await largeTable.schema()).fields;
expect(fields.map((field) => field.type.typeId)).toEqual([
arrow.Type.LargeUtf8,
arrow.Type.LargeBinary,
]);
});
it("should support a foreign Float64 vector schema end to end", async () => {
const conn = await connect(tmpDir.name);
const schema = new arrow.Schema([
@@ -2601,6 +2633,27 @@ describe("when dealing with blob columns", () => {
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
});
it("adds a LargeBinary Arrow column to a blob table", async () => {
const db = await connect(tmpDir.name);
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
const table = await db.createTable(
"blobs",
[{ id: 1n, image: Buffer.from("alpha") }],
{ schema },
);
const payload = Buffer.from("beta");
const data = new arrow18.Table({
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
});
await table.add(data);
const rows = await table.query().withRowId().toArray();
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
const [actual] = await table.fetchBlobs("image", [rowId]);
expect(actual).toEqual(payload);
});
it("reads a half-open range", async () => {
const { table, rowIds } = await openBlobTable();
const files = await table.fetchBlobFiles("image", rowIds);
+6
View File
@@ -43,6 +43,8 @@ import {
Interval,
IntervalDayTime,
IntervalYearMonth,
LargeBinary,
LargeUtf8,
List,
Map_,
Null,
@@ -465,8 +467,12 @@ function sanitizeTypeById(
return sanitizeFloat(typeLike);
case Type.Binary:
return new Binary();
case Type.LargeBinary:
return new LargeBinary();
case Type.Utf8:
return new Utf8();
case Type.LargeUtf8:
return new LargeUtf8();
case Type.Bool:
return new Bool();
case Type.Decimal: