mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-21 11:12:27 +00:00
fix(node): support LargeUtf8 and LargeBinary Arrow columns (#4287)
## Cause The Arrow type sanitizer handled Utf8 and Binary but omitted LargeUtf8 and LargeBinary, so Arrow tables from another Arrow installation failed schema sanitization before reaching LanceDB. ## Fix Reconstruct both large Arrow types during sanitization. Regression tests cover createTable and add with Arrow 15–18 and adding a LargeBinary Arrow column to a blob table. ## Validation - Confirmed the new table test failed before the fix with `Unrecognized type id in schema: 20` on all four Arrow versions. - The table, sanitizer, and Arrow Jest suites passed: 604 tests. - `pnpm lint`, `pnpm build`, and `pnpm run docs` passed. Fixes #4266 <!-- lance-gatekeeper-fix:v1 agent=1090de3c0ddeca587956632b79f0193c generation=1 --> Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
@@ -92,6 +92,38 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
await expect(table.countRows()).resolves.toBe(3);
|
||||
});
|
||||
|
||||
it("creates and adds Arrow tables with large string and binary columns", async () => {
|
||||
const data = new arrow.Table({
|
||||
text: arrow.vectorFromArray(["alpha", "beta"], new arrow.LargeUtf8()),
|
||||
bytes: arrow.vectorFromArray(
|
||||
[Buffer.from("one"), Buffer.from("two")],
|
||||
new arrow.LargeBinary(),
|
||||
),
|
||||
});
|
||||
const conn = await connect(tmpDir.name);
|
||||
const largeTable = await conn.createTable("large_columns", data);
|
||||
await largeTable.add(data);
|
||||
|
||||
const rows = await largeTable.query().toArray();
|
||||
expect(rows.map((row) => row.text)).toEqual([
|
||||
"alpha",
|
||||
"beta",
|
||||
"alpha",
|
||||
"beta",
|
||||
]);
|
||||
expect(rows.map((row) => Buffer.from(row.bytes).toString())).toEqual([
|
||||
"one",
|
||||
"two",
|
||||
"one",
|
||||
"two",
|
||||
]);
|
||||
const fields = (await largeTable.schema()).fields;
|
||||
expect(fields.map((field) => field.type.typeId)).toEqual([
|
||||
arrow.Type.LargeUtf8,
|
||||
arrow.Type.LargeBinary,
|
||||
]);
|
||||
});
|
||||
|
||||
it("should support a foreign Float64 vector schema end to end", async () => {
|
||||
const conn = await connect(tmpDir.name);
|
||||
const schema = new arrow.Schema([
|
||||
@@ -2601,6 +2633,27 @@ describe("when dealing with blob columns", () => {
|
||||
expect(Buffer.from(await files[1]!.read()).toString()).toBe("beta");
|
||||
});
|
||||
|
||||
it("adds a LargeBinary Arrow column to a blob table", async () => {
|
||||
const db = await connect(tmpDir.name);
|
||||
const schema = new Schema([new Field("id", new Int64()), blob("image")]);
|
||||
const table = await db.createTable(
|
||||
"blobs",
|
||||
[{ id: 1n, image: Buffer.from("alpha") }],
|
||||
{ schema },
|
||||
);
|
||||
const payload = Buffer.from("beta");
|
||||
const data = new arrow18.Table({
|
||||
id: arrow18.vectorFromArray([2n], new arrow18.Int64()),
|
||||
image: arrow18.vectorFromArray([payload], new arrow18.LargeBinary()),
|
||||
});
|
||||
await table.add(data);
|
||||
|
||||
const rows = await table.query().withRowId().toArray();
|
||||
const rowId = rows.find((row) => row.id === 2n)!._rowid as bigint;
|
||||
const [actual] = await table.fetchBlobs("image", [rowId]);
|
||||
expect(actual).toEqual(payload);
|
||||
});
|
||||
|
||||
it("reads a half-open range", async () => {
|
||||
const { table, rowIds } = await openBlobTable();
|
||||
const files = await table.fetchBlobFiles("image", rowIds);
|
||||
|
||||
@@ -43,6 +43,8 @@ import {
|
||||
Interval,
|
||||
IntervalDayTime,
|
||||
IntervalYearMonth,
|
||||
LargeBinary,
|
||||
LargeUtf8,
|
||||
List,
|
||||
Map_,
|
||||
Null,
|
||||
@@ -465,8 +467,12 @@ function sanitizeTypeById(
|
||||
return sanitizeFloat(typeLike);
|
||||
case Type.Binary:
|
||||
return new Binary();
|
||||
case Type.LargeBinary:
|
||||
return new LargeBinary();
|
||||
case Type.Utf8:
|
||||
return new Utf8();
|
||||
case Type.LargeUtf8:
|
||||
return new LargeUtf8();
|
||||
case Type.Bool:
|
||||
return new Bool();
|
||||
case Type.Decimal:
|
||||
|
||||
Reference in New Issue
Block a user