mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
fix(node): sanitize Map fields across Arrow versions (#3650)
## Summary - reconstruct foreign Arrow Map schemas from their single sanitized entries field - reject malformed Map types with anything other than one child - preserve the complete Map schema and `keysSorted` value through empty-table creation and IPC round trips across Arrow 15–18 ## Testing - `./node_modules/.bin/jest --runInBand __test__/arrow.test.ts __test__/sanitize.test.ts` - `pnpm lint` - `pnpm build` - `pnpm run docs` Fixes #2337
This commit is contained in:
@@ -52,6 +52,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
Float64,
|
||||
Struct,
|
||||
List,
|
||||
Map_,
|
||||
Int16,
|
||||
Int32,
|
||||
Int64,
|
||||
@@ -69,6 +70,30 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
type Schema = ApacheArrow["Schema"];
|
||||
type Table = ApacheArrow["Table"];
|
||||
|
||||
function expectValidMapField(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions
|
||||
field: any,
|
||||
): void {
|
||||
expect(DataType.isMap(field.type)).toBe(true);
|
||||
expect(field.type.keysSorted).toBe(true);
|
||||
expect(field.type.children).toHaveLength(1);
|
||||
|
||||
const entries = field.type.children[0];
|
||||
expect(entries.name).toBe("entries");
|
||||
expect(entries.nullable).toBe(false);
|
||||
expect(DataType.isStruct(entries.type)).toBe(true);
|
||||
expect(entries.type.children).toHaveLength(2);
|
||||
|
||||
const [key, value] = entries.type.children;
|
||||
expect([key.name, value.name]).toEqual(["key", "value"]);
|
||||
expect(key.nullable).toBe(false);
|
||||
expect(DataType.isUtf8(key.type)).toBe(true);
|
||||
expect(value.nullable).toBe(true);
|
||||
expect(DataType.isInt(value.type)).toBe(true);
|
||||
expect(value.type.bitWidth).toBe(32);
|
||||
expect(value.type.isSigned).toBe(true);
|
||||
}
|
||||
|
||||
// Helper method to verify various ways to create a table
|
||||
async function checkTableCreation(
|
||||
tableCreationMethod: (
|
||||
@@ -938,6 +963,34 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("will make an empty table with a Map field", async function () {
|
||||
const schema = new Schema([
|
||||
new Field(
|
||||
"attributes",
|
||||
new Map_(
|
||||
new Field(
|
||||
"entries",
|
||||
new Struct([
|
||||
new Field("key", new Utf8(), false),
|
||||
new Field("value", new Int32(), true),
|
||||
]),
|
||||
false,
|
||||
),
|
||||
true,
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const table = makeEmptyTable(schema);
|
||||
|
||||
expectValidMapField(table.schema.fields[0]);
|
||||
|
||||
const buffer = await fromTableToBuffer(table);
|
||||
const roundTripped = tableFromIPC(buffer);
|
||||
|
||||
expectValidMapField(roundTripped.schema.fields[0]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("when using two versions of arrow", function () {
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import * as arrow from "../lancedb/arrow";
|
||||
import { sanitizeField, sanitizeType } from "../lancedb/sanitize";
|
||||
import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize";
|
||||
|
||||
describe("sanitize", function () {
|
||||
describe("sanitizeType function", function () {
|
||||
@@ -181,4 +181,15 @@ describe("sanitize", function () {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sanitizeMap function", function () {
|
||||
it.each([
|
||||
["no children", []],
|
||||
["two children", [{}, {}]],
|
||||
])("should reject a Map type with %s", function (_, children) {
|
||||
expect(() => sanitizeMap({ children, keysSorted: false })).toThrow(
|
||||
"Expected a Map type to have exactly one child",
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -288,12 +288,11 @@ export function sanitizeMap(typeLike: object) {
|
||||
if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") {
|
||||
throw Error("Expected a Map type to have a `keysSorted` property");
|
||||
}
|
||||
if (typeLike.children.length !== 1) {
|
||||
throw Error("Expected a Map type to have exactly one child");
|
||||
}
|
||||
|
||||
return new Map_(
|
||||
// biome-ignore lint/suspicious/noExplicitAny: skip
|
||||
typeLike.children.map((field) => sanitizeField(field)) as any,
|
||||
typeLike.keysSorted,
|
||||
);
|
||||
return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted);
|
||||
}
|
||||
|
||||
export function sanitizeDuration(typeLike: object) {
|
||||
|
||||
Reference in New Issue
Block a user