From ac99e4dce5e02281746b9dbfd69c2554c0d899c9 Mon Sep 17 00:00:00 2001 From: kid <19265318+u70b3@users.noreply.github.com> Date: Wed, 22 Jul 2026 00:28:57 +0800 Subject: [PATCH] fix(node): sanitize Map fields across Arrow versions (#3650) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - reconstruct foreign Arrow Map schemas from their single sanitized entries field - reject malformed Map types with anything other than one child - preserve the complete Map schema and `keysSorted` value through empty-table creation and IPC round trips across Arrow 15–18 ## Testing - `./node_modules/.bin/jest --runInBand __test__/arrow.test.ts __test__/sanitize.test.ts` - `pnpm lint` - `pnpm build` - `pnpm run docs` Fixes #2337 --- nodejs/__test__/arrow.test.ts | 53 ++++++++++++++++++++++++++++++++ nodejs/__test__/sanitize.test.ts | 13 +++++++- nodejs/lancedb/sanitize.ts | 9 +++--- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 417c51c30..2735f7f72 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -52,6 +52,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( Float64, Struct, List, + Map_, Int16, Int32, Int64, @@ -69,6 +70,30 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( type Schema = ApacheArrow["Schema"]; type Table = ApacheArrow["Table"]; + function expectValidMapField( + // biome-ignore lint/suspicious/noExplicitAny: Arrow Field types vary across supported versions + field: any, + ): void { + expect(DataType.isMap(field.type)).toBe(true); + expect(field.type.keysSorted).toBe(true); + expect(field.type.children).toHaveLength(1); + + const entries = field.type.children[0]; + expect(entries.name).toBe("entries"); + expect(entries.nullable).toBe(false); + expect(DataType.isStruct(entries.type)).toBe(true); + expect(entries.type.children).toHaveLength(2); + + const [key, value] = entries.type.children; + expect([key.name, value.name]).toEqual(["key", "value"]); + expect(key.nullable).toBe(false); + expect(DataType.isUtf8(key.type)).toBe(true); + expect(value.nullable).toBe(true); + expect(DataType.isInt(value.type)).toBe(true); + expect(value.type.bitWidth).toBe(32); + expect(value.type.isSigned).toBe(true); + } + // Helper method to verify various ways to create a table async function checkTableCreation( tableCreationMethod: ( @@ -938,6 +963,34 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( false, ); }); + + it("will make an empty table with a Map field", async function () { + const schema = new Schema([ + new Field( + "attributes", + new Map_( + new Field( + "entries", + new Struct([ + new Field("key", new Utf8(), false), + new Field("value", new Int32(), true), + ]), + false, + ), + true, + ), + ), + ]); + + const table = makeEmptyTable(schema); + + expectValidMapField(table.schema.fields[0]); + + const buffer = await fromTableToBuffer(table); + const roundTripped = tableFromIPC(buffer); + + expectValidMapField(roundTripped.schema.fields[0]); + }); }); describe("when using two versions of arrow", function () { diff --git a/nodejs/__test__/sanitize.test.ts b/nodejs/__test__/sanitize.test.ts index 022b845e1..587805d53 100644 --- a/nodejs/__test__/sanitize.test.ts +++ b/nodejs/__test__/sanitize.test.ts @@ -2,7 +2,7 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors import * as arrow from "../lancedb/arrow"; -import { sanitizeField, sanitizeType } from "../lancedb/sanitize"; +import { sanitizeField, sanitizeMap, sanitizeType } from "../lancedb/sanitize"; describe("sanitize", function () { describe("sanitizeType function", function () { @@ -181,4 +181,15 @@ describe("sanitize", function () { ); }); }); + + describe("sanitizeMap function", function () { + it.each([ + ["no children", []], + ["two children", [{}, {}]], + ])("should reject a Map type with %s", function (_, children) { + expect(() => sanitizeMap({ children, keysSorted: false })).toThrow( + "Expected a Map type to have exactly one child", + ); + }); + }); }); diff --git a/nodejs/lancedb/sanitize.ts b/nodejs/lancedb/sanitize.ts index 6108ff334..9f834a7f6 100644 --- a/nodejs/lancedb/sanitize.ts +++ b/nodejs/lancedb/sanitize.ts @@ -288,12 +288,11 @@ export function sanitizeMap(typeLike: object) { if (!("keysSorted" in typeLike) || typeof typeLike.keysSorted !== "boolean") { throw Error("Expected a Map type to have a `keysSorted` property"); } + if (typeLike.children.length !== 1) { + throw Error("Expected a Map type to have exactly one child"); + } - return new Map_( - // biome-ignore lint/suspicious/noExplicitAny: skip - typeLike.children.map((field) => sanitizeField(field)) as any, - typeLike.keysSorted, - ); + return new Map_(sanitizeField(typeLike.children[0]), typeLike.keysSorted); } export function sanitizeDuration(typeLike: object) {