fix: sanitize foreign schemas (#1058)

Arrow-js uses brittle `instanceof` checks throughout the code base.
These fail unless the library instance that produced the object matches
exactly the same instance the vectordb is using. At a minimum, this
means that a user using arrow version 15 (or any version that doesn't
match exactly the version that vectordb is using) will get strange
errors when they try and use vectordb.

However, there are even cases where the versions can be perfectly
identical, and the instanceof check still fails. One such example is
when using `vite` (e.g. https://github.com/vitejs/vite/issues/3910)

This PR solves the problem in a rather brute force, but workable,
fashion. If we encounter a schema that does not pass the `instanceof`
check then we will attempt to sanitize that schema by traversing the
object and, if it has all the correct properties, constructing an
appropriate `Schema` instance via deep cloning.

nodejs/__test__/arrow.test.ts

@@ -38,9 +38,22 @@ import {
   Int64,
   Float,
   Precision,
+  MetadataVersion,
 } from "apache-arrow";
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8,
+} from "apache-arrow-old";
 import { type EmbeddingFunction } from "../dist/embedding/embedding_function";
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
 function sampleRecords(): Array<Record<string, any>> {
   return [
     {
@@ -57,8 +70,8 @@ function sampleRecords(): Array<Record<string, any>> {
 // Helper method to verify various ways to create a table
 async function checkTableCreation(
   tableCreationMethod: (
-    records: any,
-    recordsReversed: any,
+    records: Record<string, unknown>[],
+    recordsReversed: Record<string, unknown>[],
     schema: Schema,
   ) => Promise<Table>,
   infersTypes: boolean,
@@ -402,3 +415,56 @@ describe("makeEmptyTable", function () {
     );
   });
 });
+
+describe("when using two versions of arrow", function () {
+  it("can still import data", async function () {
+    const schema = new OldSchema([
+      new OldField("id", new OldInt32()),
+      new OldField(
+        "vector",
+        new OldFixedSizeList(
+          1024,
+          new OldField("item", new OldFloat32(), true),
+        ),
+      ),
+      new OldField(
+        "struct",
+        new OldStruct([
+          new OldField(
+            "nested",
+            new OldDictionary(new OldUtf8(), new OldInt32(), 1, true),
+          ),
+          new OldField("ts_with_tz", new OldTimestampNanosecond("some_tz")),
+          new OldField("ts_no_tz", new OldTimestampNanosecond(null)),
+        ]),
+      ),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    ]) as any;
+    schema.metadataVersion = MetadataVersion.V5;
+    const table = makeArrowTable([], { schema });
+
+    const buf = await fromTableToBuffer(table);
+    expect(buf.byteLength).toBeGreaterThan(0);
+    const actual = tableFromIPC(buf);
+    const actualSchema = actual.schema;
+    expect(actualSchema.fields.length).toBe(3);
+
+    // Deep equality gets hung up on some very minor unimportant differences
+    // between arrow version 13 and 15 which isn't really what we're testing for
+    // and so we do our own comparison that just checks name/type/nullability
+    function compareFields(lhs: Field, rhs: Field) {
+      expect(lhs.name).toEqual(rhs.name);
+      expect(lhs.nullable).toEqual(rhs.nullable);
+      expect(lhs.typeId).toEqual(rhs.typeId);
+      if ("children" in lhs.type && lhs.type.children !== null) {
+        const lhs_children = lhs.type.children as Field[];
+        lhs_children.forEach((child: Field, idx) => {
+          compareFields(child, rhs.type.children[idx]);
+        });
+      }
+    }
+    actualSchema.fields.forEach((field, idx) => {
+      compareFields(field, actualSchema.fields[idx]);
+    });
+  });
+});

nodejs/__test__/table.test.ts

@@ -103,12 +103,12 @@ describe("Test creating index", () => {
     // TODO: check index type.
 
     // Search without specifying the column
-    let query_vector = data.toArray()[5].vec.toJSON();
-    let rst = await tbl.query().nearestTo(query_vector).limit(2).toArrow();
+    const query_vector = data.toArray()[5].vec.toJSON();
+    const rst = await tbl.query().nearestTo(query_vector).limit(2).toArrow();
     expect(rst.numRows).toBe(2);
 
     // Search with specifying the column
-    let rst2 = await tbl.search(query_vector, "vec").limit(2).toArrow();
+    const rst2 = await tbl.search(query_vector, "vec").limit(2).toArrow();
     expect(rst2.numRows).toBe(2);
     expect(rst.toString()).toEqual(rst2.toString());
   });