feat(mv): materialize dedup results with direct representatives

nodejs/__test__/materialized_view.test.ts

@@ -52,9 +52,19 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
+    for (const [format, fn] of [
+      [3, "vector_duplicate_pairs"],
+      [4, "vector_dedup"],
+    ]) {
+      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
+      expect(read(JSON.stringify({ format, query: native })).query).toBe(
+        native,
+      );
+    }
+
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":3,"query":${JSON.stringify(query)}}`,
+      `{"format":5,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);

nodejs/lancedb/materialized_view.ts

@@ -9,9 +9,10 @@ export const DEFINITION_META_KEY = "mv.definition";
 
 /**
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
- * format 2 being a query with `GROUP BY`.
+ * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
+ * native dedup results.
  */
-export const DEFINITION_FORMAT = 2;
+export const DEFINITION_FORMAT = 4;
 
 /**
  * The query that defines a materialized view, as stored: