refactor: always fully refresh materialized views

nodejs/__test__/materialized_view.test.ts

@@ -16,15 +16,11 @@ describe("materialized views", () => {
   beforeEach(async () => {
     tmpDir = tmp.dirSync({ unsafeCleanup: true });
     db = await connect(tmpDir.name);
-    await db.createTable(
-      "people",
-      [
-        { name: "ada", age: 36 },
-        { name: "kid", age: 7 },
-        { name: "grace", age: 85 },
-      ],
-      { storageOptions: { newTableEnableStableRowIds: "true" } },
-    );
+    await db.createTable("people", [
+      { name: "ada", age: 36 },
+      { name: "kid", age: 7 },
+      { name: "grace", age: 85 },
+    ]);
   });
   afterEach(() => tmpDir.removeCallback());
 
@@ -52,19 +48,9 @@ describe("materialized views", () => {
       "SELECT * FROM people",
     );
 
-    for (const [format, fn] of [
-      [3, "vector_duplicate_pairs"],
-      [4, "vector_dedup"],
-    ]) {
-      const native = `SELECT * FROM ${fn}('images', 3, 'phash', 4)`;
-      expect(read(JSON.stringify({ format, query: native })).query).toBe(
-        native,
-      );
-    }
-
     // A newer writer's layout is reported, never guessed at.
     for (const newer of [
-      `{"format":5,"query":${JSON.stringify(query)}}`,
+      `{"format":3,"query":${JSON.stringify(query)}}`,
       '{"kind":"select_v3","source_table":"people"}',
     ]) {
       expect(() => read(newer)).toThrow(/cannot refresh/);
@@ -106,7 +92,7 @@ describe("materialized views", () => {
     );
   });
 
-  it("refreshes incrementally after an append", async () => {
+  it("fully refreshes after an append", async () => {
     const view = await db.createMaterializedView("copy", "people", {
       withNoData: true,
     });
@@ -115,11 +101,11 @@ describe("materialized views", () => {
     const people = await db.openTable("people");
     await people.add([{ name: "alan", age: 41 }]);
     const result = await view.refresh();
-    expect(result.mode).toBe("incremental");
-    expect(Number(result.rowsWritten)).toBe(1);
+    expect(result.mode).toBe("rebuild");
+    expect(Number(result.rowsWritten)).toBe(4);
     expect(await view.table().countRows()).toBe(4);
 
-    expect((await view.refresh()).mode).toBe("no_op");
+    expect((await view.refresh()).mode).toBe("rebuild");
   });
 
   it("lists views and rejects non-views", async () => {
@@ -172,9 +158,7 @@ describe("materialized views", () => {
   });
 
   it("quotes bare select names", async () => {
-    await db.createTable("odd_names", [{ "order item": "widget" }], {
-      storageOptions: { newTableEnableStableRowIds: "true" },
-    });
+    await db.createTable("odd_names", [{ "order item": "widget" }]);
     const view = await db.createMaterializedView("quoted", "odd_names", {
       select: ["order item"],
       withNoData: true,
@@ -183,10 +167,9 @@ describe("materialized views", () => {
     expect(Number(result.rowsWritten)).toBe(1);
   });
 
-  it("requires stable row ids on the source", async () => {
+  it("does not require stable row ids on the source", async () => {
     await db.createTable("plain", [{ x: 1 }]);
-    await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
-      "stable row ids",
-    );
+    const view = await db.createMaterializedView("v", "plain");
+    expect(await view.table().countRows()).toBe(1);
   });
 });

nodejs/lancedb/materialized_view.ts

@@ -9,10 +9,9 @@ export const DEFINITION_META_KEY = "mv.definition";
 
 /**
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
- * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
- * native dedup results.
+ * format 2 being a query with `GROUP BY`.
  */
-export const DEFINITION_FORMAT = 4;
+export const DEFINITION_FORMAT = 2;
 
 /**
  * The query that defines a materialized view, as stored:
@@ -188,23 +187,13 @@ export class MaterializedView {
   /**
    * Recompute the view from its source.
    *
-   * The refresh is incremental when the source's changes can be reconciled
-   * into the view -- rows added, changed or removed since the last one --
-   * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
-   * refreshes to that source version instead of the latest.
-   *
-   * Concurrent refreshes of one view do not duplicate its rows. Two that
-   * plan the same source rows conflict on commit, and the loser throws
-   * rather than writing them a second time.
+   * Every refresh rebuilds the complete view. `sourceVersion` refreshes to
+   * that source version instead of the latest.
    */
   async refresh(options?: {
-    full?: boolean;
     sourceVersion?: number;
   }): Promise<RefreshMaterializedViewResult> {
     validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
-    return await this.inner.refreshMaterializedView(
-      options?.full,
-      options?.sourceVersion,
-    );
+    return await this.inner.refreshMaterializedView(options?.sourceVersion);
   }
 }

nodejs/lancedb/table.ts

@@ -685,7 +685,6 @@ export abstract class Table {
    * @ignore
    */
   abstract refreshMaterializedView(
-    full?: boolean,
     sourceVersion?: number,
   ): Promise<RefreshMaterializedViewResult>;
 
@@ -1420,10 +1419,9 @@ export class LocalTable extends Table {
   }
 
   async refreshMaterializedView(
-    full?: boolean,
     sourceVersion?: number,
   ): Promise<RefreshMaterializedViewResult> {
-    return await this.inner.refreshMaterializedView(full, sourceVersion);
+    return await this.inner.refreshMaterializedView(sourceVersion);
   }
 
   async materializedViewDefinition(): Promise<string> {

nodejs/src/table.rs

@@ -469,13 +469,12 @@ impl Table {
     #[napi(catch_unwind)]
     pub async fn refresh_materialized_view(
         &self,
-        full: Option<bool>,
         source_version: Option<i64>,
     ) -> napi::Result<RefreshMaterializedViewResult> {
         let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
             .await
             .default_error()?;
-        let mut builder = view.refresh().full(full.unwrap_or(false));
+        let mut builder = view.refresh();
         if let Some(version) = source_version {
             let version = u64::try_from(version).map_err(|_| {
                 napi::Error::from_reason("sourceVersion must be a non-negative integer")
@@ -1608,7 +1607,7 @@ impl From<lancedb::function::FunctionErrors> for FunctionErrors {
 
 #[napi(object)]
 pub struct RefreshMaterializedViewResult {
-    /// How the view was brought up to date: "rebuild", "incremental" or "no_op".
+    /// How the view was brought up to date: "rebuild".
     pub mode: String,
     pub rows_written: i64,
     pub source_version: i64,
@@ -1619,8 +1618,6 @@ impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewRes
     fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
         let mode = match value.mode {
             lancedb::RefreshMode::Rebuild => "rebuild",
-            lancedb::RefreshMode::Incremental => "incremental",
-            lancedb::RefreshMode::NoOp => "no_op",
         };
         Self {
             mode: mode.to_string(),