feat: cascade option on materialized view refresh

A view over a view could only be brought fully up to date by refreshing
each view in the lineage by hand, in order. `cascade` on the refresh
builder, Python `refresh`/`refresh_async` and Node `refresh` does that in
one call: every upstream view is refreshed furthest first, each reading
the version the one before it left, then the named view. `full` applies
to the named view only.

Locally the lineage is walked through each view's source, and each hop is
refreshed against the incarnation read in that walk. Remote tables send
`cascade` in the refresh request, only when set, so older servers that
reject unknown fields are unaffected by plain refreshes.

python/python/lancedb/_lancedb.pyi

@@ -488,10 +488,16 @@ class Table:
         limit: Optional[int] = None,
     ) -> FunctionErrors: ...
     async def refresh_materialized_view(
-        self, full: bool = False, source_version: Optional[int] = None
+        self,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> RefreshMaterializedViewResult: ...
     async def refresh_materialized_view_async(
-        self, full: bool = False, source_version: Optional[int] = None
+        self,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> Job: ...
     async def materialized_view_definition(self) -> str: ...
     async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...

python/python/lancedb/materialized_view.py

@@ -175,7 +175,11 @@ class AsyncMaterializedView:
         return _definition_from_json(raw)
 
     async def refresh(
-        self, *, full: bool = False, source_version: Optional[int] = None
+        self,
+        *,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> "RefreshMaterializedViewResult":
         """Recompute the view from its source.
 
@@ -185,16 +189,24 @@ class AsyncMaterializedView:
         ``source_version`` refreshes to that source version instead of the
         latest.
 
+        ``cascade=True`` first refreshes every upstream view, furthest first,
+        each reading the version the one before it left; ``full`` applies to
+        this view only.
+
         Concurrent refreshes of one view do not duplicate its rows. Two that
         plan the same source rows conflict on commit, and the loser raises
         rather than writing them a second time.
         """
         return await self._table._inner.refresh_materialized_view(
-            full=full, source_version=source_version
+            full=full, source_version=source_version, cascade=cascade
         )
 
     async def refresh_async(
-        self, *, full: bool = False, source_version: Optional[int] = None
+        self,
+        *,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> "AsyncJob[RefreshMaterializedViewResult]":
         """Submit a refresh and return its job without waiting.
 
@@ -206,7 +218,7 @@ class AsyncMaterializedView:
 
         return _typed_job(
             await self._table._inner.refresh_materialized_view_async(
-                full=full, source_version=source_version
+                full=full, source_version=source_version, cascade=cascade
             ),
             RefreshMaterializedViewResult.from_json,
         )
@@ -238,14 +250,26 @@ class MaterializedView:
         return LOOP.run(self._async.definition())
 
     def refresh(
-        self, *, full: bool = False, source_version: Optional[int] = None
+        self,
+        *,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> "RefreshMaterializedViewResult":
         """Recompute the view from its source. See
         [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
-        return LOOP.run(self._async.refresh(full=full, source_version=source_version))
+        return LOOP.run(
+            self._async.refresh(
+                full=full, source_version=source_version, cascade=cascade
+            )
+        )
 
     def refresh_async(
-        self, *, full: bool = False, source_version: Optional[int] = None
+        self,
+        *,
+        full: bool = False,
+        source_version: Optional[int] = None,
+        cascade: bool = False,
     ) -> "Job[RefreshMaterializedViewResult]":
         """Submit a refresh and return its job without waiting.
 
@@ -254,6 +278,8 @@ class MaterializedView:
         """
         return Job(
             LOOP.run(
-                self._async.refresh_async(full=full, source_version=source_version)
+                self._async.refresh_async(
+                    full=full, source_version=source_version, cascade=cascade
+                )
             )
         )

python/python/tests/test_materialized_views.py

@@ -211,6 +211,25 @@ def test_create_and_refresh_jobs(tmp_path):
     assert "adults" not in db.list_materialized_views()
 
 
+def test_cascade_refreshes_upstream_views(tmp_path):
+    db = make_db(tmp_path)
+    adults = db.create_materialized_view(
+        "adults", "people", where="age >= 18", with_no_data=True
+    )
+    seniors = db.create_materialized_view(
+        "seniors", "adults", where="age >= 65", with_no_data=True
+    )
+    assert seniors.refresh().rows_written == 0
+
+    assert seniors.refresh(cascade=True).rows_written == 1
+    assert db.open_table("adults").count_rows() == 2
+
+    db.open_table("people").add([{"name": "alan", "age": 90}])
+    assert seniors.refresh_async(cascade=True).wait().mode == "incremental"
+    assert seniors.table.count_rows() == 2
+    assert adults.refresh().mode == "no_op"
+
+
 def test_definition_round_trips(tmp_path):
     db = make_db(tmp_path)
     db.create_materialized_view("adults", "people", where="age >= 18")

python/src/table.rs

@@ -1802,18 +1802,19 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (full=false, source_version=None))]
+    #[pyo3(signature = (full=false, source_version=None, cascade=false))]
     pub fn refresh_materialized_view(
         self_: PyRef<'_, Self>,
         full: bool,
         source_version: Option<u64>,
+        cascade: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let view = lancedb::MaterializedView::from_table(inner)
                 .await
                 .infer_error()?;
-            let mut builder = view.refresh().full(full);
+            let mut builder = view.refresh().full(full).cascade(cascade);
             if let Some(version) = source_version {
                 builder = builder.source_version(version);
             }
@@ -1822,18 +1823,19 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (full=false, source_version=None))]
+    #[pyo3(signature = (full=false, source_version=None, cascade=false))]
     pub fn refresh_materialized_view_async(
         self_: PyRef<'_, Self>,
         full: bool,
         source_version: Option<u64>,
+        cascade: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let view = lancedb::MaterializedView::from_table(inner)
                 .await
                 .infer_error()?;
-            let mut builder = view.refresh().full(full);
+            let mut builder = view.refresh().full(full).cascade(cascade);
             if let Some(version) = source_version {
                 builder = builder.source_version(version);
             }