fix: correct remote materialized view refresh results and options (#4672)

Remote materialized-view refreshes drained the SQL result and then
fabricated `mode=rebuild`, `rows_written=<total view rows>`, and
`source_version=0`. This made incremental refreshes report incorrect
statistics.

Make these three result fields optional across Rust, Python, and
TypeScript, and leave them unset for remote SQL refreshes. Remove the
post-refresh row count request. Local refreshes continue to populate all
metrics, and `version` remains the view version observed after
completion. Remote per-operation statistics remain unavailable until the
SQL job adapter can retrieve them.

Expose the forced-rebuild option through the Rust builder,
synchronous/asynchronous Python refresh and job methods, and TypeScript
refresh. Remote `full=true` sends `WITH (full = true)`; the default lets
the server choose its refresh mode. Local source-version pinning and
remote job cancellation are preserved.

Local refresh regression tests respect the `experimental-functions`
build gate, while remote refresh coverage also runs in default builds.

Validation:

- Rust: 20 materialized-view tests and 21 remote SQL tests passed with
`remote,experimental-functions`; `cargo fmt --all` passed.
- Python: rebuilt the extension in the uv-managed environment. The
default build passed 23 tests with 27 feature-dependent skips. The
opt-in build passed 61 tests with 3 expected default-build-only skips,
covering the materialized-view suite and 14 async query-serialization
cases from current main. Python Ruff lint and formatting checks passed.
- TypeScript: default and opt-in native builds and TypeScript
compilation passed. The default build passed 5 tests with 10 expected
feature-dependent skips; the opt-in build passed all 14
materialized-view tests with the default-build-only test skipped.
Lint/format checks and documentation generation passed.
- Repository-wide `ruff check .` reports 48 pre-existing violations in
unchanged `docs/web-tests` files; `ruff check python` passes.

Fixes #4670

<!-- lance-gatekeeper-fix:v1 agent=ece43ec4b88a48d1789f60005fa3ecfc
generation=1 -->

Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
Co-authored-by: Xuanwo <github@xuanwo.io>

python/python/lancedb/_lancedb.pyi

@@ -528,10 +528,10 @@ class Table:
         limit: Optional[int] = None,
     ) -> FunctionErrors: ...
     async def refresh_materialized_view(
-        self, source_version: Optional[int] = None
+        self, source_version: Optional[int] = None, *, full: bool = False
     ) -> RefreshMaterializedViewResult: ...
     async def refresh_materialized_view_async(
-        self, source_version: Optional[int] = None
+        self, source_version: Optional[int] = None, *, full: bool = False
     ) -> Job: ...
     async def materialized_view_definition(self) -> str: ...
     async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
@@ -921,9 +921,9 @@ class FunctionErrors:
 class RefreshMaterializedViewResult:
     @staticmethod
     def from_json(value: str) -> RefreshMaterializedViewResult: ...
-    mode: str
-    rows_written: int
-    source_version: int
+    mode: Optional[str]
+    rows_written: Optional[int]
+    source_version: Optional[int]
     version: int
 
 class AlterColumnsResult:

python/python/lancedb/materialized_view.py

@@ -184,31 +184,36 @@ class AsyncMaterializedView:
         return _definition_from_json(raw)
 
     async def refresh(
-        self, *, source_version: Optional[int] = None
+        self, *, full: bool = False, source_version: Optional[int] = None
     ) -> "RefreshMaterializedViewResult":
         """Recompute the view from its source.
 
-        Every refresh rebuilds the complete view. For local views,
-        ``source_version`` refreshes to that source version instead of the
-        latest; remote SQL refreshes do not support source-version pinning.
+        ``full=True`` forces a remote rebuild; otherwise the server chooses
+        the refresh mode. Local views always rebuild, and ``source_version``
+        selects a source version instead of the latest. Remote SQL refreshes
+        do not support source-version pinning.
+
+        Remote results leave ``mode``, ``rows_written``, and ``source_version``
+        as ``None`` because the SQL job adapter does not expose these metrics.
         """
         return await self._table._inner.refresh_materialized_view(
-            source_version=source_version
+            source_version=source_version, full=full
         )
 
     async def refresh_async(
-        self, *, source_version: Optional[int] = None
+        self, *, full: bool = False, source_version: Optional[int] = None
     ) -> "AsyncJob[RefreshMaterializedViewResult]":
         """Submit a refresh and return its job without waiting.
 
         The job may already be complete for a local view. Remote refreshes are
-        submitted through the SQL service.
+        submitted through the SQL service. Options and result metrics follow
+        [refresh][lancedb.materialized_view.AsyncMaterializedView.refresh].
         """
         from ._lancedb import RefreshMaterializedViewResult
 
         return _typed_job(
             await self._table._inner.refresh_materialized_view_async(
-                source_version=source_version
+                source_version=source_version, full=full
             ),
             RefreshMaterializedViewResult.from_json,
         )
@@ -240,18 +245,22 @@ class MaterializedView:
         return LOOP.run(self._async.definition())
 
     def refresh(
-        self, *, source_version: Optional[int] = None
+        self, *, full: bool = False, source_version: Optional[int] = None
     ) -> "RefreshMaterializedViewResult":
         """Recompute the view from its source. See
         [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
-        return LOOP.run(self._async.refresh(source_version=source_version))
+        return LOOP.run(self._async.refresh(full=full, source_version=source_version))
 
     def refresh_async(
-        self, *, source_version: Optional[int] = None
+        self, *, full: bool = False, source_version: Optional[int] = None
     ) -> "Job[RefreshMaterializedViewResult]":
         """Submit a refresh and return its job without waiting.
 
         See
         [AsyncMaterializedView.refresh_async][lancedb.materialized_view.AsyncMaterializedView.refresh_async].
         """
-        return Job(LOOP.run(self._async.refresh_async(source_version=source_version)))
+        return Job(
+            LOOP.run(
+                self._async.refresh_async(full=full, source_version=source_version)
+            )
+        )

python/python/tests/test_materialized_views.py

@@ -8,6 +8,7 @@ import threading
 
 import lancedb
 import pytest
+import pyarrow as pa
 from lancedb.db import AsyncConnection
 from lancedb.materialized_view import MaterializedViewDefinition
 from lancedb.remote.db import RemoteDBConnection
@@ -112,6 +113,98 @@ def test_remote_drop_async_returns_server_job():
     assert requests == [("/v1/materialized_view/adults/drop", {})]
 
 
+@contextlib.contextmanager
+def mock_remote_materialized_view_refresh():
+    requests = []
+
+    class Handler(http.server.BaseHTTPRequestHandler):
+        def log_message(self, *args):
+            pass
+
+        def do_POST(self):
+            requests.append(self.path)
+            if self.path == "/v1/materialized_view/view/describe":
+                response = {"query": "SELECT * FROM source"}
+            else:
+                assert self.path == "/v1/table/view/describe/"
+                response = {"version": 42, "schema": {"fields": []}}
+            encoded = json.dumps(response).encode()
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(encoded)))
+            self.end_headers()
+            self.wfile.write(encoded)
+
+    with http.server.HTTPServer(("localhost", 0), Handler) as server:
+        thread = threading.Thread(target=server.serve_forever)
+        thread.start()
+        try:
+            yield f"http://localhost:{server.server_address[1]}", requests
+        finally:
+            server.shutdown()
+            thread.join()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync", [False, True])
+@pytest.mark.parametrize("background", [False, True])
+@pytest.mark.parametrize("full", [False, True])
+async def test_remote_refresh_reports_unknown_metrics(sync, background, full):
+    flight = pytest.importorskip("pyarrow.flight")
+
+    class RefreshServer(flight.FlightServerBase):
+        def __init__(self):
+            super().__init__("grpc://127.0.0.1:0")
+            self.commands = []
+
+        def do_get(self, context, ticket):
+            self.commands.append(ticket.ticket)
+            # SQL completion carries no refresh statistics.
+            return flight.RecordBatchStream(pa.table({"status": ["ok"]}))
+
+    with (
+        mock_remote_materialized_view_refresh() as (host, requests),
+        RefreshServer() as server,
+    ):
+        options = {"full": True} if full else {}
+        connect = lancedb.connect if sync else lancedb.connect_async
+        db = connect(
+            "db://dev",
+            api_key="fake",
+            host_override=host,
+            sql_host_override=f"grpc://127.0.0.1:{server.port}",
+            client_config={"retry_config": {"retries": 0}},
+        )
+        if sync:
+            view = db.open_materialized_view("view")
+            result = (
+                view.refresh_async(**options).wait()
+                if background
+                else view.refresh(**options)
+            )
+        else:
+            db = await db
+            view = await db.open_materialized_view("view")
+            if background:
+                job = await view.refresh_async(**options)
+                result = await job.wait()
+            else:
+                result = await view.refresh(**options)
+
+        assert result.mode is None
+        assert result.rows_written is None
+        assert result.source_version is None
+        assert result.version == 42
+        assert repr(result) == (
+            "RefreshMaterializedViewResult(mode=None, rows_written=None, "
+            "source_version=None, version=42)"
+        )
+        (command,) = server.commands
+        assert b'REFRESH MATERIALIZED VIEW "view"' in command
+        assert (b"WITH (full = true)" in command) is full
+        assert not any("count_rows" in path for path in requests)
+
+
 def test_sync_remote_create_uses_public_async_connection():
     calls = []
 
@@ -226,20 +319,37 @@ def test_definition_round_trips(tmp_path):
 
 
 @pytest.mark.experimental_functions
-def test_full_refresh_after_append(tmp_path):
+@pytest.mark.parametrize("full", [False, True])
+def test_full_refresh_after_append(tmp_path, full):
     db = make_db(tmp_path)
     view = db.create_materialized_view("copy", "people", with_no_data=True)
     view.refresh()
 
     db.open_table("people").add([{"name": "alan", "age": 41}])
-    result = view.refresh()
+    result = view.refresh(full=full)
     assert result.mode == "rebuild"
     assert result.rows_written == 4
+    assert result.source_version == db.open_table("people").version
     assert view.table.count_rows() == 4
 
     assert view.refresh().mode == "rebuild"
 
 
+@pytest.mark.experimental_functions
+def test_full_refresh_can_pin_local_source_version(tmp_path):
+    db = make_db(tmp_path)
+    source = db.open_table("people")
+    version = source.version
+    view = db.create_materialized_view("copy", "people", with_no_data=True)
+    source.add([{"name": "alan", "age": 41}])
+
+    result = view.refresh(full=True, source_version=version)
+    assert result.mode == "rebuild"
+    assert result.rows_written == 3
+    assert result.source_version == version
+    assert view.table.count_rows() == 3
+
+
 @pytest.mark.experimental_functions
 def test_full_refresh_after_update(tmp_path):
     db = make_db(tmp_path)

python/src/table.rs

@@ -694,9 +694,9 @@ impl From<lancedb::function::FunctionErrors> for FunctionErrors {
 #[pyclass(get_all, from_py_object)]
 #[derive(Clone, Debug)]
 pub struct RefreshMaterializedViewResult {
-    pub mode: String,
-    pub rows_written: u64,
-    pub source_version: u64,
+    pub mode: Option<String>,
+    pub rows_written: Option<u64>,
+    pub source_version: Option<u64>,
     pub version: u64,
 }
 
@@ -716,18 +716,23 @@ impl RefreshMaterializedViewResult {
     pub fn __repr__(&self) -> String {
         format!(
             "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})",
-            self.mode, self.rows_written, self.source_version, self.version
+            self.mode.as_deref().unwrap_or("None"),
+            self.rows_written
+                .map_or_else(|| "None".into(), |value| value.to_string()),
+            self.source_version
+                .map_or_else(|| "None".into(), |value| value.to_string()),
+            self.version
         )
     }
 }
 
 impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
     fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
-        let mode = match result.mode {
+        let mode = result.mode.map(|mode| match mode {
             lancedb::RefreshMode::Rebuild => "rebuild",
-        };
+        });
         Self {
-            mode: mode.to_string(),
+            mode: mode.map(str::to_string),
             rows_written: result.rows_written,
             source_version: result.source_version,
             version: result.version,
@@ -2054,17 +2059,18 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (source_version=None))]
+    #[pyo3(signature = (source_version=None, *, full=false))]
     pub fn refresh_materialized_view(
         self_: PyRef<'_, Self>,
         source_version: Option<u64>,
+        full: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let view = lancedb::MaterializedView::from_table(inner)
                 .await
                 .infer_error()?;
-            let mut builder = view.refresh();
+            let mut builder = view.refresh().full(full);
             if let Some(version) = source_version {
                 builder = builder.source_version(version);
             }
@@ -2073,17 +2079,18 @@ impl Table {
         })
     }
 
-    #[pyo3(signature = (source_version=None))]
+    #[pyo3(signature = (source_version=None, *, full=false))]
     pub fn refresh_materialized_view_async(
         self_: PyRef<'_, Self>,
         source_version: Option<u64>,
+        full: bool,
     ) -> PyResult<Bound<'_, PyAny>> {
         let inner = self_.inner_ref()?.clone();
         future_into_py(self_.py(), async move {
             let view = lancedb::MaterializedView::from_table(inner)
                 .await
                 .infer_error()?;
-            let mut builder = view.refresh();
+            let mut builder = view.refresh().full(full);
             if let Some(version) = source_version {
                 builder = builder.source_version(version);
             }