From 622e6078c1574405c3984ab14231e1f339c8bc84 Mon Sep 17 00:00:00 2001 From: Brendan Clement Date: Tue, 6 Oct 2026 20:44:06 -0700 Subject: [PATCH] docs: describe materialized-view refresh as a full rebuild Main now rebuilds a materialized view completely on every refresh (#4373): there is no incremental mode, no `full` argument and no stable-row-ID requirement on the source. The example ran against the SDK built from this branch (0.40.0b13) failed, because a refresh after adding Bea writes both matching rows, not one. The example now creates its source with the default options and expects two rows written by the refresh, and its generated snippet is regenerated. Separate tests cover `with_no_data` (one matching row on the first refresh) and refreshing to an earlier source version. The refresh page explains the full rebuild and keeps the #incremental and #stable-row-ids anchors for the sections that replace them. Its version note now says what the published 0.39.0 does differently, as observed with that release. The management, table, database, limits, architecture and data-types pages no longer say a view's source needs stable row ids, and the feature-engineering and tutorial summaries no longer call refreshes incremental. --- docs/web-tests/py/test_materialized_view.py | 32 +++++++--- docs/web/architecture.mdx | 4 +- docs/web/basics/limits.mdx | 6 +- docs/web/feature-engineering/index.mdx | 4 +- .../materialized-views.mdx | 61 +++++++------------ docs/web/management/database.mdx | 7 +-- docs/web/management/materialized-view.mdx | 19 ++---- docs/web/management/table.mdx | 3 +- docs/web/snippets/materialized_view.mdx | 4 +- docs/web/sql/data-types.mdx | 3 - docs/web/tutorials/index.mdx | 2 +- 11 files changed, 61 insertions(+), 84 deletions(-) diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py index 74b727dbc..ec48e1033 100644 --- a/docs/web-tests/py/test_materialized_view.py +++ b/docs/web-tests/py/test_materialized_view.py @@ -10,10 +10,7 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): # --8<-- [start:materialized_view_create] import lancedb - db = lancedb.connect( - "./.lancedb", - storage_options={"new_table_enable_stable_row_ids": "true"}, - ) + db = lancedb.connect("./.lancedb") table = db.create_table( "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] ) @@ -34,9 +31,10 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): result = view.refresh() print(result.rows_written) - # 1 + # 2 # --8<-- [end:materialized_view_refresh] - assert result.rows_written == 1 + assert result.mode == "rebuild" + assert result.rows_written == 2 assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ "ADA", "BEA", @@ -46,12 +44,26 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): def test_materialized_view_with_no_data(tmp_path): import lancedb - db = lancedb.connect( - tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"} - ) - db.create_table("people", [{"name": "ada", "age": 36}]) + db = lancedb.connect(tmp_path) + db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) view = db.create_materialized_view( "adults", "people", where="age >= 18", with_no_data=True ) assert view.table.count_rows() == 0 assert view.refresh().rows_written == 1 + assert view.table.count_rows() == 1 + + +def test_materialized_view_refresh_to_source_version(tmp_path): + import lancedb + + db = lancedb.connect(tmp_path) + table = db.create_table("people", [{"name": "ada", "age": 36}]) + view = db.create_materialized_view("adults", "people", where="age >= 18") + first = table.version + table.add([{"name": "bea", "age": 41}]) + assert view.refresh().rows_written == 2 + + result = view.refresh(source_version=first) + assert (result.source_version, result.rows_written) == (first, 1) + assert view.table.count_rows() == 1 diff --git a/docs/web/architecture.mdx b/docs/web/architecture.mdx index d03b7ba5c..1a3d85d65 100644 --- a/docs/web/architecture.mdx +++ b/docs/web/architecture.mdx @@ -129,9 +129,7 @@ plus a data migration, not a rewrite. The one thing to decide early is **stable row ids**, which cannot be enabled -after a table exists and are required for -[materialized views](/management/materialized-view). See -[Limits](/basics/limits#schema). +after a table exists. See [Limits](/basics/limits#schema). ## Where to go next {#next} diff --git a/docs/web/basics/limits.mdx b/docs/web/basics/limits.mdx index 0c3647b03..d7afbe199 100644 --- a/docs/web/basics/limits.mdx +++ b/docs/web/basics/limits.mdx @@ -55,10 +55,8 @@ See [Jobs](/basics/jobs). | `IVF_RQ` vector dimensions | must be divisible by 8 | | FTS `max_token_length` | 40 by default; longer tokens are dropped | -Stable row ids **cannot be enabled after a table exists**. A table created -without them cannot become a -[materialized view](/management/materialized-view) source without being -recreated -- the one schema decision here that is genuinely irreversible. +Stable row ids **cannot be enabled after a table exists** -- the one schema +decision here that is genuinely irreversible. See [Data types](/sql/data-types). diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 53fea63c3..5a9cb45a7 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -54,8 +54,8 @@ idempotent and cheap -- but it will surprise you if you expect a spreadsheet. expression, filled by [a refresh](/feature-engineering/refresh-column). **Materialized views** derive a whole table -- a filtered, projected, or -aggregated view of a source -- and keep it current incrementally. The result is -an ordinary table: queryable, indexable, searchable. +aggregated view of a source -- and compute it again on each refresh. The result +is an ordinary table: queryable, indexable, searchable. **Functions** run Python where the data is, for work SQL cannot express: embedding a column, calling a model, decoding an image. Registered once, diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index cccab85da..eda98891e 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -1,9 +1,9 @@ --- title: "Refresh a materialized view" sidebarTitle: "Materialized View Refresh" -description: "A lakehouse object: a table defined by a query over another table, kept current incrementally." +description: "A lakehouse object: a table defined by a query over another table, recomputed by each refresh." icon: table -keywords: ["materialized view", "create_materialized_view", "refresh", "incremental"] +keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"] --- import { @@ -13,7 +13,7 @@ import { A materialized view is a lakehouse object: a table whose contents are defined by a query over another table. The query is recorded in the view's schema; -creating the view computes its rows, and a refresh brings them up to date. +creating the view computes its rows, and a refresh computes them again. The result is an ordinary table. It can be queried, indexed, and searched, and it appears in `table_names()` — nothing about reading it is special. @@ -37,9 +37,11 @@ reach the view through a refresh: -These results were verified with the development SDK 0.40.0b12, built from -lancedb main. With the published 0.39.0, `create_materialized_view` creates the -view empty and has no `with_no_data`, so the first `refresh()` writes every +These results were verified with the development SDK 0.40.0b13, built from +lancedb main. The published 0.39.0 differs: the source must be created with +stable row ids (the `new_table_enable_stable_row_ids` storage option), +`create_materialized_view` creates the view empty and has no `with_no_data`, +and `refresh()` still accepts `full`. Its first `refresh()` writes every matching row, two in this example. @@ -47,42 +49,25 @@ matching row, two in this example. Omitting it selects every source column, expanded against the schema as it was at creation time. `where` filters source rows; `limit` caps the view. -## The source needs stable row ids {#stable-row-ids} +## Any table can be the source {#stable-row-ids} - -The source table must have been created with stable row ids: +The source needs no special storage option. A refresh does not track which +source rows the view came from, so stable row ids +(`new_table_enable_stable_row_ids`) are not required: the example above creates +its source with the defaults. Earlier development builds, and the published +0.39.0, required them. -```python -db = lancedb.connect(uri, storage_options={"new_table_enable_stable_row_ids": "true"}) -``` +## Every refresh rebuilds the view {#incremental} -They are what keeps the view's provenance valid when the source is compacted, -and **they cannot be enabled after a table exists**. A table created without -them cannot later become the source of a view — it has to be recreated. - +`refresh()` runs the view's query against the source and replaces everything +the view holds with the result, in one commit. It does not reconcile changes: +`rows_written` is the size of the whole result — two after adding Bea, not +one — and a refresh with nothing new to read still rewrites the view and +commits a new table version. -This is the one decision here that is hard to undo, so it is worth setting the -option on any table you might later want to derive from. - -## Refreshes are incremental {#incremental} - -`refresh()` reconciles the source's changes into the view — rows added, changed, -or removed since the last refresh — rather than recomputing from scratch. It -falls back to a full rebuild when the changes cannot be reconciled. - -Two arguments override that: - -- `full=True` forces a rebuild. -- `source_version=N` refreshes against that source version instead of the - latest, which is how you make a view reproduce a known point in time. - -Concurrent refreshes of one view do not duplicate rows. If two plan the same -source rows, they conflict on commit and the loser raises rather than writing -them twice — so a retry loop around `refresh()` is safe. - - -**Enterprise job type:** `mat_view_refresh`. - +`source_version=N` refreshes against that source version instead of the latest, +which is how a view reproduces a known point in time. There is no `full` +argument; every refresh is a full one. ## Against a deployment {#availability} diff --git a/docs/web/management/database.mdx b/docs/web/management/database.mdx index 36c016e2e..83a249f91 100644 --- a/docs/web/management/database.mdx +++ b/docs/web/management/database.mdx @@ -106,11 +106,8 @@ const db = await lancedb.connect("s3://bucket/path", { -`new_table_enable_stable_row_ids` is worth setting deliberately. Stable row ids -cannot be enabled after a table exists, and a -[materialized view](/feature-engineering/materialized-views) requires its source -to have them. A table created without them cannot later become a view's source -without being recreated. +`new_table_enable_stable_row_ids` is worth setting deliberately: stable row ids +cannot be enabled after a table exists. See [Storage](/deployment/storage) for the full set, including per-cloud credentials. diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index ac1ebb7ac..9a03a4270 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -8,7 +8,7 @@ keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] A materialized view is a table whose contents are defined by a query over another table. The query is recorded in the view's schema, and a refresh -brings its stored rows up to date with the source. +computes its stored rows again from the source. The result is an ordinary table -- queryable, indexable, searchable, and present in `table_names()`. Nothing about reading it is special. @@ -41,18 +41,9 @@ const view = await db.createMaterializedView("recent_docs", { ``` -`source` names the table it derives from. `select`, `where` and `limit` record -the query -- there is no separate definition step. - - -The source table must have **stable row ids**, and they cannot be enabled after -a table exists. A table created without them cannot become a view's source -without being recreated. - -Set `new_table_enable_stable_row_ids` in -[storage options](/management/database#storage-options) when you create the -source, not when you create the view. - +`source` names the table it derives from; any table can be one, with no special +storage option. `select`, `where` and `limit` record the query -- there is no +separate definition step. ## Opening, listing and dropping {#manage} @@ -92,7 +83,7 @@ across the cluster rather than in your process, and SQL can drive it. - Incremental refreshes, and what they cost. + What a refresh recomputes, and what it writes. The non-materialized counterpart, defined in SQL. diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 3bd4eef58..f85bfe9b1 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -157,8 +157,7 @@ frame, or Pydantic models. [Ingesting data](/ingestion) covers each, plus An explicit schema is where nullability is decided, and where you opt into -**stable row ids** -- which cannot be enabled later and are required for a -[materialized view](/management/materialized-view) source. See +**stable row ids** -- which cannot be enabled later. See [Limits](/basics/limits#schema). diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx index 9d3a5b602..41f9def31 100644 --- a/docs/web/snippets/materialized_view.mdx +++ b/docs/web/snippets/materialized_view.mdx @@ -1,6 +1,6 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\n \"./.lancedb\",\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; -export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 1\n"; +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 2\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index e46101a3e..da106748b 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -137,9 +137,6 @@ Row metadata, available only when named — so it stays out of `SELECT *`: their validity bitmap is reused internally as a selection vector. Do not read a null there as meaningful. -`_rowid` is what [materialized views](/management/materialized-view) depend on, -which is why a source table needs stable row ids enabled at creation. - ## Casting {#casting} ```sql diff --git a/docs/web/tutorials/index.mdx b/docs/web/tutorials/index.mdx index e8e326b94..bcc9345c1 100644 --- a/docs/web/tutorials/index.mdx +++ b/docs/web/tutorials/index.mdx @@ -38,5 +38,5 @@ are runnable in Colab; the rest are written up here. | [Multi-vector search](/search/vector-search) | Several embeddings per document. | | [Reranking](/reranking) | Reordering results to improve the top-k. | | [Feature engineering](/feature-engineering) | Deriving columns and tables from data already stored. | -| [Materialized views](/management/materialized-view) | A table defined by a query, refreshed incrementally. | +| [Materialized views](/management/materialized-view) | A table defined by a query, recomputed by each refresh. | | [Training](/training/data-loading) | Reading a table into a data loader. |