diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py index 74b727dbc..ec48e1033 100644 --- a/docs/web-tests/py/test_materialized_view.py +++ b/docs/web-tests/py/test_materialized_view.py @@ -10,10 +10,7 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): # --8<-- [start:materialized_view_create] import lancedb - db = lancedb.connect( - "./.lancedb", - storage_options={"new_table_enable_stable_row_ids": "true"}, - ) + db = lancedb.connect("./.lancedb") table = db.create_table( "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] ) @@ -34,9 +31,10 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): result = view.refresh() print(result.rows_written) - # 1 + # 2 # --8<-- [end:materialized_view_refresh] - assert result.rows_written == 1 + assert result.mode == "rebuild" + assert result.rows_written == 2 assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ "ADA", "BEA", @@ -46,12 +44,26 @@ def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): def test_materialized_view_with_no_data(tmp_path): import lancedb - db = lancedb.connect( - tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"} - ) - db.create_table("people", [{"name": "ada", "age": 36}]) + db = lancedb.connect(tmp_path) + db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) view = db.create_materialized_view( "adults", "people", where="age >= 18", with_no_data=True ) assert view.table.count_rows() == 0 assert view.refresh().rows_written == 1 + assert view.table.count_rows() == 1 + + +def test_materialized_view_refresh_to_source_version(tmp_path): + import lancedb + + db = lancedb.connect(tmp_path) + table = db.create_table("people", [{"name": "ada", "age": 36}]) + view = db.create_materialized_view("adults", "people", where="age >= 18") + first = table.version + table.add([{"name": "bea", "age": 41}]) + assert view.refresh().rows_written == 2 + + result = view.refresh(source_version=first) + assert (result.source_version, result.rows_written) == (first, 1) + assert view.table.count_rows() == 1 diff --git a/docs/web/architecture.mdx b/docs/web/architecture.mdx index d03b7ba5c..1a3d85d65 100644 --- a/docs/web/architecture.mdx +++ b/docs/web/architecture.mdx @@ -129,9 +129,7 @@ plus a data migration, not a rewrite. The one thing to decide early is **stable row ids**, which cannot be enabled -after a table exists and are required for -[materialized views](/management/materialized-view). See -[Limits](/basics/limits#schema). +after a table exists. See [Limits](/basics/limits#schema). ## Where to go next {#next} diff --git a/docs/web/basics/limits.mdx b/docs/web/basics/limits.mdx index 0c3647b03..d7afbe199 100644 --- a/docs/web/basics/limits.mdx +++ b/docs/web/basics/limits.mdx @@ -55,10 +55,8 @@ See [Jobs](/basics/jobs). | `IVF_RQ` vector dimensions | must be divisible by 8 | | FTS `max_token_length` | 40 by default; longer tokens are dropped | -Stable row ids **cannot be enabled after a table exists**. A table created -without them cannot become a -[materialized view](/management/materialized-view) source without being -recreated -- the one schema decision here that is genuinely irreversible. +Stable row ids **cannot be enabled after a table exists** -- the one schema +decision here that is genuinely irreversible. See [Data types](/sql/data-types). diff --git a/docs/web/feature-engineering/index.mdx b/docs/web/feature-engineering/index.mdx index 53fea63c3..5a9cb45a7 100644 --- a/docs/web/feature-engineering/index.mdx +++ b/docs/web/feature-engineering/index.mdx @@ -54,8 +54,8 @@ idempotent and cheap -- but it will surprise you if you expect a spreadsheet. expression, filled by [a refresh](/feature-engineering/refresh-column). **Materialized views** derive a whole table -- a filtered, projected, or -aggregated view of a source -- and keep it current incrementally. The result is -an ordinary table: queryable, indexable, searchable. +aggregated view of a source -- and compute it again on each refresh. The result +is an ordinary table: queryable, indexable, searchable. **Functions** run Python where the data is, for work SQL cannot express: embedding a column, calling a model, decoding an image. Registered once, diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index cccab85da..eda98891e 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -1,9 +1,9 @@ --- title: "Refresh a materialized view" sidebarTitle: "Materialized View Refresh" -description: "A lakehouse object: a table defined by a query over another table, kept current incrementally." +description: "A lakehouse object: a table defined by a query over another table, recomputed by each refresh." icon: table -keywords: ["materialized view", "create_materialized_view", "refresh", "incremental"] +keywords: ["materialized view", "create_materialized_view", "refresh", "full refresh"] --- import { @@ -13,7 +13,7 @@ import { A materialized view is a lakehouse object: a table whose contents are defined by a query over another table. The query is recorded in the view's schema; -creating the view computes its rows, and a refresh brings them up to date. +creating the view computes its rows, and a refresh computes them again. The result is an ordinary table. It can be queried, indexed, and searched, and it appears in `table_names()` — nothing about reading it is special. @@ -37,9 +37,11 @@ reach the view through a refresh: -These results were verified with the development SDK 0.40.0b12, built from -lancedb main. With the published 0.39.0, `create_materialized_view` creates the -view empty and has no `with_no_data`, so the first `refresh()` writes every +These results were verified with the development SDK 0.40.0b13, built from +lancedb main. The published 0.39.0 differs: the source must be created with +stable row ids (the `new_table_enable_stable_row_ids` storage option), +`create_materialized_view` creates the view empty and has no `with_no_data`, +and `refresh()` still accepts `full`. Its first `refresh()` writes every matching row, two in this example. @@ -47,42 +49,25 @@ matching row, two in this example. Omitting it selects every source column, expanded against the schema as it was at creation time. `where` filters source rows; `limit` caps the view. -## The source needs stable row ids {#stable-row-ids} +## Any table can be the source {#stable-row-ids} - -The source table must have been created with stable row ids: +The source needs no special storage option. A refresh does not track which +source rows the view came from, so stable row ids +(`new_table_enable_stable_row_ids`) are not required: the example above creates +its source with the defaults. Earlier development builds, and the published +0.39.0, required them. -```python -db = lancedb.connect(uri, storage_options={"new_table_enable_stable_row_ids": "true"}) -``` +## Every refresh rebuilds the view {#incremental} -They are what keeps the view's provenance valid when the source is compacted, -and **they cannot be enabled after a table exists**. A table created without -them cannot later become the source of a view — it has to be recreated. - +`refresh()` runs the view's query against the source and replaces everything +the view holds with the result, in one commit. It does not reconcile changes: +`rows_written` is the size of the whole result — two after adding Bea, not +one — and a refresh with nothing new to read still rewrites the view and +commits a new table version. -This is the one decision here that is hard to undo, so it is worth setting the -option on any table you might later want to derive from. - -## Refreshes are incremental {#incremental} - -`refresh()` reconciles the source's changes into the view — rows added, changed, -or removed since the last refresh — rather than recomputing from scratch. It -falls back to a full rebuild when the changes cannot be reconciled. - -Two arguments override that: - -- `full=True` forces a rebuild. -- `source_version=N` refreshes against that source version instead of the - latest, which is how you make a view reproduce a known point in time. - -Concurrent refreshes of one view do not duplicate rows. If two plan the same -source rows, they conflict on commit and the loser raises rather than writing -them twice — so a retry loop around `refresh()` is safe. - - -**Enterprise job type:** `mat_view_refresh`. - +`source_version=N` refreshes against that source version instead of the latest, +which is how a view reproduces a known point in time. There is no `full` +argument; every refresh is a full one. ## Against a deployment {#availability} diff --git a/docs/web/management/database.mdx b/docs/web/management/database.mdx index 36c016e2e..83a249f91 100644 --- a/docs/web/management/database.mdx +++ b/docs/web/management/database.mdx @@ -106,11 +106,8 @@ const db = await lancedb.connect("s3://bucket/path", { -`new_table_enable_stable_row_ids` is worth setting deliberately. Stable row ids -cannot be enabled after a table exists, and a -[materialized view](/feature-engineering/materialized-views) requires its source -to have them. A table created without them cannot later become a view's source -without being recreated. +`new_table_enable_stable_row_ids` is worth setting deliberately: stable row ids +cannot be enabled after a table exists. See [Storage](/deployment/storage) for the full set, including per-cloud credentials. diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index ac1ebb7ac..9a03a4270 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -8,7 +8,7 @@ keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] A materialized view is a table whose contents are defined by a query over another table. The query is recorded in the view's schema, and a refresh -brings its stored rows up to date with the source. +computes its stored rows again from the source. The result is an ordinary table -- queryable, indexable, searchable, and present in `table_names()`. Nothing about reading it is special. @@ -41,18 +41,9 @@ const view = await db.createMaterializedView("recent_docs", { ``` -`source` names the table it derives from. `select`, `where` and `limit` record -the query -- there is no separate definition step. - - -The source table must have **stable row ids**, and they cannot be enabled after -a table exists. A table created without them cannot become a view's source -without being recreated. - -Set `new_table_enable_stable_row_ids` in -[storage options](/management/database#storage-options) when you create the -source, not when you create the view. - +`source` names the table it derives from; any table can be one, with no special +storage option. `select`, `where` and `limit` record the query -- there is no +separate definition step. ## Opening, listing and dropping {#manage} @@ -92,7 +83,7 @@ across the cluster rather than in your process, and SQL can drive it. - Incremental refreshes, and what they cost. + What a refresh recomputes, and what it writes. The non-materialized counterpart, defined in SQL. diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 3bd4eef58..f85bfe9b1 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -157,8 +157,7 @@ frame, or Pydantic models. [Ingesting data](/ingestion) covers each, plus An explicit schema is where nullability is decided, and where you opt into -**stable row ids** -- which cannot be enabled later and are required for a -[materialized view](/management/materialized-view) source. See +**stable row ids** -- which cannot be enabled later. See [Limits](/basics/limits#schema). diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx index 9d3a5b602..41f9def31 100644 --- a/docs/web/snippets/materialized_view.mdx +++ b/docs/web/snippets/materialized_view.mdx @@ -1,6 +1,6 @@ {/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} -export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\n \"./.lancedb\",\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\"./.lancedb\")\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; -export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 1\n"; +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 2\n"; diff --git a/docs/web/sql/data-types.mdx b/docs/web/sql/data-types.mdx index e46101a3e..da106748b 100644 --- a/docs/web/sql/data-types.mdx +++ b/docs/web/sql/data-types.mdx @@ -137,9 +137,6 @@ Row metadata, available only when named — so it stays out of `SELECT *`: their validity bitmap is reused internally as a selection vector. Do not read a null there as meaningful. -`_rowid` is what [materialized views](/management/materialized-view) depend on, -which is why a source table needs stable row ids enabled at creation. - ## Casting {#casting} ```sql diff --git a/docs/web/tutorials/index.mdx b/docs/web/tutorials/index.mdx index e8e326b94..bcc9345c1 100644 --- a/docs/web/tutorials/index.mdx +++ b/docs/web/tutorials/index.mdx @@ -38,5 +38,5 @@ are runnable in Colab; the rest are written up here. | [Multi-vector search](/search/vector-search) | Several embeddings per document. | | [Reranking](/reranking) | Reordering results to improve the top-k. | | [Feature engineering](/feature-engineering) | Deriving columns and tables from data already stored. | -| [Materialized views](/management/materialized-view) | A table defined by a query, refreshed incrementally. | +| [Materialized views](/management/materialized-view) | A table defined by a query, recomputed by each refresh. | | [Training](/training/data-loading) | Reading a table into a data loader. |