diff --git a/docs/web-tests/py/test_materialized_view.py b/docs/web-tests/py/test_materialized_view.py new file mode 100644 index 000000000..74b727dbc --- /dev/null +++ b/docs/web-tests/py/test_materialized_view.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Examples for docs/web/feature-engineering/materialized-views.mdx.""" + + +def test_materialized_view_create_and_refresh(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + # --8<-- [start:materialized_view_create] + import lancedb + + db = lancedb.connect( + "./.lancedb", + storage_options={"new_table_enable_stable_row_ids": "true"}, + ) + table = db.create_table( + "people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + ) + + view = db.create_materialized_view( + "adults", + "people", + select=["name", ("shout", "upper(name)")], + where="age >= 18", + ) + print(view.table.count_rows()) + # 1 + # --8<-- [end:materialized_view_create] + assert view.table.count_rows() == 1 + + # --8<-- [start:materialized_view_refresh] + table.add([{"name": "bea", "age": 41}]) + + result = view.refresh() + print(result.rows_written) + # 1 + # --8<-- [end:materialized_view_refresh] + assert result.rows_written == 1 + assert sorted(row["shout"] for row in view.table.to_arrow().to_pylist()) == [ + "ADA", + "BEA", + ] + + +def test_materialized_view_with_no_data(tmp_path): + import lancedb + + db = lancedb.connect( + tmp_path, storage_options={"new_table_enable_stable_row_ids": "true"} + ) + db.create_table("people", [{"name": "ada", "age": 36}]) + view = db.create_materialized_view( + "adults", "people", where="age >= 18", with_no_data=True + ) + assert view.table.count_rows() == 0 + assert view.refresh().rows_written == 1 diff --git a/docs/web/basics/catalog.mdx b/docs/web/basics/catalog.mdx index e90e7bd76..133599f16 100644 --- a/docs/web/basics/catalog.mdx +++ b/docs/web/basics/catalog.mdx @@ -74,8 +74,9 @@ on the bucket, which has no concept of a table. The names are close enough to mislead. Both are catalog objects defined by a query over other tables; the difference is when that query runs. -- A **materialized view** stores its rows. A refresh computes them, and reads - are reads of stored data -- fast, and as current as the last refresh. +- A **materialized view** stores its rows, and a refresh brings them up to + date. Reads are reads of stored data -- fast, and as current as the last + refresh. - A **view** stores only the query. Every read runs it, so a view is never stale and never faster than the query it wraps. diff --git a/docs/web/feature-engineering/materialized-views.mdx b/docs/web/feature-engineering/materialized-views.mdx index 8adea0b8a..c5b86d392 100644 --- a/docs/web/feature-engineering/materialized-views.mdx +++ b/docs/web/feature-engineering/materialized-views.mdx @@ -6,9 +6,14 @@ icon: table keywords: ["materialized view", "create_materialized_view", "refresh", "incremental"] --- +import { + PyMaterializedViewCreate as MaterializedViewCreate, + PyMaterializedViewRefresh as MaterializedViewRefresh, +} from '/snippets/materialized_view.mdx'; + A materialized view is a lakehouse object: a table whose contents are defined by -a query over another table. The query is recorded in the view's schema; a -refresh computes the rows. +a query over another table. The query is recorded in the view's schema; +creating the view computes its rows, and a refresh brings them up to date. The result is an ordinary table. It can be queried, indexed, and searched, and it appears in `table_names()` — nothing about reading it is special. @@ -19,27 +24,23 @@ In SQL this is [`REFRESH MATERIALIZED VIEW`](/sql/refresh#refresh-materialized-v ## Create and refresh {#create-and-refresh} -```python -import lancedb + +{MaterializedViewCreate} + -db = lancedb.connect( - "./.lancedb", - storage_options={"new_table_enable_stable_row_ids": "true"}, -) -table = db.create_table("people", [{"name": "ada", "age": 36}, - {"name": "kid", "age": 7}]) +Creating the view computes its rows, so it is ready to read. Pass +`with_no_data=True` to create it empty instead. Later changes to the source +reach the view through a refresh: -view = db.create_materialized_view( - "adults", - "people", - select=["name", ("shout", "upper(name)")], - where="age >= 18", -) + +{MaterializedViewRefresh} + -result = view.refresh() -result.rows_written -# 1 -``` + +The view is computed on creation from SDK 0.40. In 0.39 and earlier, +`create_materialized_view` always creates the view empty and the first +`refresh()` writes every matching row, two in this example. + `select` takes column names, `(alias, expression)` pairs, or a dict of the same. Omitting it selects every source column, expanded against the schema as it was @@ -94,8 +95,8 @@ process -- and that SQL can drive it: REFRESH MATERIALIZED VIEW chunks ``` -`WITH (full = true)` recomputes every row instead of only the rows the source -has added since the last refresh. `SHOW MATERIALIZED VIEWS` lists them, which is +`WITH (full = true)` recomputes every row instead of reconciling what changed +in the source since the last refresh. `SHOW MATERIALIZED VIEWS` lists them, which is cheaper than `list_materialized_views` -- that method finds views by reading every table's schema, so it costs one open per table. diff --git a/docs/web/management/materialized-view.mdx b/docs/web/management/materialized-view.mdx index 4283d7640..ac1ebb7ac 100644 --- a/docs/web/management/materialized-view.mdx +++ b/docs/web/management/materialized-view.mdx @@ -7,8 +7,8 @@ keywords: ["materialized view", "create_materialized_view", "catalog", "ddl"] --- A materialized view is a table whose contents are defined by a query over -another table. The query is recorded in the view's schema; a refresh computes -the rows. +another table. The query is recorded in the view's schema, and a refresh +brings its stored rows up to date with the source. The result is an ordinary table -- queryable, indexable, searchable, and present in `table_names()`. Nothing about reading it is special. diff --git a/docs/web/snippets/materialized_view.mdx b/docs/web/snippets/materialized_view.mdx new file mode 100644 index 000000000..9d3a5b602 --- /dev/null +++ b/docs/web/snippets/materialized_view.mdx @@ -0,0 +1,6 @@ +{/* Auto-generated by scripts/mdx_snippets_gen.py. Do not edit manually. */} + +export const PyMaterializedViewCreate = "import lancedb\n\ndb = lancedb.connect(\n \"./.lancedb\",\n storage_options={\"new_table_enable_stable_row_ids\": \"true\"},\n)\ntable = db.create_table(\n \"people\", [{\"name\": \"ada\", \"age\": 36}, {\"name\": \"kid\", \"age\": 7}]\n)\n\nview = db.create_materialized_view(\n \"adults\",\n \"people\",\n select=[\"name\", (\"shout\", \"upper(name)\")],\n where=\"age >= 18\",\n)\nprint(view.table.count_rows())\n# 1\n"; + +export const PyMaterializedViewRefresh = "table.add([{\"name\": \"bea\", \"age\": 41}])\n\nresult = view.refresh()\nprint(result.rows_written)\n# 1\n"; +