mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-24 23:18:39 +00:00
feat: refresh materialized views
A declared view holds no rows; refresh computes them. It pins one source version and brings the view to exactly the definition's result at that version, recording the version as a watermark in the view's schema metadata. The refresh is incremental when the source changed by nothing but appends and compactions since the watermark: a transaction-log walk separates the two, so only genuinely appended fragments are computed -- compaction outputs are already-materialized rows rearranged and cost nothing, which is what keeps routine background compaction from rebuilding the view. One subtlety shapes the walk: transaction files record an Append's fragments with placeholder ids (real ids are assigned at commit), while a Rewrite's ids are reserved beforehand and real, so appends are derived as new-at-head minus rewrite outputs rather than read from the log. A fragment-signature check scoped to the columns the view reads is the fallback for deltas the walk cannot classify, and passes changes to unrelated columns. Deletes and updates reconcile incrementally too: the lance delta reports the row ids the range deleted, the view evicts exactly those provenance ids, and updated rows are named by their row-version columns and recomputed. Past a fixed cap of staged ids -- or on a Legacy-storage source, which cannot serve the row-version columns -- refresh falls back to a rebuild, as it does for a vacuumed watermark version or an append a later compaction swallowed. A rebuild of an indexed view stages the new fragments uncommitted and commits one Update swapping out every old fragment, so index definitions are never absent and readers never observe an empty view; unindexed rebuilds overwrite, with the watermark riding the same commit. Refresh also accepts a pinned source version, and a row limit counts already-held rows. Concurrent refreshes of one view serialize at commit: each carries the same sentinel row id in its inserted-rows filter, so the loser's commit conflicts and lands nothing rather than double-applying. The remaining bound, stated in the module docs: on the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows on the next refresh.
This commit is contained in:
Wyatt Alt
Generated
+57
-57
@@ -141,7 +141,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1756,7 +1756,7 @@ version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3034,7 +3034,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3257,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4561,7 +4561,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4888,8 +4888,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4934,8 +4934,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4945,8 +4945,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4983,8 +4983,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5013,8 +5013,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5031,8 +5031,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5041,8 +5041,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5075,8 +5075,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5107,8 +5107,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5172,8 +5172,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5195,8 +5195,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5232,8 +5232,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5247,8 +5247,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5260,8 +5260,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5314,8 +5314,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5329,8 +5329,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5370,8 +5370,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5384,8 +5384,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
version = "11.0.0-beta.15"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#f0a9834469616e814c5ed993c269819ce1a30e4f"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -6236,7 +6236,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7629,7 +7629,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.12.1",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph",
|
||||
@@ -7648,7 +7648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.12.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -8697,7 +8697,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8768,7 +8768,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9359,7 +9359,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9466,7 +9466,7 @@ dependencies = [
|
||||
"cfg-if 1.0.4",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9758,7 +9758,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10735,7 +10735,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+30
-14
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
|
||||
rust-version = "1.91.0"
|
||||
|
||||
[workspace.dependencies]
|
||||
lance = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-core = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datagen = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-file = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-io = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-index = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-linalg = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-namespace-impls = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-table = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-testing = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-datafusion = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-encoding = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
lance-arrow = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
||||
ahash = "0.8"
|
||||
# Note that this one does not include pyarrow
|
||||
arrow = { version = "58.0.0", optional = false }
|
||||
@@ -80,3 +80,19 @@ debug = false
|
||||
debug-assertions = false
|
||||
strip = "debuginfo"
|
||||
incremental = false
|
||||
|
||||
[patch."https://github.com/lance-format/lance.git"]
|
||||
lance = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-arrow = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-core = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-datafusion = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-datagen = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-encoding = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-file = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-index = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-io = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-linalg = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-namespace = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-namespace-impls = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-table = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-testing = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
|
||||
@@ -196,4 +196,7 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
# releases are cut to crates.io. Allow that specific host.
|
||||
allow-git = [
|
||||
"https://github.com/lance-format/lance",
|
||||
# Scaffolding, paired with the [patch] section in Cargo.toml. Remove both
|
||||
# once deleted-row-id reporting is released from lance-format/lance.
|
||||
"https://github.com/wkalt/lance",
|
||||
]
|
||||
|
||||
@@ -209,7 +209,9 @@ pub use error::{Error, JobFailure, Result};
|
||||
pub use job::Job;
|
||||
use lance_index::vector::ApproxMode as LanceApproxMode;
|
||||
use lance_linalg::distance::DistanceType as LanceDistanceType;
|
||||
pub use materialized_view::{MaterializedView, MaterializedViewDefinition};
|
||||
pub use materialized_view::{
|
||||
MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
|
||||
};
|
||||
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
|
||||
/// the `metrics` feature to publish LanceDB's internal metrics; install any
|
||||
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! indexes, search -- works on it unchanged. Writes are not blocked, but a
|
||||
//! refresh that rebuilds replaces them; the definition is the source of truth.
|
||||
|
||||
pub mod refresh;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -37,6 +38,8 @@ use crate::table::refresh::quote_identifier;
|
||||
use crate::table::{ColumnDefinition, ColumnKind};
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub use refresh::{RefreshMaterializedViewResult, RefreshMode};
|
||||
|
||||
/// Schema metadata key holding the view definition, as kind-tagged JSON.
|
||||
pub const DEFINITION_META_KEY: &str = "mv.definition";
|
||||
|
||||
@@ -785,6 +788,12 @@ pub async fn prepare_declaration(
|
||||
),
|
||||
});
|
||||
}
|
||||
refresh::ensure_no_mem_wal(
|
||||
native.dataset.get().await?.as_ref(),
|
||||
"source table",
|
||||
resolved.name(),
|
||||
)
|
||||
.await?;
|
||||
let source_schema = resolved.schema().await?;
|
||||
let source_metadata = source_schema.metadata().clone();
|
||||
let (definition, mut fields, lineage) = plan(
|
||||
@@ -967,6 +976,54 @@ impl MaterializedView {
|
||||
pub fn definition(&self) -> &MaterializedViewDefinition {
|
||||
&self.definition
|
||||
}
|
||||
|
||||
/// Recompute the view from its source.
|
||||
///
|
||||
/// By default the refresh is incremental when the source's changes can be
|
||||
/// reconciled into the view, and otherwise rebuilds; see
|
||||
/// [`RefreshMaterializedViewBuilder`].
|
||||
///
|
||||
/// ```no_run
|
||||
/// # #![recursion_limit = "256"]
|
||||
/// # use lancedb::materialized_view::MaterializedView;
|
||||
/// # async fn refresh(view: &MaterializedView) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let result = view.refresh().execute().await?;
|
||||
/// println!("{:?}: {} rows", result.mode, result.rows_written);
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn refresh(&self) -> RefreshMaterializedViewBuilder {
|
||||
RefreshMaterializedViewBuilder {
|
||||
view: self.clone(),
|
||||
full: false,
|
||||
source_version: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a refresh. Created by [`MaterializedView::refresh`].
|
||||
pub struct RefreshMaterializedViewBuilder {
|
||||
view: MaterializedView,
|
||||
full: bool,
|
||||
source_version: Option<u64>,
|
||||
}
|
||||
|
||||
impl RefreshMaterializedViewBuilder {
|
||||
/// Rebuild the view even where an incremental refresh would do.
|
||||
pub fn full(mut self, full: bool) -> Self {
|
||||
self.full = full;
|
||||
self
|
||||
}
|
||||
|
||||
/// Refresh to this source table version instead of the latest.
|
||||
pub fn source_version(mut self, version: u64) -> Self {
|
||||
self.source_version = Some(version);
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn execute(self) -> Result<RefreshMaterializedViewResult> {
|
||||
refresh::execute_refresh(&self.view.table, self.full, self.source_version).await
|
||||
}
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
@@ -976,6 +1033,7 @@ impl Connection {
|
||||
/// metadata; refresh computes the rows. Local databases only.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # #![recursion_limit = "256"]
|
||||
/// # use lancedb::Connection;
|
||||
/// # async fn create(conn: &Connection) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let view = conn
|
||||
@@ -984,7 +1042,7 @@ impl Connection {
|
||||
/// .only_if("age >= 18")
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// println!("{}", view.definition().source_table);
|
||||
/// view.refresh().execute().await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
|
||||
return Err(Error::NotSupported {
|
||||
message: "an LSM write spec cannot be installed on a materialized view: \
|
||||
rows in un-compacted tiers are invisible to refresh"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
let mut builder = dataset.initialize_mem_wal();
|
||||
let writer_config_defaults = match spec {
|
||||
LsmWriteSpec::Bucket {
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
//! decides whether the fragment is staged at all -- a fragment where nothing
|
||||
//! would change stages nothing, which is what lets an expression yielding
|
||||
//! null settle instead of restaging forever. The second streams the
|
||||
//! fragment's physical rows into `write_column` a batch at a time, so peak
|
||||
//! fragment's physical rows into `write_columns` a batch at a time, so peak
|
||||
//! memory is bounded by a scan batch. The expression is evaluated by this
|
||||
//! module, never through a projection alias, and only over rows being
|
||||
//! filled: every other row -- deleted, or already holding a value -- has its
|
||||
@@ -67,7 +67,7 @@ pub(crate) async fn execute_refresh_column(
|
||||
.ok_or_else(|| Error::ColumnNotFound {
|
||||
name: column.to_string(),
|
||||
})?;
|
||||
// The dataset's own field, so the identity write_column checks against the
|
||||
// The dataset's own field, so the identity write_columns checks against the
|
||||
// manifest holds by construction.
|
||||
let column_schema = LanceSchema {
|
||||
fields: vec![field.clone()],
|
||||
@@ -83,7 +83,7 @@ pub(crate) async fn execute_refresh_column(
|
||||
}
|
||||
rows_filled += gained;
|
||||
let values = fill_stream(&dataset, &fragment, bound.clone(), column).await?;
|
||||
replacements.push(fragment.write_column(values, &column_schema).await?);
|
||||
replacements.push(fragment.write_columns(values, &column_schema).await?);
|
||||
}
|
||||
|
||||
if replacements.is_empty() {
|
||||
@@ -586,7 +586,7 @@ mod tests {
|
||||
|
||||
/// A fragment spanning several scan batches exercises the streamed fill:
|
||||
/// the probe buffers only until the first gained value and the rest flows
|
||||
/// through write_column a batch at a time.
|
||||
/// through write_columns a batch at a time.
|
||||
#[tokio::test]
|
||||
async fn test_refresh_streams_a_multi_batch_fragment() {
|
||||
let values: Vec<i32> = (0..20_000).collect();
|
||||
|
||||
Reference in New Issue
Block a user