mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 03:58:26 +00:00
feat: refresh materialized views
A declared view holds no rows; refresh computes them. It pins one source version and brings the view to exactly the definition's result at that version, recording the version as a watermark in the view's schema metadata. The refresh is incremental when the source changed by nothing but appends and compactions since the watermark: a transaction-log walk separates the two, so only genuinely appended fragments are computed -- compaction outputs are already-materialized rows rearranged and cost nothing, which is what keeps routine background compaction from rebuilding the view. One subtlety shapes the walk: transaction files record an Append's fragments with placeholder ids (real ids are assigned at commit), while a Rewrite's ids are reserved beforehand and real, so appends are derived as new-at-head minus rewrite outputs rather than read from the log. A fragment-signature check scoped to the columns the view reads is the fallback for deltas the walk cannot classify, and passes changes to unrelated columns. Anything else rebuilds: deletes, updates, a vacuumed watermark version, or an append a later compaction swallowed (its rows cannot be told apart from already-materialized ones without a provenance scan). A rebuild of an indexed view stages the new fragments uncommitted and commits one Update swapping out every old fragment, so index definitions are never absent and readers never observe an empty view; unindexed rebuilds overwrite, with the watermark riding the same commit. Refresh also accepts a pinned source version, and a row limit counts already-held rows. Two bounds, stated in the module docs: on the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows on the next refresh; and concurrent refreshes of one view can double-append, since lance appends do not conflict -- run one at a time.
This commit is contained in:
Wyatt Alt
Generated
+37
-37
@@ -1756,7 +1756,7 @@ version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3034,7 +3034,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3257,7 +3257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
[[package]]
|
||||
name = "fsst"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"rand 0.9.5",
|
||||
@@ -4561,7 +4561,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
|
||||
[[package]]
|
||||
name = "lance"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -4889,7 +4889,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4911,7 +4911,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-scalar"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4925,7 +4925,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-arrow-stats"
|
||||
version = "58.0.0"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -4935,7 +4935,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-bitpacking"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrayref",
|
||||
"crunchy",
|
||||
@@ -4946,7 +4946,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-core"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -4984,7 +4984,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-datafusion"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5014,7 +5014,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-datagen"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5032,7 +5032,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-derive"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5042,7 +5042,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-encoding"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5076,7 +5076,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-file"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-arith",
|
||||
"arrow-array",
|
||||
@@ -5108,7 +5108,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-index"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrow",
|
||||
@@ -5173,7 +5173,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-index-core"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5196,7 +5196,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-io"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5233,7 +5233,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-linalg"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5248,7 +5248,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-namespace"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -5261,7 +5261,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-namespace-impls"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -5315,7 +5315,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-select"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-buffer",
|
||||
@@ -5330,7 +5330,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-table"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-array",
|
||||
@@ -5371,7 +5371,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-testing"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"arrow-array",
|
||||
"arrow-schema",
|
||||
@@ -5385,7 +5385,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "lance-tokenizer"
|
||||
version = "11.0.0-beta.13"
|
||||
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.13#ee41152ceb9a78e5df4d2456fdbdb98542eb2059"
|
||||
source = "git+https://github.com/wkalt/lance.git?branch=ticket%2Fent-1961%2Fdelta-deleted-row-ids#ee949fbaf249e615b912e1401e7cd5b4db0670c4"
|
||||
dependencies = [
|
||||
"frostem",
|
||||
"icu_segmenter",
|
||||
@@ -6236,7 +6236,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7628,8 +7628,8 @@ version = "0.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"itertools 0.14.0",
|
||||
"heck 0.4.1",
|
||||
"itertools 0.11.0",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph",
|
||||
@@ -7648,7 +7648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
"itertools 0.11.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -7919,7 +7919,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2 0.6.3",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8697,7 +8697,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8768,7 +8768,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -9324,7 +9324,7 @@ version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -9336,7 +9336,7 @@ version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54254b8531cafa275c5e096f62d48c81435d1015405a91198ddb11e967301d40"
|
||||
dependencies = [
|
||||
"heck 0.5.0",
|
||||
"heck 0.4.1",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -9758,7 +9758,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -10735,7 +10735,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+16
@@ -80,3 +80,19 @@ debug = false
|
||||
debug-assertions = false
|
||||
strip = "debuginfo"
|
||||
incremental = false
|
||||
|
||||
[patch."https://github.com/lance-format/lance.git"]
|
||||
lance = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-arrow = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-core = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-datafusion = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-datagen = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-encoding = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-file = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-index = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-io = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-linalg = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-namespace = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-namespace-impls = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-table = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
lance-testing = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
||||
|
||||
@@ -196,4 +196,7 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
# releases are cut to crates.io. Allow that specific host.
|
||||
allow-git = [
|
||||
"https://github.com/lance-format/lance",
|
||||
# Scaffolding, paired with the [patch] section in Cargo.toml. Remove both
|
||||
# once deleted-row-id reporting is released from lance-format/lance.
|
||||
"https://github.com/wkalt/lance",
|
||||
]
|
||||
|
||||
@@ -209,7 +209,9 @@ pub use error::{Error, JobFailure, Result};
|
||||
pub use job::Job;
|
||||
use lance_index::vector::ApproxMode as LanceApproxMode;
|
||||
use lance_linalg::distance::DistanceType as LanceDistanceType;
|
||||
pub use materialized_view::{MaterializedView, MaterializedViewDefinition};
|
||||
pub use materialized_view::{
|
||||
MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
|
||||
};
|
||||
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
|
||||
/// the `metrics` feature to publish LanceDB's internal metrics; install any
|
||||
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
//! indexes, search -- works on it unchanged. Writes are not blocked, but a
|
||||
//! refresh that rebuilds replaces them; the definition is the source of truth.
|
||||
|
||||
pub mod refresh;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -37,6 +38,8 @@ use crate::table::refresh::quote_identifier;
|
||||
use crate::table::{ColumnDefinition, ColumnKind};
|
||||
use crate::{Error, Result};
|
||||
|
||||
pub use refresh::{RefreshMaterializedViewResult, RefreshMode};
|
||||
|
||||
/// Schema metadata key holding the view definition, as kind-tagged JSON.
|
||||
pub const DEFINITION_META_KEY: &str = "mv.definition";
|
||||
|
||||
@@ -785,6 +788,12 @@ pub async fn prepare_declaration(
|
||||
),
|
||||
});
|
||||
}
|
||||
refresh::ensure_no_mem_wal(
|
||||
native.dataset.get().await?.as_ref(),
|
||||
"source table",
|
||||
resolved.name(),
|
||||
)
|
||||
.await?;
|
||||
let source_schema = resolved.schema().await?;
|
||||
let source_metadata = source_schema.metadata().clone();
|
||||
let (definition, mut fields, lineage) = plan(
|
||||
@@ -967,6 +976,54 @@ impl MaterializedView {
|
||||
pub fn definition(&self) -> &MaterializedViewDefinition {
|
||||
&self.definition
|
||||
}
|
||||
|
||||
/// Recompute the view from its source.
|
||||
///
|
||||
/// By default the refresh is incremental when the source's changes can be
|
||||
/// reconciled into the view, and otherwise rebuilds; see
|
||||
/// [`RefreshMaterializedViewBuilder`].
|
||||
///
|
||||
/// ```no_run
|
||||
/// # #![recursion_limit = "256"]
|
||||
/// # use lancedb::materialized_view::MaterializedView;
|
||||
/// # async fn refresh(view: &MaterializedView) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let result = view.refresh().execute().await?;
|
||||
/// println!("{:?}: {} rows", result.mode, result.rows_written);
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn refresh(&self) -> RefreshMaterializedViewBuilder {
|
||||
RefreshMaterializedViewBuilder {
|
||||
view: self.clone(),
|
||||
full: false,
|
||||
source_version: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds a refresh. Created by [`MaterializedView::refresh`].
|
||||
pub struct RefreshMaterializedViewBuilder {
|
||||
view: MaterializedView,
|
||||
full: bool,
|
||||
source_version: Option<u64>,
|
||||
}
|
||||
|
||||
impl RefreshMaterializedViewBuilder {
|
||||
/// Rebuild the view even where an incremental refresh would do.
|
||||
pub fn full(mut self, full: bool) -> Self {
|
||||
self.full = full;
|
||||
self
|
||||
}
|
||||
|
||||
/// Refresh to this source table version instead of the latest.
|
||||
pub fn source_version(mut self, version: u64) -> Self {
|
||||
self.source_version = Some(version);
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn execute(self) -> Result<RefreshMaterializedViewResult> {
|
||||
refresh::execute_refresh(&self.view.table, self.full, self.source_version).await
|
||||
}
|
||||
}
|
||||
|
||||
impl Connection {
|
||||
@@ -976,6 +1033,7 @@ impl Connection {
|
||||
/// metadata; refresh computes the rows. Local databases only.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # #![recursion_limit = "256"]
|
||||
/// # use lancedb::Connection;
|
||||
/// # async fn create(conn: &Connection) -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let view = conn
|
||||
@@ -984,7 +1042,7 @@ impl Connection {
|
||||
/// .only_if("age >= 18")
|
||||
/// .execute()
|
||||
/// .await?;
|
||||
/// println!("{}", view.definition().source_table);
|
||||
/// view.refresh().execute().await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
|
||||
return Err(Error::NotSupported {
|
||||
message: "an LSM write spec cannot be installed on a materialized view: \
|
||||
rows in un-compacted tiers are invisible to refresh"
|
||||
.into(),
|
||||
});
|
||||
}
|
||||
let mut builder = dataset.initialize_mem_wal();
|
||||
let writer_config_defaults = match spec {
|
||||
LsmWriteSpec::Bucket {
|
||||
|
||||
Reference in New Issue
Block a user