mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-21 13:38:35 +00:00
Wyatt Alt
6fc6968ccc
A declared view holds no rows; refresh computes them. It pins one source version and brings the view to exactly the definition's result at that version, recording the version as a watermark in the view's schema metadata. The refresh is incremental when the source changed by nothing but appends and compactions since the watermark: a transaction-log walk separates the two, so only genuinely appended fragments are computed -- compaction outputs are already-materialized rows rearranged and cost nothing, which is what keeps routine background compaction from rebuilding the view. One subtlety shapes the walk: transaction files record an Append's fragments with placeholder ids (real ids are assigned at commit), while a Rewrite's ids are reserved beforehand and real, so appends are derived as new-at-head minus rewrite outputs rather than read from the log. A fragment-signature check scoped to the columns the view reads is the fallback for deltas the walk cannot classify, and passes changes to unrelated columns. Deletes and updates reconcile incrementally too: the lance delta reports the row ids the range deleted, the view evicts exactly those provenance ids, and updated rows are named by their row-version columns and recomputed. Past a fixed cap of staged ids -- or on a Legacy-storage source, which cannot serve the row-version columns -- refresh falls back to a rebuild, as it does for a vacuumed watermark version or an append a later compaction swallowed. A rebuild of an indexed view stages the new fragments uncommitted and commits one Update swapping out every old fragment, so index definitions are never absent and readers never observe an empty view; unindexed rebuilds overwrite, with the watermark riding the same commit. Refresh also accepts a pinned source version, and a row limit counts already-held rows. Concurrent refreshes of one view serialize at commit: each carries the same sentinel row id in its inserted-rows filter, so the loser's commit conflicts and lands nothing rather than double-applying. The remaining bound, stated in the module docs: on the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows on the next refresh.
99 lines
5.3 KiB
TOML
99 lines
5.3 KiB
TOML
[workspace]
|
|
members = ["rust/lancedb", "nodejs", "python"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
authors = ["LanceDB Devs <dev@lancedb.com>"]
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/lancedb/lancedb"
|
|
description = "Serverless, low-latency vector database for AI applications"
|
|
keywords = ["lancedb", "lance", "database", "vector", "search"]
|
|
categories = ["database-implementations"]
|
|
rust-version = "1.91.0"
|
|
|
|
[workspace.dependencies]
|
|
lance = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-core = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datagen = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-file = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-io = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-index = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-linalg = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace-impls = { "version" = "=11.0.0-beta.15", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-table = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-testing = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datafusion = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-encoding = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-arrow = { "version" = "=11.0.0-beta.15", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
ahash = "0.8"
|
|
# Note that this one does not include pyarrow
|
|
arrow = { version = "58.0.0", optional = false }
|
|
arrow-array = "58.0.0"
|
|
arrow-buffer = "58.0.0"
|
|
arrow-data = "58.0.0"
|
|
arrow-ipc = "58.0.0"
|
|
arrow-ord = "58.0.0"
|
|
arrow-schema = "58.0.0"
|
|
arrow-select = "58.0.0"
|
|
arrow-cast = "58.0.0"
|
|
async-trait = "0"
|
|
datafusion = { version = "54.0.0", default-features = false }
|
|
datafusion-catalog = "54.0.0"
|
|
datafusion-common = { version = "54.0.0", default-features = false }
|
|
datafusion-execution = "54.0.0"
|
|
datafusion-expr = "54.0.0"
|
|
datafusion-functions = "54.0.0"
|
|
datafusion-physical-plan = "54.0.0"
|
|
datafusion-physical-expr = "54.0.0"
|
|
datafusion-sql = "54.0.0"
|
|
env_logger = "0.11"
|
|
half = { "version" = "2.7.1", default-features = false, features = [
|
|
"num-traits",
|
|
] }
|
|
futures = "0.3"
|
|
log = "0.4"
|
|
metrics = "0.24"
|
|
metrics-util = "0.19"
|
|
moka = { version = "0.12", features = ["future"] }
|
|
object_store = "0.13.2"
|
|
pin-project = "1.0.7"
|
|
rand = "0.9"
|
|
snafu = "0.8"
|
|
url = "2"
|
|
num-traits = "0.2"
|
|
regex = "1.10"
|
|
semver = "1.0.25"
|
|
chrono = "0.4"
|
|
|
|
[profile.ci]
|
|
debug = "line-tables-only"
|
|
inherits = "dev"
|
|
incremental = false
|
|
|
|
# This rule applies to every package except workspace members (dependencies
|
|
# such as `arrow` and `tokio`). It disables debug info and related features on
|
|
# dependencies so their binaries stay smaller, improving cache reuse.
|
|
[profile.ci.package."*"]
|
|
debug = false
|
|
debug-assertions = false
|
|
strip = "debuginfo"
|
|
incremental = false
|
|
|
|
[patch."https://github.com/lance-format/lance.git"]
|
|
lance = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-arrow = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-core = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-datafusion = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-datagen = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-encoding = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-file = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-index = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-io = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-linalg = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-namespace = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-namespace-impls = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-table = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-testing = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|