mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
Wyatt Alt
4cd8e29114
A declared view holds no rows; refresh computes them. It pins one source version and brings the view to exactly the definition's result at that version, recording the version as a watermark in the view's schema metadata. The refresh is incremental when the source changed by nothing but appends and compactions since the watermark: a transaction-log walk separates the two, so only genuinely appended fragments are computed -- compaction outputs are already-materialized rows rearranged and cost nothing, which is what keeps routine background compaction from rebuilding the view. One subtlety shapes the walk: transaction files record an Append's fragments with placeholder ids (real ids are assigned at commit), while a Rewrite's ids are reserved beforehand and real, so appends are derived as new-at-head minus rewrite outputs rather than read from the log. A fragment-signature check scoped to the columns the view reads is the fallback for deltas the walk cannot classify, and passes changes to unrelated columns. Anything else rebuilds: deletes, updates, a vacuumed watermark version, or an append a later compaction swallowed (its rows cannot be told apart from already-materialized ones without a provenance scan). A rebuild of an indexed view stages the new fragments uncommitted and commits one Update swapping out every old fragment, so index definitions are never absent and readers never observe an empty view; unindexed rebuilds overwrite, with the watermark riding the same commit. Refresh also accepts a pinned source version, and a row limit counts already-held rows. Two bounds, stated in the module docs: on the append path the watermark moves in a follow-up commit, so a crash between the two re-appends those rows on the next refresh; and concurrent refreshes of one view can double-append, since lance appends do not conflict -- run one at a time.
99 lines
5.3 KiB
TOML
99 lines
5.3 KiB
TOML
[workspace]
|
|
members = ["rust/lancedb", "nodejs", "python"]
|
|
resolver = "2"
|
|
|
|
[workspace.package]
|
|
edition = "2024"
|
|
authors = ["LanceDB Devs <dev@lancedb.com>"]
|
|
license = "Apache-2.0"
|
|
repository = "https://github.com/lancedb/lancedb"
|
|
description = "Serverless, low-latency vector database for AI applications"
|
|
keywords = ["lancedb", "lance", "database", "vector", "search"]
|
|
categories = ["database-implementations"]
|
|
rust-version = "1.91.0"
|
|
|
|
[workspace.dependencies]
|
|
lance = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-core = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datagen = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-file = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-io = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-index = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-linalg = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-namespace-impls = { "version" = "=11.0.0-beta.13", default-features = false, "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-table = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-testing = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-datafusion = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-encoding = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
lance-arrow = { "version" = "=11.0.0-beta.13", "tag" = "v11.0.0-beta.13", "git" = "https://github.com/lance-format/lance.git" }
|
|
ahash = "0.8"
|
|
# Note that this one does not include pyarrow
|
|
arrow = { version = "58.0.0", optional = false }
|
|
arrow-array = "58.0.0"
|
|
arrow-buffer = "58.0.0"
|
|
arrow-data = "58.0.0"
|
|
arrow-ipc = "58.0.0"
|
|
arrow-ord = "58.0.0"
|
|
arrow-schema = "58.0.0"
|
|
arrow-select = "58.0.0"
|
|
arrow-cast = "58.0.0"
|
|
async-trait = "0"
|
|
datafusion = { version = "54.0.0", default-features = false }
|
|
datafusion-catalog = "54.0.0"
|
|
datafusion-common = { version = "54.0.0", default-features = false }
|
|
datafusion-execution = "54.0.0"
|
|
datafusion-expr = "54.0.0"
|
|
datafusion-functions = "54.0.0"
|
|
datafusion-physical-plan = "54.0.0"
|
|
datafusion-physical-expr = "54.0.0"
|
|
datafusion-sql = "54.0.0"
|
|
env_logger = "0.11"
|
|
half = { "version" = "2.7.1", default-features = false, features = [
|
|
"num-traits",
|
|
] }
|
|
futures = "0.3"
|
|
log = "0.4"
|
|
metrics = "0.24"
|
|
metrics-util = "0.19"
|
|
moka = { version = "0.12", features = ["future"] }
|
|
object_store = "0.13.2"
|
|
pin-project = "1.0.7"
|
|
rand = "0.9"
|
|
snafu = "0.8"
|
|
url = "2"
|
|
num-traits = "0.2"
|
|
regex = "1.10"
|
|
semver = "1.0.25"
|
|
chrono = "0.4"
|
|
|
|
[profile.ci]
|
|
debug = "line-tables-only"
|
|
inherits = "dev"
|
|
incremental = false
|
|
|
|
# This rule applies to every package except workspace members (dependencies
|
|
# such as `arrow` and `tokio`). It disables debug info and related features on
|
|
# dependencies so their binaries stay smaller, improving cache reuse.
|
|
[profile.ci.package."*"]
|
|
debug = false
|
|
debug-assertions = false
|
|
strip = "debuginfo"
|
|
incremental = false
|
|
|
|
[patch."https://github.com/lance-format/lance.git"]
|
|
lance = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-arrow = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-core = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-datafusion = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-datagen = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-encoding = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-file = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-index = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-io = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-linalg = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-namespace = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-namespace-impls = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-table = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|
|
lance-testing = { git = "https://github.com/wkalt/lance.git", branch = "ticket/ent-1961/delta-deleted-row-ids" }
|