rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-26 21:46:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(index): preserve layered indices during maintenance

Browse Source
This commit is contained in:
Yang Cen
2026-10-22 00:47:37 +08:00
parent 0e55bca5ba
commit f9150e8980
3 changed files with 79 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+24 -24
View File
@@ -3585,7 +3585,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -5060,7 +5060,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arc-swap",
"arrow",
@@ -5136,7 +5136,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5158,7 +5158,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5172,7 +5172,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5182,7 +5182,7 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrayref",
"crunchy",
@@ -5193,7 +5193,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5231,7 +5231,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow",
"arrow-array",
@@ -5263,7 +5263,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow",
"arrow-array",
@@ -5281,7 +5281,7 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"proc-macro2",
"quote",
@@ -5291,7 +5291,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5325,7 +5325,7 @@ dependencies = [
[[package]]
name = "lance-file"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5358,7 +5358,7 @@ dependencies = [
[[package]]
name = "lance-geo"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"datafusion",
"geo-traits",
@@ -5373,7 +5373,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arc-swap",
"arrow",
@@ -5448,7 +5448,7 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5472,7 +5472,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow",
"arrow-array",
@@ -5516,7 +5516,7 @@ dependencies = [
[[package]]
name = "lance-jsonb"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"byteorder",
"ethnum",
@@ -5534,7 +5534,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5549,7 +5549,7 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow",
"async-trait",
@@ -5564,7 +5564,7 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arc-swap",
"arrow",
@@ -5619,7 +5619,7 @@ dependencies = [
[[package]]
name = "lance-select"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5634,7 +5634,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow",
"arrow-array",
@@ -5675,7 +5675,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5689,7 +5689,7 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "14.0.0-beta.24"
source = "git+https://github.com/lance-format/lance.git?rev=5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd#5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd"
source = "git+https://github.com/lance-format/lance.git?rev=9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97#9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97"
dependencies = [
"frostem",
"icu_segmenter",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.94.0"
[workspace.dependencies]
lance = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.24", "rev" = "5f637a5ded32f0500d4113aefbe4ec4b8c5f42dd", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=14.0.0-beta.24", default-features = false, "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=14.0.0-beta.24", "rev" = "9a5e57c0ea4e66fc3c44c4a450d9733bbfd7ba97", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
python/python/tests/test_index.py
+41
View File
@@ -448,6 +448,47 @@ async def test_create_ivfrq_index(some_table: AsyncTable, num_bits, layered):
assert len(set(actual["id"].to_pylist()) & set(expected["id"].to_pylist())) >= 5
@pytest.mark.parametrize("num_bits,layered", [(5, False), (5, True), (9, True)])
def test_create_ivfrq_append_and_optimize(tmp_path, num_bits, layered):
vectors = np.random.default_rng(4252).normal(size=(512, 32)).astype("float32")
data = pa.table(
{
"id": range(len(vectors)),
"vector": pa.FixedSizeListArray.from_arrays(
pa.array(vectors.ravel()),
32,
mask=pa.array([row in (127, 383) for row in range(len(vectors))]),
),
}
)
table = connect(tmp_path).create_table(
"test", data.slice(0, 128), on_bad_vectors="null"
)
table.add(data.slice(128, 128), on_bad_vectors="null")
table.create_index(
"vector",
config=IvfRq(num_partitions=2, num_bits=num_bits, layered=layered),
)
for start in (256, 384):
table.add(data.slice(start, 128), on_bad_vectors="null")
table.optimize()
stats = table.index_stats("vector_idx")
assert stats.num_unindexed_rows == 0
assert stats.num_indexed_rows == start + 128
expected = (
table.search(vectors[start]).bypass_vector_index().limit(10).to_arrow()
)
actual = (
table.search(vectors[start])
.nprobes(2)
.refine_factor(4)
.limit(10)
.to_arrow()
)
assert len(set(actual["id"].to_pylist()) & set(expected["id"].to_pylist())) >= 5
assert table.to_arrow()["vector"].null_count == 2
@pytest.mark.parametrize("num_bits", [1, 2, 5])
@pytest.mark.parametrize("layered", [None, False, True])
def test_create_ivfrq_accelerator_preserves_layout(tmp_path, num_bits, layered):