rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-04 12:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Pin manifest-complete UUID guard

Browse Source
This commit is contained in:
ChilePiquin
2026-08-31 11:28:51 -07:00
parent 3c47c74a47
commit 00897f6635
3 changed files with 116 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+22 -22
View File
@@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arc-swap",
"arrow",
@@ -4889,7 +4889,7 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4935,7 +4935,7 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrayref",
"crunchy",
@@ -4946,7 +4946,7 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4984,7 +4984,7 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"arrow-array",
@@ -5014,7 +5014,7 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"arrow-array",
@@ -5032,7 +5032,7 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"proc-macro2",
"quote",
@@ -5042,7 +5042,7 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5076,7 +5076,7 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5108,7 +5108,7 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arc-swap",
"arrow",
@@ -5173,7 +5173,7 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5196,7 +5196,7 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"arrow-array",
@@ -5237,7 +5237,7 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5252,7 +5252,7 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"async-trait",
@@ -5265,7 +5265,7 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5319,7 +5319,7 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5334,7 +5334,7 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow",
"arrow-array",
@@ -5375,7 +5375,7 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5389,7 +5389,7 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.5"
source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082"
source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0"
dependencies = [
"frostem",
"icu_segmenter",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-core = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-file = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-index = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-table = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" }
lance = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-core = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-file = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-index = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-table = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
rust/lancedb/src/table/create_index.rs
+80
View File
@@ -473,6 +473,7 @@ mod tests {
use arrow_data::ArrayDataBuilder;
use arrow_schema::{DataType, Field, Schema};
use futures::TryStreamExt;
use lance::index::DatasetIndexExt;
use tempfile::tempdir;
use crate::connect;
@@ -1160,6 +1161,85 @@ mod tests {
assert_eq!(index.index_uuid, Some(index_uuid.to_string()));
}
#[tokio::test]
async fn test_create_index_rejects_hidden_uuid_collision_before_writing() {
use lance::dataset::transaction::{Operation, Transaction};
use lance::dataset::{CommitBuilder, WriteDestination};
let conn = connect("memory://").execute().await.unwrap();
let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap();
let table = conn
.create_table("my_table", batch)
.execute()
.await
.unwrap();
table
.create_index(&["a"], Index::BTree(BTreeIndexBuilder::default()))
.name("a_idx".to_string())
.execute()
.await
.unwrap();
let native = table.as_native().unwrap();
let mut dataset = (*native.dataset.get().await.unwrap()).clone();
let original = dataset.load_indices().await.unwrap()[0].clone();
let mut hidden = original.clone();
hidden.index_version += 1;
let version = dataset.manifest().version;
dataset = CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset)))
.execute(Transaction::new(
version,
Operation::CreateIndex {
new_indices: vec![hidden.clone()],
removed_indices: vec![original.clone()],
},
None,
))
.await
.unwrap();
native.dataset.update(dataset);
assert!(table.list_indices().await.unwrap().is_empty());
let err = table
.create_index(&["a"], Index::BTree(BTreeIndexBuilder::default()))
.name("other_idx".to_string())
.index_uuid(original.uuid)
.train(false)
.replace(false)
.execute()
.await
.unwrap_err();
assert!(err.to_string().contains("already used by index 'a_idx'"));
let dataset = (*native.dataset.get().await.unwrap()).clone();
let version = dataset.manifest().version;
let dataset = CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset)))
.execute(Transaction::new(
version,
Operation::CreateIndex {
new_indices: vec![original],
removed_indices: vec![hidden],
},
None,
))
.await
.unwrap();
native.dataset.update(dataset);
let batches = table
.query()
.only_if("a = 1")
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(batches[0].num_rows(), 1);
}
#[tokio::test]
async fn test_create_index_rejects_selected_uuid_trained_build() {
let conn = connect("memory://").execute().await.unwrap();