diff --git a/Cargo.lock b/Cargo.lock index 2f64b7865..3ca796658 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3456,7 +3456,7 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4816,7 +4816,7 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arc-swap", "arrow", @@ -4889,7 +4889,7 @@ dependencies = [ [[package]] name = "lance-arrow" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-schema", @@ -4935,7 +4935,7 @@ dependencies = [ [[package]] name = "lance-bitpacking" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrayref", "crunchy", @@ -4946,7 +4946,7 @@ dependencies = [ [[package]] name = "lance-core" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-buffer", @@ -4984,7 +4984,7 @@ dependencies = [ [[package]] name = "lance-datafusion" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "arrow-array", @@ -5014,7 +5014,7 @@ dependencies = [ [[package]] name = "lance-datagen" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "arrow-array", @@ -5032,7 +5032,7 @@ dependencies = [ [[package]] name = "lance-derive" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "proc-macro2", "quote", @@ -5042,7 +5042,7 @@ dependencies = [ [[package]] name = "lance-encoding" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-arith", "arrow-array", @@ -5076,7 +5076,7 @@ dependencies = [ [[package]] name = "lance-file" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-arith", "arrow-array", @@ -5108,7 +5108,7 @@ dependencies = [ [[package]] name = "lance-index" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arc-swap", "arrow", @@ -5173,7 +5173,7 @@ dependencies = [ [[package]] name = "lance-index-core" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-schema", @@ -5196,7 +5196,7 @@ dependencies = [ [[package]] name = "lance-io" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "arrow-array", @@ -5237,7 +5237,7 @@ dependencies = [ [[package]] name = "lance-linalg" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-schema", @@ -5252,7 +5252,7 @@ dependencies = [ [[package]] name = "lance-namespace" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "async-trait", @@ -5265,7 +5265,7 @@ dependencies = [ [[package]] name = "lance-namespace-impls" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "arrow-ipc", @@ -5319,7 +5319,7 @@ dependencies = [ [[package]] name = "lance-select" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-buffer", @@ -5334,7 +5334,7 @@ dependencies = [ [[package]] name = "lance-table" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow", "arrow-array", @@ -5375,7 +5375,7 @@ dependencies = [ [[package]] name = "lance-testing" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "arrow-array", "arrow-schema", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "lance-tokenizer" version = "12.0.0-beta.5" -source = "git+https://github.com/ChilePiquin/lance.git?rev=b16805bc83d67cfae1e6cdc25561eecb11b0c082#b16805bc83d67cfae1e6cdc25561eecb11b0c082" +source = "git+https://github.com/ChilePiquin/lance.git?rev=7b808b9b116b1afa49aff9a45254f7accb70fda0#7b808b9b116b1afa49aff9a45254f7accb70fda0" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index fa1c2ace0..7804f6407 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-core = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-file = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-index = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-table = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.5", "rev" = "b16805bc83d67cfae1e6cdc25561eecb11b0c082", "git" = "https://github.com/ChilePiquin/lance.git" } +lance = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-core = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-datagen = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-file = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-io = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-index = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-linalg = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-namespace = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-namespace-impls = { "version" = "=12.0.0-beta.5", default-features = false, "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-table = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-testing = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-datafusion = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-encoding = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } +lance-arrow = { "version" = "=12.0.0-beta.5", "rev" = "7b808b9b116b1afa49aff9a45254f7accb70fda0", "git" = "https://github.com/ChilePiquin/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 9c6f9f0b8..b69f0188f 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -473,6 +473,7 @@ mod tests { use arrow_data::ArrayDataBuilder; use arrow_schema::{DataType, Field, Schema}; use futures::TryStreamExt; + use lance::index::DatasetIndexExt; use tempfile::tempdir; use crate::connect; @@ -1160,6 +1161,85 @@ mod tests { assert_eq!(index.index_uuid, Some(index_uuid.to_string())); } + #[tokio::test] + async fn test_create_index_rejects_hidden_uuid_collision_before_writing() { + use lance::dataset::transaction::{Operation, Transaction}; + use lance::dataset::{CommitBuilder, WriteDestination}; + + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("a", Int32, [1, 2, 3])).unwrap(); + let table = conn + .create_table("my_table", batch) + .execute() + .await + .unwrap(); + + table + .create_index(&["a"], Index::BTree(BTreeIndexBuilder::default())) + .name("a_idx".to_string()) + .execute() + .await + .unwrap(); + + let native = table.as_native().unwrap(); + let mut dataset = (*native.dataset.get().await.unwrap()).clone(); + let original = dataset.load_indices().await.unwrap()[0].clone(); + let mut hidden = original.clone(); + hidden.index_version += 1; + + let version = dataset.manifest().version; + dataset = CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset))) + .execute(Transaction::new( + version, + Operation::CreateIndex { + new_indices: vec![hidden.clone()], + removed_indices: vec![original.clone()], + }, + None, + )) + .await + .unwrap(); + native.dataset.update(dataset); + assert!(table.list_indices().await.unwrap().is_empty()); + + let err = table + .create_index(&["a"], Index::BTree(BTreeIndexBuilder::default())) + .name("other_idx".to_string()) + .index_uuid(original.uuid) + .train(false) + .replace(false) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("already used by index 'a_idx'")); + + let dataset = (*native.dataset.get().await.unwrap()).clone(); + let version = dataset.manifest().version; + let dataset = CommitBuilder::new(WriteDestination::Dataset(Arc::new(dataset))) + .execute(Transaction::new( + version, + Operation::CreateIndex { + new_indices: vec![original], + removed_indices: vec![hidden], + }, + None, + )) + .await + .unwrap(); + native.dataset.update(dataset); + + let batches = table + .query() + .only_if("a = 1") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches[0].num_rows(), 1); + } + #[tokio::test] async fn test_create_index_rejects_selected_uuid_trained_build() { let conn = connect("memory://").execute().await.unwrap();