diff --git a/Cargo.lock b/Cargo.lock index 0515319b2..a2e317c22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5494,6 +5494,7 @@ dependencies = [ "path_abs", "pin-project", "prost", + "quick-xml 0.41.0", "rand 0.9.5", "reqsign-core", "reqsign-file-read-tokio", @@ -5509,8 +5510,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5528,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5543,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5558,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5613,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5628,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5669,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5683,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.15" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.15#765641dc0d7100f032fdebc00d73eef49711c4d4" +version = "14.0.0-beta.18" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.18#f80bf1d1e59f40d249bce2539b78e41d93dee9ec" dependencies = [ "frostem", "icu_segmenter", @@ -6819,8 +6820,7 @@ dependencies = [ [[package]] name = "object_store" version = "0.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" +source = "git+https://github.com/apache/arrow-rs-object-store.git?rev=c10758c207c0103355f940f0c736fb8ecb157b04#c10758c207c0103355f940f0c736fb8ecb157b04" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/Cargo.toml b/Cargo.toml index 5033b1529..c6b9e84fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.15", default-features = false, "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.15", "tag" = "v14.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.18", default-features = false, "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.18", "tag" = "v14.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow @@ -79,6 +79,10 @@ tonic = { version = "0.14", features = ["tls-native-roots", "tls-aws-lc"] } uuid = { version = "1.7.0", features = ["v4", "v7"] } chrono = { version = "0.4", default-features = false, features = ["clock"] } +[patch.crates-io] +# Lance v14.0.0-beta.18 requires multipart write modes from this upstream revision. +object_store = { git = "https://github.com/apache/arrow-rs-object-store.git", rev = "c10758c207c0103355f940f0c736fb8ecb157b04" } + [profile.ci] debug = "line-tables-only" inherits = "dev" diff --git a/deny.toml b/deny.toml index d112095ec..d5f42dbff 100644 --- a/deny.toml +++ b/deny.toml @@ -218,4 +218,6 @@ allow-registry = ["https://github.com/rust-lang/crates.io-index"] # releases are cut to crates.io. Allow that specific host. allow-git = [ "https://github.com/lance-format/lance", + # Pinned by Lance for the multipart write mode API (see [patch.crates-io] in Cargo.toml). + "https://github.com/apache/arrow-rs-object-store", ] diff --git a/java/pom.xml b/java/pom.xml index 33446040b..465554706 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.15 + 14.0.0-beta.18 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/query.rs b/rust/lancedb/src/query.rs index 07db69bb3..d66932847 100644 --- a/rust/lancedb/src/query.rs +++ b/rust/lancedb/src/query.rs @@ -3806,8 +3806,12 @@ mod tests { assert_eq!(results[0].num_columns(), 1); } + #[rstest::rstest] + #[case::single_row(1)] + #[case::two_rows(2)] + #[case::three_rows(3)] #[tokio::test] - async fn test_take_offsets_preserves_duplicate_multiplicity() { + async fn test_take_offsets_preserves_duplicate_multiplicity(#[case] max_batch_length: u32) { let tmp_dir = tempdir().unwrap(); let table = make_test_table(&tmp_dir).await; @@ -3815,7 +3819,7 @@ mod tests { .take_offsets(vec![5, 1, 5, 17]) .select(Select::Columns(vec!["id".to_string()])) .execute_with_options(QueryExecutionOptions { - max_batch_length: 2, + max_batch_length, ..Default::default() }) .await @@ -3824,7 +3828,11 @@ mod tests { .await .unwrap(); - assert_eq!(results.len(), 2); + assert!( + results + .iter() + .all(|batch| batch.num_rows() <= max_batch_length as usize) + ); assert!(results.iter().all(|batch| batch.num_columns() == 1)); let mut ids = results .iter() diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index fbe82a198..c74980c5e 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -2035,11 +2035,13 @@ mod lsm_tests { ); } - /// Claims the base table's bitmap index and keeps it as a B-tree, whose - /// flush rows are in the shape the bitmap trainer takes. + /// Claims the base table's bitmap index and keeps it as a B-tree. #[derive(Debug)] struct BitmapAsBTree; + #[derive(Debug)] + struct BitmapAsBTreeIndex(lance::dataset::mem_wal::index::BTreeMemIndex); + fn btree_plugin() -> Arc { lance::dataset::mem_wal::MemIndexRegistry::default() .plugin_for_details_url("/lance.table.BTreeIndexDetails") @@ -2059,7 +2061,8 @@ mod lsm_tests { lance_index::IndexType::Bitmap } fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria { - btree_plugin().training_criteria() + use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering}; + TrainingCriteria::new(TrainingOrdering::Values).with_row_id() } fn validate( &self, @@ -2071,7 +2074,57 @@ mod lsm_tests { &self, ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>, ) -> lance_core::Result> { - btree_plugin().create(ctx) + let (column, field_id) = ctx.single_column()?; + Ok(Arc::new(BitmapAsBTreeIndex( + lance::dataset::mem_wal::index::BTreeMemIndex::new(field_id, column.to_string()), + ))) + } + } + + #[async_trait::async_trait] + impl lance::dataset::mem_wal::index::MemIndex for BitmapAsBTreeIndex { + fn columns(&self) -> &[String] { + self.0.columns() + } + fn can_answer(&self, query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool { + self.0.can_answer(query) + } + fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> { + self.0.insert(batch, row_offset) + } + fn resident_bytes(&self) -> usize { + self.0.resident_bytes() + } + fn search( + &self, + query: &dyn lance::dataset::mem_wal::index::MemQuery, + ctx: &lance::dataset::mem_wal::index::SearchContext, + ) -> lance_core::Result> { + self.0.search(query, ctx) + } + async fn flush( + &self, + ctx: &lance::dataset::mem_wal::index::FlushContext<'_>, + ) -> lance_core::Result { + // BTree now trains on row addresses, while Bitmap still takes row IDs. + // These coincide in the flushed generation, which uses physical IDs. + assert!(!ctx.generation()?.dataset.manifest().uses_stable_row_ids()); + let batches = self + .0 + .to_training_batches(ctx.batch_size)? + .into_iter() + .map(|batch| { + assert_eq!(batch.schema().field(1).name(), lance_core::ROW_ADDR); + let schema = Arc::new(Schema::new(vec![ + batch.schema().field(0).clone(), + Field::new(lance_core::ROW_ID, DataType::UInt64, false), + ])); + RecordBatch::try_new(schema, batch.columns().to_vec()) + }) + .collect::, _>>()?; + Ok(lance::dataset::mem_wal::index::FlushOutcome::from_batches( + batches, + )) } }