From f0bfb89d9cd3ae47248f90345573b0a390c2f678 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Sat, 10 Oct 2026 23:33:56 -0700 Subject: [PATCH] chore: update lance dependency to v14.0.0-beta.11 (#4518) Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core to [v14.0.0-beta.11](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.11). Adapts the LSM resident-index guards to Lance's query-based index API and adds regression coverage for index presence and type checks. Validation: workspace Clippy with all features and warnings denied, Rust formatting, and targeted LSM query unit tests. --- Cargo.lock | 92 ++++++++++++++--------------- Cargo.toml | 28 ++++----- java/pom.xml | 2 +- rust/lancedb/src/table/query/lsm.rs | 60 ++++++++++++++++++- 4 files changed, 119 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12aab95c3..f2f4984d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3584,8 +3584,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5059,8 +5059,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5133,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5156,7 +5156,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5170,7 +5170,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5179,8 +5179,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrayref", "crunchy", @@ -5190,8 +5190,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5228,8 +5228,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5260,8 +5260,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5278,8 +5278,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "proc-macro2", "quote", @@ -5288,8 +5288,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-arith", "arrow-array", @@ -5322,8 +5322,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-arith", "arrow-array", @@ -5355,8 +5355,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "datafusion", "geo-traits", @@ -5370,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5442,8 +5442,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5466,8 +5466,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5509,8 +5509,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "byteorder", "ethnum", @@ -5527,8 +5527,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5542,8 +5542,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "async-trait", @@ -5557,8 +5557,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arc-swap", "arrow", @@ -5612,8 +5612,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -5627,8 +5627,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow", "arrow-array", @@ -5668,8 +5668,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "arrow-array", "arrow-schema", @@ -5682,8 +5682,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "14.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.10#fa3638fd816ab2ff9a2ecbba7e51465d8879958f" +version = "14.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v14.0.0-beta.11#f9c563fe1516a48b5c40d255c1d884af2f3c3d6c" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 8c9f5a91e..49d1792de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.94.0" [workspace.dependencies] -lance = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=14.0.0-beta.10", default-features = false, "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=14.0.0-beta.10", "tag" = "v14.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=14.0.0-beta.11", default-features = false, "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=14.0.0-beta.11", "tag" = "v14.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index be828e21a..40da4e36a 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 14.0.0-beta.10 + 14.0.0-beta.11 2.18.11 false 2.30.0 diff --git a/rust/lancedb/src/table/query/lsm.rs b/rust/lancedb/src/table/query/lsm.rs index 5ec053483..698b9c736 100644 --- a/rust/lancedb/src/table/query/lsm.rs +++ b/rust/lancedb/src/table/query/lsm.rs @@ -33,6 +33,7 @@ use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; use lance::Dataset; +use lance::dataset::mem_wal::index::{IndexStore, VectorMemQuery}; use lance::dataset::mem_wal::scanner::{InMemoryMemTableRef, InMemoryMemTables}; use lance::dataset::mem_wal::{ DatasetMemWalExt, LsmScanner, ShardManifestStore, ShardSnapshot, ShardWriterConfig, @@ -474,7 +475,7 @@ async fn fts_plan( // omits un-compacted documents. Reject rather than mislead. if !index_maintained(dataset, column, details, "InvertedIndexDetails").await? || !resident_memtables_carry(&in_memory, |memtable| { - memtable.index_store.get_fts_by_column(column).is_some() + !memtable.index_store.fts_granularities_on(column).is_empty() }) { return Err(Error::NotSupported { @@ -712,7 +713,7 @@ async fn vector_plan( // Reject rather than silently omit rows, mirroring the FTS arm. if !index_maintained(dataset, &column, details, "VectorIndexDetails").await? || !resident_memtables_carry(&in_memory, |memtable| { - memtable.index_store.get_hnsw_by_column(&column).is_some() + carries_vector_index(&memtable.index_store, &column) }) { return Err(Error::NotSupported { @@ -760,6 +761,22 @@ async fn vector_plan( Ok(scanner.create_plan().await?) } +/// Probe for a resident vector index without imposing a distance metric, matching +/// the presence check used before Lance introduced query-based index lookup. +fn carries_vector_index(store: &IndexStore, column: &str) -> bool { + let probe = VectorMemQuery { + vector: arrow_array::FixedSizeListArray::new_null( + Arc::new(arrow_schema::Field::new("item", DataType::Float32, true)), + 1, + 1, + ), + k: 1, + ef: None, + distance_type: None, + }; + store.index_answering(column, &probe).is_some() +} + /// Whether `column` stores binary (uint8) vectors, which the LSM vector planner /// does not support. fn is_binary_vector_column(schema: &ArrowSchema, column: &str) -> bool { @@ -804,6 +821,45 @@ mod tests { use super::*; use lance_index::mem_wal::{CompactedSsTable, IndexCatchupProgress}; + #[test] + fn resident_index_checks_distinguish_columns_and_query_types() { + use arrow_schema::Field; + use lance::dataset::mem_wal::index::MemIndexSpec; + + let schema = ArrowSchema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("text", DataType::Utf8, true), + Field::new( + "vector", + DataType::FixedSizeList(Arc::new(Field::new("item", DataType::Float32, true)), 4), + true, + ), + ]); + let schema = lance_core::datatypes::Schema::try_from(&schema).unwrap(); + let specs = vec![ + MemIndexSpec::btree("id_idx", 0, "id"), + MemIndexSpec::fts("text_idx", 1, "text"), + MemIndexSpec::hnsw( + "vector_idx", + 2, + "vector", + lance_linalg::distance::DistanceType::Cosine, + ), + ]; + let store = IndexStore::from_specs(&specs, &schema, 1_000, 16).unwrap(); + assert!(carries_vector_index(&store, "vector")); + for column in ["id", "text", "missing"] { + assert!(!carries_vector_index(&store, column)); + } + assert!(!store.fts_granularities_on("text").is_empty()); + for column in ["id", "vector", "missing"] { + assert!(store.fts_granularities_on(column).is_empty()); + } + let empty = IndexStore::from_specs(&[], &schema, 1_000, 16).unwrap(); + assert!(!carries_vector_index(&empty, "vector")); + assert!(empty.fts_granularities_on("text").is_empty()); + } + #[test] fn exclusion_watermark_gates_on_lagging_index_catchup() { let shard = Uuid::from_u128(1);