chore: update lance dependency to v14.0.0-beta.18 (#4552)

Updates the Rust workspace dependencies and Java lance-core to [Lance
v14.0.0-beta.18](https://github.com/lance-format/lance/releases/tag/v14.0.0-beta.18).
Mirrors Lance's pinned object_store patch to support the multipart write
mode API required by this release.

Validation: `cargo clippy --quiet --workspace --tests --all-features --
-D warnings`, `cargo fmt --all --quiet`, and `git diff --check` passed.

---------

Co-authored-by: Lu Qiu <luqiujob@gmail.com>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Co-authored-by: Yang Cen <bubble-cal@outlook.com>

rust/lancedb/src/query.rs

@@ -3806,8 +3806,12 @@ mod tests {
         assert_eq!(results[0].num_columns(), 1);
     }
 
+    #[rstest::rstest]
+    #[case::single_row(1)]
+    #[case::two_rows(2)]
+    #[case::three_rows(3)]
     #[tokio::test]
-    async fn test_take_offsets_preserves_duplicate_multiplicity() {
+    async fn test_take_offsets_preserves_duplicate_multiplicity(#[case] max_batch_length: u32) {
         let tmp_dir = tempdir().unwrap();
         let table = make_test_table(&tmp_dir).await;
 
@@ -3815,7 +3819,7 @@ mod tests {
             .take_offsets(vec![5, 1, 5, 17])
             .select(Select::Columns(vec!["id".to_string()]))
             .execute_with_options(QueryExecutionOptions {
-                max_batch_length: 2,
+                max_batch_length,
                 ..Default::default()
             })
             .await
@@ -3824,7 +3828,11 @@ mod tests {
             .await
             .unwrap();
 
-        assert_eq!(results.len(), 2);
+        assert!(
+            results
+                .iter()
+                .all(|batch| batch.num_rows() <= max_batch_length as usize)
+        );
         assert!(results.iter().all(|batch| batch.num_columns() == 1));
         let mut ids = results
             .iter()

rust/lancedb/src/table/merge.rs

@@ -2035,11 +2035,13 @@ mod lsm_tests {
         );
     }
 
-    /// Claims the base table's bitmap index and keeps it as a B-tree, whose
-    /// flush rows are in the shape the bitmap trainer takes.
+    /// Claims the base table's bitmap index and keeps it as a B-tree.
     #[derive(Debug)]
     struct BitmapAsBTree;
 
+    #[derive(Debug)]
+    struct BitmapAsBTreeIndex(lance::dataset::mem_wal::index::BTreeMemIndex);
+
     fn btree_plugin() -> Arc<dyn lance::dataset::mem_wal::index::MemIndexPlugin> {
         lance::dataset::mem_wal::MemIndexRegistry::default()
             .plugin_for_details_url("/lance.table.BTreeIndexDetails")
@@ -2059,7 +2061,8 @@ mod lsm_tests {
             lance_index::IndexType::Bitmap
         }
         fn training_criteria(&self) -> lance_index::scalar::registry::TrainingCriteria {
-            btree_plugin().training_criteria()
+            use lance_index::scalar::registry::{TrainingCriteria, TrainingOrdering};
+            TrainingCriteria::new(TrainingOrdering::Values).with_row_id()
         }
         fn validate(
             &self,
@@ -2071,7 +2074,57 @@ mod lsm_tests {
             &self,
             ctx: &lance::dataset::mem_wal::index::MemIndexBuildContext<'_>,
         ) -> lance_core::Result<Arc<dyn lance::dataset::mem_wal::index::MemIndex>> {
-            btree_plugin().create(ctx)
+            let (column, field_id) = ctx.single_column()?;
+            Ok(Arc::new(BitmapAsBTreeIndex(
+                lance::dataset::mem_wal::index::BTreeMemIndex::new(field_id, column.to_string()),
+            )))
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl lance::dataset::mem_wal::index::MemIndex for BitmapAsBTreeIndex {
+        fn columns(&self) -> &[String] {
+            self.0.columns()
+        }
+        fn can_answer(&self, query: &dyn lance::dataset::mem_wal::index::MemQuery) -> bool {
+            self.0.can_answer(query)
+        }
+        fn insert(&self, batch: &RecordBatch, row_offset: u64) -> lance_core::Result<()> {
+            self.0.insert(batch, row_offset)
+        }
+        fn resident_bytes(&self) -> usize {
+            self.0.resident_bytes()
+        }
+        fn search(
+            &self,
+            query: &dyn lance::dataset::mem_wal::index::MemQuery,
+            ctx: &lance::dataset::mem_wal::index::SearchContext,
+        ) -> lance_core::Result<Option<lance::dataset::mem_wal::index::MemMatches>> {
+            self.0.search(query, ctx)
+        }
+        async fn flush(
+            &self,
+            ctx: &lance::dataset::mem_wal::index::FlushContext<'_>,
+        ) -> lance_core::Result<lance::dataset::mem_wal::index::FlushOutcome> {
+            // BTree now trains on row addresses, while Bitmap still takes row IDs.
+            // These coincide in the flushed generation, which uses physical IDs.
+            assert!(!ctx.generation()?.dataset.manifest().uses_stable_row_ids());
+            let batches = self
+                .0
+                .to_training_batches(ctx.batch_size)?
+                .into_iter()
+                .map(|batch| {
+                    assert_eq!(batch.schema().field(1).name(), lance_core::ROW_ADDR);
+                    let schema = Arc::new(Schema::new(vec![
+                        batch.schema().field(0).clone(),
+                        Field::new(lance_core::ROW_ID, DataType::UInt64, false),
+                    ]));
+                    RecordBatch::try_new(schema, batch.columns().to_vec())
+                })
+                .collect::<std::result::Result<Vec<_>, _>>()?;
+            Ok(lance::dataset::mem_wal::index::FlushOutcome::from_batches(
+                batches,
+            ))
         }
     }