chore: update lance dependency to v11.0.0-beta.2 (#3886)

Updates the Lance dependencies and Java lance-core to v11.0.0-beta.2.
Includes required compatibility fixes for the LanceFileVersion module
move and the updated GooseFS/OpenDAL dependency. Lance tag:
https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.2

---------

Co-authored-by: Daniel Rammer <hamersaw@protonmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>

rust/lancedb/Cargo.toml

@@ -49,8 +49,8 @@ lance-namespace = { workspace = true }
 lance-namespace-impls = { workspace = true }
 metrics = { workspace = true, optional = true }
 metrics-util = { workspace = true, optional = true }
-# Pin the transitive GooseFS SDK until the 0.1.6 compile break is fixed upstream.
-goosefs-sdk = { version = "=0.1.5", optional = true }
+# Pin the GooseFS SDK to the version required by Lance's OpenDAL dependency.
+goosefs-sdk = { version = "=0.1.9", optional = true }
 moka = { workspace = true }
 pin-project = { workspace = true }
 tokio = { version = "1.23", features = ["rt-multi-thread", "sync"] }

rust/lancedb/src/blob.rs

@@ -17,7 +17,7 @@ use arrow_array::builder::LargeBinaryBuilder;
 use arrow_schema::{DataType, Field, Schema};
 use lance::dataset::{BlobRangeRequest as LanceBlobRangeRequest, Dataset, WriteParams};
 use lance_arrow::FieldExt;
-use lance_encoding::version::LanceFileVersion;
+use lance_file::version::LanceFileVersion;
 use lance_io::object_store::ObjectStore;
 use object_store::path::Path;
 

rust/lancedb/src/connection.rs

@@ -34,7 +34,7 @@ use crate::remote::{
     db::{OPT_REMOTE_API_KEY, OPT_REMOTE_HOST_OVERRIDE, OPT_REMOTE_REGION},
 };
 use lance::io::ObjectStoreParams;
-pub use lance_encoding::version::LanceFileVersion;
+pub use lance_file::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
 use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};

rust/lancedb/src/database/listing.rs

@@ -12,7 +12,7 @@ use lance::dataset::refs::Ref;
 use lance::dataset::{ReadParams, WriteMode, builder::DatasetBuilder};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
-use lance_encoding::version::LanceFileVersion;
+use lance_file::version::LanceFileVersion;
 use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;

rust/lancedb/src/database/namespace.rs

@@ -201,7 +201,7 @@ impl LanceNamespaceDatabase {
         &self,
         request: &DbCreateTableRequest,
     ) -> Result<(
-        Option<lance_encoding::version::LanceFileVersion>,
+        Option<lance_file::version::LanceFileVersion>,
         Option<bool>,
         Option<bool>,
     )> {
@@ -214,7 +214,7 @@ impl LanceNamespaceDatabase {
 
         let storage_version_override = storage_options
             .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
-            .map(|s| s.parse::<lance_encoding::version::LanceFileVersion>())
+            .map(|s| s.parse::<lance_file::version::LanceFileVersion>())
             .transpose()?;
 
         let v2_manifest_override = storage_options

rust/lancedb/src/remote/table.rs

@@ -2942,7 +2942,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
 }
 
 #[derive(Serialize, Clone, Debug)]
-pub(crate) struct MergeInsertRequest {
+pub struct MergeInsertRequest {
     on: String,
     when_matched_update_all: bool,
     when_matched_update_all_filt: Option<String>,
@@ -5907,16 +5907,18 @@ mod tests {
             .await
             .unwrap();
 
+        // Positions are relative to the first retained token, so dropping the
+        // leading "hello" stop word does not shift the remaining tokens.
         assert_eq!(
             tokens,
             vec![
                 FtsToken {
                     text: "こんにちは".to_string(),
-                    position: 1,
+                    position: 0,
                 },
                 FtsToken {
                     text: "世界".to_string(),
-                    position: 2,
+                    position: 1,
                 },
             ]
         );

rust/lancedb/src/remote/table/blobs.rs

@@ -90,7 +90,7 @@ struct RemoteBlobState {
 
 /// Seekable Cloud blob handle over HTTP Range.
 #[derive(Debug)]
-pub(crate) struct RemoteBlobFile {
+pub struct RemoteBlobFile {
     requester: Arc<dyn BlobRangeRequester>,
     state: Mutex<RemoteBlobState>,
     closed: AtomicBool,

rust/lancedb/src/remote/table/insert.rs

@@ -33,7 +33,7 @@ use crate::table::{AddResult, MergeResult};
 /// same Arrow-IPC streaming body and error side-channel; only the target
 /// endpoint, query parameters, and parsed result type differ.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteOp {
+pub enum WriteOp {
     /// `add`: stream to `/v1/table/{id}/insert/`, optionally overwriting.
     Insert { overwrite: bool },
     /// `merge_insert`: stream to `/v1/table/{id}/merge_insert/` with the merge
@@ -49,7 +49,7 @@ pub(crate) enum WriteOp {
 /// The parsed server response for a completed write, discriminated by the
 /// operation that produced it.
 #[derive(Debug, Clone)]
-pub(crate) enum WriteResult {
+pub enum WriteResult {
     Add(AddResult),
     Merge(MergeResult),
 }

rust/lancedb/tests/blob_integration.rs

@@ -10,7 +10,7 @@ use arrow_array::{
 use arrow_schema::{DataType, Field, Fields, Schema};
 use futures::TryStreamExt;
 use lance::Dataset;
-use lance_encoding::version::LanceFileVersion;
+use lance_file::version::LanceFileVersion;
 use lancedb::{
     Connection, Error, Result, Table,
     blob::{BlobRangeRequest, blob},