chore: update lance dependency to v1.0.3-rc.1

Cargo.toml

@@ -15,37 +15,37 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "57.2", optional = false }
+arrow = { version = "56.1", optional = false }
-arrow-array = "57.2"
+arrow-array = "56.1"
-arrow-data = "57.2"
+arrow-data = "56.1"
-arrow-ipc = "57.2"
+arrow-ipc = "56.1"
-arrow-ord = "57.2"
+arrow-ord = "56.1"
-arrow-schema = "57.2"
+arrow-schema = "56.1"
-arrow-select = "57.2"
+arrow-select = "56.1"
-arrow-cast = "57.2"
+arrow-cast = "56.1"
 async-trait = "0"
-datafusion = { version = "51.0", default-features = false }
+datafusion = { version = "50.0.0", default-features = false }
-datafusion-catalog = "51.0"
+datafusion-catalog = "50.0.0"
-datafusion-common = { version = "51.0", default-features = false }
+datafusion-common = { version = "50.0.0", default-features = false }
-datafusion-execution = "51.0"
+datafusion-execution = "50.0.0"
-datafusion-expr = "51.0"
+datafusion-expr = "50.0.0"
-datafusion-physical-plan = "51.0"
+datafusion-physical-plan = "50.0.0"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",

python/Cargo.toml

@@ -14,15 +14,15 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "57.2", features = ["pyarrow"] }
+arrow = { version = "56.1", features = ["pyarrow"] }
 async-trait = "0.1"
 lancedb = { path = "../rust/lancedb", default-features = false }
 lance-core.workspace = true
 lance-namespace.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
-pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
+pyo3 = { version = "0.25.1", features = ["extension-module", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.26", features = [
+pyo3-async-runtimes = { version = "0.25.0", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -32,7 +32,7 @@ snafu.workspace = true
 tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.26", features = [
+pyo3-build-config = { version = "0.25.0", features = [
     "extension-module",
     "abi3-py39",
 ] }

python/src/arrow.rs

@@ -36,10 +36,7 @@ impl RecordBatchStream {
 impl RecordBatchStream {
     #[getter]
     pub fn schema(&self, py: Python) -> PyResult<Py<PyAny>> {
-        (*self.schema)
+        (*self.schema).clone().into_pyarrow(py)
-            .clone()
-            .into_pyarrow(py)
-            .map(|obj| obj.unbind())
     }
 
     pub fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
@@ -57,8 +54,7 @@ impl RecordBatchStream {
                 .ok_or_else(|| PyStopAsyncIteration::new_err(""))?;
             #[allow(deprecated)]
             let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
-                let bound = inner_next.infer_error()?.to_pyarrow(py)?;
+                inner_next.infer_error()?.to_pyarrow(py)
-                Ok(bound.unbind())
             })?;
             Ok(py_obj)
         })

python/src/permutation.rs

@@ -282,10 +282,8 @@ impl PyPermutationReader {
         future_into_py(slf.py(), async move {
             let schema = reader.output_schema(selection).await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }

python/src/query.rs

@@ -455,10 +455,8 @@ impl Query {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }
@@ -539,10 +537,8 @@ impl TakeQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }
@@ -639,10 +635,8 @@ impl FTSQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }
@@ -823,10 +817,8 @@ impl VectorQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }

python/src/table.rs

@@ -288,10 +288,8 @@ impl Table {
         future_into_py(self_.py(), async move {
             let schema = inner.schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+            let py_obj: Py<PyAny> =
-                let bound = schema.to_pyarrow(py)?;
+                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
-                Ok(bound.unbind())
-            })?;
             Ok(py_obj)
         })
     }

rust/lancedb/src/connection.rs

@@ -9,6 +9,7 @@ use std::sync::Arc;
 use arrow_array::RecordBatchReader;
 use arrow_schema::{Field, SchemaRef};
 use lance::dataset::ReadParams;
+use lance::io::ObjectStoreParams;
 use lance_namespace::models::{
     CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest,
     DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest,
@@ -39,7 +40,18 @@ use crate::Table;
 pub use lance_encoding::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+
+fn update_storage_options<F>(store_params: &mut ObjectStoreParams, update: F)
+where
+    F: FnOnce(&mut HashMap<String, String>),
+{
+    let mut options = store_params.storage_options().cloned().unwrap_or_default();
+    update(&mut options);
+    store_params.storage_options_accessor = Some(Arc::new(
+        StorageOptionsAccessor::with_static_options(options),
+    ));
+}
 
 /// A builder for configuring a [`Connection::table_names`] operation
 pub struct TableNamesBuilder {
@@ -246,16 +258,16 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let store_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-        store_options.insert(key.into(), value.into());
+        update_storage_options(store_params, |options| {
+            options.insert(key.into(), value.into());
+        });
         self
     }
 
@@ -269,19 +281,18 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let store_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-
+        update_storage_options(store_params, |options| {
-        for (key, value) in pairs {
+            for (key, value) in pairs {
-            store_options.insert(key.into(), value.into());
+                options.insert(key.into(), value.into());
-        }
+            }
+        });
         self
     }
 
@@ -318,24 +329,23 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This has no effect in LanceDB Cloud.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn enable_v2_manifest_paths(mut self, use_v2_manifest_paths: bool) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
-            .get_or_insert_with(Default::default)
-            .storage_options
             .get_or_insert_with(Default::default);
-
+        update_storage_options(store_params, |options| {
-        storage_options.insert(
+            options.insert(
-            OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(),
+                OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(),
-            if use_v2_manifest_paths {
+                if use_v2_manifest_paths {
-                "true".to_string()
+                    "true".to_string()
-            } else {
+                } else {
-                "false".to_string()
+                    "false".to_string()
-            },
+                },
-        );
+            );
+        });
         self
     }
 
@@ -344,20 +354,19 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// The default is `LanceFileVersion::Stable`.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn data_storage_version(mut self, data_storage_version: LanceFileVersion) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
-            .get_or_insert_with(Default::default)
-            .storage_options
             .get_or_insert_with(Default::default);
-
+        update_storage_options(store_params, |options| {
-        storage_options.insert(
+            options.insert(
-            OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
+                OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
-            data_storage_version.to_string(),
+                data_storage_version.to_string(),
-        );
+            );
+        });
         self
     }
 
@@ -381,13 +390,17 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        self.request
+        let store_params = self
+            .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default())
+            .get_or_insert(Default::default());
-            .storage_options_provider = Some(provider);
+        let initial = store_params.storage_options().cloned().unwrap_or_default();
+        store_params.storage_options_accessor = Some(Arc::new(
+            StorageOptionsAccessor::with_initial_and_provider(initial, provider),
+        ));
         self
     }
 }
@@ -450,15 +463,15 @@ impl OpenTableBuilder {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-        storage_options.insert(key.into(), value.into());
+        update_storage_options(store_params, |options| {
+            options.insert(key.into(), value.into());
+        });
         self
     }
 
@@ -472,18 +485,17 @@ impl OpenTableBuilder {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let storage_options = self
+        let store_params = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
-            .storage_options
             .get_or_insert(Default::default());
-
+        update_storage_options(store_params, |options| {
-        for (key, value) in pairs {
+            for (key, value) in pairs {
-            storage_options.insert(key.into(), value.into());
+                options.insert(key.into(), value.into());
-        }
+            }
+        });
         self
     }
 
@@ -507,12 +519,16 @@ impl OpenTableBuilder {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        self.request
+        let store_params = self
+            .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default())
+            .get_or_insert(Default::default());
-            .storage_options_provider = Some(provider);
+        let initial = store_params.storage_options().cloned().unwrap_or_default();
+        store_params.storage_options_accessor = Some(Arc::new(
+            StorageOptionsAccessor::with_initial_and_provider(initial, provider),
+        ));
         self
     }
 

rust/lancedb/src/database/listing.rs

@@ -12,7 +12,7 @@ use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::StorageOptionsProvider;
+use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -356,7 +356,11 @@ impl ListingDatabase {
                     .clone()
                     .unwrap_or_else(|| Arc::new(lance::session::Session::default()));
                 let os_params = ObjectStoreParams {
-                    storage_options: Some(options.storage_options.clone()),
+                    storage_options_accessor: Some(Arc::new(
+                        StorageOptionsAccessor::with_static_options(
+                            options.storage_options.clone(),
+                        ),
+                    )),
                     ..Default::default()
                 };
                 let (object_store, base_path) = ObjectStore::from_uri_and_params(
@@ -492,7 +496,9 @@ impl ListingDatabase {
 
     async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
         let object_store_params = ObjectStoreParams {
-            storage_options: Some(self.storage_options.clone()),
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                self.storage_options.clone(),
+            ))),
             ..Default::default()
         };
         let mut uri = self.uri.clone();
@@ -541,7 +547,7 @@ impl ListingDatabase {
             .lance_write_params
             .as_ref()
             .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options.as_ref());
+            .and_then(|sp| sp.storage_options());
 
         let storage_version_override = storage_options
             .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
@@ -593,20 +599,25 @@ impl ListingDatabase {
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
         if !self.storage_options.is_empty() {
-            let storage_options = write_params
+            let store_params = write_params
                 .store_params
-                .get_or_insert_with(Default::default)
-                .storage_options
                 .get_or_insert_with(Default::default);
-            self.inherit_storage_options(storage_options);
+            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
+            self.inherit_storage_options(&mut storage_options);
+            store_params.storage_options_accessor = Some(Arc::new(
+                StorageOptionsAccessor::with_static_options(storage_options),
+            ));
         }
 
         // Set storage options provider if available
-        if self.storage_options_provider.is_some() {
+        if let Some(provider) = self.storage_options_provider.clone() {
-            write_params
+            let store_params = write_params
                 .store_params
-                .get_or_insert_with(Default::default)
+                .get_or_insert_with(Default::default);
-                .storage_options_provider = self.storage_options_provider.clone();
+            let initial = store_params.storage_options().cloned().unwrap_or_default();
+            store_params.storage_options_accessor = Some(Arc::new(
+                StorageOptionsAccessor::with_initial_and_provider(initial, provider),
+            ));
         }
 
         write_params.data_storage_version = self
@@ -892,7 +903,9 @@ impl Database for ListingDatabase {
         validate_table_name(&request.target_table_name)?;
 
         let storage_params = ObjectStoreParams {
-            storage_options: Some(self.storage_options.clone()),
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                self.storage_options.clone(),
+            ))),
             ..Default::default()
         };
         let read_params = ReadParams {
@@ -957,24 +970,29 @@ impl Database for ListingDatabase {
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
         if !self.storage_options.is_empty() {
-            let storage_options = request
+            let store_params = request
                 .lance_read_params
                 .get_or_insert_with(Default::default)
                 .store_options
-                .get_or_insert_with(Default::default)
-                .storage_options
                 .get_or_insert_with(Default::default);
-            self.inherit_storage_options(storage_options);
+            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
+            self.inherit_storage_options(&mut storage_options);
+            store_params.storage_options_accessor = Some(Arc::new(
+                StorageOptionsAccessor::with_static_options(storage_options),
+            ));
         }
 
         // Set storage options provider if available
-        if self.storage_options_provider.is_some() {
+        if let Some(provider) = self.storage_options_provider.clone() {
-            request
+            let store_params = request
                 .lance_read_params
                 .get_or_insert_with(Default::default)
                 .store_options
-                .get_or_insert_with(Default::default)
+                .get_or_insert_with(Default::default);
-                .storage_options_provider = self.storage_options_provider.clone();
+            let initial = store_params.storage_options().cloned().unwrap_or_default();
+            store_params.storage_options_accessor = Some(Arc::new(
+                StorageOptionsAccessor::with_initial_and_provider(initial, provider),
+            ));
         }
 
         // Some ReadParams are exposed in the OpenTableBuilder, but we also
@@ -1881,7 +1899,9 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options: Some(storage_options),
+                    storage_options_accessor: Some(Arc::new(
+                        StorageOptionsAccessor::with_static_options(storage_options),
+                    )),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -1955,7 +1975,9 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options: Some(storage_options),
+                    storage_options_accessor: Some(Arc::new(
+                        StorageOptionsAccessor::with_static_options(storage_options),
+                    )),
                     ..Default::default()
                 }),
                 ..Default::default()

rust/lancedb/src/table.rs

@@ -40,7 +40,7 @@ use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
 use lance_index::DatasetIndexExt;
 use lance_index::IndexType;
-use lance_io::object_store::LanceNamespaceStorageOptionsProvider;
+use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor};
 use lance_namespace::models::{
     QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns,
     QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery,
@@ -1425,9 +1425,7 @@ impl Table {
             })
             .collect::<Vec<_>>();
 
-        let unioned = UnionExec::try_new(projected_plans).map_err(|e| Error::Runtime {
+        let unioned = Arc::new(UnionExec::new(projected_plans));
-            message: format!("Failed to build union plan: {e}"),
-        })?;
         // We require 1 partition in the final output
         let repartitioned = RepartitionExec::try_new(
             unioned,
@@ -1668,18 +1666,14 @@ impl NativeTable {
 
         // Use DatasetBuilder::from_namespace which automatically fetches location
         // and storage options from the namespace
-        let builder = DatasetBuilder::from_namespace(
+        let builder = DatasetBuilder::from_namespace(namespace_client.clone(), table_id)
-            namespace_client.clone(),
+            .await
-            table_id,
+            .map_err(|e| match e {
-            false, // Don't ignore namespace storage options
+                lance::Error::Namespace { source, .. } => Error::Runtime {
-        )
+                    message: format!("Failed to get table info from namespace: {:?}", source),
-        .await
+                },
-        .map_err(|e| match e {
+                source => Error::Lance { source },
-            lance::Error::Namespace { source, .. } => Error::Runtime {
+            })?;
-                message: format!("Failed to get table info from namespace: {:?}", source),
-            },
-            source => Error::Lance { source },
-        })?;
 
         let dataset = builder
             .with_read_params(params)
@@ -1883,7 +1877,10 @@ impl NativeTable {
         let store_params = params
             .store_params
             .get_or_insert_with(ObjectStoreParams::default);
-        store_params.storage_options_provider = Some(storage_options_provider);
+        let initial = store_params.storage_options().cloned().unwrap_or_default();
+        store_params.storage_options_accessor = Some(Arc::new(
+            StorageOptionsAccessor::with_initial_and_provider(initial, storage_options_provider),
+        ));
 
         // Patch the params if we have a write store wrapper
         let params = match write_store_wrapper.clone() {
@@ -3244,7 +3241,7 @@ impl BaseTable for NativeTable {
             .get()
             .await
             .ok()
-            .and_then(|dataset| dataset.storage_options().cloned())
+            .and_then(|dataset| dataset.initial_storage_options().cloned())
     }
 
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {

rust/lancedb/src/table/dataset.rs

@@ -100,8 +100,7 @@ impl DatasetRef {
                 let should_checkout = match &target_ref {
                     refs::Ref::Version(_, Some(target_ver)) => version != target_ver,
                     refs::Ref::Version(_, None) => true, // No specific version, always checkout
-                    refs::Ref::VersionNumber(target_ver) => version != target_ver,
+                    refs::Ref::Tag(_) => true,           // Always checkout for tags
-                    refs::Ref::Tag(_) => true, // Always checkout for tags
                 };
 
                 if should_checkout {