fix(rust): normalize AWS credentials in built-in provider

Cargo.lock

@@ -5206,7 +5206,6 @@ dependencies = [
 [[package]]
 name = "lance-io"
 version = "11.0.0-beta.2"
-source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.2#35da5d920159b49d1b53032652f7615ab699c160"
 dependencies = [
  "arrow",
  "arrow-array",

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = ["rust/lancedb", "nodejs", "python"]
+exclude = ["vendor/lance-io"]
 resolver = "2"
 
 [workspace.package]
@@ -67,6 +68,9 @@ regex = "1.10"
 semver = "1.0.25"
 chrono = "0.4"
 
+[patch."https://github.com/lance-format/lance.git"]
+lance-io = { path = "vendor/lance-io" }
+
 [profile.ci]
 debug = "line-tables-only"
 inherits = "dev"

rust/lancedb/src/database/namespace.rs

@@ -34,7 +34,7 @@ use crate::database::read_freshness::{
     FreshnessBaselines, ReadFreshnessContextProvider, TableFreshness,
 };
 use crate::error::{Error, Result};
-use crate::io::object_store::{atomic_aws_session, install_atomic_aws_provider};
+use crate::io::object_store::atomic_aws_session;
 use crate::table::{NativeTable, map_namespace_lance_error};
 use lance::dataset::WriteMode;
 
@@ -102,9 +102,6 @@ impl LanceNamespaceDatabase {
         session: Option<Arc<lance::session::Session>>,
         namespace_client_pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
     ) -> Self {
-        if let Some(session) = &session {
-            install_atomic_aws_provider(session);
-        }
         // Client is pre-built, so we can't install the freshness provider here;
         // baselines are still tracked for a uniform bump path.
         let delimiter = resolve_delimiter(&namespace_client_properties);
@@ -157,9 +154,9 @@ impl LanceNamespaceDatabase {
         pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
         new_table_config: NewTableConfig,
     ) -> Result<Self> {
-        // Namespace construction needs a protected session even when the connection did not
-        // supply one. Keep the original option separately so per-operation sessions retain
-        // precedence when tables are opened or created later.
+        // Namespace construction needs a shared session even when the connection did not supply
+        // one. Keep the original option separately so per-operation sessions retain precedence
+        // when tables are opened or created later.
         let builder_session = atomic_aws_session(session.clone());
         let mut builder = ConnectBuilder::new(ns_impl);
         for (key, value) in ns_properties.clone() {

rust/lancedb/src/table.rs

@@ -3580,6 +3580,7 @@ mod tests {
     #[cfg(feature = "aws")]
     use lance_io::object_store::{
         ObjectStore as LanceObjectStore, ObjectStoreProvider, ObjectStoreRegistry,
+        StorageOptionsAccessor,
     };
     use tempfile::tempdir;
 
@@ -3593,7 +3594,8 @@ mod tests {
     #[cfg(feature = "aws")]
     #[derive(Debug)]
     struct RecordingS3Provider {
-        saw_atomic_credentials: Arc<AtomicBool>,
+        expected_accessor: Arc<StorageOptionsAccessor>,
+        saw_original_params: Arc<AtomicBool>,
     }
 
     #[cfg(feature = "aws")]
@@ -3604,37 +3606,49 @@ mod tests {
             _base_path: url::Url,
             params: &ObjectStoreParams,
         ) -> lance_core::Result<LanceObjectStore> {
-            self.saw_atomic_credentials
-                .store(params.aws_credentials.is_some(), Ordering::SeqCst);
+            self.saw_original_params.store(
+                params.aws_credentials.is_none()
+                    && params
+                        .storage_options_accessor
+                        .as_ref()
+                        .is_some_and(|accessor| Arc::ptr_eq(accessor, &self.expected_accessor)),
+                Ordering::SeqCst,
+            );
             Err(lance_core::Error::invalid_input("recorded test request"))
         }
     }
 
     #[cfg(feature = "aws")]
-    fn recording_s3_session() -> (Arc<lance::session::Session>, Arc<AtomicBool>) {
-        let saw_atomic_credentials = Arc::new(AtomicBool::new(false));
+    fn recording_s3_session(
+        expected_accessor: Arc<StorageOptionsAccessor>,
+    ) -> (Arc<lance::session::Session>, Arc<AtomicBool>) {
+        let saw_original_params = Arc::new(AtomicBool::new(false));
         let registry = Arc::new(ObjectStoreRegistry::default());
         registry.insert(
             "s3",
             Arc::new(RecordingS3Provider {
-                saw_atomic_credentials: saw_atomic_credentials.clone(),
+                expected_accessor,
+                saw_original_params: saw_original_params.clone(),
             }),
         );
         (
             Arc::new(lance::session::Session::new(16, 16, registry)),
-            saw_atomic_credentials,
+            saw_original_params,
         )
     }
 
     #[cfg(feature = "aws")]
-    fn explicit_s3_store_params() -> ObjectStoreParams {
-        crate::io::object_store::object_store_params_from_storage_options(HashMap::from([
-            ("aws_access_key_id".to_string(), "explicit-key".to_string()),
-            (
-                "aws_secret_access_key".to_string(),
-                "explicit-secret".to_string(),
-            ),
-        ]))
+    fn explicit_s3_store_params() -> (ObjectStoreParams, Arc<StorageOptionsAccessor>) {
+        let params =
+            crate::io::object_store::object_store_params_from_storage_options(HashMap::from([
+                ("aws_access_key_id".to_string(), "explicit-key".to_string()),
+                (
+                    "aws_secret_access_key".to_string(),
+                    "explicit-secret".to_string(),
+                ),
+            ]));
+        let accessor = params.storage_options_accessor.as_ref().unwrap().clone();
+        (params, accessor)
     }
 
     #[test]
@@ -3700,11 +3714,12 @@ mod tests {
 
     #[cfg(feature = "aws")]
     #[tokio::test]
-    async fn direct_native_open_installs_the_atomic_provider() {
-        let (session, saw_atomic_credentials) = recording_s3_session();
+    async fn direct_native_open_preserves_custom_provider_params() {
+        let (store_options, accessor) = explicit_s3_store_params();
+        let (session, saw_original_params) = recording_s3_session(accessor);
         let params = ReadParams {
             session: Some(session),
-            store_options: Some(explicit_s3_store_params()),
+            store_options: Some(store_options),
             ..Default::default()
         };
 
@@ -3724,18 +3739,19 @@ mod tests {
 
         assert!(error.to_string().contains("recorded test request"));
         assert!(
-            saw_atomic_credentials.load(Ordering::SeqCst),
-            "the public direct open path must install the wrapper on its session"
+            saw_original_params.load(Ordering::SeqCst),
+            "the public direct open path must preserve custom provider parameters"
         );
     }
 
     #[cfg(feature = "aws")]
     #[tokio::test]
-    async fn direct_native_create_installs_the_atomic_provider() {
-        let (session, saw_atomic_credentials) = recording_s3_session();
+    async fn direct_native_create_preserves_custom_provider_params() {
+        let (store_params, accessor) = explicit_s3_store_params();
+        let (session, saw_original_params) = recording_s3_session(accessor);
         let params = WriteParams {
             session: Some(session),
-            store_params: Some(explicit_s3_store_params()),
+            store_params: Some(store_params),
             ..Default::default()
         };
         let batch = make_test_batches();
@@ -3757,8 +3773,8 @@ mod tests {
 
         assert!(error.to_string().contains("recorded test request"));
         assert!(
-            saw_atomic_credentials.load(Ordering::SeqCst),
-            "the public direct create path must install the wrapper on its session"
+            saw_original_params.load(Ordering::SeqCst),
+            "the public direct create path must preserve custom provider parameters"
         );
     }
 

vendor/lance-io/Cargo.toml

@@ -0,0 +1,74 @@
+[package]
+name = "lance-io"
+version = "11.0.0-beta.2"
+edition = "2024"
+authors = ["Lance Devs <dev@lance.org>"]
+license = "Apache-2.0"
+repository = "https://github.com/lance-format/lance"
+readme = "README.md"
+description = "I/O utilities for Lance"
+keywords = ["data-format", "data-science", "machine-learning", "apache-arrow"]
+categories = ["database-implementations", "data-structures"]
+rust-version = "1.91.0"
+autobenches = false
+autotests = false
+
+[dependencies]
+object_store = "0.13.2"
+opendal = { version = "0.58.1", optional = true }
+object_store_opendal = { version = "0.58", optional = true }
+lance-arrow = { version = "=11.0.0-beta.2", tag = "v11.0.0-beta.2", git = "https://github.com/lance-format/lance.git" }
+lance-core = { version = "=11.0.0-beta.2", tag = "v11.0.0-beta.2", git = "https://github.com/lance-format/lance.git" }
+lance-namespace = { version = "=11.0.0-beta.2", tag = "v11.0.0-beta.2", git = "https://github.com/lance-format/lance.git" }
+arrow = { version = "58.0.0", features = ["ffi"] }
+arrow-array = "58.0.0"
+arrow-schema = "58.0.0"
+async-trait = "0.1"
+aws-config = { version = "1.2.0", optional = true }
+aws-credential-types = { version = "1.2.0", optional = true }
+byteorder = "1.5"
+bytes = "1.11.1"
+chrono = { version = "0.4.41", default-features = false, features = ["std", "now", "serde"] }
+futures = "0.3"
+http = "1.1.0"
+log = "0.4"
+metrics = { version = "0.24", optional = true }
+moka = { version = "0.12", features = ["future"] }
+pin-project = "1.0"
+prost = "0.14.1"
+serde = { version = "1", features = ["derive"] }
+tokio = { version = "1.23", features = ["rt-multi-thread", "macros", "fs", "sync"] }
+tracing = "0.1"
+url = "2.5.7"
+path_abs = "0.5"
+rand = "0.9.1"
+tempfile = "3"
+
+[target.'cfg(target_os = "linux")'.dependencies]
+io-uring = "0.7"
+
+[dev-dependencies]
+lance-testing = { version = "=11.0.0-beta.2", tag = "v11.0.0-beta.2", git = "https://github.com/lance-format/lance.git" }
+test-log = "0.2.15"
+mockall = "0.14.0"
+rstest = "0.26.1"
+mock_instant = "0.6.0"
+tokio = { version = "1.23", features = ["test-util"] }
+tracing-mock = "=0.1.0-beta.3"
+metrics-util = "0.19"
+
+[features]
+default = ["aws", "azure", "gcp"]
+metrics = ["dep:metrics"]
+gcs-test = []
+goosefs-test = []
+gcp = ["object_store/gcp", "dep:opendal", "opendal/services-gcs", "dep:object_store_opendal"]
+aws = ["object_store/aws", "dep:aws-config", "dep:aws-credential-types", "dep:opendal", "opendal/services-s3", "dep:object_store_opendal"]
+azure = ["object_store/azure", "dep:opendal", "opendal/services-azblob", "opendal/services-azdls", "dep:object_store_opendal"]
+oss = ["dep:opendal", "opendal/services-oss", "dep:object_store_opendal"]
+goosefs = ["dep:opendal", "opendal/services-goosefs", "dep:object_store_opendal"]
+tencent = ["dep:opendal", "opendal/services-cos", "dep:object_store_opendal"]
+huggingface = ["dep:opendal", "opendal/services-huggingface", "dep:object_store_opendal"]
+tos = ["dep:opendal", "opendal/services-tos", "dep:object_store_opendal"]
+tos-test = ["tos"]
+test-util = []

vendor/lance-io/LANCEDB_PATCH.md

@@ -0,0 +1,13 @@
+# LanceDB patch provenance
+
+This directory vendors `lance-io` 11.0.0-beta.2 from Lance commit
+`35da5d920159b49d1b53032652f7615ab699c160`.
+
+`Cargo.toml` uses the equivalent standalone dependency metadata from the published crate. Upstream
+benchmark and integration-test targets are omitted because this copy is compiled only as a patched
+dependency; the library sources are otherwise retained.
+
+The local patch makes AWS credential-family merging atomic before backend selection and teaches
+the built-in OpenDAL S3 path to refresh credential-only storage options. Keeping the change inside
+`AwsStoreProvider` leaves arbitrary registry providers and their complete `ObjectStore` results
+untouched. Remove this patch when the same behavior is available in the pinned Lance release.

vendor/lance-io/README.md

@@ -0,0 +1,9 @@
+# lance-io
+
+`lance-io` is an internal sub-crate, containing various utilities for
+reading and writing data.  It includes reader/writer traits that
+define what Lance expects from a filesystem, encoders and decoders to
+convert to/from Arrow data and various layouts, and misc. utilities such
+as routines for reading protobuf data from files.
+
+**Important Note**: This crate is **not intended for external usage**.

vendor/lance-io/src/ffi.rs

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use arrow::ffi_stream::FFI_ArrowArrayStream;
+use arrow_array::RecordBatch;
+use arrow_schema::{ArrowError, SchemaRef};
+use futures::StreamExt;
+use lance_core::Result;
+
+use crate::stream::RecordBatchStream;
+
+#[pin_project::pin_project]
+struct RecordBatchIteratorAdaptor<S: RecordBatchStream> {
+    schema: SchemaRef,
+
+    #[pin]
+    stream: S,
+
+    handle: tokio::runtime::Handle,
+}
+
+impl<S: RecordBatchStream> RecordBatchIteratorAdaptor<S> {
+    fn new(stream: S, schema: SchemaRef, handle: tokio::runtime::Handle) -> Self {
+        Self {
+            schema,
+            stream,
+            handle,
+        }
+    }
+}
+
+impl<S: RecordBatchStream + Unpin> arrow::record_batch::RecordBatchReader
+    for RecordBatchIteratorAdaptor<S>
+{
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+impl<S: RecordBatchStream + Unpin> Iterator for RecordBatchIteratorAdaptor<S> {
+    type Item = std::result::Result<RecordBatch, ArrowError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.handle
+            .block_on(async { self.stream.next().await })
+            .map(|r| r.map_err(|e| ArrowError::ExternalError(Box::new(e))))
+    }
+}
+
+/// Wrap a [`RecordBatchStream`] into an [FFI_ArrowArrayStream].
+pub fn to_ffi_arrow_array_stream(
+    stream: impl RecordBatchStream + std::marker::Unpin + 'static,
+    handle: tokio::runtime::Handle,
+) -> Result<FFI_ArrowArrayStream> {
+    let schema = stream.schema();
+    let arrow_stream = RecordBatchIteratorAdaptor::new(stream, schema, handle);
+    let reader = FFI_ArrowArrayStream::new(Box::new(arrow_stream));
+
+    Ok(reader)
+}

vendor/lance-io/src/lib.rs

@@ -0,0 +1,387 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+#![recursion_limit = "512"]
+use std::{
+    ops::{Range, RangeFrom, RangeFull, RangeTo},
+    sync::Arc,
+};
+
+use arrow::datatypes::UInt32Type;
+use arrow_array::{PrimitiveArray, UInt32Array};
+
+use lance_core::{Error, Result};
+
+pub mod ffi;
+pub mod local;
+pub mod object_reader;
+pub mod object_store;
+pub mod object_writer;
+pub mod scheduler;
+pub mod spill;
+pub mod stream;
+#[cfg(test)]
+pub mod testing;
+pub mod traits;
+#[cfg(target_os = "linux")]
+pub mod uring;
+pub mod utils;
+
+pub use scheduler::{bytes_read_counter, iops_counter};
+
+/// Defines a selection of rows to read from a file/batch
+#[derive(Debug, Clone, PartialEq, Default)]
+pub enum ReadBatchParams {
+    /// Select a contiguous range of rows
+    Range(Range<usize>),
+    /// Select multiple contiguous ranges of rows
+    Ranges(Arc<[Range<u64>]>),
+    /// Select all rows (this is the default)
+    #[default]
+    RangeFull,
+    /// Select all rows up to a given index
+    RangeTo(RangeTo<usize>),
+    /// Select all rows starting at a given index
+    RangeFrom(RangeFrom<usize>),
+    /// Select scattered non-contiguous rows
+    Indices(UInt32Array),
+}
+
+impl std::fmt::Display for ReadBatchParams {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            Self::Range(r) => write!(f, "Range({}..{})", r.start, r.end),
+            Self::Ranges(ranges) => {
+                let mut ranges_str = ranges.iter().fold(String::new(), |mut acc, r| {
+                    acc.push_str(&format!("{}..{}", r.start, r.end));
+                    acc.push(',');
+                    acc
+                });
+                // Remove the trailing comma
+                if !ranges_str.is_empty() {
+                    ranges_str.pop();
+                }
+                write!(f, "Ranges({})", ranges_str)
+            }
+            Self::RangeFull => write!(f, "RangeFull"),
+            Self::RangeTo(r) => write!(f, "RangeTo({})", r.end),
+            Self::RangeFrom(r) => write!(f, "RangeFrom({})", r.start),
+            Self::Indices(indices) => {
+                let mut indices_str = indices.values().iter().fold(String::new(), |mut acc, v| {
+                    acc.push_str(&v.to_string());
+                    acc.push(',');
+                    acc
+                });
+                if !indices_str.is_empty() {
+                    indices_str.pop();
+                }
+                write!(f, "Indices({})", indices_str)
+            }
+        }
+    }
+}
+
+impl From<&[u32]> for ReadBatchParams {
+    fn from(value: &[u32]) -> Self {
+        Self::Indices(UInt32Array::from_iter_values(value.iter().copied()))
+    }
+}
+
+impl From<UInt32Array> for ReadBatchParams {
+    fn from(value: UInt32Array) -> Self {
+        Self::Indices(value)
+    }
+}
+
+impl From<RangeFull> for ReadBatchParams {
+    fn from(_: RangeFull) -> Self {
+        Self::RangeFull
+    }
+}
+
+impl From<Range<usize>> for ReadBatchParams {
+    fn from(r: Range<usize>) -> Self {
+        Self::Range(r)
+    }
+}
+
+impl From<RangeTo<usize>> for ReadBatchParams {
+    fn from(r: RangeTo<usize>) -> Self {
+        Self::RangeTo(r)
+    }
+}
+
+impl From<RangeFrom<usize>> for ReadBatchParams {
+    fn from(r: RangeFrom<usize>) -> Self {
+        Self::RangeFrom(r)
+    }
+}
+
+impl From<&Self> for ReadBatchParams {
+    fn from(params: &Self) -> Self {
+        params.clone()
+    }
+}
+
+impl ReadBatchParams {
+    /// Validate that the selection is valid given the length of the batch
+    pub fn valid_given_len(&self, len: usize) -> bool {
+        match self {
+            Self::Indices(indices) => indices.iter().all(|i| i.unwrap_or(0) < len as u32),
+            Self::Range(r) => r.start < len && r.end <= len,
+            Self::Ranges(ranges) => ranges.iter().all(|r| r.end <= len as u64),
+            Self::RangeFull => true,
+            Self::RangeTo(r) => r.end <= len,
+            Self::RangeFrom(r) => r.start < len,
+        }
+    }
+
+    /// Slice the selection
+    ///
+    /// For example, given ReadBatchParams::RangeFull and slice(10, 20), the output will be
+    /// ReadBatchParams::Range(10..20)
+    ///
+    /// Given ReadBatchParams::Range(10..20) and slice(5, 3), the output will be
+    /// ReadBatchParams::Range(15..18)
+    ///
+    /// Given ReadBatchParams::RangeTo(20) and slice(10, 5), the output will be
+    /// ReadBatchParams::Range(10..15)
+    ///
+    /// Given ReadBatchParams::RangeFrom(20) and slice(10, 5), the output will be
+    /// ReadBatchParams::Range(30..35)
+    ///
+    /// Given ReadBatchParams::Indices([1, 3, 5, 7, 9]) and slice(1, 3), the output will be
+    /// ReadBatchParams::Indices([3, 5, 7])
+    ///
+    /// You cannot slice beyond the bounds of the selection and an attempt to do so will
+    /// return an error.
+    pub fn slice(&self, start: usize, length: usize) -> Result<Self> {
+        let out_of_bounds = |size: usize| {
+            Err(Error::invalid_input_source(
+                format!(
+                    "Cannot slice from {} with length {} given a selection of size {}",
+                    start, length, size
+                )
+                .into(),
+            ))
+        };
+
+        match self {
+            Self::Indices(indices) => {
+                if start + length > indices.len() {
+                    return out_of_bounds(indices.len());
+                }
+                Ok(Self::Indices(indices.slice(start, length)))
+            }
+            Self::Range(r) => {
+                if (r.start + start + length) > r.end {
+                    return out_of_bounds(r.end - r.start);
+                }
+                Ok(Self::Range((r.start + start)..(r.start + start + length)))
+            }
+            Self::Ranges(ranges) => {
+                let mut new_ranges = Vec::with_capacity(ranges.len());
+                let mut to_skip = start as u64;
+                let mut to_take = length as u64;
+                let mut total_num_rows = 0;
+                for r in ranges.as_ref() {
+                    let num_rows = r.end - r.start;
+                    total_num_rows += num_rows;
+                    if to_skip > num_rows {
+                        to_skip -= num_rows;
+                        continue;
+                    }
+                    let new_start = r.start + to_skip;
+                    let to_take_this_range = (num_rows - to_skip).min(to_take);
+                    new_ranges.push(new_start..(new_start + to_take_this_range));
+                    to_skip = 0;
+                    to_take -= to_take_this_range;
+                    if to_take == 0 {
+                        break;
+                    }
+                }
+                if to_take > 0 {
+                    out_of_bounds(total_num_rows as usize)
+                } else {
+                    Ok(Self::Ranges(new_ranges.into()))
+                }
+            }
+            Self::RangeFull => Ok(Self::Range(start..(start + length))),
+            Self::RangeTo(range) => {
+                if start + length > range.end {
+                    return out_of_bounds(range.end);
+                }
+                Ok(Self::Range(start..(start + length)))
+            }
+            Self::RangeFrom(r) => {
+                // No way to validate out_of_bounds, assume caller will do so
+                Ok(Self::Range((r.start + start)..(r.start + start + length)))
+            }
+        }
+    }
+
+    /// Convert a read range into a vector of row offsets
+    ///
+    /// RangeFull and RangeFrom are unbounded and cannot be converted into row offsets
+    /// and any attempt to do so will return an error.  Call slice first
+    pub fn to_offsets(&self) -> Result<PrimitiveArray<UInt32Type>> {
+        match self {
+            Self::Indices(indices) => Ok(indices.clone()),
+            Self::Range(r) => Ok(UInt32Array::from(Vec::from_iter(
+                r.start as u32..r.end as u32,
+            ))),
+            Self::Ranges(ranges) => {
+                let num_rows = ranges
+                    .iter()
+                    .map(|r| (r.end - r.start) as usize)
+                    .sum::<usize>();
+                let mut offsets = Vec::with_capacity(num_rows);
+                for r in ranges.as_ref() {
+                    offsets.extend(r.start as u32..r.end as u32);
+                }
+                Ok(UInt32Array::from(offsets))
+            }
+            Self::RangeFull => Err(Error::invalid_input("cannot materialize RangeFull")),
+            Self::RangeTo(r) => Ok(UInt32Array::from(Vec::from_iter(0..r.end as u32))),
+            Self::RangeFrom(_) => Err(Error::invalid_input("cannot materialize RangeFrom")),
+        }
+    }
+
+    pub fn iter_offset_ranges<'a>(
+        &'a self,
+    ) -> Result<Box<dyn Iterator<Item = Range<u32>> + Send + 'a>> {
+        match self {
+            Self::Indices(indices) => Ok(Box::new(indices.values().iter().map(|i| *i..(*i + 1)))),
+            Self::Range(r) => Ok(Box::new(std::iter::once(r.start as u32..r.end as u32))),
+            Self::Ranges(ranges) => Ok(Box::new(
+                ranges.iter().map(|r| r.start as u32..r.end as u32),
+            )),
+            Self::RangeFull => Err(Error::invalid_input("cannot materialize RangeFull")),
+            Self::RangeTo(r) => Ok(Box::new(std::iter::once(0..r.end as u32))),
+            Self::RangeFrom(_) => Err(Error::invalid_input("cannot materialize RangeFrom")),
+        }
+    }
+
+    /// Convert a read range into a vector of row ranges
+    pub fn to_ranges(&self) -> Result<Vec<Range<u64>>> {
+        match self {
+            Self::Indices(indices) => Ok(indices
+                .values()
+                .iter()
+                .map(|i| *i as u64..(*i + 1) as u64)
+                .collect()),
+            Self::Range(r) => Ok(vec![r.start as u64..r.end as u64]),
+            Self::Ranges(ranges) => Ok(ranges.to_vec()),
+            Self::RangeFull => Err(Error::invalid_input("cannot materialize RangeFull")),
+            Self::RangeTo(r) => Ok(vec![0..r.end as u64]),
+            Self::RangeFrom(_) => Err(Error::invalid_input("cannot materialize RangeFrom")),
+        }
+    }
+
+    /// Same thing as to_offsets but the caller knows the total number of rows in the file
+    ///
+    /// This makes it possible to materialize RangeFull / RangeFrom
+    pub fn to_offsets_total(&self, total: u32) -> PrimitiveArray<UInt32Type> {
+        match self {
+            Self::Indices(indices) => indices.clone(),
+            Self::Range(r) => UInt32Array::from_iter_values(r.start as u32..r.end as u32),
+            Self::Ranges(ranges) => {
+                let num_rows = ranges
+                    .iter()
+                    .map(|r| (r.end - r.start) as usize)
+                    .sum::<usize>();
+                let mut offsets = Vec::with_capacity(num_rows);
+                for r in ranges.as_ref() {
+                    offsets.extend(r.start as u32..r.end as u32);
+                }
+                UInt32Array::from(offsets)
+            }
+            Self::RangeFull => UInt32Array::from_iter_values(0_u32..total),
+            Self::RangeTo(r) => UInt32Array::from_iter_values(0..r.end as u32),
+            Self::RangeFrom(r) => UInt32Array::from_iter_values(r.start as u32..total),
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::ops::{RangeFrom, RangeTo};
+
+    use arrow_array::UInt32Array;
+
+    use crate::ReadBatchParams;
+
+    #[test]
+    fn test_params_slice() {
+        let params = ReadBatchParams::Ranges(vec![0..15, 20..40].into());
+        let sliced = params.slice(10, 10).unwrap();
+        assert_eq!(sliced, ReadBatchParams::Ranges(vec![10..15, 20..25].into()));
+    }
+
+    #[test]
+    fn test_params_to_offsets() {
+        let check = |params: ReadBatchParams, base_offset, length, expected: Vec<u32>| {
+            let offsets = params
+                .slice(base_offset, length)
+                .unwrap()
+                .to_offsets()
+                .unwrap();
+            let expected = UInt32Array::from(expected);
+            assert_eq!(offsets, expected);
+        };
+
+        check(ReadBatchParams::RangeFull, 0, 100, (0..100).collect());
+        check(ReadBatchParams::RangeFull, 50, 100, (50..150).collect());
+        check(
+            ReadBatchParams::RangeFrom(RangeFrom { start: 500 }),
+            0,
+            100,
+            (500..600).collect(),
+        );
+        check(
+            ReadBatchParams::RangeFrom(RangeFrom { start: 500 }),
+            100,
+            100,
+            (600..700).collect(),
+        );
+        check(
+            ReadBatchParams::RangeTo(RangeTo { end: 800 }),
+            0,
+            100,
+            (0..100).collect(),
+        );
+        check(
+            ReadBatchParams::RangeTo(RangeTo { end: 800 }),
+            200,
+            100,
+            (200..300).collect(),
+        );
+        check(
+            ReadBatchParams::Indices(UInt32Array::from(vec![1, 3, 5, 7, 9])),
+            0,
+            2,
+            vec![1, 3],
+        );
+        check(
+            ReadBatchParams::Indices(UInt32Array::from(vec![1, 3, 5, 7, 9])),
+            2,
+            2,
+            vec![5, 7],
+        );
+
+        let check_error = |params: ReadBatchParams, base_offset, length| {
+            assert!(params.slice(base_offset, length).is_err());
+        };
+
+        check_error(ReadBatchParams::Indices(UInt32Array::from(vec![1])), 0, 2);
+        check_error(ReadBatchParams::Indices(UInt32Array::from(vec![1])), 1, 1);
+        check_error(ReadBatchParams::Range(0..10), 5, 6);
+        check_error(ReadBatchParams::RangeTo(RangeTo { end: 10 }), 5, 6);
+
+        assert!(ReadBatchParams::RangeFull.to_offsets().is_err());
+        assert!(
+            ReadBatchParams::RangeFrom(RangeFrom { start: 10 })
+                .to_offsets()
+                .is_err()
+        );
+    }
+}

vendor/lance-io/src/local.rs

@@ -0,0 +1,331 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Optimized local I/Os
+
+use std::fs::File;
+use std::io::{ErrorKind, Read, SeekFrom};
+use std::ops::Range;
+use std::sync::Arc;
+
+// TODO: Clean up windows/unix stuff
+#[cfg(unix)]
+use std::os::unix::fs::FileExt;
+#[cfg(windows)]
+use std::os::windows::fs::FileExt;
+
+use async_trait::async_trait;
+use bytes::{Bytes, BytesMut};
+use futures::future::BoxFuture;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result};
+use object_store::path::Path;
+use tokio::io::AsyncSeekExt;
+use tokio::sync::OnceCell;
+use tracing::instrument;
+
+use crate::object_reader::stream_local_range;
+use crate::object_store::DEFAULT_LOCAL_IO_PARALLELISM;
+use crate::object_writer::WriteResult;
+use crate::traits::{ByteStream, Reader, Writer};
+use crate::utils::tracking_store::IOTracker;
+
+/// Convert an [`object_store::path::Path`] to a [`std::path::Path`].
+pub fn to_local_path(path: &Path) -> String {
+    if cfg!(windows) {
+        path.to_string()
+    } else {
+        format!("/{path}")
+    }
+}
+
+/// Recursively remove a directory, specified by [`object_store::path::Path`].
+pub fn remove_dir_all(path: &Path) -> Result<()> {
+    let local_path = to_local_path(path);
+    std::fs::remove_dir_all(local_path).map_err(|err| match err.kind() {
+        ErrorKind::NotFound => Error::not_found(path.to_string()),
+        _ => Error::from(err),
+    })?;
+    Ok(())
+}
+
+/// Copy a file from one location to another, supporting cross-filesystem copies.
+///
+/// Unlike hard links, this function works across filesystem boundaries.
+pub fn copy_file(from: &Path, to: &Path) -> Result<()> {
+    let from_path = to_local_path(from);
+    let to_path = to_local_path(to);
+
+    // Ensure the parent directory exists
+    if let Some(parent) = std::path::Path::new(&to_path).parent() {
+        std::fs::create_dir_all(parent).map_err(Error::from)?;
+    }
+
+    std::fs::copy(&from_path, &to_path).map_err(|err| match err.kind() {
+        ErrorKind::NotFound => Error::not_found(from.to_string()),
+        _ => Error::from(err),
+    })?;
+    Ok(())
+}
+
+/// Await a filesystem operation running on a blocking thread, flattening the
+/// join and IO errors into a single `object_store` error.
+///
+/// Deliberately not written as `handle.await?` at the call sites: a `JoinError`
+/// means the operation panicked, and short-circuiting on it would skip the
+/// caller's metrics recording for exactly the failure worth counting.
+pub(crate) async fn join_local_io<T>(
+    handle: tokio::task::JoinHandle<std::io::Result<T>>,
+) -> object_store::Result<T> {
+    match handle.await {
+        Ok(result) => result.map_err(|err| object_store::Error::Generic {
+            store: "LocalFileSystem",
+            source: err.into(),
+        }),
+        Err(err) => Err(err.into()),
+    }
+}
+
+/// Object reader for local file system.
+#[derive(Debug)]
+pub struct LocalObjectReader {
+    /// File handler.
+    file: Arc<File>,
+
+    /// Fie path.
+    path: Path,
+
+    /// Known size of the file. This is either passed in on construction or
+    /// cached on the first metadata call.
+    size: OnceCell<usize>,
+
+    /// Block size, in bytes.
+    block_size: usize,
+
+    /// IO tracker for monitoring read operations.
+    io_tracker: Arc<IOTracker>,
+}
+
+impl DeepSizeOf for LocalObjectReader {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // Skipping `file` as it should just be a file handle
+        self.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl LocalObjectReader {
+    pub async fn open_local_path(
+        path: impl AsRef<std::path::Path>,
+        block_size: usize,
+        known_size: Option<usize>,
+    ) -> Result<Box<dyn Reader>> {
+        let path = path.as_ref().to_owned();
+        let object_store_path = Path::from_filesystem_path(&path)?;
+        Self::open(&object_store_path, block_size, known_size).await
+    }
+
+    /// Open a local object reader, with default prefetch size.
+    ///
+    /// For backward compatibility with existing code that doesn't need tracking.
+    #[instrument(level = "debug")]
+    pub async fn open(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+    ) -> Result<Box<dyn Reader>> {
+        Self::open_with_tracker(path, block_size, known_size, Default::default()).await
+    }
+
+    /// Open a local object reader with optional IO tracking.
+    #[instrument(level = "debug")]
+    pub(crate) async fn open_with_tracker(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<Box<dyn Reader>> {
+        let path = path.clone();
+        let local_path = to_local_path(&path);
+        tokio::task::spawn_blocking(move || {
+            let file = File::open(&local_path).map_err(|e| match e.kind() {
+                ErrorKind::NotFound => Error::not_found(path.to_string()),
+                _ => e.into(),
+            })?;
+            let size = OnceCell::new_with(known_size);
+            Ok(Box::new(Self {
+                file: Arc::new(file),
+                block_size,
+                size,
+                path,
+                io_tracker,
+            }) as Box<dyn Reader>)
+        })
+        .await?
+    }
+}
+
+impl Reader for LocalObjectReader {
+    fn path(&self) -> &Path {
+        &self.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        DEFAULT_LOCAL_IO_PARALLELISM
+    }
+
+    /// Returns the file size.
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move {
+            let file = self.file.clone();
+            self.size
+                .get_or_try_init(|| async move {
+                    // The metadata lookup is this reader's equivalent of the HEAD
+                    // request a cloud reader makes to learn the object size.
+                    let metrics = self.io_tracker.begin_io("head");
+                    let result =
+                        join_local_io(tokio::task::spawn_blocking(move || file.metadata())).await;
+                    metrics.record(&result, 0);
+                    Ok(result?.len() as usize)
+                })
+                .await
+                .cloned()
+        })
+    }
+
+    /// Reads a range of data.
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let file = self.file.clone();
+        let io_tracker = self.io_tracker.clone();
+        let path = self.path.clone();
+        let num_bytes = range.len() as u64;
+        let range_u64 = (range.start as u64)..(range.end as u64);
+
+        Box::pin(async move {
+            let metrics = io_tracker.begin_io("get");
+            let result = join_local_io(tokio::task::spawn_blocking(move || {
+                let mut buf = BytesMut::with_capacity(range.len());
+                // Safety: `buf` is set with appropriate capacity above. It is
+                // written to below and we check all data is initialized at that point.
+                unsafe { buf.set_len(range.len()) };
+                #[cfg(unix)]
+                file.read_exact_at(buf.as_mut(), range.start as u64)?;
+                #[cfg(windows)]
+                read_exact_at(file, buf.as_mut(), range.start as u64)?;
+
+                Ok(buf.freeze())
+            }))
+            .await;
+
+            metrics.record(&result, num_bytes);
+            if result.is_ok() {
+                io_tracker.record_read("get_range", path, num_bytes, Some(range_u64));
+            }
+
+            result
+        })
+    }
+
+    /// Reads the entire file.
+    #[instrument(level = "debug", skip(self))]
+    fn get_all(&self) -> BoxFuture<'_, object_store::Result<Bytes>> {
+        Box::pin(async move {
+            let mut file = self.file.clone();
+            let io_tracker = self.io_tracker.clone();
+            let path = self.path.clone();
+
+            let metrics = io_tracker.begin_io("get");
+            let result = join_local_io(tokio::task::spawn_blocking(move || {
+                let mut buf = Vec::new();
+                file.read_to_end(buf.as_mut())?;
+                Ok(Bytes::from(buf))
+            }))
+            .await;
+
+            let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64);
+            metrics.record(&result, num_bytes);
+            if let Ok(bytes) = &result {
+                io_tracker.record_read("get_all", path, bytes.len() as u64, None);
+            }
+
+            result
+        })
+    }
+
+    fn get_stream(&self) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let size = self.size().await?;
+            Ok(stream_local_range(
+                self.file.clone(),
+                self.path.clone(),
+                self.io_tracker.clone(),
+                0..size,
+                self.block_size.max(8 * 1024),
+            ))
+        })
+    }
+
+    fn get_range_stream(
+        &self,
+        range: Range<usize>,
+    ) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        let file = self.file.clone();
+        let path = self.path.clone();
+        let io_tracker = self.io_tracker.clone();
+        let chunk_size = self.block_size.max(8 * 1024);
+        Box::pin(async move {
+            Ok(stream_local_range(
+                file, path, io_tracker, range, chunk_size,
+            ))
+        })
+    }
+}
+
+#[cfg(windows)]
+pub(crate) fn read_exact_at(
+    file: Arc<File>,
+    mut buf: &mut [u8],
+    mut offset: u64,
+) -> std::io::Result<()> {
+    let expected_len = buf.len();
+    while !buf.is_empty() {
+        match file.seek_read(buf, offset) {
+            Ok(0) => break,
+            Ok(n) => {
+                let tmp = buf;
+                buf = &mut tmp[n..];
+                offset += n as u64;
+            }
+            Err(ref e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+            Err(e) => return Err(e),
+        }
+    }
+    if !buf.is_empty() {
+        Err(std::io::Error::new(
+            std::io::ErrorKind::UnexpectedEof,
+            format!(
+                "failed to fill whole buffer. Expected {} bytes, got {}",
+                expected_len, offset
+            ),
+        ))
+    } else {
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl Writer for tokio::fs::File {
+    async fn tell(&mut self) -> Result<usize> {
+        Ok(self.seek(SeekFrom::Current(0)).await? as usize)
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        let size = self.seek(SeekFrom::Current(0)).await? as usize;
+        tokio::io::AsyncWriteExt::shutdown(self).await?;
+        Ok(WriteResult { size, e_tag: None })
+    }
+}

vendor/lance-io/src/object_reader.rs

@@ -0,0 +1,464 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::fs::File;
+use std::ops::Range;
+use std::sync::Arc;
+
+use crate::local::join_local_io;
+#[cfg(windows)]
+use crate::local::read_exact_at;
+#[cfg(unix)]
+use std::os::unix::fs::FileExt;
+
+use bytes::Bytes;
+use futures::{
+    FutureExt,
+    future::{BoxFuture, Shared},
+    stream::{self, StreamExt},
+};
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result, error::CloneableError};
+use object_store::ObjectStoreExt;
+use object_store::{GetOptions, GetResult, ObjectStore, Result as OSResult, path::Path};
+use tokio::sync::OnceCell;
+use tracing::instrument;
+
+use crate::{
+    object_store::DEFAULT_CLOUD_IO_PARALLELISM,
+    traits::{ByteStream, Reader},
+};
+
+trait StaticGetRange {
+    fn path(&self) -> &Path;
+    fn get_range(&self) -> BoxFuture<'static, OSResult<GetResult>>;
+}
+
+/// A wrapper around an object store and a path that implements a static
+/// get_range method by assuming self is stored in an Arc.
+struct GetRequest {
+    object_store: Arc<dyn ObjectStore>,
+    path: Path,
+    options: GetOptions,
+}
+
+impl StaticGetRange for Arc<GetRequest> {
+    fn path(&self) -> &Path {
+        &self.path
+    }
+
+    fn get_range(&self) -> BoxFuture<'static, OSResult<GetResult>> {
+        let store_and_path = self.clone();
+        Box::pin(async move {
+            store_and_path
+                .object_store
+                .get_opts(&store_and_path.path, store_and_path.options.clone())
+                .await
+        })
+    }
+}
+
+/// Object Reader
+///
+/// Object Store + Base Path
+#[derive(Debug)]
+pub struct CloudObjectReader {
+    // Object Store.
+    pub object_store: Arc<dyn ObjectStore>,
+    // File path
+    pub path: Path,
+    // File size, if known.
+    size: OnceCell<usize>,
+
+    block_size: usize,
+    download_retry_count: usize,
+}
+
+impl DeepSizeOf for CloudObjectReader {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // Skipping object_store because there is no easy way to do that and it shouldn't be too big
+        self.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl CloudObjectReader {
+    /// Create an ObjectReader from URI
+    pub fn new(
+        object_store: Arc<dyn ObjectStore>,
+        path: Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        download_retry_count: usize,
+    ) -> Result<Self> {
+        Ok(Self {
+            object_store,
+            path,
+            size: OnceCell::new_with(known_size),
+            block_size,
+            download_retry_count,
+        })
+    }
+}
+
+// Retries for the initial request are handled by object store, but
+// there are no retries for failures that occur during the streaming
+// of the response body. Thus we add an outer retry loop here.
+async fn do_with_retry<'a, O>(f: impl Fn() -> BoxFuture<'a, OSResult<O>> + Clone) -> OSResult<O> {
+    let mut retries = 3;
+    loop {
+        let f = f.clone();
+        match f().await {
+            Ok(val) => return Ok(val),
+            Err(err) => {
+                if retries == 0 {
+                    return Err(err);
+                }
+                retries -= 1;
+            }
+        }
+    }
+}
+
+// We have a separate retry loop here.  This is because object_store does not
+// attempt retries on downloads that fail during streaming of the response body.
+//
+// However, this failure is pretty common (e.g. timeout) and we want to retry in these
+// situations.  In addition, we provide additional logging information in these
+// failures cases.
+async fn do_get_with_outer_retry(
+    download_retry_count: usize,
+    get_request: Arc<GetRequest>,
+    desc: impl Fn() -> String,
+) -> OSResult<Bytes> {
+    let mut retries = download_retry_count;
+    loop {
+        let get_request_clone = get_request.clone();
+        let get_result = do_with_retry(move || get_request_clone.get_range()).await?;
+        match get_result.bytes().await {
+            Ok(bytes) => return Ok(bytes),
+            Err(err) => {
+                if retries == 0 {
+                    log::warn!(
+                        "Failed to download {} from {} after {} attempts.  This may indicate that cloud storage is overloaded or your timeout settings are too restrictive.  Error details: {:?}",
+                        desc(),
+                        get_request.path(),
+                        download_retry_count,
+                        err
+                    );
+                    return Err(err);
+                }
+                log::debug!(
+                    "Retrying {} from {} (remaining retries: {}).  Error details: {:?}",
+                    desc(),
+                    get_request.path(),
+                    retries,
+                    err
+                );
+                retries -= 1;
+            }
+        }
+    }
+}
+
+impl Reader for CloudObjectReader {
+    fn path(&self) -> &Path {
+        &self.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        DEFAULT_CLOUD_IO_PARALLELISM
+    }
+
+    /// Object/File Size.
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move {
+            self.size
+                .get_or_try_init(|| async move {
+                    let meta =
+                        do_with_retry(|| Box::pin(self.object_store.head(&self.path))).await?;
+                    Ok(meta.size as usize)
+                })
+                .await
+                .cloned()
+        })
+    }
+
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, OSResult<Bytes>> {
+        let object_store = self.object_store.clone();
+        let path = self.path.clone();
+        let get_range = Range {
+            start: range.start as u64,
+            end: range.end as u64,
+        };
+        Box::pin(async move {
+            let bytes = do_with_retry(move || {
+                let object_store = object_store.clone();
+                let path = path.clone();
+                let get_range = get_range.clone();
+                Box::pin(async move { object_store.get_ranges(&path, &[get_range]).await })
+            })
+            .await?;
+
+            bytes
+                .into_iter()
+                .next()
+                .ok_or_else(|| object_store::Error::Generic {
+                    store: "CloudObjectReader",
+                    source: "get_ranges returned no bytes".into(),
+                })
+        })
+    }
+
+    #[instrument(level = "debug", skip_all)]
+    fn get_all(&self) -> BoxFuture<'_, OSResult<Bytes>> {
+        let get_request = Arc::new(GetRequest {
+            object_store: self.object_store.clone(),
+            path: self.path.clone(),
+            options: GetOptions::default(),
+        });
+        Box::pin(async move {
+            do_get_with_outer_retry(self.download_retry_count, get_request, || {
+                "read_all".to_string()
+            })
+            .await
+        })
+    }
+
+    fn get_stream(&self) -> BoxFuture<'_, OSResult<ByteStream>> {
+        let get_request = Arc::new(GetRequest {
+            object_store: self.object_store.clone(),
+            path: self.path.clone(),
+            options: GetOptions::default(),
+        });
+        Box::pin(async move {
+            let get_request_clone = get_request.clone();
+            let get_result = do_with_retry(move || get_request_clone.get_range()).await?;
+            Ok(get_result.into_stream())
+        })
+    }
+
+    fn get_range_stream(&self, range: Range<usize>) -> BoxFuture<'_, OSResult<ByteStream>> {
+        let get_request = Arc::new(GetRequest {
+            object_store: self.object_store.clone(),
+            path: self.path.clone(),
+            options: GetOptions {
+                range: Some(
+                    Range {
+                        start: range.start as u64,
+                        end: range.end as u64,
+                    }
+                    .into(),
+                ),
+                ..Default::default()
+            },
+        });
+        Box::pin(async move {
+            let get_request_clone = get_request.clone();
+            let get_result = do_with_retry(move || get_request_clone.get_range()).await?;
+            Ok(get_result.into_stream())
+        })
+    }
+}
+
+#[derive(Debug)]
+pub struct SmallReaderInner {
+    path: Path,
+    size: usize,
+    state: std::sync::Mutex<SmallReaderState>,
+}
+
+/// A reader for a file so small, we just eagerly read it all into memory.
+///
+/// When created, it represents a future that will read the whole file into memory.
+///
+/// On the first read call, it will start the read. Multiple threads can call read at the same time.
+///
+/// Once the read is complete, any thread can call read again to get the result.
+#[derive(Clone, Debug)]
+pub struct SmallReader {
+    inner: Arc<SmallReaderInner>,
+}
+
+enum SmallReaderState {
+    Loading(Shared<BoxFuture<'static, std::result::Result<Bytes, CloneableError>>>),
+    Finished(std::result::Result<Bytes, CloneableError>),
+}
+
+impl std::fmt::Debug for SmallReaderState {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Loading(_) => write!(f, "Loading"),
+            Self::Finished(Ok(data)) => {
+                write!(f, "Finished({} bytes)", data.len())
+            }
+            Self::Finished(Err(err)) => {
+                write!(f, "Finished({})", err.0)
+            }
+        }
+    }
+}
+
+impl SmallReader {
+    pub fn new(
+        store: Arc<dyn ObjectStore>,
+        path: Path,
+        download_retry_count: usize,
+        size: usize,
+    ) -> Self {
+        let path_ref = path.clone();
+        let state = SmallReaderState::Loading(
+            Box::pin(async move {
+                let object_reader =
+                    CloudObjectReader::new(store, path_ref, 0, None, download_retry_count)
+                        .map_err(CloneableError)?;
+                object_reader
+                    .get_all()
+                    .await
+                    .map_err(|err| CloneableError(Error::from(err)))
+            })
+            .boxed()
+            .shared(),
+        );
+        Self {
+            inner: Arc::new(SmallReaderInner {
+                path,
+                size,
+                state: std::sync::Mutex::new(state),
+            }),
+        }
+    }
+}
+
+impl SmallReaderInner {
+    async fn wait(&self) -> OSResult<Bytes> {
+        let future = {
+            let state = self.state.lock().unwrap();
+            match &*state {
+                SmallReaderState::Loading(future) => future.clone(),
+                SmallReaderState::Finished(result) => {
+                    return result.clone().map_err(|err| err.0.into());
+                }
+            }
+        };
+
+        let result = future.await;
+        let result_to_return = result.clone().map_err(|err| err.0.into());
+        let mut state = self.state.lock().unwrap();
+        if matches!(*state, SmallReaderState::Loading(_)) {
+            *state = SmallReaderState::Finished(result);
+        }
+        result_to_return
+    }
+}
+
+impl Reader for SmallReader {
+    fn path(&self) -> &Path {
+        &self.inner.path
+    }
+
+    fn block_size(&self) -> usize {
+        64 * 1024
+    }
+
+    fn io_parallelism(&self) -> usize {
+        1024
+    }
+
+    /// Object/File Size.
+    fn size(&self) -> BoxFuture<'_, OSResult<usize>> {
+        let size = self.inner.size;
+        Box::pin(async move { Ok(size) })
+    }
+
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, OSResult<Bytes>> {
+        let inner = self.inner.clone();
+        Box::pin(async move {
+            let bytes = inner.wait().await?;
+            let start = range.start;
+            let end = range.end;
+            if start >= bytes.len() || end > bytes.len() {
+                return Err(object_store::Error::Generic {
+                    store: "memory",
+                    source: format!(
+                        "Invalid range {}..{} for object of size {} bytes",
+                        start,
+                        end,
+                        bytes.len()
+                    )
+                    .into(),
+                });
+            }
+            Ok(bytes.slice(range))
+        })
+    }
+
+    fn get_all(&self) -> BoxFuture<'_, OSResult<Bytes>> {
+        Box::pin(async move { self.inner.wait().await })
+    }
+}
+
+pub(crate) fn stream_local_range(
+    file: Arc<File>,
+    path: Path,
+    io_tracker: Arc<crate::utils::tracking_store::IOTracker>,
+    range: Range<usize>,
+    chunk_size: usize,
+) -> ByteStream {
+    stream::try_unfold(
+        (file, path, io_tracker, range.start, range.end),
+        move |state| async move {
+            let (file, path, io_tracker, start, end) = state;
+            if start >= end {
+                return Ok(None);
+            }
+
+            let next = (start + chunk_size).min(end);
+            let file_clone = file.clone();
+            let path_clone = path.clone();
+            let num_bytes = (next - start) as u64;
+            let metrics = io_tracker.begin_io("get");
+            let result = join_local_io(tokio::task::spawn_blocking(move || {
+                let mut buf = bytes::BytesMut::with_capacity(next - start);
+                // Safety: buffer capacity matches the exact number of bytes we read below.
+                unsafe { buf.set_len(next - start) };
+                #[cfg(unix)]
+                file_clone.read_exact_at(buf.as_mut(), start as u64)?;
+                #[cfg(windows)]
+                read_exact_at(file_clone, buf.as_mut(), start as u64)?;
+                Ok::<_, std::io::Error>(buf.freeze())
+            }))
+            .await;
+            metrics.record(&result, num_bytes);
+            let bytes = result?;
+
+            io_tracker.record_read(
+                "get_range_stream",
+                path_clone,
+                num_bytes,
+                Some(start as u64..next as u64),
+            );
+
+            Ok(Some((bytes, (file, path, io_tracker, next, end))))
+        },
+    )
+    .boxed()
+}
+
+impl DeepSizeOf for SmallReader {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        let mut size = self.inner.path.as_ref().deep_size_of_children(context);
+
+        if let Ok(guard) = self.inner.state.try_lock()
+            && let SmallReaderState::Finished(Ok(data)) = &*guard
+        {
+            size += data.len();
+        }
+
+        size
+    }
+}

vendor/lance-io/src/object_store/dynamic_credentials.rs

@@ -0,0 +1,411 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::fmt;
+use std::marker::PhantomData;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use lance_core::error::{Error, Result};
+use object_store::{CredentialProvider, Result as ObjectStoreResult};
+
+use crate::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+
+#[cfg(feature = "aws")]
+use object_store::aws::AwsCredential as ObjectStoreAwsCredential;
+#[cfg(feature = "azure")]
+use object_store::azure::{AzureAccessKey, AzureCredential};
+#[cfg(feature = "gcp")]
+use object_store::gcp::GcpCredential;
+
+/// Raw dynamic storage options fetched from a credential-vending source.
+///
+/// Callers must convert this bag into a cloud-specific credential type via
+/// `TryFrom<DynamicCredentials>`.
+#[derive(Clone)]
+pub struct DynamicCredentials(pub HashMap<String, String>);
+
+impl fmt::Debug for DynamicCredentials {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_tuple("DynamicCredentials")
+            .field(&format_args!("[{} keys redacted]", self.0.len()))
+            .finish()
+    }
+}
+
+#[derive(Clone)]
+pub struct NamespaceCredentialsProvider<T> {
+    accessor: Arc<StorageOptionsAccessor>,
+    _credential: PhantomData<T>,
+}
+
+impl<T> fmt::Debug for NamespaceCredentialsProvider<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("NamespaceCredentialsProvider")
+            .field("accessor", &self.accessor)
+            .field("credential_type", &std::any::type_name::<T>())
+            .finish()
+    }
+}
+
+impl<T> NamespaceCredentialsProvider<T> {
+    pub fn new(accessor: Arc<StorageOptionsAccessor>) -> Self {
+        Self {
+            accessor,
+            _credential: PhantomData,
+        }
+    }
+
+    pub fn from_provider(provider: Arc<dyn StorageOptionsProvider>) -> Self {
+        Self::new(Arc::new(StorageOptionsAccessor::with_provider(provider)))
+    }
+
+    pub fn from_provider_with_initial(
+        provider: Arc<dyn StorageOptionsProvider>,
+        initial_options: HashMap<String, String>,
+    ) -> Self {
+        Self::new(Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+            initial_options,
+            provider,
+        )))
+    }
+}
+
+/// Build a dynamic credential provider for any cloud type, returning `None`
+/// if the accessor has no provider or the provider options are incompatible with `T`.
+pub async fn build_dynamic_credential_provider<T>(
+    accessor: Option<Arc<StorageOptionsAccessor>>,
+) -> Result<Option<Arc<dyn CredentialProvider<Credential = T>>>>
+where
+    T: TryFrom<DynamicCredentials, Error = Error> + fmt::Debug + Send + Sync + 'static,
+{
+    let Some(accessor) = accessor.filter(|a| a.has_provider()) else {
+        return Ok(None);
+    };
+
+    let compatible = if let Some(initial) = accessor.initial_storage_options()
+        && T::try_from(DynamicCredentials(initial.clone())).is_ok()
+    {
+        true
+    } else {
+        let fetched = accessor.refresh_storage_options().await?.0;
+        T::try_from(DynamicCredentials(fetched)).is_ok()
+    };
+
+    if !compatible {
+        return Ok(None);
+    }
+
+    Ok(Some(
+        Arc::new(NamespaceCredentialsProvider::<T>::new(accessor))
+            as Arc<dyn CredentialProvider<Credential = T>>,
+    ))
+}
+
+fn map_credential_error(error: Error) -> object_store::Error {
+    object_store::Error::Generic {
+        store: "NamespaceCredentialsProvider",
+        source: Box::new(error),
+    }
+}
+
+#[async_trait]
+impl<T> CredentialProvider for NamespaceCredentialsProvider<T>
+where
+    T: TryFrom<DynamicCredentials, Error = Error> + fmt::Debug + Send + Sync + 'static,
+{
+    type Credential = T;
+
+    async fn get_credential(&self) -> ObjectStoreResult<Arc<Self::Credential>> {
+        let storage_options = self
+            .accessor
+            .get_storage_options()
+            .await
+            .map_err(map_credential_error)?;
+
+        let credential = match T::try_from(DynamicCredentials(storage_options.0)) {
+            Ok(credential) => credential,
+            Err(_) if self.accessor.has_provider() => {
+                let storage_options = self
+                    .accessor
+                    .refresh_storage_options()
+                    .await
+                    .map_err(map_credential_error)?;
+                T::try_from(DynamicCredentials(storage_options.0)).map_err(map_credential_error)?
+            }
+            Err(error) => return Err(map_credential_error(error)),
+        };
+
+        Ok(Arc::new(credential))
+    }
+}
+
+fn missing_dynamic_credential(kind: &str) -> Error {
+    Error::invalid_input(format!(
+        "Missing required {kind} credential fields in dynamic storage options"
+    ))
+}
+
+#[cfg(feature = "azure")]
+fn split_azure_sas(sas: &str) -> Result<Vec<(String, String)>> {
+    let pairs = url::form_urlencoded::parse(sas.trim_start_matches('?').as_bytes())
+        .map(|(key, value)| (key.into_owned(), value.into_owned()))
+        .collect::<Vec<_>>();
+
+    if pairs.is_empty() {
+        return Err(Error::invalid_input(
+            "Azure SAS token is empty or invalid in dynamic storage options",
+        ));
+    }
+
+    Ok(pairs)
+}
+
+#[cfg(feature = "aws")]
+impl TryFrom<DynamicCredentials> for ObjectStoreAwsCredential {
+    type Error = Error;
+
+    fn try_from(credentials: DynamicCredentials) -> Result<Self> {
+        let key_id = credentials
+            .0
+            .get("aws_access_key_id")
+            .or_else(|| credentials.0.get("access_key_id"))
+            .cloned();
+        let secret_key = credentials
+            .0
+            .get("aws_secret_access_key")
+            .or_else(|| credentials.0.get("secret_access_key"))
+            .cloned();
+        let token = credentials
+            .0
+            .get("aws_session_token")
+            .or_else(|| credentials.0.get("aws_token"))
+            .or_else(|| credentials.0.get("aws_security_token"))
+            .or_else(|| credentials.0.get("session_token"))
+            .or_else(|| credentials.0.get("token"))
+            .cloned();
+
+        match (key_id, secret_key) {
+            (Some(key_id), Some(secret_key)) => Ok(Self {
+                key_id,
+                secret_key,
+                token,
+            }),
+            _ => Err(missing_dynamic_credential("AWS")),
+        }
+    }
+}
+
+#[cfg(feature = "azure")]
+impl TryFrom<DynamicCredentials> for AzureCredential {
+    type Error = Error;
+
+    fn try_from(credentials: DynamicCredentials) -> Result<Self> {
+        if let Some(sas) = credentials
+            .0
+            .get("azure_storage_sas_token")
+            .or_else(|| credentials.0.get("azure_storage_sas_key"))
+            .or_else(|| credentials.0.get("sas_token"))
+            .or_else(|| credentials.0.get("sas_key"))
+        {
+            return Ok(Self::SASToken(split_azure_sas(sas)?));
+        }
+
+        if let Some(token) = credentials
+            .0
+            .get("azure_storage_token")
+            .or_else(|| credentials.0.get("bearer_token"))
+            .or_else(|| credentials.0.get("token"))
+        {
+            return Ok(Self::BearerToken(token.clone()));
+        }
+
+        if let Some(access_key) = credentials
+            .0
+            .get("azure_storage_account_key")
+            .or_else(|| credentials.0.get("azure_storage_access_key"))
+            .or_else(|| credentials.0.get("azure_storage_master_key"))
+            .or_else(|| credentials.0.get("access_key"))
+            .or_else(|| credentials.0.get("master_key"))
+            .or_else(|| credentials.0.get("account_key"))
+        {
+            return Ok(Self::AccessKey(
+                AzureAccessKey::try_new(access_key).map_err(|source| {
+                    Error::invalid_input(format!("Invalid Azure access key: {source}"))
+                })?,
+            ));
+        }
+
+        Err(missing_dynamic_credential("Azure"))
+    }
+}
+
+#[cfg(feature = "gcp")]
+impl TryFrom<DynamicCredentials> for GcpCredential {
+    type Error = Error;
+
+    fn try_from(credentials: DynamicCredentials) -> Result<Self> {
+        let bearer = credentials
+            .0
+            .get("google_storage_token")
+            .cloned()
+            .ok_or_else(|| missing_dynamic_credential("GCP"))?;
+
+        Ok(Self { bearer })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::*;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+
+    #[cfg(feature = "aws")]
+    #[tokio::test]
+    async fn test_dynamic_aws_credentials() {
+        let provider = Arc::new(StaticMockStorageOptionsProvider {
+            options: HashMap::from([
+                ("aws_access_key_id".to_string(), "AKID".to_string()),
+                ("aws_secret_access_key".to_string(), "SECRET".to_string()),
+                ("aws_session_token".to_string(), "TOKEN".to_string()),
+            ]),
+        });
+
+        let credentials =
+            NamespaceCredentialsProvider::<ObjectStoreAwsCredential>::from_provider(provider)
+                .get_credential()
+                .await
+                .expect("aws credentials should convert");
+
+        assert_eq!(credentials.key_id, "AKID");
+        assert_eq!(credentials.secret_key, "SECRET");
+        assert_eq!(credentials.token.as_deref(), Some("TOKEN"));
+    }
+
+    #[cfg(feature = "aws")]
+    #[tokio::test]
+    async fn test_dynamic_aws_credentials_aws_token_alias() {
+        let provider = Arc::new(StaticMockStorageOptionsProvider {
+            options: HashMap::from([
+                ("aws_access_key_id".to_string(), "AKID".to_string()),
+                ("aws_secret_access_key".to_string(), "SECRET".to_string()),
+                ("aws_token".to_string(), "TOKEN".to_string()),
+            ]),
+        });
+
+        let credentials =
+            NamespaceCredentialsProvider::<ObjectStoreAwsCredential>::from_provider(provider)
+                .get_credential()
+                .await
+                .expect("aws credentials should convert");
+
+        assert_eq!(credentials.token.as_deref(), Some("TOKEN"));
+    }
+
+    #[cfg(feature = "aws")]
+    #[tokio::test]
+    async fn test_dynamic_credentials_fetch_provider_when_initial_has_metadata_only() {
+        let provider = Arc::new(StaticMockStorageOptionsProvider {
+            options: HashMap::from([
+                ("aws_access_key_id".to_string(), "AKID".to_string()),
+                ("aws_secret_access_key".to_string(), "SECRET".to_string()),
+            ]),
+        });
+        let accessor = Arc::new(StorageOptionsAccessor::with_initial_and_provider(
+            HashMap::from([("region".to_string(), "us-west-2".to_string())]),
+            provider,
+        ));
+
+        let credentials =
+            build_dynamic_credential_provider::<ObjectStoreAwsCredential>(Some(accessor))
+                .await
+                .expect("dynamic credential provider should build")
+                .expect("provider should be returned")
+                .get_credential()
+                .await
+                .expect("provider-vended aws credentials should convert");
+
+        assert_eq!(credentials.key_id, "AKID");
+        assert_eq!(credentials.secret_key, "SECRET");
+    }
+
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_dynamic_azure_credentials() {
+        let provider = Arc::new(StaticMockStorageOptionsProvider {
+            options: HashMap::from([(
+                "azure_storage_sas_token".to_string(),
+                "?sv=2022-11-02&sp=rl&sig=test".to_string(),
+            )]),
+        });
+
+        let credentials = NamespaceCredentialsProvider::<AzureCredential>::from_provider(provider)
+            .get_credential()
+            .await
+            .expect("azure credentials should convert");
+
+        match credentials.as_ref() {
+            AzureCredential::SASToken(pairs) => {
+                assert!(
+                    pairs
+                        .iter()
+                        .any(|(key, value)| key == "sv" && value == "2022-11-02")
+                );
+                assert!(
+                    pairs
+                        .iter()
+                        .any(|(key, value)| key == "sig" && value == "test")
+                );
+            }
+            other => panic!("expected SAS token, got {other:?}"),
+        }
+    }
+
+    #[cfg(feature = "azure")]
+    #[tokio::test]
+    async fn test_dynamic_azure_credentials_short_sas_aliases() {
+        for key in ["sas_token", "sas_key"] {
+            let provider = Arc::new(StaticMockStorageOptionsProvider {
+                options: HashMap::from([(
+                    key.to_string(),
+                    "?sv=2022-11-02&sp=rl&sig=short".to_string(),
+                )]),
+            });
+
+            let credentials =
+                NamespaceCredentialsProvider::<AzureCredential>::from_provider(provider)
+                    .get_credential()
+                    .await
+                    .unwrap_or_else(|_| panic!("azure credentials should convert for key '{key}'"));
+
+            match credentials.as_ref() {
+                AzureCredential::SASToken(pairs) => {
+                    assert!(
+                        pairs.iter().any(|(k, v)| k == "sig" && v == "short"),
+                        "SAS token from key '{key}' should contain sig=short"
+                    );
+                }
+                other => panic!("expected SAS token for key '{key}', got {other:?}"),
+            }
+        }
+    }
+
+    #[cfg(feature = "gcp")]
+    #[tokio::test]
+    async fn test_dynamic_gcp_credentials() {
+        let provider = Arc::new(StaticMockStorageOptionsProvider {
+            options: HashMap::from([("google_storage_token".to_string(), "gcp-token".to_string())]),
+        });
+
+        let credentials = NamespaceCredentialsProvider::<GcpCredential>::from_provider(provider)
+            .get_credential()
+            .await
+            .expect("gcp credentials should convert");
+
+        assert_eq!(credentials.bearer, "gcp-token");
+    }
+}

vendor/lance-io/src/object_store/dynamic_opendal.rs

@@ -0,0 +1,368 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::fmt;
+use std::ops::Range;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use futures::{StreamExt, TryStreamExt, stream, stream::BoxStream};
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    ObjectStore as OSObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,
+    RenameOptions,
+};
+use object_store_opendal::OpendalStore;
+use tokio::sync::RwLock;
+
+use crate::object_store::StorageOptionsAccessor;
+use lance_core::Result;
+
+type NormalizeConfigFn = fn(&HashMap<String, String>) -> Result<HashMap<String, String>>;
+type BuildStoreFn = fn(HashMap<String, String>) -> Result<OpendalStore>;
+type FilterDynamicOptionsFn = fn(&HashMap<String, String>) -> HashMap<String, String>;
+
+#[derive(Debug, Clone)]
+struct CachedOpenDalStore {
+    config: HashMap<String, String>,
+    store: Arc<OpendalStore>,
+}
+
+#[derive(Clone)]
+pub(in crate::object_store) struct DynamicOpenDalStore {
+    name: Arc<str>,
+    base_options: Arc<HashMap<String, String>>,
+    accessor: Arc<StorageOptionsAccessor>,
+    normalize_config: NormalizeConfigFn,
+    build_store: BuildStoreFn,
+    filter_dynamic_options: Option<FilterDynamicOptionsFn>,
+    atomic_key_groups: Vec<Vec<&'static str>>,
+    protected_keys: Vec<&'static str>,
+    cache: Arc<RwLock<Option<CachedOpenDalStore>>>,
+}
+
+impl fmt::Debug for DynamicOpenDalStore {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("DynamicOpenDalStore")
+            .field("name", &self.name)
+            .field("accessor", &self.accessor)
+            .finish()
+    }
+}
+
+impl DynamicOpenDalStore {
+    pub(in crate::object_store) fn new(
+        name: impl Into<Arc<str>>,
+        base_options: HashMap<String, String>,
+        accessor: Arc<StorageOptionsAccessor>,
+        normalize_config: NormalizeConfigFn,
+        build_store: BuildStoreFn,
+    ) -> Self {
+        Self {
+            name: name.into(),
+            base_options: Arc::new(base_options),
+            accessor,
+            normalize_config,
+            build_store,
+            filter_dynamic_options: None,
+            atomic_key_groups: Vec::new(),
+            protected_keys: Vec::new(),
+            cache: Arc::new(RwLock::new(None)),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub(in crate::object_store) fn with_protected_keys(
+        mut self,
+        keys: impl IntoIterator<Item = &'static str>,
+    ) -> Self {
+        self.protected_keys = keys.into_iter().collect();
+        self
+    }
+
+    /// Restrict provider-vended updates before they are merged into the fixed store config.
+    pub(in crate::object_store) fn with_dynamic_options_filter(
+        mut self,
+        filter: FilterDynamicOptionsFn,
+    ) -> Self {
+        self.filter_dynamic_options = Some(filter);
+        self
+    }
+
+    /// Treat a set of related options as one authority when provider values are present.
+    pub(in crate::object_store) fn with_atomic_key_group(
+        mut self,
+        keys: impl IntoIterator<Item = &'static str>,
+    ) -> Self {
+        self.atomic_key_groups.push(keys.into_iter().collect());
+        self
+    }
+
+    fn merge_options(
+        &self,
+        mut dynamic_options: HashMap<String, String>,
+    ) -> HashMap<String, String> {
+        if let Some(filter) = self.filter_dynamic_options {
+            dynamic_options = filter(&dynamic_options);
+        }
+        for key in &self.protected_keys {
+            dynamic_options.remove(*key);
+        }
+        let mut merged = self.base_options.as_ref().clone();
+        for group in &self.atomic_key_groups {
+            if group.iter().any(|key| dynamic_options.contains_key(*key)) {
+                for key in group {
+                    merged.remove(*key);
+                }
+            }
+        }
+        merged.extend(dynamic_options);
+        merged
+    }
+
+    pub(in crate::object_store) async fn current_store(&self) -> Result<Arc<OpendalStore>> {
+        let merged_options = self.merge_options(self.accessor.get_storage_options().await?.0);
+        let config = (self.normalize_config)(&merged_options)?;
+
+        // Cache reuse depends on exact normalized config equality. Providers
+        // should return stable, canonicalized values for semantically identical
+        // configurations to avoid unnecessary store rebuilds.
+        {
+            let cache = self.cache.read().await;
+            if let Some(cached) = cache.as_ref()
+                && cached.config == config
+            {
+                return Ok(cached.store.clone());
+            }
+        }
+
+        let store = Arc::new((self.build_store)(config.clone())?);
+        let mut cache = self.cache.write().await;
+        if let Some(cached) = cache.as_ref()
+            && cached.config == config
+        {
+            return Ok(cached.store.clone());
+        }
+
+        *cache = Some(CachedOpenDalStore {
+            config,
+            store: store.clone(),
+        });
+        Ok(store)
+    }
+
+    fn map_store_error(&self, error: lance_core::Error) -> object_store::Error {
+        object_store::Error::Generic {
+            store: "DynamicOpenDalStore",
+            source: Box::new(error),
+        }
+    }
+}
+
+impl fmt::Display for DynamicOpenDalStore {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "DynamicOpenDalStore({})", self.name)
+    }
+}
+
+#[async_trait::async_trait]
+impl OSObjectStore for DynamicOpenDalStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        payload: PutPayload,
+        opts: PutOptions,
+    ) -> object_store::Result<PutResult> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .put_opts(location, payload, opts)
+            .await
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> object_store::Result<Box<dyn MultipartUpload>> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .put_multipart_opts(location, opts)
+            .await
+    }
+
+    async fn get_opts(
+        &self,
+        location: &Path,
+        options: GetOptions,
+    ) -> object_store::Result<GetResult> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .get_opts(location, options)
+            .await
+    }
+
+    async fn get_ranges(
+        &self,
+        location: &Path,
+        ranges: &[Range<u64>],
+    ) -> object_store::Result<Vec<Bytes>> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .get_ranges(location, ranges)
+            .await
+    }
+
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, object_store::Result<Path>>,
+    ) -> BoxStream<'static, object_store::Result<Path>> {
+        let this = self.clone();
+        stream::once(async move {
+            let store = this
+                .current_store()
+                .await
+                .map_err(|e| this.map_store_error(e))?;
+            Ok::<_, object_store::Error>((store, locations))
+        })
+        .map_ok(|(store, locations)| store.delete_stream(locations))
+        .try_flatten()
+        .boxed()
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result<ObjectMeta>> {
+        let prefix = prefix.cloned();
+        let this = self.clone();
+        stream::once(async move {
+            this.current_store()
+                .await
+                .map_err(|e| this.map_store_error(e))
+        })
+        .map_ok(move |store| store.list(prefix.as_ref()))
+        .try_flatten()
+        .boxed()
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result<ListResult> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .list_with_delimiter(prefix)
+            .await
+    }
+
+    async fn copy_opts(
+        &self,
+        from: &Path,
+        to: &Path,
+        opts: CopyOptions,
+    ) -> object_store::Result<()> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .copy_opts(from, to, opts)
+            .await
+    }
+
+    async fn rename_opts(
+        &self,
+        from: &Path,
+        to: &Path,
+        opts: RenameOptions,
+    ) -> object_store::Result<()> {
+        self.current_store()
+            .await
+            .map_err(|e| self.map_store_error(e))?
+            .rename_opts(from, to, opts)
+            .await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use opendal::{Operator, services::Memory};
+
+    use super::*;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+
+    #[tokio::test]
+    async fn test_dynamic_store_caches_by_normalized_config() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([("token".to_string(), "value".to_string())]),
+            },
+        )));
+
+        let store = DynamicOpenDalStore::new(
+            "memory",
+            HashMap::new(),
+            accessor,
+            |options| Ok(options.clone()),
+            |_| {
+                let operator = Operator::new(Memory::default()).map_err(|e| {
+                    lance_core::Error::invalid_input(format!(
+                        "Failed to create memory operator: {e:?}"
+                    ))
+                })?;
+                Ok(OpendalStore::new(operator))
+            },
+        );
+
+        let first = store
+            .current_store()
+            .await
+            .expect("first store should build");
+        let second = store
+            .current_store()
+            .await
+            .expect("second store should reuse cache");
+
+        assert!(Arc::ptr_eq(&first, &second));
+    }
+
+    #[test]
+    fn test_merge_options_preserves_protected_base_keys() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::new(),
+            },
+        )));
+        let store = DynamicOpenDalStore::new(
+            "memory",
+            HashMap::from([
+                ("bucket".to_string(), "url-bucket".to_string()),
+                ("root".to_string(), "/".to_string()),
+                ("token".to_string(), "base-token".to_string()),
+            ]),
+            accessor,
+            |options| Ok(options.clone()),
+            |_| {
+                let operator = Operator::new(Memory::default()).map_err(|e| {
+                    lance_core::Error::invalid_input(format!(
+                        "Failed to create memory operator: {e:?}"
+                    ))
+                })?;
+                Ok(OpendalStore::new(operator))
+            },
+        )
+        .with_protected_keys(["bucket", "root"]);
+
+        let merged = store.merge_options(HashMap::from([
+            ("bucket".to_string(), "provider-bucket".to_string()),
+            ("root".to_string(), "/provider-root".to_string()),
+            ("token".to_string(), "provider-token".to_string()),
+        ]));
+
+        assert_eq!(merged.get("bucket").unwrap(), "url-bucket");
+        assert_eq!(merged.get("root").unwrap(), "/");
+        assert_eq!(merged.get("token").unwrap(), "provider-token");
+    }
+}

vendor/lance-io/src/object_store/list_retry.rs

@@ -0,0 +1,403 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{future::Future, pin::Pin, sync::Arc, task::Poll, time::Duration};
+
+use futures::stream::BoxStream;
+use futures::{Stream, StreamExt};
+use object_store::{ObjectMeta, ObjectStore, path::Path};
+use rand::Rng;
+use tokio::time::Sleep;
+
+const DEFAULT_BASE_RETRY_DELAY: Duration = Duration::from_millis(100);
+const DEFAULT_MAX_RETRY_DELAY: Duration = Duration::from_secs(5);
+
+/// A stream that does outer retries on list operations.
+///
+/// This is to handle request responses that ObjectStore doesn't handle, such as
+/// the error `error decoding response body` from queries to GCS.
+pub struct ListRetryStream {
+    object_store: Arc<dyn ObjectStore>,
+    current_stream: BoxStream<'static, object_store::Result<ObjectMeta>>,
+    prefix: Option<Path>,
+    last_successful_key: Option<Path>,
+    max_retries: usize,
+    current_retries: usize,
+    retry_sleep: Option<Pin<Box<Sleep>>>,
+    base_retry_delay: Duration,
+    max_retry_delay: Duration,
+}
+
+impl ListRetryStream {
+    pub fn new(
+        object_store: Arc<dyn ObjectStore>,
+        prefix: Option<Path>,
+        max_retries: usize,
+    ) -> Self {
+        let current_stream = object_store.list(prefix.as_ref());
+        Self {
+            object_store,
+            current_stream,
+            prefix,
+            last_successful_key: None,
+            max_retries,
+            current_retries: 0,
+            retry_sleep: None,
+            base_retry_delay: DEFAULT_BASE_RETRY_DELAY,
+            max_retry_delay: DEFAULT_MAX_RETRY_DELAY,
+        }
+    }
+
+    #[cfg(test)]
+    fn new_with_backoff(
+        object_store: Arc<dyn ObjectStore>,
+        prefix: Option<Path>,
+        max_retries: usize,
+        base_retry_delay: Duration,
+        max_retry_delay: Duration,
+    ) -> Self {
+        let current_stream = object_store.list(prefix.as_ref());
+        Self {
+            object_store,
+            current_stream,
+            prefix,
+            last_successful_key: None,
+            max_retries,
+            current_retries: 0,
+            retry_sleep: None,
+            base_retry_delay,
+            max_retry_delay,
+        }
+    }
+
+    fn is_retryable(error: &object_store::Error) -> bool {
+        !matches!(
+            error,
+            object_store::Error::NotFound { .. }
+                | object_store::Error::InvalidPath { .. }
+                | object_store::Error::NotSupported { .. }
+                | object_store::Error::NotImplemented { .. }
+        )
+    }
+
+    fn retry_delay(&self) -> Duration {
+        let exponent = self.current_retries.saturating_sub(1).min(16) as u32;
+        let base_ms = self.base_retry_delay.as_millis().max(1);
+        let max_ms = self.max_retry_delay.as_millis().max(base_ms);
+        let cap_ms = base_ms.saturating_mul(1_u128 << exponent).min(max_ms);
+        let min_ms = (cap_ms / 2).max(1);
+        let delay_ms = if cap_ms > min_ms {
+            rand::rng().random_range(min_ms..=cap_ms)
+        } else {
+            cap_ms
+        };
+        Duration::from_millis(delay_ms.min(u64::MAX as u128) as u64)
+    }
+
+    fn recreate_stream(&mut self) {
+        self.current_stream = if let Some(offset) = self.last_successful_key.clone() {
+            self.object_store
+                .list_with_offset(self.prefix.as_ref(), &offset)
+        } else {
+            self.object_store.list(self.prefix.as_ref())
+        };
+    }
+}
+
+impl Stream for ListRetryStream {
+    type Item = Result<ObjectMeta, object_store::Error>;
+
+    fn poll_next(
+        self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Option<Self::Item>> {
+        let this = self.get_mut();
+        loop {
+            if let Some(sleep) = this.retry_sleep.as_mut() {
+                match sleep.as_mut().poll(cx) {
+                    Poll::Ready(()) => {
+                        this.retry_sleep = None;
+                        this.recreate_stream();
+                    }
+                    Poll::Pending => return Poll::Pending,
+                }
+            }
+
+            match this.current_stream.poll_next_unpin(cx) {
+                Poll::Ready(Some(Ok(meta))) => {
+                    this.last_successful_key = Some(meta.location.clone());
+                    return Poll::Ready(Some(Ok(meta)));
+                }
+                Poll::Ready(None) => {
+                    // If the stream is done, return None
+                    return Poll::Ready(None);
+                }
+                Poll::Ready(Some(Err(error))) if Self::is_retryable(&error) => {
+                    if this.current_retries < this.max_retries {
+                        this.current_retries += 1;
+                        this.retry_sleep = Some(Box::pin(tokio::time::sleep(this.retry_delay())));
+
+                        continue;
+                    } else {
+                        return Poll::Ready(Some(Err(error)));
+                    }
+                }
+                Poll::Ready(Some(Err(error))) => {
+                    return Poll::Ready(Some(Err(error)));
+                }
+                Poll::Pending => {
+                    return Poll::Pending;
+                }
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::collections::VecDeque;
+    use std::fmt::{Debug, Display, Formatter};
+    use std::ops::Range;
+    use std::sync::Mutex;
+    use std::sync::atomic::{AtomicUsize, Ordering};
+    use std::time::Instant;
+
+    use async_trait::async_trait;
+    use bytes::Bytes;
+    use futures::stream;
+    use object_store::memory::InMemory;
+    use object_store::{
+        CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, PutMultipartOptions,
+        PutOptions, PutPayload, PutResult, Result as OSResult,
+    };
+
+    fn assert_send<T: Send>() {}
+
+    #[test]
+    fn test_list_retry_stream_send() {
+        // Ensure that ListRetryStream is Send
+        assert_send::<ListRetryStream>();
+    }
+
+    fn object_meta(path: &str) -> ObjectMeta {
+        ObjectMeta {
+            location: Path::from(path),
+            last_modified: chrono::Utc::now(),
+            size: 1,
+            e_tag: None,
+            version: None,
+        }
+    }
+
+    fn retryable_error() -> object_store::Error {
+        object_store::Error::Generic {
+            store: "scripted",
+            source: "retryable list error".into(),
+        }
+    }
+
+    fn not_found_error() -> object_store::Error {
+        object_store::Error::NotFound {
+            path: "missing".to_string(),
+            source: "missing".into(),
+        }
+    }
+
+    struct ScriptedListStore {
+        inner: InMemory,
+        list_streams: Mutex<VecDeque<Vec<OSResult<ObjectMeta>>>>,
+        offset_streams: Mutex<VecDeque<Vec<OSResult<ObjectMeta>>>>,
+        list_calls: AtomicUsize,
+        offset_calls: AtomicUsize,
+        last_offset: Mutex<Option<Path>>,
+    }
+
+    impl ScriptedListStore {
+        fn new(
+            list_streams: Vec<Vec<OSResult<ObjectMeta>>>,
+            offset_streams: Vec<Vec<OSResult<ObjectMeta>>>,
+        ) -> Self {
+            Self {
+                inner: InMemory::new(),
+                list_streams: Mutex::new(list_streams.into()),
+                offset_streams: Mutex::new(offset_streams.into()),
+                list_calls: AtomicUsize::new(0),
+                offset_calls: AtomicUsize::new(0),
+                last_offset: Mutex::new(None),
+            }
+        }
+
+        fn list_calls(&self) -> usize {
+            self.list_calls.load(Ordering::SeqCst)
+        }
+
+        fn offset_calls(&self) -> usize {
+            self.offset_calls.load(Ordering::SeqCst)
+        }
+
+        fn last_offset(&self) -> Option<Path> {
+            self.last_offset.lock().unwrap().clone()
+        }
+    }
+
+    impl Display for ScriptedListStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            write!(f, "ScriptedListStore")
+        }
+    }
+
+    impl Debug for ScriptedListStore {
+        fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+            f.debug_struct("ScriptedListStore").finish()
+        }
+    }
+
+    #[async_trait]
+    impl ObjectStore for ScriptedListStore {
+        async fn put_opts(
+            &self,
+            location: &Path,
+            bytes: PutPayload,
+            opts: PutOptions,
+        ) -> OSResult<PutResult> {
+            self.inner.put_opts(location, bytes, opts).await
+        }
+
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>> {
+            self.inner.put_multipart_opts(location, opts).await
+        }
+
+        async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+            self.inner.get_opts(location, options).await
+        }
+
+        async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+            self.inner.get_ranges(location, ranges).await
+        }
+
+        fn delete_stream(
+            &self,
+            locations: BoxStream<'static, OSResult<Path>>,
+        ) -> BoxStream<'static, OSResult<Path>> {
+            self.inner.delete_stream(locations)
+        }
+
+        fn list(&self, _prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.list_calls.fetch_add(1, Ordering::SeqCst);
+            let results = self
+                .list_streams
+                .lock()
+                .unwrap()
+                .pop_front()
+                .unwrap_or_default();
+            stream::iter(results).boxed()
+        }
+
+        fn list_with_offset(
+            &self,
+            _prefix: Option<&Path>,
+            offset: &Path,
+        ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+            self.offset_calls.fetch_add(1, Ordering::SeqCst);
+            *self.last_offset.lock().unwrap() = Some(offset.clone());
+            let results = self
+                .offset_streams
+                .lock()
+                .unwrap()
+                .pop_front()
+                .unwrap_or_default();
+            stream::iter(results).boxed()
+        }
+
+        async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+            self.inner.list_with_delimiter(prefix).await
+        }
+
+        async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+            self.inner.copy_opts(from, to, opts).await
+        }
+    }
+
+    #[tokio::test]
+    async fn test_list_retry_stream_retries_after_backoff() {
+        let store = Arc::new(ScriptedListStore::new(
+            vec![
+                vec![Err(retryable_error())],
+                vec![Ok(object_meta("prefix/file"))],
+            ],
+            vec![],
+        ));
+        let stream = ListRetryStream::new_with_backoff(
+            store.clone(),
+            Some(Path::from("prefix")),
+            1,
+            Duration::from_millis(20),
+            Duration::from_millis(20),
+        );
+
+        let start = Instant::now();
+        let items = stream.collect::<Vec<_>>().await;
+
+        assert_eq!(items.len(), 1);
+        assert!(items[0].is_ok());
+        assert_eq!(store.list_calls(), 2);
+        assert!(
+            start.elapsed() >= Duration::from_millis(10),
+            "retry should wait before recreating the list stream"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_list_retry_stream_resumes_after_last_successful_key() {
+        let store = Arc::new(ScriptedListStore::new(
+            vec![vec![Ok(object_meta("prefix/a")), Err(retryable_error())]],
+            vec![vec![Ok(object_meta("prefix/b"))]],
+        ));
+        let stream = ListRetryStream::new_with_backoff(
+            store.clone(),
+            Some(Path::from("prefix")),
+            1,
+            Duration::from_millis(1),
+            Duration::from_millis(1),
+        );
+
+        let items = stream.collect::<Vec<_>>().await;
+
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].as_ref().unwrap().location, Path::from("prefix/a"));
+        assert_eq!(items[1].as_ref().unwrap().location, Path::from("prefix/b"));
+        assert_eq!(store.list_calls(), 1);
+        assert_eq!(store.offset_calls(), 1);
+        assert_eq!(store.last_offset(), Some(Path::from("prefix/a")));
+    }
+
+    #[tokio::test]
+    async fn test_list_retry_stream_non_retryable_errors_return_immediately() {
+        let store = Arc::new(ScriptedListStore::new(
+            vec![vec![Err(not_found_error())]],
+            vec![],
+        ));
+        let stream = ListRetryStream::new_with_backoff(
+            store.clone(),
+            Some(Path::from("prefix")),
+            5,
+            Duration::from_millis(1),
+            Duration::from_millis(1),
+        );
+
+        let items = stream.collect::<Vec<_>>().await;
+
+        assert_eq!(items.len(), 1);
+        assert!(matches!(
+            items.into_iter().next().unwrap(),
+            Err(object_store::Error::NotFound { .. })
+        ));
+        assert_eq!(store.list_calls(), 1);
+        assert_eq!(store.offset_calls(), 0);
+    }
+}

vendor/lance-io/src/object_store/providers.rs

@@ -0,0 +1,520 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{
+    collections::HashMap,
+    sync::{
+        Arc, RwLock, Weak,
+        atomic::{AtomicU64, Ordering},
+    },
+};
+
+use object_store::path::Path;
+use url::Url;
+
+use crate::object_store::WrappingObjectStore;
+use crate::object_store::uri_to_url;
+
+use super::{ObjectStore, ObjectStoreParams, tracing::ObjectStoreTracingExt};
+use lance_core::error::{Error, LanceOptionExt, Result};
+
+#[cfg(feature = "aws")]
+pub mod aws;
+#[cfg(feature = "azure")]
+pub mod azure;
+#[cfg(feature = "gcp")]
+pub mod gcp;
+#[cfg(feature = "goosefs")]
+pub mod goosefs;
+#[cfg(feature = "huggingface")]
+pub mod huggingface;
+pub mod local;
+pub mod memory;
+#[cfg(feature = "oss")]
+pub mod oss;
+pub mod shared_memory;
+#[cfg(feature = "tencent")]
+pub mod tencent;
+#[cfg(feature = "tos")]
+pub mod tos;
+
+#[async_trait::async_trait]
+pub trait ObjectStoreProvider: std::fmt::Debug + Sync + Send {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore>;
+
+    /// Extract the path relative to the base of the store.
+    ///
+    /// For example, in S3 the path is relative to the bucket. So a URL of
+    /// `s3://bucket/path/to/file` would return `path/to/file`.
+    ///
+    /// Meanwhile, for a file store, the path is relative to the filesystem root.
+    /// So a URL of `file:///path/to/file` would return `/path/to/file`.
+    fn extract_path(&self, url: &Url) -> Result<Path> {
+        // url.path() returns a percent-encoded string (per the WHATWG URL spec).
+        // Path::from_url_path decodes it first so the Path internal representation
+        // holds the raw UTF-8 string. This prevents double-encoding when the
+        // object store client later percent-encodes the path for HTTP requests.
+        Path::from_url_path(url.path()).map_err(|e| {
+            Error::invalid_input(format!("Invalid path in URL '{}': {}", url.path(), e))
+        })
+    }
+
+    /// Calculate the unique prefix that should be used for this object store.
+    ///
+    /// For object stores that don't have the concept of buckets, this will just be something like
+    /// 'file' or 'memory'.
+    ///
+    /// In object stores where all bucket names are unique, like s3, this will be
+    /// simply 's3$my_bucket_name' or similar.
+    ///
+    /// In Azure, only the combination of (account name, container name) is unique, so
+    /// this will be something like 'az$account_name@container'
+    ///
+    /// Providers should override this if they have special requirements like Azure's.
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        _storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        Ok(format!("{}${}", url.scheme(), url.authority()))
+    }
+}
+
+/// Statistics for the object store registry cache.
+#[derive(Debug, Clone, Default)]
+pub struct ObjectStoreRegistryStats {
+    /// Number of cache hits (store was already cached and reused).
+    pub hits: u64,
+    /// Number of cache misses (new store had to be created).
+    pub misses: u64,
+    /// Number of currently active object stores in the cache.
+    pub active_stores: usize,
+}
+
+/// A registry of object store providers.
+///
+/// Use [`Self::default()`] to create one with the available default providers.
+/// This includes (depending on features enabled):
+/// - `memory`: An in-memory object store.
+/// - `file`: A local file object store, with optimized code paths.
+/// - `file-object-store`: A local file object store that uses the ObjectStore API,
+///   for all operations. Used for testing with ObjectStore wrappers.
+/// - `file+uring`: A local file object store using io_uring (Linux only).
+/// - `s3`: An S3 object store.
+/// - `s3+ddb`: An S3 object store with DynamoDB for metadata.
+/// - `az`: An Azure Blob Storage object store.
+/// - `gs`: A Google Cloud Storage object store.
+/// - `tos`: A Volcengine TOS object store.
+///
+/// Use [`Self::empty()`] to create an empty registry, with no providers registered.
+///
+/// The registry also caches object stores that are currently in use. It holds
+/// weak references to the object stores, so they are not held onto. If an object
+/// store is no longer in use, it will be removed from the cache on the next
+/// call to either [`Self::active_stores()`] or [`Self::get_store()`].
+#[derive(Debug)]
+pub struct ObjectStoreRegistry {
+    providers: RwLock<HashMap<String, Arc<dyn ObjectStoreProvider>>>,
+    // Cache of object stores currently in use. We use a weak reference so the
+    // cache itself doesn't keep them alive if no object store is actually using
+    // it.
+    active_stores: RwLock<HashMap<(String, ObjectStoreParams), Weak<ObjectStore>>>,
+    // Cache statistics
+    hits: AtomicU64,
+    misses: AtomicU64,
+}
+
+impl ObjectStoreRegistry {
+    /// Create a new registry with no providers registered.
+    ///
+    /// Typically, you want to use [`Self::default()`] instead, so you get the
+    /// default providers.
+    pub fn empty() -> Self {
+        Self {
+            providers: RwLock::new(HashMap::new()),
+            active_stores: RwLock::new(HashMap::new()),
+            hits: AtomicU64::new(0),
+            misses: AtomicU64::new(0),
+        }
+    }
+
+    /// Get the object store provider for a given scheme.
+    pub fn get_provider(&self, scheme: &str) -> Option<Arc<dyn ObjectStoreProvider>> {
+        self.providers
+            .read()
+            .expect("ObjectStoreRegistry lock poisoned")
+            .get(scheme)
+            .cloned()
+    }
+
+    /// Get a list of all active object stores.
+    ///
+    /// Calling this will also clean up any weak references to object stores that
+    /// are no longer valid.
+    pub fn active_stores(&self) -> Vec<Arc<ObjectStore>> {
+        let mut found_inactive = false;
+        let output = self
+            .active_stores
+            .read()
+            .expect("ObjectStoreRegistry lock poisoned")
+            .values()
+            .filter_map(|weak| match weak.upgrade() {
+                Some(store) => Some(store),
+                None => {
+                    found_inactive = true;
+                    None
+                }
+            })
+            .collect();
+
+        if found_inactive {
+            // Clean up the cache by removing any weak references that are no longer valid
+            let mut cache_lock = self
+                .active_stores
+                .write()
+                .expect("ObjectStoreRegistry lock poisoned");
+            cache_lock.retain(|_, weak| weak.upgrade().is_some());
+        }
+        output
+    }
+
+    /// Get cache statistics for monitoring and debugging.
+    ///
+    /// Returns the number of cache hits, misses, and currently active stores.
+    /// This is useful for detecting configuration issues that cause excessive
+    /// cache misses (e.g., storage options that vary per-request).
+    pub fn stats(&self) -> ObjectStoreRegistryStats {
+        let active_stores = self
+            .active_stores
+            .read()
+            .map(|s| s.values().filter(|w| w.strong_count() > 0).count())
+            .unwrap_or(0);
+        ObjectStoreRegistryStats {
+            hits: self.hits.load(Ordering::Relaxed),
+            misses: self.misses.load(Ordering::Relaxed),
+            active_stores,
+        }
+    }
+
+    fn scheme_not_found_error(&self, scheme: &str) -> Error {
+        let mut message = format!("No object store provider found for scheme: '{}'", scheme);
+        if let Ok(providers) = self.providers.read() {
+            let valid_schemes = providers.keys().cloned().collect::<Vec<_>>().join(", ");
+            message.push_str(&format!("\nValid schemes: {}", valid_schemes));
+        }
+        Error::invalid_input(message)
+    }
+
+    /// Get an object store for a given base path and parameters.
+    ///
+    /// If the object store is already in use, it will return a strong reference
+    /// to the object store. If the object store is not in use, it will create a
+    /// new object store and return a strong reference to it.
+    pub async fn get_store(
+        &self,
+        base_path: Url,
+        params: &ObjectStoreParams,
+    ) -> Result<Arc<ObjectStore>> {
+        // Base-scoped storage options (`base_<id>.<key>`) are directives for
+        // other registered base paths; resolve them away before building or
+        // caching a store for this location. Params already resolved for a
+        // base contain no scoped entries, so this is a no-op for them.
+        let params = params.scoped_to_base(None);
+        let params = params.as_ref();
+        let scheme = base_path.scheme();
+        let Some(provider) = self.get_provider(scheme) else {
+            return Err(self.scheme_not_found_error(scheme));
+        };
+
+        let cache_path =
+            provider.calculate_object_store_prefix(&base_path, params.storage_options())?;
+        let cache_key = (cache_path.clone(), params.clone());
+
+        // Check if we have a cached store for this base path and params
+        {
+            let maybe_store = self
+                .active_stores
+                .read()
+                .ok()
+                .expect_ok()?
+                .get(&cache_key)
+                .cloned();
+            if let Some(store) = maybe_store {
+                if let Some(store) = store.upgrade() {
+                    self.hits.fetch_add(1, Ordering::Relaxed);
+                    return Ok(store);
+                } else {
+                    // Remove the weak reference if it is no longer valid
+                    let mut cache_lock = self
+                        .active_stores
+                        .write()
+                        .expect("ObjectStoreRegistry lock poisoned");
+                    if let Some(store) = cache_lock.get(&cache_key)
+                        && store.upgrade().is_none()
+                    {
+                        // Remove the weak reference if it is no longer valid
+                        cache_lock.remove(&cache_key);
+                    }
+                }
+            }
+        }
+
+        self.misses.fetch_add(1, Ordering::Relaxed);
+
+        let mut store = provider.new_store(base_path, params).await?;
+
+        store.inner = store.inner.traced();
+
+        // Label metrics by the store's unique prefix (e.g. `s3$bucket`,
+        // `az$container@account`) so multiple stores on one cloud differ.
+        crate::object_store::meter_store(&mut store.inner, &mut store.io_tracker, &cache_path);
+
+        if let Some(wrapper) = &params.object_store_wrapper {
+            store.inner = wrapper.wrap(&cache_path, store.inner);
+        }
+
+        // Always wrap with IO tracking
+        store.inner = store.io_tracker.wrap("", store.inner);
+
+        let store = Arc::new(store);
+
+        {
+            // Insert the store into the cache
+            let mut cache_lock = self.active_stores.write().ok().expect_ok()?;
+            cache_lock.insert(cache_key, Arc::downgrade(&store));
+        }
+
+        Ok(store)
+    }
+
+    /// Calculate the datastore prefix based on the URI and the storage options.
+    /// The data store prefix should uniquely identify the datastore.
+    pub fn calculate_object_store_prefix(
+        &self,
+        uri: &str,
+        storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        let url = uri_to_url(uri)?;
+        match self.get_provider(url.scheme()) {
+            None => {
+                if url.scheme() == "file" || url.scheme().len() == 1 {
+                    Ok("file".to_string())
+                } else {
+                    Err(self.scheme_not_found_error(url.scheme()))
+                }
+            }
+            Some(provider) => provider.calculate_object_store_prefix(&url, storage_options),
+        }
+    }
+}
+
+impl Default for ObjectStoreRegistry {
+    fn default() -> Self {
+        let mut providers: HashMap<String, Arc<dyn ObjectStoreProvider>> = HashMap::new();
+
+        providers.insert("memory".into(), Arc::new(memory::MemoryStoreProvider));
+        providers.insert(
+            "shared-memory".into(),
+            Arc::new(shared_memory::SharedMemoryStoreProvider::default()),
+        );
+        providers.insert("file".into(), Arc::new(local::FileStoreProvider));
+        // The "file" scheme has special optimized code paths that bypass
+        // the ObjectStore API for better performance. However, this can make it
+        // hard to test when using ObjectStore wrappers, such as IOTrackingStore.
+        // So we provide a "file-object-store" scheme that uses the ObjectStore API.
+        // The specialized code paths are differentiated by the scheme name.
+        providers.insert(
+            "file-object-store".into(),
+            Arc::new(local::FileStoreProvider),
+        );
+        #[cfg(target_os = "linux")]
+        providers.insert("file+uring".into(), Arc::new(local::FileStoreProvider));
+
+        #[cfg(feature = "aws")]
+        {
+            let aws = Arc::new(aws::AwsStoreProvider);
+            providers.insert("s3".into(), aws.clone());
+            providers.insert("s3+ddb".into(), aws);
+        }
+        #[cfg(feature = "azure")]
+        {
+            let azure = Arc::new(azure::AzureBlobStoreProvider);
+            providers.insert("az".into(), azure.clone());
+            providers.insert("abfss".into(), azure);
+        }
+        #[cfg(feature = "gcp")]
+        providers.insert("gs".into(), Arc::new(gcp::GcsStoreProvider));
+        #[cfg(feature = "goosefs")]
+        providers.insert("goosefs".into(), Arc::new(goosefs::GooseFsStoreProvider));
+        #[cfg(feature = "oss")]
+        providers.insert("oss".into(), Arc::new(oss::OssStoreProvider));
+        #[cfg(feature = "tencent")]
+        providers.insert("cos".into(), Arc::new(tencent::TencentStoreProvider));
+        #[cfg(feature = "huggingface")]
+        providers.insert("hf".into(), Arc::new(huggingface::HuggingfaceStoreProvider));
+        #[cfg(feature = "tos")]
+        providers.insert("tos".into(), Arc::new(tos::TosStoreProvider));
+        Self {
+            providers: RwLock::new(providers),
+            active_stores: RwLock::new(HashMap::new()),
+            hits: AtomicU64::new(0),
+            misses: AtomicU64::new(0),
+        }
+    }
+}
+
+impl ObjectStoreRegistry {
+    /// Add a new object store provider to the registry. The provider will be used
+    /// in [`Self::get_store()`] when a URL is passed with a matching scheme.
+    pub fn insert(&self, scheme: &str, provider: Arc<dyn ObjectStoreProvider>) {
+        self.providers
+            .write()
+            .expect("ObjectStoreRegistry lock poisoned")
+            .insert(scheme.into(), provider);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::*;
+
+    #[derive(Debug)]
+    struct DummyProvider;
+
+    #[async_trait::async_trait]
+    impl ObjectStoreProvider for DummyProvider {
+        async fn new_store(
+            &self,
+            _base_path: Url,
+            _params: &ObjectStoreParams,
+        ) -> Result<ObjectStore> {
+            unreachable!("This test doesn't create stores")
+        }
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix() {
+        let provider = DummyProvider;
+        let url = Url::parse("dummy://blah/path").unwrap();
+        assert_eq!(
+            "dummy$blah",
+            provider.calculate_object_store_prefix(&url, None).unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_get_store_resolves_base_scoped_options() {
+        use crate::object_store::StorageOptionsAccessor;
+
+        let registry = ObjectStoreRegistry::default();
+        let url = Url::parse("memory://test").unwrap();
+
+        let with_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("shared".to_string(), "value".to_string()),
+                    ("base_1.account_key".to_string(), "base1-key".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+        let without_scoped = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([("shared".to_string(), "value".to_string())]),
+            ))),
+            ..Default::default()
+        };
+
+        // Base-scoped entries are resolved away before the store is built and
+        // cached, so params with and without them yield the same cached store.
+        let store_scoped = registry.get_store(url.clone(), &with_scoped).await.unwrap();
+        let store_plain = registry.get_store(url, &without_scoped).await.unwrap();
+        assert!(Arc::ptr_eq(&store_scoped, &store_plain));
+    }
+
+    #[test]
+    fn test_calculate_object_store_scheme_not_found() {
+        let registry = ObjectStoreRegistry::empty();
+        registry.insert("dummy", Arc::new(DummyProvider));
+        let s = "Invalid user input: No object store provider found for scheme: 'dummy2'\nValid schemes: dummy";
+        let result = registry
+            .calculate_object_store_prefix("dummy2://mybucket/my/long/path", None)
+            .expect_err("expected error")
+            .to_string();
+        assert_eq!(s, &result[..s.len()]);
+    }
+
+    // Test that paths without a scheme get treated as local paths.
+    #[test]
+    fn test_calculate_object_store_prefix_for_local() {
+        let registry = ObjectStoreRegistry::empty();
+        assert_eq!(
+            "file",
+            registry
+                .calculate_object_store_prefix("/tmp/foobar", None)
+                .unwrap()
+        );
+    }
+
+    // Test that paths with a single-letter scheme that is not registered for anything get treated as local paths.
+    #[test]
+    fn test_calculate_object_store_prefix_for_local_windows_path() {
+        let registry = ObjectStoreRegistry::empty();
+        assert_eq!(
+            "file",
+            registry
+                .calculate_object_store_prefix("c://dos/path", None)
+                .unwrap()
+        );
+    }
+
+    // Test that paths with a given scheme get mapped to that storage provider.
+    #[test]
+    fn test_calculate_object_store_prefix_for_dummy_path() {
+        let registry = ObjectStoreRegistry::empty();
+        registry.insert("dummy", Arc::new(DummyProvider));
+        assert_eq!(
+            "dummy$mybucket",
+            registry
+                .calculate_object_store_prefix("dummy://mybucket/my/long/path", None)
+                .unwrap()
+        );
+    }
+
+    #[tokio::test]
+    async fn test_stats_hit_miss_tracking() {
+        use crate::object_store::StorageOptionsAccessor;
+        let registry = ObjectStoreRegistry::default();
+        let url = Url::parse("memory://test").unwrap();
+
+        let params1 = ObjectStoreParams::default();
+        let params2 = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([("k".into(), "v".into())]),
+            ))),
+            ..Default::default()
+        };
+
+        // (hits, misses, active)
+        let cases: &[(&ObjectStoreParams, (u64, u64, usize))] = &[
+            (&params1, (0, 1, 1)), // miss: new params
+            (&params1, (1, 1, 1)), // hit: same params
+            (&params2, (1, 2, 2)), // miss: different storage_options
+        ];
+
+        let mut stores = vec![]; // retain the stores
+        for (params, (hits, misses, active)) in cases {
+            stores.push(registry.get_store(url.clone(), params).await.unwrap());
+            let s = registry.stats();
+            assert_eq!(
+                (s.hits, s.misses, s.active_stores),
+                (*hits, *misses, *active)
+            );
+        }
+
+        // Same params returns same instance
+        assert!(Arc::ptr_eq(&stores[0], &stores[1]));
+    }
+}

vendor/lance-io/src/object_store/providers/azure.rs

@@ -0,0 +1,674 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{
+    collections::HashMap,
+    str::FromStr,
+    sync::{Arc, LazyLock},
+    time::Duration,
+};
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Azblob, services::Azdls};
+
+use object_store::{
+    RetryConfig,
+    azure::{AzureConfigKey, AzureCredential, MicrosoftAzureBuilder},
+};
+use url::Url;
+
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor,
+    dynamic_credentials::build_dynamic_credential_provider,
+    throttle::{AimdThrottleConfig, AimdThrottleState, AimdThrottledStore, cloud_http_connector},
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct AzureBlobStoreProvider;
+
+impl AzureBlobStoreProvider {
+    /// Normalize Azure storage options for OpenDAL, resolving aliases for
+    /// well-known keys while passing through all other options (e.g.
+    /// `client_id`, `tenant_id`, `encryption_key`, etc.) so that OpenDAL
+    /// can use them directly.
+    fn normalize_opendal_azure_options(
+        options: &HashMap<String, String>,
+    ) -> HashMap<String, String> {
+        // Start with all options so unknown keys are forwarded to OpenDAL.
+        let mut config_map = options.clone();
+
+        // Normalize well-known aliases into canonical OpenDAL key names.
+        // Remove the alias after resolving to avoid duplicate/conflicting entries.
+        let alias_groups: &[(&str, &[&str])] = &[
+            ("account_name", &["azure_storage_account_name"]),
+            ("endpoint", &["azure_storage_endpoint", "azure_endpoint"]),
+            (
+                "account_key",
+                &[
+                    "azure_storage_account_key",
+                    "azure_storage_access_key",
+                    "azure_storage_master_key",
+                    "access_key",
+                    "master_key",
+                ],
+            ),
+            (
+                "sas_token",
+                &[
+                    "azure_storage_sas_token",
+                    "azure_storage_sas_key",
+                    "sas_key",
+                ],
+            ),
+        ];
+
+        for (canonical, aliases) in alias_groups {
+            if !config_map.contains_key(*canonical) {
+                for alias in *aliases {
+                    if let Some(value) = config_map.remove(*alias) {
+                        config_map.insert(canonical.to_string(), value);
+                        break;
+                    }
+                }
+            } else {
+                // Canonical key exists; remove aliases to avoid conflicts.
+                for alias in *aliases {
+                    config_map.remove(*alias);
+                }
+            }
+        }
+
+        config_map
+    }
+
+    fn build_opendal_operator(
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<Operator> {
+        // Start with all storage options as the config map
+        // OpenDAL will handle environment variables through its default credentials chain
+        let mut config_map = Self::normalize_opendal_azure_options(&storage_options.0);
+
+        match base_path.scheme() {
+            "az" => {
+                let container = base_path
+                    .host_str()
+                    .ok_or_else(|| Error::invalid_input("Azure URL must contain container name"))?
+                    .to_string();
+
+                config_map.insert("container".to_string(), container);
+
+                let prefix = base_path.path().trim_start_matches('/');
+                if !prefix.is_empty() {
+                    config_map.insert("root".to_string(), format!("/{}", prefix));
+                }
+
+                Operator::from_iter::<Azblob>(config_map).map_err(|e| {
+                    Error::invalid_input(format!("Failed to create Azure Blob operator: {:?}", e))
+                })
+            }
+            "abfss" => {
+                let filesystem = base_path.username();
+                if filesystem.is_empty() {
+                    return Err(Error::invalid_input(
+                        "abfss:// URL must include account: abfss://<filesystem>@<account>.dfs.core.windows.net/path",
+                    ));
+                }
+                let host = base_path.host_str().ok_or_else(|| {
+                    Error::invalid_input(
+                        "abfss:// URL must include account: abfss://<filesystem>@<account>.dfs.core.windows.net/path"
+                    )
+                })?;
+
+                config_map.insert("filesystem".to_string(), filesystem.to_string());
+                config_map.insert("endpoint".to_string(), format!("https://{}", host));
+                config_map
+                    .entry("account_name".to_string())
+                    .or_insert_with(|| host.split('.').next().unwrap_or(host).to_string());
+
+                let root_path = base_path.path().trim_start_matches('/');
+                if !root_path.is_empty() {
+                    config_map.insert("root".to_string(), format!("/{}", root_path));
+                }
+
+                Operator::from_iter::<Azdls>(config_map).map_err(|e| {
+                    Error::invalid_input(format!(
+                        "Failed to create Azure DFS (ADLS Gen2) operator: {:?}",
+                        e
+                    ))
+                })
+            }
+            _ => Err(Error::invalid_input(format!(
+                "Unsupported Azure scheme: {}",
+                base_path.scheme()
+            ))),
+        }
+    }
+
+    async fn build_opendal_azure_store(
+        &self,
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<Arc<dyn OSObjectStore>> {
+        let operator = Self::build_opendal_operator(base_path, storage_options)?;
+        Ok(Arc::new(OpendalStore::new(operator)))
+    }
+
+    async fn build_microsoft_azure_store(
+        &self,
+        base_path: &Url,
+        storage_options: &StorageOptions,
+        accessor: Option<Arc<StorageOptionsAccessor>>,
+        throttle_state: Option<&AimdThrottleState>,
+    ) -> Result<Arc<dyn OSObjectStore>> {
+        // Use a low retry count since the AIMD throttle layer handles
+        // throttle recovery with its own retry loop.
+        let retry_config = RetryConfig {
+            backoff: Default::default(),
+            max_retries: storage_options.client_max_retries(),
+            retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()),
+        };
+
+        let mut builder = MicrosoftAzureBuilder::new()
+            .with_url(base_path.as_ref())
+            .with_retry(retry_config)
+            .with_client_options(storage_options.client_options()?);
+        for (key, value) in storage_options.as_azure_options() {
+            builder = builder.with_config(key, value);
+        }
+
+        if let Some(credentials) =
+            build_dynamic_credential_provider::<AzureCredential>(accessor).await?
+        {
+            builder = builder.with_credentials(credentials);
+        }
+
+        let store_prefix =
+            self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?;
+        builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix));
+
+        Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
+    }
+
+    fn calculate_object_store_prefix_with_env(
+        url: &Url,
+        storage_options: Option<&HashMap<String, String>>,
+        env_options: &HashMap<String, String>,
+    ) -> Result<String> {
+        let authority = url.authority();
+        let (container, account) = match authority.find("@") {
+            Some(at_index) => {
+                // The URI has an:
+                // - az:// schema type and is similar to 'az://container@account.dfs.core.windows.net/path-part/file
+                //         or possibly 'az://container@account/path-part/file' (the short version).
+                // - abfss:// schema type and is similar to 'abfss://filesystem@account.dfs.core.windows.net/path-part/file'.
+                let container = &authority[..at_index];
+                let account = &authority[at_index + 1..];
+                (
+                    container,
+                    account.split(".").next().unwrap_or_default().to_string(),
+                )
+            }
+            None => {
+                // The URI looks like 'az://container/path-part/file'.
+                // We must look at the storage options to find the account.
+                let mut account = match storage_options {
+                    Some(opts) => StorageOptions::find_configured_storage_account(opts),
+                    None => None,
+                };
+                if account.is_none() {
+                    account = StorageOptions::find_configured_storage_account(env_options);
+                }
+                let account = account.ok_or(Error::invalid_input("Unable to find object store prefix: no Azure account name in URI, and no storage account configured."))?;
+                (authority, account)
+            }
+        };
+        Ok(format!("{}${}@{}", url.scheme(), container, account))
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for AzureBlobStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let scheme = base_path.scheme().to_string();
+        if scheme != "az" && scheme != "abfss" {
+            return Err(Error::invalid_input(format!(
+                "Unsupported Azure scheme '{}', expected 'az' or 'abfss'",
+                scheme
+            )));
+        }
+
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let mut storage_options =
+            StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
+        storage_options.with_env_azure();
+        let download_retry_count = storage_options.download_retry_count();
+
+        let use_opendal = storage_options
+            .0
+            .get("use_opendal")
+            .map(|v| v.as_str() == "true")
+            .unwrap_or(false);
+
+        let accessor = params.get_accessor();
+
+        let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?;
+        let throttle_state = if throttle_config.is_disabled() {
+            None
+        } else {
+            Some(AimdThrottleState::new(throttle_config)?)
+        };
+
+        let inner: Arc<dyn OSObjectStore> = if use_opendal {
+            // OpenDAL Azure intentionally uses static/environment-backed configuration only.
+            // Namespace-vended dynamic credentials are supported on the native object_store path.
+            self.build_opendal_azure_store(&base_path, &storage_options)
+                .await?
+        } else {
+            self.build_microsoft_azure_store(
+                &base_path,
+                &storage_options,
+                accessor,
+                throttle_state.as_ref(),
+            )
+            .await?
+        };
+        let inner = if let Some(throttle_state) = throttle_state {
+            Arc::new(AimdThrottledStore::new_with_state(
+                inner,
+                throttle_state,
+                !use_opendal,
+            )) as Arc<dyn OSObjectStore>
+        } else {
+            inner
+        };
+
+        Ok(ObjectStore {
+            inner,
+            scheme,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: false,
+            list_is_lexically_ordered: true,
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count,
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        Self::calculate_object_store_prefix_with_env(url, storage_options, &ENV_OPTIONS.0)
+    }
+}
+
+static ENV_OPTIONS: LazyLock<StorageOptions> = LazyLock::new(StorageOptions::from_env);
+
+impl StorageOptions {
+    /// Iterate over all environment variables, looking for anything related to Azure.
+    fn from_env() -> Self {
+        let mut opts = HashMap::<String, String>::new();
+        for (os_key, os_value) in std::env::vars_os() {
+            if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str())
+                && let Ok(config_key) = AzureConfigKey::from_str(&key.to_ascii_lowercase())
+            {
+                opts.insert(config_key.as_ref().to_string(), value.to_string());
+            }
+        }
+        Self(opts)
+    }
+
+    /// Add values from the environment to storage options
+    pub fn with_env_azure(&mut self) {
+        for (os_key, os_value) in &ENV_OPTIONS.0 {
+            if !self.0.contains_key(os_key) {
+                self.0.insert(os_key.clone(), os_value.clone());
+            }
+        }
+    }
+
+    /// Subset of options relevant for azure storage
+    pub fn as_azure_options(&self) -> HashMap<AzureConfigKey, String> {
+        self.0
+            .iter()
+            .filter_map(|(key, value)| {
+                let az_key = AzureConfigKey::from_str(&key.to_ascii_lowercase()).ok()?;
+                Some((az_key, value.clone()))
+            })
+            .collect()
+    }
+
+    #[allow(clippy::manual_map)]
+    fn find_configured_storage_account(map: &HashMap<String, String>) -> Option<String> {
+        if let Some(account) = map.get("azure_storage_account_name") {
+            Some(account.clone())
+        } else if let Some(account) = map.get("account_name") {
+            Some(account.clone())
+        } else {
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+    use crate::object_store::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::collections::HashMap;
+
+    #[test]
+    fn test_azure_store_path() {
+        let provider = AzureBlobStoreProvider;
+
+        let url = Url::parse("az://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[tokio::test]
+    async fn test_use_opendal_flag() {
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("az://test-container/path").unwrap();
+        let params_with_flag = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("use_opendal".to_string(), "true".to_string()),
+                    ("account_name".to_string(), "test_account".to_string()),
+                    (
+                        "endpoint".to_string(),
+                        "https://test_account.blob.core.windows.net".to_string(),
+                    ),
+                    (
+                        "account_key".to_string(),
+                        "dGVzdF9hY2NvdW50X2tleQ==".to_string(),
+                    ),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider
+            .new_store(url.clone(), &params_with_flag)
+            .await
+            .unwrap();
+        assert_eq!(store.scheme, "az");
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("Opendal") && inner_desc.contains("azblob"),
+            "az:// with use_opendal=true should use OpenDAL Azblob, got: {}",
+            inner_desc
+        );
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_azure_credentials_provider() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([(
+                    "azure_storage_sas_token".to_string(),
+                    "?sv=2022-11-02&sp=rl&sig=test".to_string(),
+                )]),
+            },
+        )));
+
+        let credentials = build_dynamic_credential_provider::<AzureCredential>(Some(accessor))
+            .await
+            .expect("dynamic azure credentials should build")
+            .expect("expected credential provider")
+            .get_credential()
+            .await
+            .expect("expected azure credential");
+
+        match credentials.as_ref() {
+            AzureCredential::SASToken(pairs) => {
+                assert!(
+                    pairs
+                        .iter()
+                        .any(|(key, value)| key == "sig" && value == "test")
+                );
+            }
+            other => panic!("expected SAS token, got {other:?}"),
+        }
+    }
+
+    #[test]
+    fn test_find_configured_storage_account() {
+        assert_eq!(
+            Some("myaccount".to_string()),
+            StorageOptions::find_configured_storage_account(&HashMap::from_iter(
+                [
+                    ("access_key".to_string(), "myaccesskey".to_string()),
+                    (
+                        "azure_storage_account_name".to_string(),
+                        "myaccount".to_string()
+                    )
+                ]
+                .into_iter()
+            ))
+        );
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_from_url_and_options() {
+        let provider = AzureBlobStoreProvider;
+        let options = HashMap::from_iter([("account_name".to_string(), "bob".to_string())]);
+        assert_eq!(
+            "az$container@bob",
+            provider
+                .calculate_object_store_prefix(
+                    &Url::parse("az://container/path").unwrap(),
+                    Some(&options)
+                )
+                .unwrap()
+        );
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_from_url_and_ignored_options() {
+        let provider = AzureBlobStoreProvider;
+        let options = HashMap::from_iter([("account_name".to_string(), "bob".to_string())]);
+        assert_eq!(
+            "az$container@account",
+            provider
+                .calculate_object_store_prefix(
+                    &Url::parse("az://container@account.dfs.core.windows.net/path").unwrap(),
+                    Some(&options)
+                )
+                .unwrap()
+        );
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_from_url_short_account() {
+        let provider = AzureBlobStoreProvider;
+        let options = HashMap::from_iter([("account_name".to_string(), "bob".to_string())]);
+        assert_eq!(
+            "az$container@account",
+            provider
+                .calculate_object_store_prefix(
+                    &Url::parse("az://container@account/path").unwrap(),
+                    Some(&options)
+                )
+                .unwrap()
+        );
+    }
+
+    #[test]
+    fn test_fail_to_calculate_object_store_prefix_from_url() {
+        let options = HashMap::from_iter([("access_key".to_string(), "myaccesskey".to_string())]);
+        let expected = "Invalid user input: Unable to find object store prefix: no Azure account name in URI, and no storage account configured.";
+        let result = AzureBlobStoreProvider::calculate_object_store_prefix_with_env(
+            &Url::parse("az://container/path").unwrap(),
+            Some(&options),
+            &HashMap::new(),
+        )
+        .expect_err("expected error")
+        .to_string();
+        assert_eq!(expected, &result[..expected.len()]);
+    }
+
+    // --- abfss:// tests ---
+
+    #[test]
+    fn test_abfss_extract_path() {
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path/to/dataset.lance")
+            .unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(
+            path,
+            object_store::path::Path::from("path/to/dataset.lance")
+        );
+    }
+
+    #[test]
+    fn test_calculate_abfss_prefix() {
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path/to/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "abfss$myfs@myaccount");
+    }
+
+    #[test]
+    fn test_calculate_abfss_prefix_ignores_storage_options() {
+        let provider = AzureBlobStoreProvider;
+        let options =
+            HashMap::from_iter([("account_name".to_string(), "other_account".to_string())]);
+        let url = Url::parse("abfss://myfs@myaccount.dfs.core.windows.net/path").unwrap();
+        let prefix = provider
+            .calculate_object_store_prefix(&url, Some(&options))
+            .unwrap();
+        assert_eq!(prefix, "abfss$myfs@myaccount");
+    }
+
+    #[tokio::test]
+    async fn test_abfss_default_uses_microsoft_builder() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), "testaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider.new_store(url, &params).await.unwrap();
+        assert_eq!(store.scheme, "abfss");
+        assert!(!store.is_local());
+        assert!(store.is_cloud());
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("MicrosoftAzure"),
+            "abfss:// without use_opendal should use MicrosoftAzureBuilder, got: {}",
+            inner_desc
+        );
+    }
+
+    #[tokio::test]
+    async fn test_unsupported_scheme_rejected() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("wasbs://container@myaccount.blob.core.windows.net/path").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("account_name".to_string(), "myaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let err = provider
+            .new_store(url, &params)
+            .await
+            .expect_err("expected error for unsupported scheme");
+        assert!(
+            err.to_string().contains("Unsupported Azure scheme"),
+            "unexpected error: {}",
+            err
+        );
+    }
+
+    #[tokio::test]
+    async fn test_abfss_with_opendal_uses_azdls() {
+        use crate::object_store::StorageOptionsAccessor;
+        let provider = AzureBlobStoreProvider;
+        let url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("use_opendal".to_string(), "true".to_string()),
+                    ("account_name".to_string(), "testaccount".to_string()),
+                    ("account_key".to_string(), "dGVzdA==".to_string()),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider.new_store(url, &params).await.unwrap();
+        assert_eq!(store.scheme, "abfss");
+        assert!(!store.is_local());
+        assert!(store.is_cloud());
+        let inner_desc = store.inner.to_string();
+        assert!(
+            inner_desc.contains("Opendal") && inner_desc.contains("azdls"),
+            "abfss:// with use_opendal=true should use OpenDAL Azdls, got: {}",
+            inner_desc
+        );
+    }
+
+    #[test]
+    fn test_azdls_capabilities_differ_from_azblob() {
+        let common_opts = StorageOptions(HashMap::from([
+            ("account_name".to_string(), "testaccount".to_string()),
+            ("account_key".to_string(), "dGVzdA==".to_string()),
+            (
+                "endpoint".to_string(),
+                "https://testaccount.blob.core.windows.net".to_string(),
+            ),
+        ]));
+
+        // Build az:// operator (uses Azblob backend)
+        let az_url = Url::parse("az://test-container/path").unwrap();
+        let az_operator =
+            AzureBlobStoreProvider::build_opendal_operator(&az_url, &common_opts).unwrap();
+
+        // Build abfss:// operator (uses Azdls backend)
+        let abfss_url = Url::parse("abfss://testfs@testaccount.dfs.core.windows.net/data").unwrap();
+        let abfss_operator =
+            AzureBlobStoreProvider::build_opendal_operator(&abfss_url, &common_opts).unwrap();
+
+        let azblob_cap = az_operator.info().capability();
+        let azdls_cap = abfss_operator.info().capability();
+
+        // Both support basic operations
+        assert!(azblob_cap.read);
+        assert!(azdls_cap.read);
+        assert!(azblob_cap.write);
+        assert!(azdls_cap.write);
+        assert!(azblob_cap.list);
+        assert!(azdls_cap.list);
+
+        // Azdls supports rename and create_dir (HNS features); Azblob does not
+        assert!(azdls_cap.rename, "Azdls should support rename");
+        assert!(azdls_cap.create_dir, "Azdls should support create_dir");
+        assert!(!azblob_cap.rename, "Azblob should not support rename");
+    }
+}

vendor/lance-io/src/object_store/providers/gcp.rs

@@ -0,0 +1,262 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{collections::HashMap, str::FromStr, sync::Arc, time::Duration};
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Gcs};
+
+use object_store::{
+    RetryConfig, StaticCredentialProvider,
+    gcp::{GcpCredential, GoogleCloudStorageBuilder, GoogleConfigKey},
+};
+use url::Url;
+
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions, StorageOptionsAccessor,
+    dynamic_credentials::build_dynamic_credential_provider,
+    throttle::{AimdThrottleConfig, AimdThrottleState, AimdThrottledStore, cloud_http_connector},
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct GcsStoreProvider;
+
+impl GcsStoreProvider {
+    async fn build_opendal_gcs_store(
+        &self,
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<Arc<dyn OSObjectStore>> {
+        let bucket = base_path
+            .host_str()
+            .ok_or_else(|| Error::invalid_input("GCS URL must contain bucket name"))?
+            .to_string();
+
+        let prefix = base_path.path().trim_start_matches('/').to_string();
+
+        // Start with all storage options as the config map
+        // OpenDAL will handle environment variables through its default credentials chain
+        let mut config_map: HashMap<String, String> = storage_options.0.clone();
+
+        // Set required OpenDAL configuration
+        config_map.insert("bucket".to_string(), bucket);
+
+        if !prefix.is_empty() {
+            config_map.insert("root".to_string(), format!("/{}", prefix));
+        }
+
+        let operator = Operator::from_iter::<Gcs>(config_map)
+            .map_err(|e| Error::invalid_input(format!("Failed to create GCS operator: {:?}", e)))?;
+
+        Ok(Arc::new(OpendalStore::new(operator)) as Arc<dyn OSObjectStore>)
+    }
+
+    async fn build_google_cloud_store(
+        &self,
+        base_path: &Url,
+        storage_options: &StorageOptions,
+        accessor: Option<Arc<StorageOptionsAccessor>>,
+        throttle_state: Option<&AimdThrottleState>,
+    ) -> Result<Arc<dyn OSObjectStore>> {
+        // Use a low retry count since the AIMD throttle layer handles
+        // throttle recovery with its own retry loop.
+        let retry_config = RetryConfig {
+            backoff: Default::default(),
+            max_retries: storage_options.client_max_retries(),
+            retry_timeout: Duration::from_secs(storage_options.client_retry_timeout()),
+        };
+
+        let mut builder = GoogleCloudStorageBuilder::new()
+            .with_url(base_path.as_ref())
+            .with_retry(retry_config)
+            .with_client_options(storage_options.client_options()?);
+        for (key, value) in storage_options.as_gcs_options() {
+            builder = builder.with_config(key, value);
+        }
+
+        if let Some(credentials) =
+            build_dynamic_credential_provider::<GcpCredential>(accessor).await?
+        {
+            builder = builder.with_credentials(credentials);
+        } else if let Some(storage_token) = storage_options.get("google_storage_token") {
+            let credential = GcpCredential {
+                bearer: storage_token.clone(),
+            };
+            let credential_provider = Arc::new(StaticCredentialProvider::new(credential)) as _;
+            builder = builder.with_credentials(credential_provider);
+        }
+
+        let store_prefix =
+            self.calculate_object_store_prefix(base_path, Some(&storage_options.0))?;
+        builder = builder.with_http_connector(cloud_http_connector(throttle_state, store_prefix));
+
+        Ok(Arc::new(builder.build()?) as Arc<dyn OSObjectStore>)
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for GcsStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let mut storage_options =
+            StorageOptions::new(params.storage_options().cloned().unwrap_or_default());
+        storage_options.with_env_gcs();
+        let download_retry_count = storage_options.download_retry_count();
+
+        let use_opendal = storage_options
+            .0
+            .get("use_opendal")
+            .map(|v| v.as_str() == "true")
+            .unwrap_or(false);
+
+        let accessor = params.get_accessor();
+
+        let throttle_config = AimdThrottleConfig::from_storage_options(params.storage_options())?;
+        let throttle_state = if throttle_config.is_disabled() {
+            None
+        } else {
+            Some(AimdThrottleState::new(throttle_config)?)
+        };
+
+        let inner = if use_opendal {
+            // OpenDAL GCS intentionally uses static/environment-backed configuration only.
+            // Namespace-vended dynamic credentials are supported on the native object_store path.
+            self.build_opendal_gcs_store(&base_path, &storage_options)
+                .await?
+        } else {
+            self.build_google_cloud_store(
+                &base_path,
+                &storage_options,
+                accessor,
+                throttle_state.as_ref(),
+            )
+            .await?
+        };
+        let inner = if let Some(throttle_state) = throttle_state {
+            Arc::new(AimdThrottledStore::new_with_state(
+                inner,
+                throttle_state,
+                !use_opendal,
+            )) as Arc<dyn OSObjectStore>
+        } else {
+            inner
+        };
+
+        Ok(ObjectStore {
+            inner,
+            scheme: String::from("gs"),
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: false,
+            list_is_lexically_ordered: true,
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count,
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+}
+
+impl StorageOptions {
+    /// Add values from the environment to storage options
+    pub fn with_env_gcs(&mut self) {
+        for (os_key, os_value) in std::env::vars_os() {
+            if let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) {
+                let lowercase_key = key.to_ascii_lowercase();
+                let token_key = "google_storage_token";
+
+                if let Ok(config_key) = GoogleConfigKey::from_str(&lowercase_key) {
+                    if !self.0.contains_key(config_key.as_ref()) {
+                        self.0
+                            .insert(config_key.as_ref().to_string(), value.to_string());
+                    }
+                }
+                // Check for GOOGLE_STORAGE_TOKEN until GoogleConfigKey supports storage token
+                else if lowercase_key == token_key && !self.0.contains_key(token_key) {
+                    self.0.insert(token_key.to_string(), value.to_string());
+                }
+            }
+        }
+    }
+
+    /// Subset of options relevant for gcs storage
+    pub fn as_gcs_options(&self) -> HashMap<GoogleConfigKey, String> {
+        self.0
+            .iter()
+            .filter_map(|(key, value)| {
+                let gcs_key = GoogleConfigKey::from_str(&key.to_ascii_lowercase()).ok()?;
+                Some((gcs_key, value.clone()))
+            })
+            .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+    use crate::object_store::{ObjectStoreParams, StorageOptionsAccessor};
+    use std::collections::HashMap;
+
+    #[test]
+    fn test_gcs_store_path() {
+        let provider = GcsStoreProvider;
+
+        let url = Url::parse("gs://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[tokio::test]
+    async fn test_use_opendal_flag() {
+        let provider = GcsStoreProvider;
+        let url = Url::parse("gs://test-bucket/path").unwrap();
+        let params_with_flag = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([
+                    ("use_opendal".to_string(), "true".to_string()),
+                    (
+                        "service_account".to_string(),
+                        "test@example.iam.gserviceaccount.com".to_string(),
+                    ),
+                ]),
+            ))),
+            ..Default::default()
+        };
+
+        let store = provider
+            .new_store(url.clone(), &params_with_flag)
+            .await
+            .unwrap();
+        assert_eq!(store.scheme, "gs");
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_gcp_credentials_provider() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([(
+                    "google_storage_token".to_string(),
+                    "gcp-token".to_string(),
+                )]),
+            },
+        )));
+
+        let credentials = build_dynamic_credential_provider::<GcpCredential>(Some(accessor))
+            .await
+            .expect("dynamic gcp credentials should build")
+            .expect("expected credential provider")
+            .get_credential()
+            .await
+            .expect("expected gcp credential");
+
+        assert_eq!(credentials.bearer, "gcp-token");
+    }
+}

vendor/lance-io/src/object_store/providers/goosefs.rs

@@ -0,0 +1,379 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::GooseFs};
+use url::Url;
+
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+/// Default GooseFS Master gRPC port.
+const DEFAULT_GOOSEFS_PORT: u16 = 9200;
+
+/// GooseFS object store provider.
+///
+/// Uses OpenDAL's GooseFs service to access GooseFS via gRPC.
+/// URL format: `goosefs://host:port/path`
+///
+/// Where:
+/// - `host:port` is the GooseFS Master address (default port: 9200)
+/// - `/path` is the filesystem path within GooseFS
+///
+/// Path handling model (S3-style):
+/// - The OpenDAL `root` is fixed to `/` (or a user-supplied cluster-wide base)
+///   so that a single `Operator` can serve every dataset under the same
+///   master. This keeps the `ObjectStoreRegistry` cache correct: two URLs
+///   like `goosefs://host:9200/a.lance` and `goosefs://host:9200/b.lance`
+///   share one store and each request carries its own object key.
+/// - Path extraction relies on the default [`ObjectStoreProvider::extract_path`]
+///   implementation, which returns the URL path (percent-decoded) as the key
+///   passed to `ObjectStore::get`, `put`, etc. — mirroring how `s3://bucket/k`
+///   yields key `k`.
+///
+/// Supported configuration keys (via `storage_options` or environment variables,
+/// resolved with priority: `storage_options` > env var > URL authority > default):
+///
+/// | storage_options key       | env var                 | purpose                                                                                       |
+/// |---------------------------|-------------------------|-----------------------------------------------------------------------------------------------|
+/// | `goosefs_master_addr`     | `GOOSEFS_MASTER_ADDR`   | Master gRPC address, e.g. `host:9200`. Supports HA: `addr1:port,addr2:port`.                  |
+/// | `goosefs_root`            | `GOOSEFS_ROOT`          | Cluster-wide OpenDAL root shared by all datasets under the same master. Defaults to `/`.      |
+/// | `goosefs_write_type`      | `GOOSEFS_WRITE_TYPE`    | GooseFS write type (e.g. `MUST_CACHE`, `CACHE_THROUGH`, `THROUGH`, `ASYNC_THROUGH`).          |
+/// | `goosefs_block_size`      | `GOOSEFS_BLOCK_SIZE`    | GooseFS block size (bytes). Distinct from Lance's own `block_size`.                           |
+/// | `goosefs_chunk_size`      | `GOOSEFS_CHUNK_SIZE`    | GooseFS chunk size (bytes) used by the client.                                                |
+/// | `goosefs_auth_type`       | `GOOSEFS_AUTH_TYPE`     | Authentication mode: `nosasl` or `simple`.                                                    |
+/// | `goosefs_auth_username`   | `GOOSEFS_AUTH_USERNAME` | Username for `simple` auth mode.                                                              |
+///
+/// Note on `goosefs_root`: it is deliberately cluster-wide (not per-URL) so
+/// that many datasets under the same master share a single cached `Operator`.
+/// A custom root also participates in the `ObjectStoreRegistry` cache prefix,
+/// so stores rooted at different subtrees do not collide.
+#[derive(Default, Debug)]
+pub struct GooseFsStoreProvider;
+
+impl GooseFsStoreProvider {
+    /// Resolve the GooseFS Master address from storage_options, environment, or URL.
+    ///
+    /// Priority:
+    /// 1. `storage_options["goosefs_master_addr"]` (supports HA: "addr1:port,addr2:port")
+    /// 2. `GOOSEFS_MASTER_ADDR` environment variable
+    /// 3. URL authority (host:port from the URL)
+    fn resolve_master_addr(url: &Url, storage_options: &StorageOptions) -> Result<String> {
+        // 1. storage_options
+        if let Some(addr) = storage_options
+            .0
+            .get("goosefs_master_addr")
+            .filter(|v| !v.is_empty())
+        {
+            return Ok(addr.clone());
+        }
+
+        // 2. Environment variable
+        if let Ok(addr) = std::env::var("GOOSEFS_MASTER_ADDR")
+            && !addr.is_empty()
+        {
+            return Ok(addr);
+        }
+
+        // 3. URL authority
+        let host = url.host_str().ok_or_else(|| {
+            Error::invalid_input(
+                "GooseFS URL must contain a master address (host), e.g. goosefs://host:port/path",
+            )
+        })?;
+
+        let port = url.port().unwrap_or(DEFAULT_GOOSEFS_PORT);
+        Ok(format!("{}:{}", host, port))
+    }
+
+    /// Resolve a storage option from storage_options or environment variable.
+    fn resolve_option(
+        storage_options: &StorageOptions,
+        option_key: &str,
+        env_key: &str,
+    ) -> Option<String> {
+        storage_options
+            .0
+            .get(option_key)
+            .cloned()
+            .or_else(|| std::env::var(env_key).ok())
+            .filter(|v| !v.is_empty())
+    }
+
+    /// Resolve the OpenDAL `root` for this Operator. See the file-level docs on
+    /// [`GooseFsStoreProvider`] for the semantics of `goosefs_root`.
+    fn resolve_root(storage_options: &StorageOptions) -> String {
+        Self::resolve_option(storage_options, "goosefs_root", "GOOSEFS_ROOT")
+            .unwrap_or_else(|| "/".to_string())
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for GooseFsStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        // Resolve master address
+        let master_addr = Self::resolve_master_addr(&base_path, &storage_options)?;
+
+        // Resolve a stable cluster-wide root. The URL path is *not* used here
+        // because it varies per dataset; per-request keys are supplied by
+        // `extract_path` instead.
+        let root = Self::resolve_root(&storage_options);
+
+        // Build OpenDAL config map
+        let mut config_map: HashMap<String, String> = HashMap::new();
+        config_map.insert("master_addr".to_string(), master_addr);
+        config_map.insert("root".to_string(), root);
+
+        // Optional: write_type
+        if let Some(wt) =
+            Self::resolve_option(&storage_options, "goosefs_write_type", "GOOSEFS_WRITE_TYPE")
+        {
+            config_map.insert("write_type".to_string(), wt);
+        }
+
+        // Optional: block_size (for GooseFS, not Lance block_size)
+        if let Some(bs) =
+            Self::resolve_option(&storage_options, "goosefs_block_size", "GOOSEFS_BLOCK_SIZE")
+        {
+            config_map.insert("block_size".to_string(), bs);
+        }
+
+        // Optional: chunk_size
+        if let Some(cs) =
+            Self::resolve_option(&storage_options, "goosefs_chunk_size", "GOOSEFS_CHUNK_SIZE")
+        {
+            config_map.insert("chunk_size".to_string(), cs);
+        }
+
+        // Optional: auth_type (nosasl / simple)
+        if let Some(at) =
+            Self::resolve_option(&storage_options, "goosefs_auth_type", "GOOSEFS_AUTH_TYPE")
+        {
+            config_map.insert("auth_type".to_string(), at);
+        }
+
+        // Optional: auth_username (used in SIMPLE auth mode)
+        if let Some(au) = Self::resolve_option(
+            &storage_options,
+            "goosefs_auth_username",
+            "GOOSEFS_AUTH_USERNAME",
+        ) {
+            config_map.insert("auth_username".to_string(), au);
+        }
+
+        // Create OpenDAL Operator with GooseFS service
+        let operator = Operator::from_iter::<GooseFs>(config_map).map_err(|e| {
+            Error::invalid_input(format!("Failed to create GooseFS operator: {:?}", e))
+        })?;
+
+        // Wrap as object_store::ObjectStore via OpendalStore bridge
+        let opendal_store = Arc::new(OpendalStore::new(operator));
+
+        Ok(ObjectStore {
+            scheme: "goosefs".to_string(),
+            inner: opendal_store,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(false),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    // `extract_path` uses the default `ObjectStoreProvider` trait implementation:
+    // it percent-decodes the URL path and returns it as the object key, exactly
+    // like S3 does for `s3://bucket/key`. Overriding it here would only
+    // duplicate that behavior. See the file-level doc comment above for the
+    // full path-handling model.
+
+    /// Calculate the object store prefix used as the registry cache key.
+    ///
+    /// Format: `goosefs$host:port`. Because the OpenDAL root is now cluster-
+    /// wide (not per-URL), all datasets under the same master intentionally
+    /// share the same cached [`ObjectStore`]; the URL path is disambiguated
+    /// by [`Self::extract_path`] on each request. This is analogous to how
+    /// two `s3://bucket/a` and `s3://bucket/b` URLs share one store.
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        // If a custom `goosefs_root` is provided, include it in the prefix so
+        // that stores built with different roots don't accidentally collide.
+        let opts = StorageOptions(storage_options.cloned().unwrap_or_default());
+        let root = Self::resolve_root(&opts);
+        if root == "/" {
+            Ok(format!("{}${}", url.scheme(), url.authority()))
+        } else {
+            Ok(format!("{}${}#{}", url.scheme(), url.authority(), root))
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_goosefs_extract_path_basic() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/data/embeddings.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "data/embeddings.lance");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_deep() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://master:9200/a/b/c/d.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "a/b/c/d.lance");
+    }
+
+    #[test]
+    fn test_goosefs_extract_path_percent_decoded() {
+        // The URL contains a percent-encoded space; extract_path must decode
+        // it once so the ObjectStore layer does not double-encode later.
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://master:9200/dir/with%20space/f.lance").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "dir/with space/f.lance");
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_default_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "goosefs$10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_with_hostname() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://myhost:9200/data").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "goosefs$myhost:9200");
+    }
+
+    /// Regression test: two URLs pointing at different datasets under the
+    /// same master must produce the *same* cache prefix so they share one
+    /// Operator, and correctness must come from `extract_path` returning
+    /// distinct keys — never from a per-URL root baked into the prefix.
+    #[test]
+    fn test_prefix_shared_across_datasets_same_master() {
+        let provider = GooseFsStoreProvider;
+        let url_a = Url::parse("goosefs://10.0.0.1:9200/repro/a.lance").unwrap();
+        let url_b = Url::parse("goosefs://10.0.0.1:9200/repro/b.lance").unwrap();
+
+        let pa = provider
+            .calculate_object_store_prefix(&url_a, None)
+            .unwrap();
+        let pb = provider
+            .calculate_object_store_prefix(&url_b, None)
+            .unwrap();
+        assert_eq!(pa, pb, "same master must share one cache prefix");
+
+        // Extracted keys must differ so the shared Operator can route
+        // requests to the correct dataset.
+        assert_ne!(
+            provider.extract_path(&url_a).unwrap(),
+            provider.extract_path(&url_b).unwrap(),
+            "distinct URLs must yield distinct object keys",
+        );
+    }
+
+    /// Different masters must never share a cache entry.
+    #[test]
+    fn test_prefix_isolated_across_masters() {
+        let provider = GooseFsStoreProvider;
+        let u1 = Url::parse("goosefs://host-a:9200/x.lance").unwrap();
+        let u2 = Url::parse("goosefs://host-b:9200/x.lance").unwrap();
+        assert_ne!(
+            provider.calculate_object_store_prefix(&u1, None).unwrap(),
+            provider.calculate_object_store_prefix(&u2, None).unwrap(),
+        );
+    }
+
+    /// A user-supplied `goosefs_root` participates in the cache prefix so
+    /// stores rooted at different subtrees don't collide.
+    #[test]
+    fn test_prefix_includes_custom_root() {
+        let provider = GooseFsStoreProvider;
+        let url = Url::parse("goosefs://host:9200/x.lance").unwrap();
+
+        let default_prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        let custom_opts: HashMap<String, String> =
+            HashMap::from([("goosefs_root".to_string(), "/tenant-a".to_string())]);
+        let custom_prefix = provider
+            .calculate_object_store_prefix(&url, Some(&custom_opts))
+            .unwrap();
+
+        assert_eq!(default_prefix, "goosefs$host:9200");
+        assert_eq!(custom_prefix, "goosefs$host:9200#/tenant-a");
+        assert_ne!(default_prefix, custom_prefix);
+    }
+
+    #[test]
+    fn test_resolve_master_addr_from_url() {
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let storage_options = StorageOptions(HashMap::new());
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_resolve_master_addr_default_port() {
+        let url = Url::parse("goosefs://10.0.0.1/data").unwrap();
+        let storage_options = StorageOptions(HashMap::new());
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.1:9200");
+    }
+
+    #[test]
+    fn test_resolve_master_addr_from_storage_options() {
+        let url = Url::parse("goosefs://10.0.0.1:9200/data").unwrap();
+        let storage_options = StorageOptions(HashMap::from([(
+            "goosefs_master_addr".to_string(),
+            "10.0.0.2:9200,10.0.0.3:9200".to_string(),
+        )]));
+        let addr = GooseFsStoreProvider::resolve_master_addr(&url, &storage_options).unwrap();
+        assert_eq!(addr, "10.0.0.2:9200,10.0.0.3:9200");
+    }
+
+    #[test]
+    fn test_resolve_root_defaults_to_slash() {
+        let opts = StorageOptions(HashMap::new());
+        assert_eq!(GooseFsStoreProvider::resolve_root(&opts), "/");
+    }
+
+    #[test]
+    fn test_resolve_root_from_storage_options() {
+        let opts = StorageOptions(HashMap::from([(
+            "goosefs_root".to_string(),
+            "/tenant-a".to_string(),
+        )]));
+        assert_eq!(GooseFsStoreProvider::resolve_root(&opts), "/tenant-a");
+    }
+}

vendor/lance-io/src/object_store/providers/huggingface.rs

@@ -0,0 +1,440 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store::path::Path;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Huggingface};
+use url::Url;
+
+use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+use crate::object_store::parse_hf_repo_id;
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+/// Hugging Face object store provider backed by OpenDAL.
+#[derive(Default, Debug)]
+pub struct HuggingfaceStoreProvider;
+
+/// Parsed components from a Hugging Face URL.
+#[derive(Debug, PartialEq, Eq)]
+struct ParsedHfUrl {
+    repo_type: String,
+    repo_id: String,
+    relative_path: String,
+}
+
+fn parse_hf_url(url: &Url) -> Result<ParsedHfUrl> {
+    let mut repo_type = url
+        .host_str()
+        .ok_or_else(|| Error::invalid_input("Huggingface URL must contain repo type"))?
+        .to_string();
+    // OpenDAL expects `dataset` instead of `datasets`; keep the workaround here and adapt tests.
+    if repo_type == "datasets" {
+        repo_type = "dataset".to_string();
+    }
+
+    let mut segments = url.path().trim_start_matches('/').split('/');
+    let owner = segments
+        .next()
+        .ok_or_else(|| Error::invalid_input("Huggingface URL must contain owner"))?;
+    let repo_name = segments
+        .next()
+        .ok_or_else(|| Error::invalid_input("Huggingface URL must contain repository name"))?;
+
+    let relative_path = segments.collect::<Vec<_>>().join("/");
+
+    Ok(ParsedHfUrl {
+        repo_type,
+        repo_id: format!("{owner}/{repo_name}"),
+        relative_path,
+    })
+}
+
+fn build_hf_base_options(
+    repo_type: &str,
+    repo_id: &str,
+    storage_options: &StorageOptions,
+) -> HashMap<String, String> {
+    let mut options = storage_options.0.clone();
+    options.insert("repo_type".to_string(), repo_type.to_string());
+    options.insert("repo_id".to_string(), repo_id.to_string());
+    options
+}
+
+fn normalize_download_mode(download_mode: String) -> Result<String> {
+    match download_mode.to_lowercase().as_str() {
+        "xet" => Ok("xet".to_string()),
+        "http" => Ok("http".to_string()),
+        _ => Err(Error::invalid_input(format!(
+            "Invalid Huggingface download_mode: {download_mode}. Expected one of: xet, http"
+        ))),
+    }
+}
+
+fn normalize_hf_config(options: &HashMap<String, String>) -> Result<HashMap<String, String>> {
+    let mut config_map = HashMap::new();
+
+    let repo_type = options
+        .get("repo_type")
+        .cloned()
+        .ok_or_else(|| Error::invalid_input("Huggingface repo_type is required"))?;
+    let repo_id = options
+        .get("repo_id")
+        .cloned()
+        .ok_or_else(|| Error::invalid_input("Huggingface repo_id is required"))?;
+
+    config_map.insert("repo_type".to_string(), repo_type);
+    config_map.insert("repo_id".to_string(), repo_id);
+
+    if let Some(revision) = options
+        .get("hf_revision")
+        .cloned()
+        .or_else(|| options.get("revision").cloned())
+    {
+        config_map.insert("revision".to_string(), revision);
+    }
+
+    if let Some(root) = options
+        .get("hf_root")
+        .cloned()
+        .or_else(|| options.get("root").cloned())
+        && !root.is_empty()
+    {
+        config_map.insert("root".to_string(), root);
+    }
+
+    if let Some(token) = options
+        .get("hf_token")
+        .cloned()
+        .or_else(|| options.get("token").cloned())
+        && !token.is_empty()
+    {
+        config_map.insert("token".to_string(), token);
+    }
+
+    let download_mode = options
+        .get("hf_download_mode")
+        .filter(|download_mode| !download_mode.is_empty())
+        .cloned()
+        .or_else(|| {
+            options
+                .get("download_mode")
+                .filter(|download_mode| !download_mode.is_empty())
+                .cloned()
+        })
+        .unwrap_or_else(|| "http".to_string());
+    config_map.insert(
+        "download_mode".to_string(),
+        normalize_download_mode(download_mode)?,
+    );
+
+    Ok(config_map)
+}
+
+fn build_hf_store(config_map: HashMap<String, String>) -> Result<OpendalStore> {
+    let repo_type = config_map
+        .get("repo_type")
+        .ok_or_else(|| Error::invalid_input("Huggingface repo_type is required"))?;
+    let repo_id = config_map
+        .get("repo_id")
+        .ok_or_else(|| Error::invalid_input("Huggingface repo_id is required"))?;
+
+    let mut builder = Huggingface::default().repo_type(repo_type).repo_id(repo_id);
+    if let Some(revision) = config_map.get("revision") {
+        builder = builder.revision(revision);
+    }
+    if let Some(root) = config_map.get("root") {
+        builder = builder.root(root);
+    }
+    if let Some(token) = config_map.get("token") {
+        builder = builder.token(token);
+    }
+    if let Some(download_mode) = config_map.get("download_mode") {
+        builder = builder.download_mode(download_mode);
+    }
+
+    let operator = Operator::new(builder).map_err(|e| {
+        Error::invalid_input(format!("Failed to create Huggingface operator: {:?}", e))
+    })?;
+
+    Ok(OpendalStore::new(operator))
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for HuggingfaceStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let ParsedHfUrl {
+            repo_type, repo_id, ..
+        } = parse_hf_url(&base_path)?;
+
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+        let download_retry_count = storage_options.download_retry_count();
+
+        let mut base_options = build_hf_base_options(&repo_type, &repo_id, &storage_options);
+        if !base_options.contains_key("hf_token") && !base_options.contains_key("token") {
+            if let Ok(token) = std::env::var("HF_TOKEN") {
+                base_options.insert("hf_token".to_string(), token);
+            } else if let Ok(token) = std::env::var("HUGGINGFACE_TOKEN") {
+                base_options.insert("hf_token".to_string(), token);
+            }
+        }
+
+        let accessor = params.get_accessor();
+        let inner: Arc<dyn OSObjectStore> =
+            if let Some(accessor) = accessor.filter(|a| a.has_provider()) {
+                Arc::new(
+                    DynamicOpenDalStore::new(
+                        format!("hf:{}", base_path),
+                        base_options,
+                        accessor,
+                        normalize_hf_config,
+                        build_hf_store,
+                    )
+                    .with_protected_keys(["repo_type", "repo_id"]),
+                )
+            } else {
+                Arc::new(build_hf_store(normalize_hf_config(&base_options)?)?)
+            };
+
+        Ok(ObjectStore {
+            scheme: "hf".to_string(),
+            inner,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(true),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count,
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    fn extract_path(&self, url: &Url) -> Result<Path> {
+        let parsed = parse_hf_url(url)?;
+        Path::from_url_path(&parsed.relative_path).map_err(|_| {
+            Error::invalid_input(format!("Invalid path in Huggingface URL: {}", url.path()))
+        })
+    }
+
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        _storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        let repo_id = parse_hf_repo_id(url)?;
+        Ok(format!("{}${}@{}", url.scheme(), url.authority(), repo_id))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::sync::Arc;
+
+    use crate::object_store::StorageOptionsAccessor;
+    use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+
+    #[test]
+    fn parse_basic_url() {
+        let url = Url::parse("hf://datasets/acme/repo/path/to/table.lance").unwrap();
+        let parsed = parse_hf_url(&url).unwrap();
+        assert_eq!(
+            parsed,
+            ParsedHfUrl {
+                repo_type: "dataset".to_string(),
+                repo_id: "acme/repo".to_string(),
+                relative_path: "path/to/table.lance".to_string(),
+            }
+        );
+    }
+
+    #[test]
+    fn storage_option_revision_takes_precedence() {
+        use crate::object_store::StorageOptionsAccessor;
+        use std::sync::Arc;
+        let url = Url::parse("hf://datasets/acme/repo/data/file").unwrap();
+        let params = ObjectStoreParams {
+            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
+                HashMap::from([(String::from("hf_revision"), String::from("stable"))]),
+            ))),
+            ..Default::default()
+        };
+        // new_store should accept without creating operator; test precedence via builder config
+        let ParsedHfUrl {
+            repo_type, repo_id, ..
+        } = parse_hf_url(&url).unwrap();
+
+        // Build config map the same way new_store would to assert precedence logic.
+        let mut config_map: HashMap<String, String> = HashMap::new();
+        config_map.insert("repo_type".to_string(), repo_type);
+        config_map.insert("repo".to_string(), repo_id);
+        if let Some(rev) = params
+            .storage_options()
+            .unwrap()
+            .get("hf_revision")
+            .cloned()
+        {
+            config_map.insert("revision".to_string(), rev);
+        }
+        assert_eq!(config_map.get("revision").unwrap(), "stable");
+    }
+
+    #[test]
+    fn storage_options_cannot_override_url_repo_identity() {
+        let config = normalize_hf_config(&build_hf_base_options(
+            "dataset",
+            "acme/repo",
+            &crate::object_store::StorageOptions(HashMap::from([
+                ("repo_type".to_string(), "model".to_string()),
+                ("repo_id".to_string(), "other/repo".to_string()),
+                ("hf_revision".to_string(), "stable".to_string()),
+            ])),
+        ))
+        .unwrap();
+
+        assert_eq!(config.get("repo_type").unwrap(), "dataset");
+        assert_eq!(config.get("repo_id").unwrap(), "acme/repo");
+        assert_eq!(config.get("revision").unwrap(), "stable");
+    }
+
+    #[test]
+    fn storage_option_download_mode_takes_hf_prefix_precedence() {
+        let config = normalize_hf_config(&build_hf_base_options(
+            "dataset",
+            "acme/repo",
+            &crate::object_store::StorageOptions(HashMap::from([
+                ("download_mode".to_string(), "xet".to_string()),
+                ("hf_download_mode".to_string(), "http".to_string()),
+            ])),
+        ))
+        .unwrap();
+
+        assert_eq!(config.get("download_mode").unwrap(), "http");
+    }
+
+    #[test]
+    fn storage_option_download_mode_defaults_to_http() {
+        let config = normalize_hf_config(&build_hf_base_options(
+            "dataset",
+            "acme/repo",
+            &crate::object_store::StorageOptions(HashMap::new()),
+        ))
+        .unwrap();
+
+        assert_eq!(config.get("download_mode").unwrap(), "http");
+    }
+
+    #[test]
+    fn storage_option_download_mode_rejects_invalid_value() {
+        let err = normalize_hf_config(&build_hf_base_options(
+            "dataset",
+            "acme/repo",
+            &crate::object_store::StorageOptions(HashMap::from([(
+                "hf_download_mode".to_string(),
+                "invalid".to_string(),
+            )])),
+        ))
+        .unwrap_err();
+
+        assert!(
+            err.to_string().contains("download_mode"),
+            "unexpected error: {}",
+            err
+        );
+    }
+
+    #[test]
+    fn parse_hf_repo_id_with_type_and_owner_repo() {
+        let url = Url::parse("hf://models/owner/repo/path/to/file").unwrap();
+        let repo = crate::object_store::parse_hf_repo_id(&url).unwrap();
+        assert_eq!(repo, "owner/repo");
+    }
+
+    #[test]
+    fn parse_hf_repo_id_legacy_without_type() {
+        let url = Url::parse("hf://owner/repo/path/to/file").unwrap();
+        let repo = crate::object_store::parse_hf_repo_id(&url).unwrap();
+        assert_eq!(repo, "owner/repo");
+    }
+
+    #[test]
+    fn parse_hf_repo_id_strips_revision() {
+        let url = Url::parse("hf://datasets/owner/repo@main/data").unwrap();
+        let repo = crate::object_store::parse_hf_repo_id(&url).unwrap();
+        assert_eq!(repo, "owner/repo");
+    }
+
+    #[test]
+    fn parse_hf_repo_id_missing_segments_errors() {
+        let url = Url::parse("hf://datasets/only-owner").unwrap();
+        let err = crate::object_store::parse_hf_repo_id(&url).unwrap_err();
+        assert!(
+            err.to_string().contains("owner/repo"),
+            "unexpected error: {}",
+            err
+        );
+    }
+
+    #[test]
+    fn extract_path_returns_relative() {
+        let url = Url::parse("hf://datasets/acme/repo/sub/dir/table.lance").unwrap();
+        let provider = HuggingfaceStoreProvider;
+        let path = provider.extract_path(&url).unwrap();
+        assert_eq!(path.to_string(), "sub/dir/table.lance");
+    }
+
+    #[test]
+    fn calculate_prefix_uses_repo_id() {
+        let provider = HuggingfaceStoreProvider;
+        let url = Url::parse("hf://datasets/acme/repo/path").unwrap();
+        let prefix = provider.calculate_object_store_prefix(&url, None).unwrap();
+        assert_eq!(prefix, "hf$datasets@acme/repo");
+    }
+
+    #[test]
+    fn parse_invalid_url_errors() {
+        let url = Url::parse("hf://datasets/only-owner").unwrap();
+        let err = parse_hf_url(&url).unwrap_err();
+        assert!(err.to_string().contains("repository name"));
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_opendal_hf_store_uses_provider_token() {
+        let parsed = parse_hf_url(&Url::parse("hf://datasets/acme/repo/path").unwrap()).unwrap();
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([("hf_token".to_string(), "dynamic-token".to_string())]),
+            },
+        )));
+
+        let store = DynamicOpenDalStore::new(
+            "hf",
+            build_hf_base_options(
+                &parsed.repo_type,
+                &parsed.repo_id,
+                &crate::object_store::StorageOptions(HashMap::new()),
+            ),
+            accessor,
+            normalize_hf_config,
+            build_hf_store,
+        );
+
+        let current_store = store
+            .current_store()
+            .await
+            .expect("dynamic OpenDAL HuggingFace store should build");
+
+        assert!(current_store.to_string().contains("Opendal"));
+    }
+}

vendor/lance-io/src/object_store/providers/local.rs

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::object_store::{
+    DEFAULT_LOCAL_BLOCK_SIZE, DEFAULT_LOCAL_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::Error;
+use lance_core::error::Result;
+use object_store::{local::LocalFileSystem, path::Path};
+use url::Url;
+
+#[derive(Default, Debug)]
+pub struct FileStoreProvider;
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for FileStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_LOCAL_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+        let download_retry_count = storage_options.download_retry_count();
+        Ok(ObjectStore {
+            inner: Arc::new(LocalFileSystem::new()),
+            scheme: base_path.scheme().to_owned(),
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: false,
+            list_is_lexically_ordered: false,
+            io_parallelism: DEFAULT_LOCAL_IO_PARALLELISM,
+            download_retry_count,
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    fn extract_path(&self, url: &Url) -> Result<Path> {
+        if let Ok(file_path) = url.to_file_path()
+            && let Ok(path) = Path::from_absolute_path(&file_path)
+        {
+            return Ok(path);
+        }
+
+        Path::from_url_path(url.path()).map_err(|e| {
+            Error::invalid_input(format!("Failed to parse path '{}': {}", url.path(), e))
+        })
+    }
+
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        _storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        Ok(url.scheme().to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::object_store::uri_to_url;
+
+    use super::*;
+
+    #[test]
+    fn test_file_store_path() {
+        let provider = FileStoreProvider;
+
+        let cases = [
+            ("file:///", ""),
+            ("file:///usr/local/bin", "usr/local/bin"),
+            ("file-object-store:///path/to/file", "path/to/file"),
+            ("file:///path/to/foo/../bar", "path/to/bar"),
+        ];
+
+        for (uri, expected_path) in cases {
+            let url = uri_to_url(uri).unwrap();
+            let path = provider.extract_path(&url).unwrap();
+            assert_eq!(path.as_ref(), expected_path, "uri: '{}'", uri);
+        }
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix() {
+        let provider = FileStoreProvider;
+        assert_eq!(
+            "file",
+            provider
+                .calculate_object_store_prefix(&Url::parse("file:///etc").unwrap(), None)
+                .unwrap()
+        );
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix_for_file_object_store() {
+        let provider = FileStoreProvider;
+        assert_eq!(
+            "file-object-store",
+            provider
+                .calculate_object_store_prefix(
+                    &Url::parse("file-object-store:///etc").unwrap(),
+                    None
+                )
+                .unwrap()
+        );
+    }
+
+    #[test]
+    #[cfg(windows)]
+    fn test_file_store_path_windows() {
+        let provider = FileStoreProvider;
+
+        let cases = [
+            (
+                "C:\\Users\\ADMINI~1\\AppData\\Local\\",
+                "C:/Users/ADMINI~1/AppData/Local",
+            ),
+            (
+                "C:\\Users\\ADMINI~1\\AppData\\Local\\..\\",
+                "C:/Users/ADMINI~1/AppData",
+            ),
+            (
+                "file-object-store:///C:/Users/ADMINI~1/AppData/Local",
+                "C:/Users/ADMINI~1/AppData/Local",
+            ),
+            (
+                "file:///C:/Users/RUNNER~1/AppData/Local/Temp/tmpm49j_w0f",
+                "C:/Users/RUNNER~1/AppData/Local/Temp/tmpm49j_w0f",
+            ),
+        ];
+
+        for (uri, expected_path) in cases {
+            let url = uri_to_url(uri).unwrap();
+            let path = provider.extract_path(&url).unwrap();
+            assert_eq!(path.as_ref(), expected_path);
+        }
+    }
+}

vendor/lance-io/src/object_store/providers/memory.rs

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::object_store::{
+    DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_LOCAL_BLOCK_SIZE, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::Result;
+use object_store::{memory::InMemory, path::Path};
+use url::Url;
+
+/// Provides a fresh in-memory object store for each call to `new_store`.
+#[derive(Default, Debug)]
+pub struct MemoryStoreProvider;
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for MemoryStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_LOCAL_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+        let download_retry_count = storage_options.download_retry_count();
+        Ok(ObjectStore {
+            inner: Arc::new(InMemory::new()),
+            scheme: String::from("memory"),
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: false,
+            list_is_lexically_ordered: true,
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count,
+            io_tracker: Default::default(),
+            store_prefix: self
+                .calculate_object_store_prefix(&base_path, params.storage_options())?,
+        })
+    }
+
+    fn extract_path(&self, url: &Url) -> Result<Path> {
+        let mut output = String::new();
+        if let Some(domain) = url.domain() {
+            output.push_str(domain);
+        }
+        output.push_str(url.path());
+        // The in-memory store uses the Path directly as a key with no HTTP layer,
+        // so there is no re-encoding step and thus no double-encoding to avoid.
+        // Path::from also tolerates the empty segments that local temp paths embed.
+        Ok(Path::from(output))
+    }
+
+    fn calculate_object_store_prefix(
+        &self,
+        _url: &Url,
+        _storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        Ok("memory".to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_memory_store_path() {
+        let provider = MemoryStoreProvider;
+
+        let url = Url::parse("memory://path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[test]
+    fn test_calculate_object_store_prefix() {
+        let provider = MemoryStoreProvider;
+        assert_eq!(
+            "memory",
+            provider
+                .calculate_object_store_prefix(&Url::parse("memory://etc").unwrap(), None)
+                .unwrap()
+        );
+    }
+}

vendor/lance-io/src/object_store/providers/oss.rs

@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Oss};
+use url::Url;
+
+use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct OssStoreProvider;
+
+impl OssStoreProvider {
+    fn base_oss_options(
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<HashMap<String, String>> {
+        let bucket = base_path
+            .host_str()
+            .ok_or_else(|| Error::invalid_input("OSS URL must contain bucket name"))?
+            .to_string();
+
+        let prefix = base_path.path().trim_start_matches('/').to_string();
+
+        // Snapshot env-backed OSS defaults at store construction time. Dynamic provider
+        // options can still override these values during per-request config merging.
+        let mut config_map: HashMap<String, String> = std::env::vars()
+            .filter(|(key, _)| {
+                key.starts_with("OSS_")
+                    || key.starts_with("AWS_")
+                    || key.starts_with("ALIBABA_CLOUD_")
+            })
+            .map(|(key, value)| {
+                let normalized_key = key
+                    .to_lowercase()
+                    .replace("oss_", "")
+                    .replace("aws_", "")
+                    .replace("alibaba_cloud_", "");
+                (normalized_key, value)
+            })
+            .collect();
+
+        config_map.extend(storage_options.0.clone());
+
+        config_map.insert("bucket".to_string(), bucket);
+        if prefix.is_empty() {
+            config_map.remove("root");
+        } else {
+            config_map.insert("root".to_string(), "/".to_string());
+        }
+
+        Ok(config_map)
+    }
+
+    /// Normalize OSS storage options, resolving aliases for well-known keys
+    /// while passing through all other options (e.g. `role_arn`,
+    /// `sts_endpoint`, `allow_anonymous`, etc.) so that OpenDAL can use them.
+    fn normalize_oss_config(options: &HashMap<String, String>) -> Result<HashMap<String, String>> {
+        let mut config_map = options.clone();
+
+        let alias_groups: &[(&str, &[&str])] = &[
+            ("endpoint", &["oss_endpoint"]),
+            ("access_key_id", &["oss_access_key_id"]),
+            ("access_key_secret", &["oss_secret_access_key"]),
+            ("region", &["oss_region"]),
+            ("security_token", &["oss_security_token"]),
+        ];
+
+        for (canonical, aliases) in alias_groups {
+            for alias in *aliases {
+                if let Some(value) = config_map.remove(*alias) {
+                    config_map.insert(canonical.to_string(), value);
+                    break;
+                }
+            }
+        }
+
+        if !config_map.contains_key("endpoint") {
+            return Err(Error::invalid_input(
+                "OSS endpoint is required. Please provide 'oss_endpoint' in storage options or set OSS_ENDPOINT environment variable",
+            ));
+        }
+
+        Ok(config_map)
+    }
+
+    fn build_oss_store(config_map: HashMap<String, String>) -> Result<OpendalStore> {
+        let operator = Operator::from_iter::<Oss>(config_map)
+            .map_err(|e| Error::invalid_input(format!("Failed to create OSS operator: {:?}", e)))?;
+
+        Ok(OpendalStore::new(operator))
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for OssStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        let base_options = Self::base_oss_options(&base_path, &storage_options)?;
+        let accessor = params.get_accessor();
+
+        let inner: Arc<dyn OSObjectStore> =
+            if let Some(accessor) = accessor.filter(|a| a.has_provider()) {
+                Arc::new(
+                    DynamicOpenDalStore::new(
+                        format!("oss:{}", base_path),
+                        base_options,
+                        accessor,
+                        Self::normalize_oss_config,
+                        Self::build_oss_store,
+                    )
+                    .with_protected_keys(["bucket", "root"]),
+                )
+            } else {
+                Arc::new(Self::build_oss_store(Self::normalize_oss_config(
+                    &base_options,
+                )?)?)
+            };
+
+        let mut url = base_path;
+        if !url.path().ends_with('/') {
+            url.set_path(&format!("{}/", url.path()));
+        }
+
+        Ok(ObjectStore {
+            scheme: "oss".to_string(),
+            inner,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(true),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::OssStoreProvider;
+    use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+    use crate::object_store::{ObjectStoreProvider, StorageOptionsAccessor};
+    use url::Url;
+
+    #[test]
+    fn test_oss_store_path() {
+        let provider = OssStoreProvider;
+
+        let url = Url::parse("oss://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[test]
+    fn test_oss_alias_options_override_canonical_env_options() {
+        let config = OssStoreProvider::normalize_oss_config(&HashMap::from([
+            (
+                "endpoint".to_string(),
+                "https://env.example.com".to_string(),
+            ),
+            (
+                "oss_endpoint".to_string(),
+                "https://user.example.com".to_string(),
+            ),
+            ("access_key_id".to_string(), "env-akid".to_string()),
+            ("oss_access_key_id".to_string(), "user-akid".to_string()),
+            ("access_key_secret".to_string(), "env-secret".to_string()),
+            (
+                "oss_secret_access_key".to_string(),
+                "user-secret".to_string(),
+            ),
+            ("region".to_string(), "env-region".to_string()),
+            ("oss_region".to_string(), "user-region".to_string()),
+            ("security_token".to_string(), "env-token".to_string()),
+            ("oss_security_token".to_string(), "user-token".to_string()),
+            ("bucket".to_string(), "bucket".to_string()),
+        ]))
+        .unwrap();
+
+        assert_eq!(config.get("endpoint").unwrap(), "https://user.example.com");
+        assert_eq!(config.get("access_key_id").unwrap(), "user-akid");
+        assert_eq!(config.get("access_key_secret").unwrap(), "user-secret");
+        assert_eq!(config.get("region").unwrap(), "user-region");
+        assert_eq!(config.get("security_token").unwrap(), "user-token");
+        assert!(!config.contains_key("oss_endpoint"));
+        assert!(!config.contains_key("oss_security_token"));
+    }
+
+    #[test]
+    fn test_oss_url_bucket_and_root_are_authoritative() {
+        let storage_options = crate::object_store::StorageOptions(HashMap::from([
+            (
+                "oss_endpoint".to_string(),
+                "https://oss-cn-hangzhou.aliyuncs.com".to_string(),
+            ),
+            ("bucket".to_string(), "storage-options-bucket".to_string()),
+            ("root".to_string(), "/storage-options-root".to_string()),
+        ]));
+        let base_options = OssStoreProvider::base_oss_options(
+            &Url::parse("oss://url-bucket/path").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = OssStoreProvider::normalize_oss_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert_eq!(config.get("root").unwrap(), "/");
+    }
+
+    #[test]
+    fn test_oss_empty_url_path_removes_storage_option_root() {
+        let storage_options = crate::object_store::StorageOptions(HashMap::from([
+            (
+                "oss_endpoint".to_string(),
+                "https://oss-cn-hangzhou.aliyuncs.com".to_string(),
+            ),
+            ("root".to_string(), "/storage-options-root".to_string()),
+        ]));
+        let base_options = OssStoreProvider::base_oss_options(
+            &Url::parse("oss://url-bucket").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = OssStoreProvider::normalize_oss_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert!(!config.contains_key("root"));
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_opendal_oss_store_uses_provider_credentials() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([
+                    (
+                        "oss_endpoint".to_string(),
+                        "https://oss-cn-hangzhou.aliyuncs.com".to_string(),
+                    ),
+                    ("oss_access_key_id".to_string(), "akid".to_string()),
+                    ("oss_secret_access_key".to_string(), "secret".to_string()),
+                    ("oss_security_token".to_string(), "token".to_string()),
+                ]),
+            },
+        )));
+
+        let base_options = OssStoreProvider::base_oss_options(
+            &Url::parse("oss://bucket/path").unwrap(),
+            &crate::object_store::StorageOptions(HashMap::new()),
+        )
+        .unwrap();
+
+        let store = DynamicOpenDalStore::new(
+            "oss",
+            base_options,
+            accessor,
+            OssStoreProvider::normalize_oss_config,
+            OssStoreProvider::build_oss_store,
+        );
+
+        let current_store = store
+            .current_store()
+            .await
+            .expect("dynamic OpenDAL OSS store should build");
+
+        assert!(current_store.to_string().contains("Opendal"));
+    }
+}

vendor/lance-io/src/object_store/providers/shared_memory.rs

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{
+    collections::HashMap,
+    sync::{Arc, LazyLock, Mutex},
+};
+
+use crate::object_store::{
+    ObjectStore, ObjectStoreParams, ObjectStoreProvider, providers::memory::MemoryStoreProvider,
+};
+use lance_core::error::Result;
+use object_store::{memory::InMemory, path::Path};
+use url::Url;
+
+/// Process-global pool of in-memory backends keyed by URL authority.
+///
+/// Different authorities map to different backends (act as "buckets"); same
+/// authority across any caller in the process resolves to the same `Arc<InMemory>`.
+/// The pool grows for the lifetime of the process — entries are never evicted.
+static SHARED_BACKENDS: LazyLock<Mutex<HashMap<String, Arc<InMemory>>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+
+fn shared_backend_for(url: &Url) -> Arc<InMemory> {
+    SHARED_BACKENDS
+        .lock()
+        .expect("SHARED_BACKENDS mutex poisoned")
+        .entry(url.authority().to_string())
+        .or_insert_with(|| Arc::new(InMemory::new()))
+        .clone()
+}
+
+/// Like [`MemoryStoreProvider`], but every caller resolving a `shared-memory://<authority>/...`
+/// URL with the same `<authority>` sees the same backing bytes — across `ObjectStoreRegistry`
+/// instances, threads, and unrelated components in the same process.
+///
+/// Intended for tests and harnesses that need multiple actors to coordinate through a
+/// common in-memory object store (e.g. a writer and an independent reader, multi-pod
+/// fence simulations). Choose distinct authorities for isolation
+/// (`shared-memory://test-a` vs `shared-memory://test-b`).
+///
+/// Unlike `memory://` — which mints a fresh `InMemory` per `new_store` call — this
+/// provider is opt-in precisely so existing tests relying on per-call isolation are
+/// unaffected.
+#[derive(Default, Debug)]
+pub struct SharedMemoryStoreProvider {
+    inner: MemoryStoreProvider,
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for SharedMemoryStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let mut store = self.inner.new_store(base_path.clone(), params).await?;
+        store.inner = shared_backend_for(&base_path);
+        store.scheme = String::from("shared-memory");
+        store.store_prefix = self.calculate_object_store_prefix(&base_path, None)?;
+        Ok(store)
+    }
+
+    fn extract_path(&self, url: &Url) -> Result<Path> {
+        // The authority is the bucket; the URL path is the object path within it.
+        Ok(Path::from(url.path().trim_start_matches('/')))
+    }
+
+    fn calculate_object_store_prefix(
+        &self,
+        url: &Url,
+        _storage_options: Option<&HashMap<String, String>>,
+    ) -> Result<String> {
+        Ok(format!("shared-memory${}", url.authority()))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::object_store::ObjectStoreRegistry;
+    use bytes::Bytes;
+    use object_store::{ObjectStoreExt as _, PutPayload};
+
+    async fn store_for(uri: &str) -> (Arc<ObjectStore>, Path) {
+        let registry = Arc::new(ObjectStoreRegistry::default());
+        let (store, path) = ObjectStore::from_uri_and_params(registry, uri, &Default::default())
+            .await
+            .unwrap();
+        (store, path)
+    }
+
+    #[tokio::test]
+    async fn same_authority_shares_bytes_across_registries() {
+        let (writer, _) = store_for("shared-memory://bucket-a/").await;
+        writer
+            .inner
+            .put(&Path::from("file"), PutPayload::from_static(b"hello"))
+            .await
+            .unwrap();
+
+        // Build a *separate* registry — no shared state at the registry layer.
+        let (reader, _) = store_for("shared-memory://bucket-a/").await;
+        let bytes = reader.inner.get(&Path::from("file")).await.unwrap();
+        assert_eq!(bytes.bytes().await.unwrap(), Bytes::from_static(b"hello"));
+    }
+
+    #[tokio::test]
+    async fn different_authorities_are_isolated() {
+        let (a, _) = store_for("shared-memory://iso-a/").await;
+        let (b, _) = store_for("shared-memory://iso-b/").await;
+        a.inner
+            .put(&Path::from("k"), PutPayload::from_static(b"in-a"))
+            .await
+            .unwrap();
+        assert!(b.inner.get(&Path::from("k")).await.is_err());
+    }
+
+    #[tokio::test]
+    async fn extract_path_strips_authority() {
+        let provider = SharedMemoryStoreProvider::default();
+        let url = Url::parse("shared-memory://bucket/foo/bar").unwrap();
+        assert_eq!(provider.extract_path(&url).unwrap(), Path::from("foo/bar"));
+    }
+
+    #[tokio::test]
+    async fn from_uri_and_params_resolves_path_correctly() {
+        let (store, path) = store_for("shared-memory://path-test/sub/dir/obj").await;
+        assert_eq!(path, Path::from("sub/dir/obj"));
+        store
+            .inner
+            .put(&path, PutPayload::from_static(b"payload"))
+            .await
+            .unwrap();
+
+        let (peer, peer_path) = store_for("shared-memory://path-test/sub/dir/obj").await;
+        let bytes = peer.inner.get(&peer_path).await.unwrap();
+        assert_eq!(bytes.bytes().await.unwrap(), Bytes::from_static(b"payload"));
+    }
+
+    #[test]
+    fn calculate_prefix_is_per_authority() {
+        let provider = SharedMemoryStoreProvider::default();
+        let a = provider
+            .calculate_object_store_prefix(&Url::parse("shared-memory://x/p").unwrap(), None)
+            .unwrap();
+        let b = provider
+            .calculate_object_store_prefix(&Url::parse("shared-memory://y/p").unwrap(), None)
+            .unwrap();
+        assert_ne!(a, b);
+        assert_eq!(a, "shared-memory$x");
+    }
+}

vendor/lance-io/src/object_store/providers/tencent.rs

@@ -0,0 +1,122 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Cos};
+use url::Url;
+
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct TencentStoreProvider;
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for TencentStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        let bucket = base_path
+            .host_str()
+            .ok_or_else(|| Error::invalid_input("Tencent Cos URL must contain bucket name"))?
+            .to_string();
+
+        let prefix = base_path.path().trim_start_matches('/').to_string();
+
+        // Start with environment variables as base configuration
+        let mut config_map: HashMap<String, String> = std::env::vars()
+            .filter(|(k, _)| k.starts_with("COS_") || k.starts_with("TENCENTCLOUD_"))
+            .map(|(k, v)| {
+                // Convert env var names to opendal config keys
+                let key = k
+                    .to_lowercase()
+                    .replace("cos_", "")
+                    .replace("tencentcloud_", "");
+                (key, v)
+            })
+            .collect();
+
+        config_map.insert("bucket".to_string(), bucket);
+
+        if !prefix.is_empty() {
+            config_map.insert("root".to_string(), "/".to_string());
+        }
+
+        // Override with storage options if provided
+        if let Some(endpoint) = storage_options.0.get("cos_endpoint") {
+            config_map.insert("endpoint".to_string(), endpoint.clone());
+        }
+
+        if let Some(secret_id) = storage_options.0.get("cos_secret_id") {
+            config_map.insert("secret_id".to_string(), secret_id.clone());
+        }
+
+        if let Some(secret_key) = storage_options.0.get("cos_secret_key") {
+            config_map.insert("secret_key".to_string(), secret_key.clone());
+        }
+
+        if let Some(enable_versioning) = storage_options.0.get("cos_enable_versioning") {
+            config_map.insert("enable_versioning".to_string(), enable_versioning.clone());
+        }
+
+        // Currently, the configuration options for CosConfig in OpenDAL are very limited.
+        // Most configurations need to be entered via environment variables, such as TENCENTCLOUD_SECURITY_TOKEN, TENCENTCLOUD_REGION, etc.
+        // (more env config details: https://github.com/apache/opendal-reqsign/blob/v0.16.5/src/tencent/config.rs)
+        // Therefore, we need to keep `disable_config_load` always false to allow configurations to be loaded from environment variables.
+        // TODO: improve CosConfig in opendal and add more storage_option here
+        config_map.insert("disable_config_load".to_string(), "false".to_string());
+
+        if !config_map.contains_key("endpoint") {
+            return Err(Error::invalid_input(
+                "COS endpoint is required. Please provide 'cos_endpoint' in storage options or set COS_ENDPOINT environment variable",
+            ));
+        }
+
+        let operator = Operator::from_iter::<Cos>(config_map)
+            .map_err(|e| Error::invalid_input(format!("Failed to create COS operator: {:?}", e)))?;
+
+        let opendal_store = Arc::new(OpendalStore::new(operator));
+
+        let mut url = base_path;
+        if !url.path().ends_with('/') {
+            url.set_path(&format!("{}/", url.path()));
+        }
+
+        Ok(ObjectStore {
+            scheme: "cos".to_string(),
+            inner: opendal_store,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(true),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::TencentStoreProvider;
+    use crate::object_store::ObjectStoreProvider;
+    use url::Url;
+
+    #[test]
+    fn test_cos_store_path() {
+        let provider = TencentStoreProvider;
+
+        let url = Url::parse("cos://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+}

vendor/lance-io/src/object_store/providers/tos.rs

@@ -0,0 +1,300 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use object_store::ObjectStore as OSObjectStore;
+use object_store_opendal::OpendalStore;
+use opendal::{Operator, services::Tos};
+use url::Url;
+
+use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+use crate::object_store::{
+    DEFAULT_CLOUD_BLOCK_SIZE, DEFAULT_CLOUD_IO_PARALLELISM, DEFAULT_MAX_IOP_SIZE, ObjectStore,
+    ObjectStoreParams, ObjectStoreProvider, StorageOptions,
+};
+use lance_core::error::{Error, Result};
+
+#[derive(Default, Debug)]
+pub struct TosStoreProvider;
+
+impl TosStoreProvider {
+    fn tos_env_options_from_iter<I, K, V>(vars: I) -> HashMap<String, String>
+    where
+        I: IntoIterator<Item = (K, V)>,
+        K: Into<String>,
+        V: Into<String>,
+    {
+        let vars = vars
+            .into_iter()
+            .map(|(key, value)| (key.into(), value.into()))
+            .collect::<Vec<_>>();
+        let mut config_map = HashMap::new();
+
+        for prefix in ["VOLCENGINE_", "TOS_"] {
+            for (key, value) in &vars {
+                if let Some(stripped_key) = key.strip_prefix(prefix) {
+                    config_map.insert(stripped_key.to_ascii_lowercase(), value.clone());
+                }
+            }
+        }
+
+        config_map
+    }
+
+    fn base_tos_options(
+        base_path: &Url,
+        storage_options: &StorageOptions,
+    ) -> Result<HashMap<String, String>> {
+        let bucket = base_path
+            .host_str()
+            .ok_or_else(|| Error::invalid_input("TOS URL must contain bucket name"))?
+            .to_string();
+
+        let prefix = base_path.path().trim_start_matches('/').to_string();
+
+        let mut config_map = Self::tos_env_options_from_iter(std::env::vars());
+
+        config_map.extend(storage_options.0.clone());
+
+        config_map.insert("bucket".to_string(), bucket);
+        if prefix.is_empty() {
+            config_map.remove("root");
+        } else {
+            config_map.insert("root".to_string(), "/".to_string());
+        }
+
+        Ok(config_map)
+    }
+
+    /// Normalize TOS storage options, resolving aliases for well-known keys
+    /// while passing through all other options so that OpenDAL can use them.
+    fn normalize_tos_config(options: &HashMap<String, String>) -> Result<HashMap<String, String>> {
+        let mut config_map = options.clone();
+
+        let alias_groups: &[(&str, &[&str])] = &[
+            ("endpoint", &["tos_endpoint"]),
+            ("region", &["tos_region"]),
+            ("access_key_id", &["tos_access_key_id"]),
+            ("secret_access_key", &["tos_secret_access_key"]),
+            ("security_token", &["tos_security_token"]),
+        ];
+
+        for (canonical, aliases) in alias_groups {
+            for alias in *aliases {
+                if let Some(value) = config_map.remove(*alias) {
+                    config_map.insert(canonical.to_string(), value);
+                    break;
+                }
+            }
+        }
+
+        if !config_map.contains_key("endpoint") {
+            return Err(Error::invalid_input(
+                "TOS endpoint is required. Please provide 'tos_endpoint' in storage options or set TOS_ENDPOINT environment variable",
+            ));
+        }
+
+        Ok(config_map)
+    }
+
+    fn build_tos_store(config_map: HashMap<String, String>) -> Result<OpendalStore> {
+        let operator = Operator::from_iter::<Tos>(config_map)
+            .map_err(|e| Error::invalid_input(format!("Failed to create TOS operator: {:?}", e)))?;
+
+        Ok(OpendalStore::new(operator))
+    }
+}
+
+#[async_trait::async_trait]
+impl ObjectStoreProvider for TosStoreProvider {
+    async fn new_store(&self, base_path: Url, params: &ObjectStoreParams) -> Result<ObjectStore> {
+        let block_size = params.block_size.unwrap_or(DEFAULT_CLOUD_BLOCK_SIZE);
+        let storage_options = StorageOptions(params.storage_options().cloned().unwrap_or_default());
+
+        let base_options = Self::base_tos_options(&base_path, &storage_options)?;
+        let accessor = params.get_accessor();
+
+        let inner: Arc<dyn OSObjectStore> =
+            if let Some(accessor) = accessor.filter(|a| a.has_provider()) {
+                Arc::new(
+                    DynamicOpenDalStore::new(
+                        format!("tos:{}", base_path),
+                        base_options,
+                        accessor,
+                        Self::normalize_tos_config,
+                        Self::build_tos_store,
+                    )
+                    .with_protected_keys(["bucket", "root"]),
+                )
+            } else {
+                Arc::new(Self::build_tos_store(Self::normalize_tos_config(
+                    &base_options,
+                )?)?)
+            };
+
+        let mut url = base_path;
+        if !url.path().ends_with('/') {
+            url.set_path(&format!("{}/", url.path()));
+        }
+
+        Ok(ObjectStore {
+            scheme: "tos".to_string(),
+            inner,
+            block_size,
+            max_iop_size: *DEFAULT_MAX_IOP_SIZE,
+            use_constant_size_upload_parts: params.use_constant_size_upload_parts,
+            list_is_lexically_ordered: params.list_is_lexically_ordered.unwrap_or(true),
+            io_parallelism: DEFAULT_CLOUD_IO_PARALLELISM,
+            download_retry_count: storage_options.download_retry_count(),
+            io_tracker: Default::default(),
+            store_prefix: self.calculate_object_store_prefix(&url, params.storage_options())?,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use super::TosStoreProvider;
+    use crate::object_store::dynamic_opendal::DynamicOpenDalStore;
+    use crate::object_store::test_utils::StaticMockStorageOptionsProvider;
+    use crate::object_store::{ObjectStoreProvider, StorageOptionsAccessor};
+    use url::Url;
+
+    #[test]
+    fn test_tos_store_path() {
+        let provider = TosStoreProvider;
+
+        let url = Url::parse("tos://bucket/path/to/file").unwrap();
+        let path = provider.extract_path(&url).unwrap();
+        let expected_path = object_store::path::Path::from("path/to/file");
+        assert_eq!(path, expected_path);
+    }
+
+    #[test]
+    fn test_tos_env_options_normalize_supported_prefixes() {
+        let config = TosStoreProvider::tos_env_options_from_iter([
+            ("VOLCENGINE_ENDPOINT", "https://tos-cn-beijing.volces.com"),
+            ("TOS_ACCESS_KEY_ID", "tos-akid"),
+            ("TOS_SECRET_ACCESS_KEY", "tos-secret"),
+        ]);
+
+        assert_eq!(
+            config.get("endpoint").unwrap(),
+            "https://tos-cn-beijing.volces.com"
+        );
+        assert_eq!(config.get("access_key_id").unwrap(), "tos-akid");
+        assert_eq!(config.get("secret_access_key").unwrap(), "tos-secret");
+    }
+
+    #[test]
+    fn test_tos_alias_options_override_canonical_env_options() {
+        let config = TosStoreProvider::normalize_tos_config(&HashMap::from([
+            (
+                "endpoint".to_string(),
+                "https://env.example.com".to_string(),
+            ),
+            (
+                "tos_endpoint".to_string(),
+                "https://user.example.com".to_string(),
+            ),
+            ("region".to_string(), "env-region".to_string()),
+            ("tos_region".to_string(), "user-region".to_string()),
+            ("access_key_id".to_string(), "env-akid".to_string()),
+            ("tos_access_key_id".to_string(), "user-akid".to_string()),
+            ("secret_access_key".to_string(), "env-secret".to_string()),
+            (
+                "tos_secret_access_key".to_string(),
+                "user-secret".to_string(),
+            ),
+            ("security_token".to_string(), "env-token".to_string()),
+            ("tos_security_token".to_string(), "user-token".to_string()),
+            ("bucket".to_string(), "bucket".to_string()),
+        ]))
+        .unwrap();
+
+        assert_eq!(config.get("endpoint").unwrap(), "https://user.example.com");
+        assert_eq!(config.get("region").unwrap(), "user-region");
+        assert_eq!(config.get("access_key_id").unwrap(), "user-akid");
+        assert_eq!(config.get("secret_access_key").unwrap(), "user-secret");
+        assert_eq!(config.get("security_token").unwrap(), "user-token");
+        assert!(!config.contains_key("tos_endpoint"));
+        assert!(!config.contains_key("tos_secret_access_key"));
+        assert!(!config.contains_key("tos_security_token"));
+    }
+
+    #[test]
+    fn test_tos_url_bucket_and_root_are_authoritative() {
+        let storage_options = crate::object_store::StorageOptions(HashMap::from([
+            (
+                "tos_endpoint".to_string(),
+                "https://tos-cn-beijing.volces.com".to_string(),
+            ),
+            ("bucket".to_string(), "storage-options-bucket".to_string()),
+            ("root".to_string(), "/storage-options-root".to_string()),
+        ]));
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket/path").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = TosStoreProvider::normalize_tos_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert_eq!(config.get("root").unwrap(), "/");
+
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket").unwrap(),
+            &storage_options,
+        )
+        .unwrap();
+        let config = TosStoreProvider::normalize_tos_config(&base_options).unwrap();
+
+        assert_eq!(config.get("bucket").unwrap(), "url-bucket");
+        assert!(!config.contains_key("root"));
+    }
+
+    #[tokio::test]
+    async fn test_dynamic_opendal_tos_store_uses_provider_credentials() {
+        let accessor = Arc::new(StorageOptionsAccessor::with_provider(Arc::new(
+            StaticMockStorageOptionsProvider {
+                options: HashMap::from([
+                    (
+                        "tos_endpoint".to_string(),
+                        "https://tos-cn-beijing.volces.com".to_string(),
+                    ),
+                    ("tos_region".to_string(), "cn-beijing".to_string()),
+                    ("tos_access_key_id".to_string(), "akid".to_string()),
+                    ("tos_secret_access_key".to_string(), "secret".to_string()),
+                    ("tos_security_token".to_string(), "token".to_string()),
+                ]),
+            },
+        )));
+
+        let base_options = TosStoreProvider::base_tos_options(
+            &Url::parse("tos://url-bucket/path").unwrap(),
+            &crate::object_store::StorageOptions(HashMap::new()),
+        )
+        .unwrap();
+
+        let store = DynamicOpenDalStore::new(
+            "tos",
+            base_options,
+            accessor,
+            TosStoreProvider::normalize_tos_config,
+            TosStoreProvider::build_tos_store,
+        )
+        .with_protected_keys(["bucket", "root"]);
+
+        let current_store = store
+            .current_store()
+            .await
+            .expect("dynamic OpenDAL TOS store should build");
+
+        assert!(current_store.to_string().contains("Opendal"));
+    }
+}

vendor/lance-io/src/object_store/test_utils.rs

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::collections::HashMap;
+
+use async_trait::async_trait;
+
+use super::StorageOptionsProvider;
+use lance_core::Result;
+
+#[derive(Debug)]
+pub struct StaticMockStorageOptionsProvider {
+    pub options: HashMap<String, String>,
+}
+
+#[async_trait]
+impl StorageOptionsProvider for StaticMockStorageOptionsProvider {
+    async fn fetch_storage_options(&self) -> Result<Option<HashMap<String, String>>> {
+        Ok(Some(self.options.clone()))
+    }
+
+    fn provider_id(&self) -> String {
+        "StaticMockStorageOptionsProvider".to_string()
+    }
+}

vendor/lance-io/src/object_store/tracing.rs

@@ -0,0 +1,401 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Wrappers around object_store that apply tracing
+
+use std::ops::Range;
+use std::sync::Arc;
+
+use bytes::Bytes;
+use futures::StreamExt;
+use futures::stream::BoxStream;
+use lance_core::utils::tracing::StreamTracingExt;
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
+    UploadPart,
+};
+use tracing::{Instrument, Span, instrument};
+
+#[derive(Debug)]
+pub struct TracedMultipartUpload {
+    write_span: Span,
+    target: Box<dyn MultipartUpload>,
+    write_size: usize,
+}
+
+#[async_trait::async_trait]
+impl MultipartUpload for TracedMultipartUpload {
+    fn put_part(&mut self, data: PutPayload) -> UploadPart {
+        let write_span = self.write_span.clone();
+        self.write_size += data.content_length();
+        let fut = self.target.put_part(data);
+        Box::pin(fut.instrument(write_span))
+    }
+
+    #[instrument(level = "debug", skip_all)]
+    async fn complete(&mut self) -> OSResult<PutResult> {
+        let res = self.target.complete().await?;
+        self.write_span.record("size", self.write_size);
+        Ok(res)
+    }
+
+    #[instrument(level = "debug", skip_all)]
+    async fn abort(&mut self) -> OSResult<()> {
+        self.target.abort().await
+    }
+}
+
+#[derive(Debug)]
+pub struct TracedObjectStore {
+    target: Arc<dyn object_store::ObjectStore>,
+}
+
+impl std::fmt::Display for TracedObjectStore {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("TracedObjectStore({})", self.target))
+    }
+}
+
+#[async_trait::async_trait]
+#[deny(clippy::missing_trait_methods)]
+impl object_store::ObjectStore for TracedObjectStore {
+    #[instrument(level = "debug", skip(self, bytes, location, opts), fields(path = location.as_ref(), size = bytes.content_length()))]
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        self.target.put_opts(location, bytes, opts).await
+    }
+
+    #[instrument(level = "debug", skip(self, location, opts), fields(path = location.as_ref(), size = tracing::field::Empty))]
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn object_store::MultipartUpload>> {
+        let upload = self.target.put_multipart_opts(location, opts).await?;
+        Ok(Box::new(TracedMultipartUpload {
+            target: upload,
+            write_span: tracing::Span::current(),
+            write_size: 0,
+        }))
+    }
+
+    #[instrument(level = "debug", skip(self, options, location), fields(path = location.as_ref(), size = tracing::field::Empty))]
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        let res = self.target.get_opts(location, options).await?;
+
+        let span = tracing::Span::current();
+        span.record("size", res.range.end - res.range.start);
+
+        Ok(res)
+    }
+
+    #[instrument(level = "debug", skip(self, location), fields(path = location.as_ref(), size = ranges.iter().map(|r| r.end - r.start).sum::<u64>()))]
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        self.target.get_ranges(location, ranges).await
+    }
+
+    #[instrument(level = "debug", skip_all)]
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OSResult<Path>>,
+    ) -> BoxStream<'static, OSResult<Path>> {
+        self.target
+            .delete_stream(locations)
+            .stream_in_current_span()
+            .boxed()
+    }
+
+    #[instrument(level = "debug", skip(self, prefix), fields(prefix = prefix.map(|p| p.as_ref())))]
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.target.list(prefix).stream_in_current_span().boxed()
+    }
+
+    #[instrument(level = "debug", skip(self, prefix, offset), fields(prefix = prefix.map(|p| p.as_ref()), offset = offset.as_ref()))]
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.target
+            .list_with_offset(prefix, offset)
+            .stream_in_current_span()
+            .boxed()
+    }
+
+    #[instrument(level = "debug", skip(self, prefix), fields(prefix = prefix.map(|p| p.as_ref())))]
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        self.target.list_with_delimiter(prefix).await
+    }
+
+    #[instrument(level = "debug", skip(self, from, to, opts), fields(from = from.as_ref(), to = to.as_ref()))]
+    async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+        self.target.copy_opts(from, to, opts).await
+    }
+
+    #[instrument(level = "debug", skip(self, from, to, opts), fields(from = from.as_ref(), to = to.as_ref()))]
+    async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+        self.target.rename_opts(from, to, opts).await
+    }
+}
+
+pub trait ObjectStoreTracingExt {
+    fn traced(self) -> Arc<dyn object_store::ObjectStore>;
+}
+
+impl ObjectStoreTracingExt for Arc<dyn object_store::ObjectStore> {
+    fn traced(self) -> Arc<dyn object_store::ObjectStore> {
+        Arc::new(TracedObjectStore { target: self })
+    }
+}
+
+impl<T: object_store::ObjectStore> ObjectStoreTracingExt for Arc<T> {
+    fn traced(self) -> Arc<dyn object_store::ObjectStore> {
+        Arc::new(TracedObjectStore { target: self })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use bytes::Bytes;
+    use object_store::memory::InMemory;
+    use object_store::path::Path;
+    use object_store::{ObjectStoreExt, PutPayload};
+    use tracing_mock::{expect, subscriber};
+
+    fn payload(data: &[u8]) -> PutPayload {
+        PutPayload::from_bytes(Bytes::copy_from_slice(data))
+    }
+
+    fn make_store() -> Arc<dyn object_store::ObjectStore> {
+        Arc::new(InMemory::new()).traced()
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_put_records_path_and_size() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+
+        let span = expect::span().named("put_opts");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                span.clone().with_fields(
+                    expect::field("path")
+                        .with_value(&"a/b.bin")
+                        .and(expect::field("size").with_value(&data.len()))
+                        .only(),
+                ),
+            )
+            .enter(span.clone())
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        make_store().put(&path, payload(data)).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_get_records_path_and_size() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+        let size = data.len() as u64; // meta.size is u64
+
+        // Seed without an active mock subscriber.
+        let store = make_store();
+        store.put(&path, payload(data)).await.unwrap();
+
+        let span = expect::span().named("get_opts");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                // size = Empty at span creation, so only path is visited.
+                span.clone()
+                    .with_fields(expect::field("path").with_value(&"a/b.bin").only()),
+            )
+            .enter(span.clone())
+            .record(span.clone(), expect::field("size").with_value(&size))
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.get(&path).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_get_range_records_path_and_size() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+
+        let store = make_store();
+        store.put(&path, payload(data)).await.unwrap();
+
+        let range = 2u64..7u64;
+        let size = range.end - range.start;
+
+        let span = expect::span().named("get_opts");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                span.clone()
+                    .with_fields(expect::field("path").with_value(&"a/b.bin").only()),
+            )
+            .enter(span.clone())
+            .record(span.clone(), expect::field("size").with_value(&size))
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.get_range(&path, range).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_get_ranges_records_path_and_total_size() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+
+        let store = make_store();
+        store.put(&path, payload(data)).await.unwrap();
+
+        let ranges = [2u64..5u64, 6u64..9u64];
+        let size: u64 = ranges.iter().map(|r| r.end - r.start).sum();
+
+        let span = expect::span().named("get_ranges");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                // `ranges` is also captured automatically as a debug field since
+                // it is not in the skip list, so we don't use `.only()` here.
+                span.clone().with_fields(
+                    expect::field("path")
+                        .with_value(&"a/b.bin")
+                        .and(expect::field("size").with_value(&size)),
+                ),
+            )
+            .enter(span.clone())
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.get_ranges(&path, &ranges).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_head_records_path() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+        let size = data.len() as u64;
+
+        let store = make_store();
+        store.put(&path, payload(data)).await.unwrap();
+
+        let span = expect::span().named("get_opts");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                span.clone()
+                    .with_fields(expect::field("path").with_value(&"a/b.bin").only()),
+            )
+            .enter(span.clone())
+            .record(span.clone(), expect::field("size").with_value(&size))
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.head(&path).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_delete_records_path() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+
+        let store = make_store();
+        store.put(&path, payload(data)).await.unwrap();
+
+        let span = expect::span().named("delete_stream");
+        let (sub, handle) = subscriber::mock()
+            .new_span(span.clone())
+            .enter(span.clone())
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.delete(&path).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_copy_records_from_and_to() {
+        let from = Path::from("a/src.bin");
+        let to = Path::from("a/dst.bin");
+        let data = b"hello world";
+
+        let store = make_store();
+        store.put(&from, payload(data)).await.unwrap();
+
+        let span = expect::span().named("copy_opts");
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                span.clone().with_fields(
+                    expect::field("from")
+                        .with_value(&"a/src.bin")
+                        .and(expect::field("to").with_value(&"a/dst.bin"))
+                        .only(),
+                ),
+            )
+            .enter(span.clone())
+            .exit(span.clone())
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        store.copy(&from, &to).await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+
+    #[tokio::test(flavor = "current_thread")]
+    async fn test_put_multipart_records_path() {
+        let path = Path::from("a/b.bin");
+        let data = b"hello world";
+
+        let put_mp_span = expect::span().named("put_multipart_opts");
+        // Expect only the span creation; any subsequent enter/exit/record
+        // events are not in the queue so they are silently ignored.
+        let (sub, handle) = subscriber::mock()
+            .new_span(
+                // size = Empty at span creation, so only path is visited.
+                put_mp_span.with_fields(expect::field("path").with_value(&"a/b.bin").only()),
+            )
+            .run_with_handle();
+
+        let _guard = tracing::subscriber::set_default(sub);
+        let store = make_store();
+        let mut upload = store.put_multipart(&path).await.unwrap();
+        upload.put_part(payload(data)).await.unwrap();
+        upload.complete().await.unwrap();
+        drop(_guard);
+
+        handle.assert_finished();
+    }
+}

vendor/lance-io/src/object_writer.rs

@@ -0,0 +1,806 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::io;
+use std::pin::Pin;
+use std::sync::{Arc, OnceLock};
+use std::task::Poll;
+
+use crate::object_store::ObjectStore as LanceObjectStore;
+use async_trait::async_trait;
+use bytes::Bytes;
+use futures::FutureExt;
+use futures::future::BoxFuture;
+use object_store::{MultipartUpload, ObjectStoreExt};
+use object_store::{ObjectStore, Result as OSResult, path::Path};
+use tokio::io::{AsyncWrite, AsyncWriteExt};
+use tokio::task::JoinSet;
+
+use lance_core::{Error, Result};
+use tracing::Instrument;
+
+use crate::traits::Writer;
+use crate::utils::tracking_store::{IOTracker, IoMetricsGuard};
+use tokio::runtime::Handle;
+
+/// Start at 5MB.
+const INITIAL_UPLOAD_STEP: usize = 1024 * 1024 * 5;
+
+fn max_upload_parallelism() -> usize {
+    static MAX_UPLOAD_PARALLELISM: OnceLock<usize> = OnceLock::new();
+    *MAX_UPLOAD_PARALLELISM.get_or_init(|| {
+        std::env::var("LANCE_UPLOAD_CONCURRENCY")
+            .ok()
+            .and_then(|s| s.parse::<usize>().ok())
+            .unwrap_or(10)
+    })
+}
+
+/// Maximum body size for a single S3 PUT: strictly less than 5 GiB.
+/// AWS rejects single-PUT bodies of exactly 5 GiB (= 5 * 1024^3) with
+/// `EntityTooLarge`, so we clamp `LANCE_INITIAL_UPLOAD_SIZE` one byte
+/// below that threshold to keep the buffer-fills-to-clamp single-PUT
+/// path safe. See lance#6750 for the related txn-file write fix.
+const MAX_UPLOAD_PART_SIZE: usize = 1024 * 1024 * 1024 * 5 - 1;
+
+/// Clamps a requested upload part size to the valid [5MB, 5GB] range.
+/// Returns the clamped value and whether clamping was necessary.
+fn clamp_initial_upload_size(raw: usize) -> (usize, bool) {
+    let clamped = raw.clamp(INITIAL_UPLOAD_STEP, MAX_UPLOAD_PART_SIZE);
+    (clamped, clamped != raw)
+}
+
+fn initial_upload_size() -> usize {
+    static LANCE_INITIAL_UPLOAD_SIZE: OnceLock<usize> = OnceLock::new();
+    *LANCE_INITIAL_UPLOAD_SIZE.get_or_init(|| {
+        let Some(raw) = std::env::var("LANCE_INITIAL_UPLOAD_SIZE")
+            .ok()
+            .and_then(|s| s.parse::<usize>().ok())
+        else {
+            return INITIAL_UPLOAD_STEP;
+        };
+        let (clamped, was_clamped) = clamp_initial_upload_size(raw);
+        if was_clamped {
+            // OnceLock caches the result, so this warning fires at most once per process.
+            tracing::warn!(
+                requested = raw,
+                clamped,
+                "LANCE_INITIAL_UPLOAD_SIZE must be between 5MB and 5GB; clamping to valid range"
+            );
+        }
+        clamped
+    })
+}
+
+/// Writer to an object in an object store.
+///
+/// If the object is small enough, the writer will upload the object in a single
+/// PUT request. If the object is larger, the writer will create a multipart
+/// upload and upload parts in parallel.
+///
+/// This implements the `AsyncWrite` trait.
+pub struct ObjectWriter {
+    state: UploadState,
+    path: Arc<Path>,
+    cursor: usize,
+    buffer: Vec<u8>,
+    // TODO: use constant size to support R2
+    use_constant_size_upload_parts: bool,
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct WriteResult {
+    pub size: usize,
+    pub e_tag: Option<String>,
+}
+
+enum UploadState {
+    /// The writer has been opened but no data has been written yet. Will be in
+    /// this state until the buffer is full or the writer is shut down.
+    Started(Arc<dyn ObjectStore>),
+    /// The writer is in the process of creating a multipart upload.
+    CreatingUpload(BoxFuture<'static, OSResult<Box<dyn MultipartUpload>>>),
+    /// The writer is in the process of uploading parts.
+    InProgress {
+        part_idx: u16,
+        upload: Box<dyn MultipartUpload>,
+        futures: JoinSet<OSResult<()>>,
+    },
+    /// The writer is in the process of uploading data in a single PUT request.
+    /// This happens when shutdown is called before the buffer is full.
+    PuttingSingle(BoxFuture<'static, OSResult<WriteResult>>),
+    /// The writer is in the process of completing the multipart upload.
+    Completing(BoxFuture<'static, OSResult<WriteResult>>),
+    /// The writer has been shut down and all data has been written.
+    Done(WriteResult),
+}
+
+/// Methods for state transitions.
+impl UploadState {
+    fn started_to_putting_single(&mut self, path: Arc<Path>, buffer: Vec<u8>) {
+        // To get owned self, we temporarily swap with Done.
+        let this = std::mem::replace(self, Self::Done(WriteResult::default()));
+        *self = match this {
+            Self::Started(store) => {
+                let fut = async move {
+                    let size = buffer.len();
+                    let res = store.put(&path, buffer.into()).await?;
+                    Ok(WriteResult {
+                        size,
+                        e_tag: res.e_tag,
+                    })
+                };
+                Self::PuttingSingle(Box::pin(fut))
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn in_progress_to_completing(&mut self) {
+        // To get owned self, we temporarily swap with Done.
+        let this = std::mem::replace(self, Self::Done(WriteResult::default()));
+        *self = match this {
+            Self::InProgress {
+                mut upload,
+                futures,
+                ..
+            } => {
+                debug_assert!(futures.is_empty());
+                let fut = async move {
+                    let res = upload.complete().await?;
+                    Ok(WriteResult {
+                        size: 0, // This will be set properly later.
+                        e_tag: res.e_tag,
+                    })
+                };
+                Self::Completing(Box::pin(fut))
+            }
+            _ => unreachable!(),
+        };
+    }
+}
+
+impl ObjectWriter {
+    pub async fn new(object_store: &LanceObjectStore, path: &Path) -> Result<Self> {
+        Ok(Self {
+            state: UploadState::Started(object_store.inner.clone()),
+            cursor: 0,
+            path: Arc::new(path.clone()),
+            buffer: Vec::with_capacity(initial_upload_size()),
+            use_constant_size_upload_parts: object_store.use_constant_size_upload_parts,
+        })
+    }
+
+    /// Returns the contents of `buffer` as a `Bytes` object and resets `buffer`.
+    /// The new capacity of `buffer` is determined by the current part index.
+    fn next_part_buffer(buffer: &mut Vec<u8>, part_idx: u16, constant_upload_size: bool) -> Bytes {
+        let new_capacity = if constant_upload_size {
+            // The store does not support variable part sizes, so use the initial size.
+            initial_upload_size()
+        } else {
+            // Increase the upload size every 100 parts. This gives maximum part size of 2.5TB.
+            initial_upload_size().max(((part_idx / 100) as usize + 1) * INITIAL_UPLOAD_STEP)
+        };
+        let new_buffer = Vec::with_capacity(new_capacity);
+        let part = std::mem::replace(buffer, new_buffer);
+        Bytes::from(part)
+    }
+
+    fn put_part(
+        upload: &mut dyn MultipartUpload,
+        buffer: Bytes,
+    ) -> BoxFuture<'static, OSResult<()>> {
+        log::debug!(
+            "MultipartUpload submitting part with {} bytes",
+            buffer.len()
+        );
+        upload.put_part(buffer.into())
+    }
+
+    fn poll_tasks(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::result::Result<(), io::Error> {
+        let mut_self = &mut *self;
+        loop {
+            match &mut mut_self.state {
+                UploadState::Started(_) | UploadState::Done(_) => break,
+                UploadState::CreatingUpload(fut) => match fut.poll_unpin(cx) {
+                    Poll::Ready(Ok(mut upload)) => {
+                        let mut futures = JoinSet::new();
+
+                        let data = Self::next_part_buffer(
+                            &mut mut_self.buffer,
+                            0,
+                            mut_self.use_constant_size_upload_parts,
+                        );
+                        futures.spawn(Self::put_part(upload.as_mut(), data));
+
+                        mut_self.state = UploadState::InProgress {
+                            part_idx: 1, // We just used 0
+                            futures,
+                            upload,
+                        };
+                    }
+                    Poll::Ready(Err(e)) => return Err(std::io::Error::other(e)),
+                    Poll::Pending => break,
+                },
+                UploadState::InProgress { futures, .. } => {
+                    while let Poll::Ready(Some(res)) = futures.poll_join_next(cx) {
+                        match res {
+                            Ok(Ok(())) => {}
+                            Err(err) => return Err(std::io::Error::other(err)),
+                            Ok(Err(err)) => return Err(err.into()),
+                        }
+                    }
+                    break;
+                }
+                UploadState::PuttingSingle(fut) | UploadState::Completing(fut) => {
+                    match fut.poll_unpin(cx) {
+                        Poll::Ready(Ok(mut res)) => {
+                            res.size = mut_self.cursor;
+                            mut_self.state = UploadState::Done(res)
+                        }
+                        Poll::Ready(Err(e)) => return Err(std::io::Error::other(e)),
+                        Poll::Pending => break,
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub async fn abort(&mut self) {
+        let state = std::mem::replace(&mut self.state, UploadState::Done(WriteResult::default()));
+        if let UploadState::InProgress { mut upload, .. } = state {
+            let _ = upload.abort().await;
+        }
+    }
+}
+
+impl Drop for ObjectWriter {
+    fn drop(&mut self) {
+        // If there is a multipart upload started but not finished, we should abort it.
+        if matches!(self.state, UploadState::InProgress { .. }) {
+            // Take ownership of the state.
+            let state =
+                std::mem::replace(&mut self.state, UploadState::Done(WriteResult::default()));
+            if let UploadState::InProgress { mut upload, .. } = state
+                && let Ok(handle) = Handle::try_current()
+            {
+                handle.spawn(async move {
+                    let _ = upload.abort().await;
+                });
+            }
+        }
+    }
+}
+
+impl AsyncWrite for ObjectWriter {
+    fn poll_write(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> std::task::Poll<std::result::Result<usize, std::io::Error>> {
+        self.as_mut().poll_tasks(cx)?;
+
+        // Fill buffer up to remaining capacity.
+        let remaining_capacity = self.buffer.capacity() - self.buffer.len();
+        let bytes_to_write = std::cmp::min(remaining_capacity, buf.len());
+        self.buffer.extend_from_slice(&buf[..bytes_to_write]);
+        self.cursor += bytes_to_write;
+
+        // Rust needs a little help to borrow self mutably and immutably at the same time
+        // through a Pin.
+        let mut_self = &mut *self;
+
+        // Instantiate next request, if available.
+        if mut_self.buffer.capacity() == mut_self.buffer.len() {
+            match &mut mut_self.state {
+                UploadState::Started(store) => {
+                    let path = mut_self.path.clone();
+                    let store = store.clone();
+                    let fut = Box::pin(async move { store.put_multipart(path.as_ref()).await });
+                    self.state = UploadState::CreatingUpload(fut);
+                }
+                // TODO: Make max concurrency configurable from storage options.
+                UploadState::InProgress {
+                    upload,
+                    part_idx,
+                    futures,
+                    ..
+                } if futures.len() < max_upload_parallelism() => {
+                    let data = Self::next_part_buffer(
+                        &mut mut_self.buffer,
+                        *part_idx,
+                        mut_self.use_constant_size_upload_parts,
+                    );
+                    futures.spawn(
+                        Self::put_part(upload.as_mut(), data).instrument(tracing::Span::current()),
+                    );
+                    *part_idx += 1;
+                }
+                _ => {}
+            }
+        }
+
+        self.poll_tasks(cx)?;
+
+        match bytes_to_write {
+            0 => Poll::Pending,
+            _ => Poll::Ready(Ok(bytes_to_write)),
+        }
+    }
+
+    fn poll_flush(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+        self.as_mut().poll_tasks(cx)?;
+
+        match &self.state {
+            UploadState::Started(_) | UploadState::Done(_) => Poll::Ready(Ok(())),
+            UploadState::CreatingUpload(_)
+            | UploadState::Completing(_)
+            | UploadState::PuttingSingle(_) => Poll::Pending,
+            UploadState::InProgress { futures, .. } => {
+                if futures.is_empty() {
+                    Poll::Ready(Ok(()))
+                } else {
+                    Poll::Pending
+                }
+            }
+        }
+    }
+
+    fn poll_shutdown(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<std::result::Result<(), std::io::Error>> {
+        loop {
+            self.as_mut().poll_tasks(cx)?;
+
+            // Rust needs a little help to borrow self mutably and immutably at the same time
+            // through a Pin.
+            let mut_self = &mut *self;
+            match &mut mut_self.state {
+                UploadState::Done(_) => return Poll::Ready(Ok(())),
+                UploadState::CreatingUpload(_)
+                | UploadState::PuttingSingle(_)
+                | UploadState::Completing(_) => return Poll::Pending,
+                UploadState::Started(_) => {
+                    // If we didn't start a multipart upload, we can just do a single put.
+                    let part = std::mem::take(&mut mut_self.buffer);
+                    let path = mut_self.path.clone();
+                    self.state.started_to_putting_single(path, part);
+                }
+                UploadState::InProgress {
+                    upload, futures, ..
+                } => {
+                    // Flush final batch
+                    if !mut_self.buffer.is_empty() && futures.len() < max_upload_parallelism() {
+                        // We can just use `take` since we don't need the buffer anymore.
+                        let data = Bytes::from(std::mem::take(&mut mut_self.buffer));
+                        futures.spawn(
+                            Self::put_part(upload.as_mut(), data)
+                                .instrument(tracing::Span::current()),
+                        );
+                        // We need to go back to beginning of loop to poll the
+                        // new feature and get the waker registered on the ctx.
+                        continue;
+                    }
+
+                    // We handle the transition from in progress to completing here.
+                    if futures.is_empty() {
+                        self.state.in_progress_to_completing();
+                    } else {
+                        return Poll::Pending;
+                    }
+                }
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl Writer for ObjectWriter {
+    async fn tell(&mut self) -> Result<usize> {
+        Ok(self.cursor)
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        AsyncWriteExt::shutdown(self).await.map_err(|e| {
+            Error::io(format!(
+                "failed to shutdown object writer for {}: {}",
+                self.path, e
+            ))
+        })?;
+        if let UploadState::Done(result) = &self.state {
+            Ok(result.clone())
+        } else {
+            unreachable!()
+        }
+    }
+}
+
+pub struct LocalWriter {
+    path: Path,
+    state: LocalWriteState,
+}
+
+#[derive(Default)]
+enum LocalWriteState {
+    Writing(Box<WritingState>),
+    Finishing {
+        size: usize,
+        future: BoxFuture<'static, Result<WriteResult>>,
+    },
+    Done(WriteResult),
+    #[default]
+    Poisoned,
+}
+
+struct WritingState {
+    writer: tokio::io::BufWriter<tokio::fs::File>,
+    cursor: usize,
+    /// Temp path that auto-deletes on drop. Set to `None` after `persist()`.
+    temp_path: tempfile::TempPath,
+    io_tracker: Arc<IOTracker>,
+    /// The whole file is reported as a single `put`, so this covers everything
+    /// from opening the file to it being durable under its final path. A writer
+    /// dropped before `persist()` records nothing, like an aborted upload.
+    metrics: IoMetricsGuard,
+}
+
+impl LocalWriter {
+    pub fn new(
+        file: tokio::fs::File,
+        path: Path,
+        temp_path: tempfile::TempPath,
+        io_tracker: Arc<IOTracker>,
+    ) -> Self {
+        Self {
+            path,
+            state: LocalWriteState::Writing(Box::new(WritingState {
+                writer: tokio::io::BufWriter::new(file),
+                cursor: 0,
+                temp_path,
+                metrics: io_tracker.begin_io("put"),
+                io_tracker,
+            })),
+        }
+    }
+
+    fn already_closed_err(path: &Path) -> io::Error {
+        io::Error::other(format!(
+            "cannot write to LocalWriter for {} after shutdown",
+            path
+        ))
+    }
+
+    fn poisoned_err(path: &Path) -> io::Error {
+        io::Error::other(format!("LocalWriter for {} is in poisoned state", path))
+    }
+
+    async fn persist(
+        temp_path: tempfile::TempPath,
+        final_path: Path,
+        size: usize,
+        io_tracker: Arc<IOTracker>,
+        metrics: IoMetricsGuard,
+    ) -> Result<WriteResult> {
+        let local_path = crate::local::to_local_path(&final_path);
+        let persisted = tokio::task::spawn_blocking(move || -> Result<String> {
+            temp_path.persist(&local_path).map_err(|e| {
+                Error::io(format!(
+                    "failed to persist temp file to {}: {}",
+                    local_path, e.error
+                ))
+            })?;
+
+            let metadata = std::fs::metadata(&local_path).map_err(|e| {
+                Error::io(format!("failed to read metadata for {}: {}", local_path, e))
+            })?;
+            Ok(get_etag(&metadata))
+        })
+        .await
+        .map_err(|e| Error::io(format!("spawn_blocking failed: {}", e)))
+        .and_then(|e_tag| e_tag);
+
+        metrics.record(&persisted, size as u64);
+        let e_tag = persisted?;
+
+        io_tracker.record_write("put", final_path, size as u64);
+
+        Ok(WriteResult {
+            size,
+            e_tag: Some(e_tag),
+        })
+    }
+}
+
+impl AsyncWrite for LocalWriter {
+    fn poll_write(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+        buf: &[u8],
+    ) -> Poll<std::result::Result<usize, std::io::Error>> {
+        if let LocalWriteState::Writing(state) = &mut self.state {
+            let poll = Pin::new(&mut state.writer).poll_write(cx, buf);
+            if let Poll::Ready(Ok(n)) = &poll {
+                state.cursor += *n;
+            }
+            poll
+        } else {
+            Poll::Ready(Err(Self::already_closed_err(&self.path)))
+        }
+    }
+
+    fn poll_flush(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<std::result::Result<(), std::io::Error>> {
+        if let LocalWriteState::Writing(state) = &mut self.state {
+            Pin::new(&mut state.writer).poll_flush(cx)
+        } else {
+            Poll::Ready(Err(Self::already_closed_err(&self.path)))
+        }
+    }
+
+    fn poll_shutdown(
+        mut self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<std::result::Result<(), std::io::Error>> {
+        let mut_self = &mut *self;
+        loop {
+            match &mut mut_self.state {
+                LocalWriteState::Writing(state) => {
+                    if Pin::new(&mut state.writer).poll_shutdown(cx).is_pending() {
+                        return Poll::Pending;
+                    }
+
+                    // Write is complete, we can transition to persisting.
+                    let LocalWriteState::Writing(state) =
+                        std::mem::replace(&mut mut_self.state, LocalWriteState::Poisoned)
+                    else {
+                        unreachable!()
+                    };
+                    let size = state.cursor;
+                    mut_self.state = LocalWriteState::Finishing {
+                        size,
+                        future: Box::pin(Self::persist(
+                            state.temp_path,
+                            mut_self.path.clone(),
+                            size,
+                            state.io_tracker,
+                            state.metrics,
+                        )),
+                    };
+                }
+                LocalWriteState::Finishing { future, .. } => match future.poll_unpin(cx) {
+                    Poll::Ready(Ok(result)) => mut_self.state = LocalWriteState::Done(result),
+                    Poll::Ready(Err(e)) => {
+                        return Poll::Ready(Err(io::Error::other(e)));
+                    }
+                    Poll::Pending => return Poll::Pending,
+                },
+                LocalWriteState::Done(_) => return Poll::Ready(Ok(())),
+                LocalWriteState::Poisoned => {
+                    return Poll::Ready(Err(Self::poisoned_err(&self.path)));
+                }
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl Writer for LocalWriter {
+    async fn tell(&mut self) -> Result<usize> {
+        match &mut self.state {
+            LocalWriteState::Writing(state) => Ok(state.cursor),
+            LocalWriteState::Finishing { size, .. } => Ok(*size),
+            LocalWriteState::Done(result) => Ok(result.size),
+            LocalWriteState::Poisoned => Err(Self::poisoned_err(&self.path).into()),
+        }
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        AsyncWriteExt::shutdown(self).await.map_err(|e| {
+            Error::io(format!(
+                "failed to shutdown local writer for {}: {}",
+                self.path, e
+            ))
+        })?;
+
+        match &self.state {
+            LocalWriteState::Done(result) => Ok(result.clone()),
+            _ => unreachable!(),
+        }
+    }
+}
+
+// Based on object store's implementation.
+pub fn get_etag(metadata: &std::fs::Metadata) -> String {
+    let inode = get_inode(metadata);
+    let size = metadata.len();
+    let mtime = metadata
+        .modified()
+        .ok()
+        .and_then(|mtime| mtime.duration_since(std::time::SystemTime::UNIX_EPOCH).ok())
+        .unwrap_or_default()
+        .as_micros();
+
+    // Use an ETag scheme based on that used by many popular HTTP servers
+    // <https://httpd.apache.org/docs/2.2/mod/core.html#fileetag>
+    format!("{inode:x}-{mtime:x}-{size:x}")
+}
+
+#[cfg(unix)]
+fn get_inode(metadata: &std::fs::Metadata) -> u64 {
+    std::os::unix::fs::MetadataExt::ino(metadata)
+}
+
+#[cfg(not(unix))]
+fn get_inode(_metadata: &std::fs::Metadata) -> u64 {
+    0
+}
+
+#[cfg(test)]
+mod tests {
+    use tokio::io::AsyncWriteExt;
+
+    use super::*;
+
+    #[tokio::test]
+    async fn test_write() {
+        let store = LanceObjectStore::memory();
+
+        let mut object_writer = ObjectWriter::new(&store, &Path::from("/foo"))
+            .await
+            .unwrap();
+        assert_eq!(object_writer.tell().await.unwrap(), 0);
+
+        let buf = vec![0; 256];
+        assert_eq!(object_writer.write(buf.as_slice()).await.unwrap(), 256);
+        assert_eq!(object_writer.tell().await.unwrap(), 256);
+
+        assert_eq!(object_writer.write(buf.as_slice()).await.unwrap(), 256);
+        assert_eq!(object_writer.tell().await.unwrap(), 512);
+
+        assert_eq!(object_writer.write(buf.as_slice()).await.unwrap(), 256);
+        assert_eq!(object_writer.tell().await.unwrap(), 256 * 3);
+
+        let res = Writer::shutdown(&mut object_writer).await.unwrap();
+        assert_eq!(res.size, 256 * 3);
+
+        // Trigger multi part upload
+        let mut object_writer = ObjectWriter::new(&store, &Path::from("/bar"))
+            .await
+            .unwrap();
+        let buf = vec![0; INITIAL_UPLOAD_STEP / 3 * 2];
+        for i in 0..5 {
+            // Write more data to trigger the multipart upload
+            // This should be enough to trigger a multipart upload
+            object_writer.write_all(buf.as_slice()).await.unwrap();
+            // Check the cursor
+            assert_eq!(object_writer.tell().await.unwrap(), (i + 1) * buf.len());
+        }
+        let res = Writer::shutdown(&mut object_writer).await.unwrap();
+        assert_eq!(res.size, buf.len() * 5);
+    }
+
+    #[tokio::test]
+    async fn test_abort_write() {
+        let store = LanceObjectStore::memory();
+
+        let mut object_writer = ObjectWriter::new(&store, &Path::from("/foo"))
+            .await
+            .unwrap();
+        object_writer.abort().await;
+    }
+
+    #[tokio::test]
+    async fn test_local_writer_shutdown() {
+        let tmp = lance_core::utils::tempfile::TempStdDir::default();
+        let file_path = tmp.join("test_local_writer.bin");
+        let os_path = Path::from_absolute_path(&file_path).unwrap();
+        let io_tracker = Arc::new(IOTracker::default());
+
+        let named_temp = tempfile::NamedTempFile::new_in(&*tmp).unwrap();
+        let temp_file_path = named_temp.path().to_owned();
+        let (std_file, temp_path) = named_temp.into_parts();
+        let file = tokio::fs::File::from_std(std_file);
+        let mut writer = LocalWriter::new(file, os_path, temp_path, io_tracker.clone());
+
+        let data = b"hello local writer";
+        writer.write_all(data).await.unwrap();
+
+        // Before shutdown, the final path should not exist
+        assert!(!file_path.exists());
+        // But the temp file should exist
+        assert!(temp_file_path.exists());
+
+        let result = Writer::shutdown(&mut writer).await.unwrap();
+        assert_eq!(result.size, data.len());
+        assert!(result.e_tag.is_some());
+        assert!(!result.e_tag.as_ref().unwrap().is_empty());
+
+        // After shutdown, the final path should exist and temp should be gone
+        assert!(file_path.exists());
+        assert!(!temp_file_path.exists());
+
+        let stats = io_tracker.stats();
+        assert_eq!(stats.write_iops, 1);
+        assert_eq!(stats.written_bytes, data.len() as u64);
+    }
+
+    #[tokio::test]
+    async fn test_local_writer_drop_cleans_up() {
+        let tmp = lance_core::utils::tempfile::TempStdDir::default();
+        let file_path = tmp.join("test_drop.bin");
+        let os_path = Path::from_absolute_path(&file_path).unwrap();
+        let io_tracker = Arc::new(IOTracker::default());
+
+        let named_temp = tempfile::NamedTempFile::new_in(&*tmp).unwrap();
+        let temp_file_path = named_temp.path().to_owned();
+        let (std_file, temp_path) = named_temp.into_parts();
+        let file = tokio::fs::File::from_std(std_file);
+        let mut writer = LocalWriter::new(file, os_path, temp_path, io_tracker);
+
+        writer.write_all(b"some data").await.unwrap();
+        assert!(temp_file_path.exists());
+
+        // Drop without shutdown should clean up the temp file
+        drop(writer);
+        assert!(!temp_file_path.exists());
+        assert!(!file_path.exists());
+    }
+
+    #[test]
+    fn clamp_initial_upload_size_below_min_is_clamped_up() {
+        assert_eq!(clamp_initial_upload_size(0), (INITIAL_UPLOAD_STEP, true));
+        assert_eq!(
+            clamp_initial_upload_size(INITIAL_UPLOAD_STEP - 1),
+            (INITIAL_UPLOAD_STEP, true)
+        );
+    }
+
+    #[test]
+    fn clamp_initial_upload_size_within_range_is_unchanged() {
+        assert_eq!(
+            clamp_initial_upload_size(INITIAL_UPLOAD_STEP),
+            (INITIAL_UPLOAD_STEP, false)
+        );
+        assert_eq!(
+            clamp_initial_upload_size(MAX_UPLOAD_PART_SIZE),
+            (MAX_UPLOAD_PART_SIZE, false)
+        );
+        let mid = INITIAL_UPLOAD_STEP * 8; // 40MB, in range
+        assert_eq!(clamp_initial_upload_size(mid), (mid, false));
+    }
+
+    #[test]
+    fn clamp_initial_upload_size_above_max_is_clamped_down() {
+        assert_eq!(
+            clamp_initial_upload_size(MAX_UPLOAD_PART_SIZE + 1),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+        assert_eq!(
+            clamp_initial_upload_size(usize::MAX),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+    }
+
+    /// Regression for the foot-gun where `LANCE_INITIAL_UPLOAD_SIZE=5368709120`
+    /// (exactly 5 GiB, Pucheng's setting) caused a single-PUT of 5 GiB on
+    /// shutdown — which S3 rejects with `EntityTooLarge`. After tightening
+    /// `MAX_UPLOAD_PART_SIZE` to 5 GiB - 1, raw 5 GiB must clamp DOWN.
+    #[test]
+    fn clamp_initial_upload_size_at_5gib_clamps_down() {
+        let exactly_5_gib: usize = 5 * 1024 * 1024 * 1024;
+        assert_eq!(
+            clamp_initial_upload_size(exactly_5_gib),
+            (MAX_UPLOAD_PART_SIZE, true)
+        );
+    }
+}

vendor/lance-io/src/scheduler/lite.rs

@@ -0,0 +1,955 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! A lightweight I/O scheduler primarily intended for use with I/O uring.
+//!
+//! This scheduler attempts to avoid any kind of task switching whenever possible
+//! to minimize context switching overhead.
+//!
+//! There are a few limitations compared to the standard scheduler:
+//!
+//! * There is no concurrency limit.  The scheduler will allow as many IOPS to run
+//!   as possible as long as the backpressure throttle is not exceeded.
+//! * There is no "babysitting" of IOPS.  An I/O task will only be polled when its
+//!   future is polled.  The standard scheduler will `spawn` I/O tasks and so they
+//!   are always polled by tokio's runtime.  This is important for operations like
+//!   cloud requests where intermittent polling is required to clear out network
+//!   buffers and keep the TCP connection moving.
+
+use std::{
+    collections::{BinaryHeap, HashMap},
+    fmt::Debug,
+    future::Future,
+    ops::Range,
+    pin::Pin,
+    sync::{
+        Arc, Mutex, MutexGuard,
+        atomic::{AtomicU64, Ordering},
+    },
+    task::{Context, Poll, Waker},
+    time::Instant,
+};
+
+use bytes::Bytes;
+use lance_core::{Error, Result};
+
+use super::{
+    BACKPRESSURE_DEBOUNCE, BACKPRESSURE_MIN, IoStats, SCHEDULER_STATE_EVENT_TARGET,
+    SchedulerStateEvent, emit_scheduler_state_event,
+};
+
+type RunFn = Box<dyn FnOnce() -> Pin<Box<dyn Future<Output = Result<Bytes>> + Send>> + Send>;
+
+/// The state of an I/O task
+///
+/// The state machine is as follows:
+///
+/// * `Broken` - The task is in an error state and cannot be run, should never happen
+/// * `Initial` - The task has been submitted but does not have a backpressure reservation
+/// * `Reserved` - The task has a backpressure reservation
+/// * `Running` - The task is running and has a future to poll
+/// * `Finished` - The task has finished and has a result
+enum TaskState {
+    Broken,
+    Initial {
+        idle_waker: Option<Waker>,
+        run_fn: RunFn,
+    },
+    Reserved {
+        idle_waker: Option<Waker>,
+        backpressure_reservation: BackpressureReservation,
+        run_fn: RunFn,
+    },
+    Running {
+        backpressure_reservation: BackpressureReservation,
+        inner: Pin<Box<dyn Future<Output = Result<Bytes>> + Send>>,
+    },
+    Finished {
+        backpressure_reservation: BackpressureReservation,
+        data: Result<Bytes>,
+    },
+}
+
+impl TaskState {
+    fn backpressure_reservation(&self) -> Option<BackpressureReservation> {
+        match self {
+            Self::Reserved {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Running {
+                backpressure_reservation,
+                ..
+            }
+            | Self::Finished {
+                backpressure_reservation,
+                ..
+            } => Some(*backpressure_reservation),
+            Self::Initial { .. } | Self::Broken => None,
+        }
+    }
+}
+
+/// A custom error type that might have a backpressure reservation
+///
+/// This is used instead of Lance's standard error type so we can ensure
+/// we release the reservation before returning the error.
+struct BrokenTaskError {
+    message: String,
+    backpressure_reservation: Option<BackpressureReservation>,
+}
+
+/// The result type corresponding to BrokenTaskError
+type TaskResult = std::result::Result<(), BrokenTaskError>;
+
+impl BrokenTaskError {
+    // Create a BrokenTaskError from a task state
+    //
+    // This will capture any backpressure reservation the task has and put it into the
+    // error so we make sure to release it when returning the error.
+    fn new(task_state: TaskState, message: String) -> Self {
+        match task_state.backpressure_reservation() {
+            None => Self {
+                message,
+                backpressure_reservation: None,
+            },
+            Some(reservation) => Self {
+                message,
+                backpressure_reservation: Some(reservation),
+            },
+        }
+    }
+}
+
+/// An I/O task represents a single read operation
+struct IoTask {
+    /// The unique identifier of the task (only used for debugging)
+    id: u64,
+    /// The number of bytes to read
+    num_bytes: u64,
+    /// The priority of the task, lower values are higher priority
+    priority: u128,
+    /// The current state of the task
+    state: TaskState,
+    /// When true, the task bypasses backpressure
+    bypass_backpressure: bool,
+}
+
+impl IoTask {
+    fn is_reserved(&self) -> bool {
+        !matches!(self.state, TaskState::Initial { .. })
+    }
+
+    fn cancel(&mut self) -> bool {
+        let was_running = matches!(self.state, TaskState::Running { .. });
+        self.state = TaskState::Finished {
+            backpressure_reservation: BackpressureReservation {
+                num_bytes: 0,
+                priority: 0,
+            },
+            data: Err(Error::io_source(Box::new(Error::io_source(
+                "I/O Task cancelled".to_string().into(),
+            )))),
+        };
+        was_running
+    }
+
+    fn reserve(&mut self, backpressure_reservation: BackpressureReservation) -> TaskResult {
+        let state = std::mem::replace(&mut self.state, TaskState::Broken);
+        let TaskState::Initial { idle_waker, run_fn } = state else {
+            return Err(BrokenTaskError::new(
+                state,
+                format!("Task with id {} not in initial state", self.id),
+            ));
+        };
+        self.state = TaskState::Reserved {
+            idle_waker,
+            backpressure_reservation,
+            run_fn,
+        };
+        Ok(())
+    }
+
+    fn start(&mut self) -> TaskResult {
+        let state = std::mem::replace(&mut self.state, TaskState::Broken);
+        let TaskState::Reserved {
+            backpressure_reservation,
+            idle_waker,
+            run_fn,
+        } = state
+        else {
+            return Err(BrokenTaskError::new(
+                state,
+                format!("Task with id {} not in reserved state", self.id),
+            ));
+        };
+        let inner = run_fn();
+        self.state = TaskState::Running {
+            backpressure_reservation,
+            inner,
+        };
+
+        // If someone is already waiting for this task let them know it is now running
+        // so they can poll it
+        if let Some(idle_waker) = idle_waker {
+            idle_waker.wake();
+        }
+        Ok(())
+    }
+
+    fn poll(&mut self, cx: &mut Context<'_>) -> Poll<()> {
+        match &mut self.state {
+            TaskState::Broken => Poll::Ready(()),
+            TaskState::Initial { idle_waker, .. } | TaskState::Reserved { idle_waker, .. } => {
+                idle_waker.replace(cx.waker().clone());
+                Poll::Pending
+            }
+            TaskState::Running {
+                inner,
+                backpressure_reservation,
+            } => match inner.as_mut().poll(cx) {
+                Poll::Ready(data) => {
+                    self.state = TaskState::Finished {
+                        data,
+                        backpressure_reservation: *backpressure_reservation,
+                    };
+                    Poll::Ready(())
+                }
+                Poll::Pending => Poll::Pending,
+            },
+            TaskState::Finished { .. } => Poll::Ready(()),
+        }
+    }
+
+    fn consume(self) -> Result<(Result<Bytes>, BackpressureReservation)> {
+        let TaskState::Finished {
+            data,
+            backpressure_reservation,
+        } = self.state
+        else {
+            return Err(Error::internal(format!(
+                "Task with id {} not in finished state",
+                self.id
+            )));
+        };
+        Ok((data, backpressure_reservation))
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BackpressureReservation {
+    num_bytes: u64,
+    priority: u128,
+}
+
+/// A throttle to control how many bytes can be read before we pause to let compute catch up
+trait BackpressureThrottle: Send {
+    fn try_acquire(&mut self, num_bytes: u64, priority: u128) -> Option<BackpressureReservation>;
+    fn release(&mut self, reservation: BackpressureReservation);
+    /// Unconditionally acquire a zero-cost reservation, tracking only the priority.
+    /// Used for bypass tasks that must never be blocked by backpressure.
+    fn force_acquire(&mut self, priority: u128) -> BackpressureReservation;
+    fn state(&self) -> BackpressureState;
+}
+
+// We want to allow requests that have a lower priority than any
+// currently in-flight request.  This helps avoid potential deadlocks
+// related to backpressure.  Unfortunately, it is quite expensive to
+// keep track of which priorities are in-flight.
+//
+// TODO: At some point it would be nice if we can optimize this away but
+// in_flight should remain relatively small (generally less than 256 items)
+// and has not shown itself to be a bottleneck yet.
+struct PrioritiesInFlight {
+    in_flight: Vec<u128>,
+}
+
+impl PrioritiesInFlight {
+    fn new(capacity: u64) -> Self {
+        Self {
+            in_flight: Vec::with_capacity(capacity as usize * 2),
+        }
+    }
+
+    fn min_in_flight(&self) -> u128 {
+        self.in_flight.first().copied().unwrap_or(u128::MAX)
+    }
+
+    fn contains(&self, prio: u128) -> bool {
+        self.in_flight.binary_search(&prio).is_ok()
+    }
+
+    fn push(&mut self, prio: u128) {
+        let pos = match self.in_flight.binary_search(&prio) {
+            Ok(pos) => pos,
+            Err(pos) => pos,
+        };
+        self.in_flight.insert(pos, prio);
+    }
+
+    fn remove(&mut self, prio: u128) {
+        if let Ok(pos) = self.in_flight.binary_search(&prio) {
+            self.in_flight.remove(pos);
+        }
+    }
+
+    fn len(&self) -> usize {
+        self.in_flight.len()
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+struct BackpressureState {
+    max_bytes: u64,
+    bytes_available: i64,
+    priorities_in_flight: u64,
+    no_backpressure: bool,
+}
+
+struct SimpleBackpressureThrottle {
+    max_bytes: u64,
+    start: Instant,
+    last_warn: AtomicU64,
+    bytes_available: i64,
+    priorities_in_flight: PrioritiesInFlight,
+    // When true, skip all byte-based backpressure checks (set when max_bytes == 0)
+    no_backpressure: bool,
+}
+
+impl SimpleBackpressureThrottle {
+    fn new(max_bytes: u64, max_concurrency: u64) -> Self {
+        if max_bytes > i64::MAX as u64 {
+            // This is unlikely to ever be an issue
+            panic!("Max bytes must be less than {}", i64::MAX);
+        }
+        Self {
+            max_bytes,
+            start: Instant::now(),
+            last_warn: AtomicU64::new(0),
+            bytes_available: max_bytes as i64,
+            priorities_in_flight: PrioritiesInFlight::new(max_concurrency),
+            no_backpressure: max_bytes == 0,
+        }
+    }
+
+    fn warn_if_needed(&self) {
+        let seconds_elapsed = self.start.elapsed().as_secs();
+        let last_warn = self.last_warn.load(Ordering::Acquire);
+        let since_last_warn = seconds_elapsed - last_warn;
+        if (last_warn == 0
+            && seconds_elapsed > BACKPRESSURE_MIN
+            && seconds_elapsed < BACKPRESSURE_DEBOUNCE)
+            || since_last_warn > BACKPRESSURE_DEBOUNCE
+        {
+            tracing::event!(tracing::Level::DEBUG, "Backpressure throttle exceeded");
+            log::debug!(
+                "Backpressure throttle is full, I/O will pause until buffer is drained.  Max I/O bandwidth will not be achieved because CPU is falling behind"
+            );
+            self.last_warn
+                .store(seconds_elapsed.max(1), Ordering::Release);
+        }
+    }
+}
+
+impl BackpressureThrottle for SimpleBackpressureThrottle {
+    fn try_acquire(&mut self, num_bytes: u64, priority: u128) -> Option<BackpressureReservation> {
+        if self.no_backpressure
+            || self.bytes_available >= num_bytes as i64
+            || self.priorities_in_flight.min_in_flight() >= priority
+            // Chunks from an admitted logical request must keep moving.  A
+            // higher-priority request may be scheduled later and remain
+            // unconsumed while the caller awaits this request.
+            || self.priorities_in_flight.contains(priority)
+        {
+            self.bytes_available -= num_bytes as i64;
+            self.priorities_in_flight.push(priority);
+            Some(BackpressureReservation {
+                num_bytes,
+                priority,
+            })
+        } else {
+            self.warn_if_needed();
+            None
+        }
+    }
+
+    fn release(&mut self, reservation: BackpressureReservation) {
+        self.bytes_available += reservation.num_bytes as i64;
+        self.priorities_in_flight.remove(reservation.priority);
+    }
+
+    fn force_acquire(&mut self, priority: u128) -> BackpressureReservation {
+        self.priorities_in_flight.push(priority);
+        BackpressureReservation {
+            num_bytes: 0,
+            priority,
+        }
+    }
+
+    fn state(&self) -> BackpressureState {
+        BackpressureState {
+            max_bytes: self.max_bytes,
+            bytes_available: self.bytes_available,
+            priorities_in_flight: self.priorities_in_flight.len() as u64,
+            no_backpressure: self.no_backpressure,
+        }
+    }
+}
+
+struct TaskEntry {
+    task_id: u64,
+    priority: u128,
+    reserved: bool,
+}
+
+impl Ord for TaskEntry {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        // Prefer reserved tasks over unreserved tasks and then highest priority tasks over lowest
+        // priority tasks.
+        //
+        // This is a max-heap so we sort by reserved in normal order (true > false) and priority
+        // in reverse order (lowest priority first)
+        self.reserved
+            .cmp(&other.reserved)
+            .then(other.priority.cmp(&self.priority))
+    }
+}
+
+impl PartialOrd for TaskEntry {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+
+impl PartialEq for TaskEntry {
+    fn eq(&self, other: &Self) -> bool {
+        self.priority == other.priority
+    }
+}
+
+impl Eq for TaskEntry {}
+
+struct IoQueueState {
+    backpressure_throttle: Box<dyn BackpressureThrottle>,
+    pending_tasks: BinaryHeap<TaskEntry>,
+    tasks: HashMap<u64, IoTask>,
+    next_task_id: u64,
+}
+
+impl IoQueueState {
+    fn new(max_concurrency: u64, max_bytes: u64) -> Self {
+        Self {
+            backpressure_throttle: Box::new(SimpleBackpressureThrottle::new(
+                max_bytes,
+                max_concurrency,
+            )),
+            pending_tasks: BinaryHeap::new(),
+            tasks: HashMap::new(),
+            next_task_id: 0,
+        }
+    }
+
+    // If a task is in an unexpected state then we need to release any reservations that were made
+    // before we return an error.
+    //
+    // Note: this is perhaps a bit paranoid as a task should never be in an unexpected state.
+    fn handle_result(&mut self, result: TaskResult) -> Result<()> {
+        if let Err(error) = result {
+            if let Some(reservation) = error.backpressure_reservation {
+                self.backpressure_throttle.release(reservation);
+            }
+            Err(Error::internal(error.message))
+        } else {
+            Ok(())
+        }
+    }
+
+    fn scheduler_state_event(&self) -> Option<SchedulerStateEvent> {
+        if !tracing::enabled!(target: SCHEDULER_STATE_EVENT_TARGET, tracing::Level::TRACE) {
+            return None;
+        }
+
+        let backpressure = self.backpressure_throttle.state();
+        let pending_bytes = self
+            .pending_tasks
+            .iter()
+            .filter_map(|entry| self.tasks.get(&entry.task_id))
+            .map(|task| task.num_bytes)
+            .sum::<u64>();
+        let active_iops = self
+            .tasks
+            .values()
+            .filter(|task| matches!(task.state, TaskState::Running { .. }))
+            .count() as u64;
+
+        Some(SchedulerStateEvent {
+            queue_kind: "lite",
+            io_capacity: 0,
+            iops_available: 0,
+            active_iops,
+            pending_iops: self.pending_tasks.len() as u64,
+            pending_bytes,
+            bytes_available: backpressure.bytes_available,
+            bytes_reserved: backpressure.max_bytes as i64 - backpressure.bytes_available,
+            io_buffer_size_bytes: backpressure.max_bytes,
+            priorities_in_flight: backpressure.priorities_in_flight,
+            no_backpressure: backpressure.no_backpressure,
+            head_task_bytes: None,
+            head_task_priority_high: None,
+            head_task_priority_low: None,
+            min_in_flight_priority_high: None,
+            min_in_flight_priority_low: None,
+            head_task_can_deliver: None,
+            head_task_priority_bypass: None,
+            head_task_blocked_by_iops: None,
+            head_task_blocked_by_bytes: None,
+        })
+    }
+}
+
+/// A queue of I/O tasks to be shared between the I/O scheduler and the I/O decoder.
+///
+/// The queue is protected by two different throttles.  The first controls memory backpressure, and
+/// will only allow a certain number of bytes to be allocated for reads.  This throttle is released
+/// as soon as the decoder consumes the bytes (not when the bytes have been fully processed).  This
+/// throttle is currently scoped to the scheduler and not shared across the process.  This will likely
+/// change in the future.
+///
+/// The second throttle controls how many IOPS can be issued concurrently.  This throttle is released
+/// as soon as the IOP is finished.  This throttle has both a local per-scheduler limit and also a
+/// process-wide limit.
+///
+/// Note: unlike the standard scheduler, there is no dedicated I/O loop thread.  If the decoder is not
+/// polling the I/O tasks then nothing else will.  This scheduler is currently intended for use with I/O
+/// uring where I/O tasks are bunched together and polling one task advances all outstanding I/O.  It
+/// would not be suitable for cloud storage where each task is an independent HTTP request and needs to
+/// be polled individually (though presumably one could use I/O uring for networked cloud storage some
+/// day as well)
+pub(super) struct IoQueue {
+    state: Arc<Mutex<IoQueueState>>,
+    stats: IoStats,
+}
+
+impl IoQueue {
+    pub fn new(max_concurrency: u64, max_bytes: u64, stats: IoStats) -> Self {
+        Self {
+            state: Arc::new(Mutex::new(IoQueueState::new(max_concurrency, max_bytes))),
+            stats,
+        }
+    }
+
+    fn push(&self, mut task: IoTask, mut state: MutexGuard<IoQueueState>) -> Result<()> {
+        let task_id = task.id;
+        let maybe_reservation = if task.bypass_backpressure {
+            Some(state.backpressure_throttle.force_acquire(task.priority))
+        } else {
+            state
+                .backpressure_throttle
+                .try_acquire(task.num_bytes, task.priority)
+        };
+        if let Some(reservation) = maybe_reservation {
+            state.handle_result(task.reserve(reservation))?;
+            state.handle_result(task.start())?;
+            state.tasks.insert(task_id, task);
+            let event = state.scheduler_state_event();
+            drop(state);
+            emit_scheduler_state_event(event, &self.stats);
+            return Ok(());
+        }
+
+        state.pending_tasks.push(TaskEntry {
+            task_id,
+            priority: task.priority,
+            reserved: task.is_reserved(),
+        });
+        state.tasks.insert(task_id, task);
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
+        Ok(())
+    }
+
+    pub(super) fn submit(
+        self: Arc<Self>,
+        range: Range<u64>,
+        priority: u128,
+        run_fn: RunFn,
+        bypass_backpressure: bool,
+    ) -> Result<TaskHandle> {
+        log::trace!(
+            "Submitting I/O task with range {:?}, priority {:?}",
+            range,
+            priority
+        );
+        let mut state = self.state.lock().unwrap();
+        let task_id = state.next_task_id;
+        state.next_task_id += 1;
+
+        let task = IoTask {
+            id: task_id,
+            num_bytes: range.end - range.start,
+            priority,
+            bypass_backpressure,
+            state: TaskState::Initial {
+                idle_waker: None,
+                run_fn,
+            },
+        };
+        self.push(task, state)?;
+        Ok(TaskHandle {
+            task_id,
+            queue: self,
+        })
+    }
+
+    // When a task completes we should check to see if any other tasks are now runnable
+    fn on_task_complete(&self, mut state: MutexGuard<IoQueueState>) -> Result<()> {
+        let result = {
+            let state_ref = &mut *state;
+            let mut task_result = TaskResult::Ok(());
+            while !state_ref.pending_tasks.is_empty() {
+                // Unwrap safe here since we just checked the queue is not empty
+                let task_id = state_ref.pending_tasks.peek().unwrap().task_id;
+                let Some(task) = state_ref.tasks.get_mut(&task_id) else {
+                    // The caller dropped this task's handle (see `abandon`); discard the
+                    // stale queue entry instead of spinning on it.
+                    state_ref.pending_tasks.pop();
+                    continue;
+                };
+                if !task.is_reserved() {
+                    let Some(reservation) = state_ref
+                        .backpressure_throttle
+                        .try_acquire(task.num_bytes, task.priority)
+                    else {
+                        break;
+                    };
+                    if let Err(e) = task.reserve(reservation) {
+                        task_result = Err(e);
+                        break;
+                    }
+                }
+                state_ref.pending_tasks.pop();
+                if let Err(e) = task.start() {
+                    task_result = Err(e);
+                    break;
+                }
+            }
+            state_ref.handle_result(task_result)
+        };
+        let event = state.scheduler_state_event();
+        drop(state);
+        emit_scheduler_state_event(event, &self.stats);
+        result
+    }
+
+    fn poll(&self, task_id: u64, cx: &mut Context<'_>) -> Poll<Result<Bytes>> {
+        let mut state = self.state.lock().unwrap();
+        let Some(task) = state.tasks.get_mut(&task_id) else {
+            // This should never happen and indicates a bug
+            return Poll::Ready(Err(Error::internal(format!(
+                "Task with id {} was lost",
+                task_id
+            ))));
+        };
+        match task.poll(cx) {
+            Poll::Ready(_) => {
+                let task = state.tasks.remove(&task_id).unwrap();
+                let (bytes, reservation) = task.consume()?;
+                state.backpressure_throttle.release(reservation);
+                // We run on_task_complete even if not newly finished because we released the backpressure reservation
+                match self.on_task_complete(state) {
+                    Ok(_) => Poll::Ready(bytes),
+                    Err(e) => Poll::Ready(Err(e)),
+                }
+            }
+            Poll::Pending => Poll::Pending,
+        }
+    }
+
+    pub(super) fn close(&self) {
+        let event = {
+            let mut state = self.state.lock().unwrap();
+            for task in std::mem::take(&mut state.tasks).values_mut() {
+                task.cancel();
+            }
+            state.scheduler_state_event()
+        };
+        emit_scheduler_state_event(event, &self.stats);
+    }
+
+    // Called when a caller drops a task's handle before the task finishes.  Removes
+    // the task and returns any backpressure reservation it holds to the budget, then
+    // re-checks the queue so newly-affordable tasks can start.  Unlike the standard
+    // release path (`poll`), this runs without the task being polled to completion,
+    // so a cancelled read does not leak its reservation.
+    fn abandon(&self, task_id: u64) {
+        let mut state = self.state.lock().unwrap();
+        let Some(task) = state.tasks.remove(&task_id) else {
+            // Already consumed by `poll`; nothing to release.
+            return;
+        };
+
+        if let Some(reservation) = task.state.backpressure_reservation() {
+            state.backpressure_throttle.release(reservation);
+        }
+        // Freed budget may make queued tasks runnable; there is no caller to surface
+        // an error to here.
+        let _ = self.on_task_complete(state);
+    }
+}
+
+pub(super) struct TaskHandle {
+    task_id: u64,
+    queue: Arc<IoQueue>,
+}
+
+impl Future for TaskHandle {
+    type Output = Result<Bytes>;
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        self.queue.poll(self.task_id, cx)
+    }
+}
+
+impl Drop for TaskHandle {
+    fn drop(&mut self) {
+        self.queue.abandon(self.task_id);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tokio::sync::oneshot;
+
+    #[tokio::test]
+    async fn test_priority_ordering() {
+        // Backpressure budget of 10 bytes: only one 10-byte task runs at a time.
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
+
+        // Records the priority of each task when its run_fn is invoked (i.e. when
+        // the task transitions to Running).
+        let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
+
+        // Helper: builds a RunFn that records `prio` in start_order and then
+        // waits on the oneshot receiver for its result bytes.
+        let make_run_fn =
+            |prio: u128, rx: oneshot::Receiver<Bytes>, order: Arc<Mutex<Vec<u128>>>| -> RunFn {
+                Box::new(move || {
+                    order.lock().unwrap().push(prio);
+                    Box::pin(async move { Ok(rx.await.unwrap()) })
+                })
+            };
+
+        // Submit a blocker task (priority 0, 10 bytes).
+        // It starts immediately because there is enough backpressure budget.
+        let (blocker_tx, blocker_rx) = oneshot::channel();
+        let blocker = queue
+            .clone()
+            .submit(
+                0..10,
+                0,
+                make_run_fn(0, blocker_rx, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        // Submit four tasks with out-of-order priorities.
+        // All are queued because the blocker consumed the full budget.
+        let (tx_30, rx_30) = oneshot::channel();
+        let h30 = queue
+            .clone()
+            .submit(
+                0..10,
+                30,
+                make_run_fn(30, rx_30, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        let (tx_10, rx_10) = oneshot::channel();
+        let h10 = queue
+            .clone()
+            .submit(
+                0..10,
+                10,
+                make_run_fn(10, rx_10, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        let (tx_50, rx_50) = oneshot::channel();
+        let h50 = queue
+            .clone()
+            .submit(
+                0..10,
+                50,
+                make_run_fn(50, rx_50, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        let (tx_20, rx_20) = oneshot::channel();
+        let h20 = queue
+            .clone()
+            .submit(
+                0..10,
+                20,
+                make_run_fn(20, rx_20, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        // Only the blocker has started so far.
+        assert_eq!(*start_order.lock().unwrap(), vec![0]);
+
+        // Complete the blocker -> frees budget -> starts priority 10 (lowest value = highest priority).
+        blocker_tx.send(Bytes::from_static(b"x")).unwrap();
+        blocker.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 10]);
+
+        // Complete priority 10 -> starts priority 20.
+        tx_10.send(Bytes::from_static(b"x")).unwrap();
+        h10.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 10, 20]);
+
+        // Complete priority 20 -> starts priority 30.
+        tx_20.send(Bytes::from_static(b"x")).unwrap();
+        h20.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 10, 20, 30]);
+
+        // Complete priority 30 -> starts priority 50.
+        tx_30.send(Bytes::from_static(b"x")).unwrap();
+        h30.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 10, 20, 30, 50]);
+
+        // Complete priority 50 -> no more pending tasks.
+        tx_50.send(Bytes::from_static(b"x")).unwrap();
+        h50.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 10, 20, 30, 50]);
+    }
+
+    #[tokio::test]
+    async fn test_zero_buffer_bypasses_backpressure() {
+        // Budget = 0 sets no_backpressure = true, so all tasks start immediately
+        // regardless of how many bytes are "outstanding".
+        let queue = Arc::new(IoQueue::new(128, 0, IoStats::default()));
+        let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
+
+        let make_run_fn =
+            |prio: u128, rx: oneshot::Receiver<Bytes>, order: Arc<Mutex<Vec<u128>>>| -> RunFn {
+                Box::new(move || {
+                    order.lock().unwrap().push(prio);
+                    Box::pin(async move { Ok(rx.await.unwrap()) })
+                })
+            };
+
+        let (tx0, rx0) = oneshot::channel();
+        let h0 = queue
+            .clone()
+            .submit(0..10, 0, make_run_fn(0, rx0, start_order.clone()), false)
+            .unwrap();
+        let (tx1, rx1) = oneshot::channel();
+        let h1 = queue
+            .clone()
+            .submit(0..10, 1, make_run_fn(1, rx1, start_order.clone()), false)
+            .unwrap();
+        let (tx2, rx2) = oneshot::channel();
+        let h2 = queue
+            .clone()
+            .submit(0..10, 2, make_run_fn(2, rx2, start_order.clone()), false)
+            .unwrap();
+
+        // All three tasks start immediately — no backpressure budget check when max_bytes=0.
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 1, 2]);
+
+        tx0.send(Bytes::from_static(b"done")).unwrap();
+        tx1.send(Bytes::from_static(b"done")).unwrap();
+        tx2.send(Bytes::from_static(b"done")).unwrap();
+        h0.await.unwrap();
+        h1.await.unwrap();
+        h2.await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_bypass_flag_proceeds_past_exhausted_budget() {
+        // Budget of 10 bytes. A blocker task fills it. A task with bypass=true starts
+        // immediately despite the exhausted budget; a normal task stays queued.
+        let queue = Arc::new(IoQueue::new(128, 10, IoStats::default()));
+        let start_order: Arc<Mutex<Vec<u128>>> = Arc::new(Mutex::new(Vec::new()));
+
+        let make_run_fn =
+            |prio: u128, rx: oneshot::Receiver<Bytes>, order: Arc<Mutex<Vec<u128>>>| -> RunFn {
+                Box::new(move || {
+                    order.lock().unwrap().push(prio);
+                    Box::pin(async move { Ok(rx.await.unwrap()) })
+                })
+            };
+
+        // Blocker (priority 0, 10 bytes): fills the budget.
+        let (blocker_tx, blocker_rx) = oneshot::channel();
+        let blocker = queue
+            .clone()
+            .submit(
+                0..10,
+                0,
+                make_run_fn(0, blocker_rx, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        // Normal (priority 1, 10 bytes): blocked — budget exhausted, no priority bypass.
+        let (normal_tx, normal_rx) = oneshot::channel();
+        let normal = queue
+            .clone()
+            .submit(
+                0..10,
+                1,
+                make_run_fn(1, normal_rx, start_order.clone()),
+                false,
+            )
+            .unwrap();
+
+        // Bypass (priority 2, 10 bytes): starts immediately via force_acquire.
+        let (bypass_tx, bypass_rx) = oneshot::channel();
+        let bypass = queue
+            .clone()
+            .submit(
+                0..10,
+                2,
+                make_run_fn(2, bypass_rx, start_order.clone()),
+                true,
+            )
+            .unwrap();
+
+        // Blocker (0) and bypass (2) have started; normal (1) is still queued.
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 2]);
+
+        // Completing the blocker frees the budget and unblocks the normal task.
+        blocker_tx.send(Bytes::from_static(b"done")).unwrap();
+        blocker.await.unwrap();
+        assert_eq!(*start_order.lock().unwrap(), vec![0, 2, 1]);
+
+        bypass_tx.send(Bytes::from_static(b"done")).unwrap();
+        bypass.await.unwrap();
+        normal_tx.send(Bytes::from_static(b"done")).unwrap();
+        normal.await.unwrap();
+    }
+
+    #[test]
+    fn test_same_priority_reservation_continues_after_higher_priority() {
+        let mut throttle = SimpleBackpressureThrottle::new(10, 128);
+
+        let low_priority_first = throttle.try_acquire(6, 10).unwrap();
+        let high_priority = throttle.try_acquire(4, 0).unwrap();
+        let low_priority_next = throttle.try_acquire(6, 10);
+
+        assert!(
+            low_priority_next.is_some(),
+            "chunks from an already admitted logical request should continue"
+        );
+
+        throttle.release(low_priority_first);
+        throttle.release(high_priority);
+        throttle.release(low_priority_next.unwrap());
+    }
+}

vendor/lance-io/src/spill.rs

@@ -0,0 +1,543 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Reclaimable scratch storage.
+//!
+//! A [`SpillStore`] hands out scratch space for temporary state that is too
+//! large to keep in memory and is read back later in the same process (for
+//! example, posting lists or shuffle runs accumulated while building an index).
+//! The backing storage is reclaimed automatically when the handle is dropped.
+//!
+//! [`SpillStore::new_spill`] returns a [`Writer`] paired with a [`Spill`]
+//! handle: the writer is the byte sink (feed it to `FileWriter::try_new`, or
+//! write to it directly); the [`Spill`] reads the bytes back (via
+//! [`crate::scheduler::ScanScheduler::open_reader`] for a v2 `FileReader`) and
+//! owns the file's lifetime.
+//!
+//! # Lifecycle
+//!
+//! - **Write-once.** The only way to obtain a writer is `new_spill`, and each
+//!   call allocates a fresh unit of storage, so a single spill cannot be
+//!   written twice — there is no second-writer path to guard against.
+//! - **Write-before-read.** [`Spill::reader`] fails until the writer has been
+//!   shut down, so partially written bytes are never read back.
+//! - **RAII.** Dropping the [`Spill`] deletes the file and releases its bytes
+//!   back to the store's disk budget. The store's temp directory is the
+//!   backstop for anything leaked if a handle is forgotten.
+//!
+//! # Disk cap
+//!
+//! [`LocalSpillStore::with_cap`] enforces a byte budget shared across all live
+//! handles, returning a typed [`lance_core::Error::DiskCapExceeded`] rather than
+//! silently filling the disk. Accounting is reserve-on-write + release-on-drop
+//! (by stat), which is exact for the write-once contract. Two minor
+//! inexactnesses are not engineered around: a write aborted at the cap leaks its
+//! reservation until the store is dropped, and a file whose size cannot be
+//! stat-ed on drop is not released.
+
+use std::io;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use std::task::{Context, Poll};
+
+use async_trait::async_trait;
+use object_store::path::Path;
+use tokio::io::AsyncWrite;
+
+use lance_core::{Error, Result};
+
+use crate::object_store::ObjectStore;
+use crate::object_writer::WriteResult;
+use crate::traits::{Reader, Writer};
+
+/// A factory for scratch storage.
+///
+/// The trait is object-safe and `Send + Sync` so it can be held behind an
+/// `Arc<dyn SpillStore>` (e.g. inside a `Session`). Implementations need not be
+/// backed by local files (e.g. in-memory buffers, remote object stores).
+#[async_trait]
+pub trait SpillStore: Send + Sync + 'static {
+    /// Allocate a unit of scratch storage.
+    ///
+    /// Returns the byte sink to write it with and a [`Spill`] handle to read it
+    /// back. For a capped store, writes that would exceed the cap fail with
+    /// [`lance_core::Error::DiskCapExceeded`]. The storage is reclaimed when the
+    /// [`Spill`] is dropped.
+    async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)>;
+}
+
+/// The readable half of a spill, and the owner of its backing storage.
+///
+/// Dropping it reclaims the storage. The trait is object-safe so it can be
+/// returned as `Box<dyn Spill>` from [`SpillStore::new_spill`].
+#[async_trait]
+pub trait Spill: Send + Sync {
+    /// Open a reader over the spilled bytes.
+    ///
+    /// Fails until the paired writer has been shut down, since the bytes are not
+    /// complete before then.
+    async fn reader(&self) -> Result<Box<dyn Reader>>;
+}
+
+/// A shared, cloneable byte budget.
+///
+/// Cloning produces another handle to the *same* underlying counter, so a quota
+/// shared across many writers enforces a single combined cap.
+#[derive(Debug, Clone)]
+struct DiskQuota {
+    cap_bytes: u64,
+    used: Arc<Mutex<u64>>,
+}
+
+impl DiskQuota {
+    fn new(cap_bytes: u64) -> Self {
+        Self {
+            cap_bytes,
+            used: Arc::new(Mutex::new(0)),
+        }
+    }
+
+    /// Try to reserve `n` bytes, failing with [`Error::DiskCapExceeded`] if the
+    /// reservation would push total usage past the cap.
+    fn try_reserve(&self, n: u64) -> Result<()> {
+        // The lock is held only for a couple of arithmetic ops and never across
+        // an `.await`, so a std `Mutex` is the simplest correct choice.
+        let mut used = self.used.lock().unwrap();
+        let next = used.saturating_add(n);
+        if next > self.cap_bytes {
+            return Err(Error::disk_cap_exceeded(self.cap_bytes, *used));
+        }
+        *used = next;
+        Ok(())
+    }
+
+    /// Release `n` previously reserved bytes back to the budget.
+    fn release(&self, n: u64) {
+        // Saturating sub keeps a stray double-release from underflowing.
+        let mut used = self.used.lock().unwrap();
+        *used = used.saturating_sub(n);
+    }
+}
+
+/// The byte sink handed out by [`SpillStore::new_spill`].
+///
+/// It optionally reserves a [`DiskQuota`] as bytes are written (keeping cap
+/// enforcement inside the spill store rather than in [`ObjectStore`], and
+/// working for any backend the store opens), and flips a shared `finished` flag
+/// on shutdown so the paired [`Spill`] knows the bytes are complete.
+struct SpillWriter {
+    inner: Box<dyn Writer>,
+    quota: Option<DiskQuota>,
+    finished: Arc<AtomicBool>,
+}
+
+impl AsyncWrite for SpillWriter {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<io::Result<usize>> {
+        let this = self.get_mut();
+        let Some(quota) = &this.quota else {
+            return Pin::new(this.inner.as_mut()).poll_write(cx, buf);
+        };
+        // Reserve up-front for the bytes we intend to write, then release the
+        // remainder the inner writer did not accept so the reservation tracks
+        // bytes actually buffered (and, for a write-once file, the file size).
+        if let Err(e) = quota.try_reserve(buf.len() as u64) {
+            return Poll::Ready(Err(io::Error::other(e)));
+        }
+        let poll = Pin::new(this.inner.as_mut()).poll_write(cx, buf);
+        match &poll {
+            Poll::Ready(Ok(n)) => quota.release((buf.len() - *n) as u64),
+            _ => quota.release(buf.len() as u64),
+        }
+        poll
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        Pin::new(self.get_mut().inner.as_mut()).poll_flush(cx)
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+        let this = self.get_mut();
+        let poll = Pin::new(this.inner.as_mut()).poll_shutdown(cx);
+        if matches!(poll, Poll::Ready(Ok(()))) {
+            // Mirrors `Writer::shutdown` so the flag is set whichever shutdown
+            // surface the consumer drives (`AsyncWrite` vs the `Writer` trait).
+            this.finished.store(true, Ordering::Relaxed);
+        }
+        poll
+    }
+}
+
+#[async_trait]
+impl Writer for SpillWriter {
+    async fn tell(&mut self) -> Result<usize> {
+        self.inner.tell().await
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        let result = self.inner.shutdown().await?;
+        // Signal the paired `Spill` that the bytes are now complete. `Relaxed`
+        // is sufficient: this only flags that shutdown happened; the file
+        // contents are synchronized through the filesystem, not this flag.
+        self.finished.store(true, Ordering::Relaxed);
+        Ok(result)
+    }
+}
+
+/// A [`SpillStore`] that writes temporary files to a local temp directory.
+///
+/// By default there is no disk cap. Use [`LocalSpillStore::with_cap`] to
+/// configure one shared across every handle this store produces.
+///
+/// The temp directory is deleted when the store is dropped, cleaning up any
+/// files whose handles have already been dropped.
+pub struct LocalSpillStore {
+    store: Arc<ObjectStore>,
+    /// Backstop cleanup: removes the whole scratch directory on drop.
+    temp_dir: Arc<tempfile::TempDir>,
+    file_counter: Arc<AtomicU64>,
+    /// Byte budget shared across every handle, enforced while writing.
+    quota: Option<DiskQuota>,
+}
+
+impl LocalSpillStore {
+    /// Create a store with no disk cap.
+    pub fn new() -> Result<Self> {
+        Ok(Self {
+            store: Arc::new(ObjectStore::local()),
+            temp_dir: Arc::new(tempfile::tempdir()?),
+            file_counter: Arc::new(AtomicU64::new(0)),
+            quota: None,
+        })
+    }
+
+    /// Create a store that returns [`lance_core::Error::DiskCapExceeded`] once
+    /// total bytes written across all live handles would exceed `cap_bytes`.
+    pub fn with_cap(cap_bytes: u64) -> Result<Self> {
+        Ok(Self {
+            store: Arc::new(ObjectStore::local()),
+            temp_dir: Arc::new(tempfile::tempdir()?),
+            file_counter: Arc::new(AtomicU64::new(0)),
+            quota: Some(DiskQuota::new(cap_bytes)),
+        })
+    }
+}
+
+impl Default for LocalSpillStore {
+    fn default() -> Self {
+        Self::new().expect("failed to create temp directory for LocalSpillStore")
+    }
+}
+
+#[async_trait]
+impl SpillStore for LocalSpillStore {
+    async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)> {
+        let idx = self.file_counter.fetch_add(1, Ordering::Relaxed);
+        let fs_path = self.temp_dir.path().join(format!("spill_{idx:06}.bin"));
+        let os_path = Path::from_absolute_path(&fs_path)?;
+        let finished = Arc::new(AtomicBool::new(false));
+
+        let writer = Box::new(SpillWriter {
+            inner: self.store.create(&os_path).await?,
+            quota: self.quota.clone(),
+            finished: finished.clone(),
+        });
+        let spill = Box::new(LocalSpill {
+            store: self.store.clone(),
+            os_path,
+            fs_path,
+            quota: self.quota.clone(),
+            finished,
+            _temp_dir: self.temp_dir.clone(),
+        });
+        Ok((writer, spill))
+    }
+}
+
+/// The readable half of a [`LocalSpillStore`] spill; reclaims the file on drop.
+struct LocalSpill {
+    store: Arc<ObjectStore>,
+    os_path: Path,
+    fs_path: PathBuf,
+    quota: Option<DiskQuota>,
+    /// Set by the paired [`SpillWriter`] once it has been shut down.
+    finished: Arc<AtomicBool>,
+    /// Keep the store's temp directory alive for at least this file's lifetime.
+    _temp_dir: Arc<tempfile::TempDir>,
+}
+
+#[async_trait]
+impl Spill for LocalSpill {
+    async fn reader(&self) -> Result<Box<dyn Reader>> {
+        // `Relaxed` is sufficient: the flag only gates "has the writer shut
+        // down"; the bytes themselves are synchronized through the filesystem,
+        // not this load.
+        if !self.finished.load(Ordering::Relaxed) {
+            return Err(Error::invalid_input(
+                "spill reader requested before the writer was shut down",
+            ));
+        }
+        self.store.open(&self.os_path).await
+    }
+}
+
+impl Drop for LocalSpill {
+    fn drop(&mut self) {
+        // Release the bytes this file occupied back to the budget. We stat the
+        // persisted file rather than tracking writes, which is exact for the
+        // write-once contract.
+        if let Some(quota) = &self.quota
+            && let Ok(metadata) = std::fs::metadata(&self.fs_path)
+        {
+            quota.release(metadata.len());
+        }
+        // Best-effort removal; the temp dir is the backstop.
+        let _ = std::fs::remove_file(&self.fs_path);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tokio::io::AsyncWriteExt;
+
+    /// Write `data` to a fresh writer and shut it down.
+    async fn finish_writer(mut writer: Box<dyn Writer>, data: &[u8]) -> Result<()> {
+        writer.write_all(data).await?;
+        Writer::shutdown(writer.as_mut()).await?;
+        Ok(())
+    }
+
+    #[test]
+    fn test_disk_quota_reserve_release() {
+        let quota = DiskQuota::new(100);
+        quota.try_reserve(60).unwrap();
+        assert!(quota.try_reserve(60).is_err());
+        quota.release(60);
+        quota.try_reserve(60).unwrap();
+        // Reserving exactly up to the cap succeeds; one byte past it fails.
+        quota.try_reserve(40).unwrap();
+        assert!(quota.try_reserve(1).is_err());
+    }
+
+    #[tokio::test]
+    async fn test_write_then_read() {
+        let store = LocalSpillStore::new().unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+
+        let data = b"hello spill world";
+        finish_writer(writer, data).await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        let read_back = reader.get_all().await.unwrap();
+        assert_eq!(read_back.as_ref(), data);
+    }
+
+    #[tokio::test]
+    async fn test_reader_requires_finished_writer() {
+        let store = LocalSpillStore::new().unwrap();
+        let (mut writer, spill) = store.new_spill().await.unwrap();
+        writer.write_all(b"partial").await.unwrap();
+
+        // Reading before the writer is shut down is rejected.
+        let Err(err) = spill.reader().await else {
+            panic!("reader before shutdown should be rejected");
+        };
+        assert!(
+            matches!(err, Error::InvalidInput { .. }),
+            "expected InvalidInput, got {err:?}"
+        );
+
+        // After shutdown the reader sees the bytes.
+        Writer::shutdown(writer.as_mut()).await.unwrap();
+        let reader = spill.reader().await.unwrap();
+        assert_eq!(reader.get_all().await.unwrap().as_ref(), b"partial");
+    }
+
+    #[tokio::test]
+    async fn test_reader_ready_after_async_shutdown() {
+        // Shutting down through the `AsyncWrite` surface (not the `Writer`
+        // trait) must also mark the spill readable — covers poll_shutdown's
+        // flag set, the path the `Writer::shutdown` tests don't reach.
+        let store = LocalSpillStore::new().unwrap();
+        let (mut writer, spill) = store.new_spill().await.unwrap();
+        writer.write_all(b"async").await.unwrap();
+        AsyncWriteExt::shutdown(&mut writer).await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        assert_eq!(reader.get_all().await.unwrap().as_ref(), b"async");
+    }
+
+    #[tokio::test]
+    async fn test_empty_spill() {
+        // A spill written with no bytes round-trips empty, and the capped path
+        // handles the zero-byte reserve/stat without error.
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+        finish_writer(writer, b"").await.unwrap();
+
+        let reader = spill.reader().await.unwrap();
+        assert!(reader.get_all().await.unwrap().is_empty());
+    }
+
+    #[tokio::test]
+    async fn test_raii_cleanup() {
+        let store = LocalSpillStore::new().unwrap();
+        let (writer, spill) = store.new_spill().await.unwrap();
+        finish_writer(writer, b"some bytes").await.unwrap();
+
+        // The first spill gets a deterministic name under the store's temp dir.
+        let path = store.temp_dir.path().join("spill_000000.bin");
+        assert!(path.exists());
+        drop(spill);
+        assert!(!path.exists(), "spill file should be deleted on drop");
+    }
+
+    #[tokio::test]
+    async fn test_cap_exceeded() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer, _spill) = store.new_spill().await.unwrap();
+        let err = finish_writer(writer, &[0u8; 101]).await.unwrap_err();
+        assert!(
+            matches!(err, Error::DiskCapExceeded { cap_bytes: 100, .. }),
+            "expected DiskCapExceeded, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cap_shared_across_files() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+        let (writer_a, _spill_a) = store.new_spill().await.unwrap();
+        let (writer_b, _spill_b) = store.new_spill().await.unwrap();
+
+        finish_writer(writer_a, &[0u8; 60]).await.unwrap();
+        // 60 already reserved by `a`; writing 60 more would reach 120 > 100.
+        let err = finish_writer(writer_b, &[0u8; 60]).await.unwrap_err();
+        assert!(
+            matches!(err, Error::DiskCapExceeded { cap_bytes: 100, .. }),
+            "expected DiskCapExceeded, got {err:?}"
+        );
+    }
+
+    #[tokio::test]
+    async fn test_cap_freed_on_drop() {
+        let store = LocalSpillStore::with_cap(100).unwrap();
+
+        {
+            let (writer, spill) = store.new_spill().await.unwrap();
+            finish_writer(writer, &[0u8; 80]).await.unwrap();
+            // `spill` drops at the end of this block, releasing its 80 bytes.
+            drop(spill);
+        }
+
+        let (writer, _spill) = store.new_spill().await.unwrap();
+        // Succeeds because the cap is no longer under pressure.
+        finish_writer(writer, &[0u8; 80]).await.unwrap();
+    }
+
+    #[tokio::test]
+    async fn test_custom_implementation() {
+        // A custom store can satisfy the traits without a local file.
+        struct MemStore;
+        struct MemSpill;
+
+        #[async_trait]
+        impl Spill for MemSpill {
+            async fn reader(&self) -> Result<Box<dyn Reader>> {
+                ObjectStore::memory().open(&Path::from("/mem")).await
+            }
+        }
+
+        #[async_trait]
+        impl SpillStore for MemStore {
+            async fn new_spill(&self) -> Result<(Box<dyn Writer>, Box<dyn Spill>)> {
+                let writer = ObjectStore::memory().create(&Path::from("/mem")).await?;
+                Ok((writer, Box::new(MemSpill)))
+            }
+        }
+
+        let store = MemStore;
+        // Exercise the factory + trait objects; the in-memory store is a fresh
+        // instance per call so we don't round-trip data here.
+        let (_writer, _spill) = store.new_spill().await.unwrap();
+    }
+
+    /// A [`Writer`] whose `poll_write` accepts a fixed number of bytes per call,
+    /// or fails, so we can drive the [`SpillWriter`] release arms that the local
+    /// backend (which accepts every write in full) never hits.
+    struct ControlledWriter {
+        outcome: Poll<io::Result<usize>>,
+    }
+
+    impl AsyncWrite for ControlledWriter {
+        fn poll_write(
+            self: Pin<&mut Self>,
+            _cx: &mut Context<'_>,
+            buf: &[u8],
+        ) -> Poll<io::Result<usize>> {
+            match &self.outcome {
+                Poll::Ready(Ok(n)) => Poll::Ready(Ok((*n).min(buf.len()))),
+                Poll::Ready(Err(e)) => Poll::Ready(Err(io::Error::new(e.kind(), e.to_string()))),
+                Poll::Pending => Poll::Pending,
+            }
+        }
+        fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+        fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
+            Poll::Ready(Ok(()))
+        }
+    }
+
+    #[async_trait]
+    impl Writer for ControlledWriter {
+        async fn tell(&mut self) -> Result<usize> {
+            Ok(0)
+        }
+        async fn shutdown(&mut self) -> Result<WriteResult> {
+            Ok(WriteResult::default())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_spill_writer_releases_unaccepted_bytes() {
+        // Short write: the inner writer accepts only 10 of the 40 reserved bytes,
+        // so the 30-byte remainder must be returned to the budget.
+        let quota = DiskQuota::new(100);
+        let mut writer = SpillWriter {
+            inner: Box::new(ControlledWriter {
+                outcome: Poll::Ready(Ok(10)),
+            }),
+            quota: Some(quota.clone()),
+            finished: Arc::new(AtomicBool::new(false)),
+        };
+        let n = writer.write(&[0u8; 40]).await.unwrap();
+        assert_eq!(n, 10);
+        assert_eq!(
+            *quota.used.lock().unwrap(),
+            10,
+            "only the accepted bytes should remain reserved"
+        );
+
+        // Failed write: the full reservation must be released.
+        let quota = DiskQuota::new(100);
+        let mut writer = SpillWriter {
+            inner: Box::new(ControlledWriter {
+                outcome: Poll::Ready(Err(io::Error::other("boom"))),
+            }),
+            quota: Some(quota.clone()),
+            finished: Arc::new(AtomicBool::new(false)),
+        };
+        writer.write(&[0u8; 40]).await.unwrap_err();
+        assert_eq!(
+            *quota.used.lock().unwrap(),
+            0,
+            "a failed write should release its entire reservation"
+        );
+    }
+}

vendor/lance-io/src/stream.rs

@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use arrow_array::RecordBatch;
+use arrow_schema::{ArrowError, SchemaRef};
+use futures::stream::BoxStream;
+use futures::{Stream, StreamExt};
+use pin_project::pin_project;
+
+use lance_core::Result;
+
+pub type BatchStream = BoxStream<'static, Result<RecordBatch>>;
+
+pub fn arrow_stream_to_lance_stream(
+    arrow_stream: BoxStream<'static, std::result::Result<RecordBatch, ArrowError>>,
+) -> BatchStream {
+    arrow_stream.map(|r| r.map_err(Into::into)).boxed()
+}
+
+/// RecordBatch Stream trait.
+pub trait RecordBatchStream: Stream<Item = Result<RecordBatch>> + Send {
+    /// Returns the schema of the stream.
+    fn schema(&self) -> SchemaRef;
+}
+
+/// Combines a [`Stream`] with a [`SchemaRef`] implementing
+/// [`RecordBatchStream`] for the combination
+#[pin_project]
+pub struct RecordBatchStreamAdapter<S> {
+    schema: SchemaRef,
+
+    #[pin]
+    stream: S,
+}
+
+impl<S> RecordBatchStreamAdapter<S> {
+    /// Creates a new [`RecordBatchStreamAdapter`] from the provided schema and stream
+    pub fn new(schema: SchemaRef, stream: S) -> Self {
+        Self { schema, stream }
+    }
+}
+
+impl<S> std::fmt::Debug for RecordBatchStreamAdapter<S> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RecordBatchStreamAdapter")
+            .field("schema", &self.schema)
+            .finish()
+    }
+}
+
+impl<S> RecordBatchStream for RecordBatchStreamAdapter<S>
+where
+    S: Stream<Item = Result<RecordBatch>> + Send + 'static,
+{
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+impl<S> Stream for RecordBatchStreamAdapter<S>
+where
+    S: Stream<Item = Result<RecordBatch>>,
+{
+    type Item = Result<RecordBatch>;
+
+    fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        self.project().stream.poll_next(cx)
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.stream.size_hint()
+    }
+}

vendor/lance-io/src/testing.rs

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+use std::fmt::{self, Display, Formatter};
+
+use async_trait::async_trait;
+use futures::stream::BoxStream;
+use mockall::mock;
+use object_store::{
+    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    ObjectStore as OSObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult,
+    Result as OSResult, path::Path,
+};
+use std::future::Future;
+
+mock! {
+    pub ObjectStore {}
+
+    #[async_trait]
+    impl OSObjectStore for ObjectStore {
+        async fn put_opts(&self, location: &Path, bytes: PutPayload, opts: PutOptions) -> OSResult<PutResult>;
+        async fn put_multipart_opts(
+            &self,
+            location: &Path,
+            opts: PutMultipartOptions,
+        ) -> OSResult<Box<dyn MultipartUpload>>;
+        fn get_opts<'life0, 'life1, 'async_trait>(
+            &'life0 self,
+            location: &'life1 Path,
+            options: GetOptions
+        ) -> std::pin::Pin<Box<dyn Future<Output=OSResult<GetResult> > +Send+'async_trait> > where
+        Self: 'async_trait,
+        'life0: 'async_trait,
+        'life1: 'async_trait;
+        fn delete_stream(&self, locations: BoxStream<'static, OSResult<Path>>) -> BoxStream<'static, OSResult<Path>>;
+        fn list<'a>(&'a self, prefix: Option<&'a Path>) -> BoxStream<'_, OSResult<ObjectMeta>>;
+        async fn list_with_delimiter<'a, 'b>(&'a self, prefix: Option<&'b Path>) -> OSResult<ListResult>;
+        async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()>;
+    }
+}
+
+impl std::fmt::Debug for MockObjectStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "MockObjectStore")
+    }
+}
+
+impl Display for MockObjectStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "MockObjectStore")
+    }
+}

vendor/lance-io/src/traits.rs

@@ -0,0 +1,177 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::ops::Range;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use futures::{StreamExt, future::BoxFuture, stream::BoxStream};
+use lance_core::deepsize::DeepSizeOf;
+use object_store::path::Path;
+use prost::Message;
+use tokio::io::{AsyncWrite, AsyncWriteExt};
+
+use lance_core::Result;
+
+use crate::object_writer::WriteResult;
+
+pub trait ProtoStruct {
+    type Proto: Message;
+}
+
+pub type ByteStream = BoxStream<'static, object_store::Result<Bytes>>;
+
+/// A trait for writing to a file on local file system or object store.
+#[async_trait]
+pub trait Writer: AsyncWrite + Unpin + Send {
+    /// Tell the current offset.
+    async fn tell(&mut self) -> Result<usize>;
+
+    /// Flush all buffered data and finalize the write, returning metadata about
+    /// the written object.
+    async fn shutdown(&mut self) -> Result<WriteResult>;
+}
+
+#[async_trait]
+impl Writer for Box<dyn Writer> {
+    async fn tell(&mut self) -> Result<usize> {
+        self.as_mut().tell().await
+    }
+
+    async fn shutdown(&mut self) -> Result<WriteResult> {
+        self.as_mut().shutdown().await
+    }
+}
+
+/// Lance Write Extension.
+#[async_trait]
+pub trait WriteExt {
+    /// Write a Protobuf message to the [Writer], and returns the file position
+    /// where the protobuf is written.
+    async fn write_protobuf(&mut self, msg: &impl Message) -> Result<usize>;
+
+    async fn write_struct<
+        'b,
+        M: Message + From<&'b T>,
+        T: ProtoStruct<Proto = M> + Send + Sync + 'b,
+    >(
+        &mut self,
+        obj: &'b T,
+    ) -> Result<usize> {
+        let msg: M = M::from(obj);
+        self.write_protobuf(&msg).await
+    }
+    /// Write magics to the tail of a file before closing the file.
+    async fn write_magics(
+        &mut self,
+        pos: usize,
+        major_version: i16,
+        minor_version: i16,
+        magic: &[u8],
+    ) -> Result<()>;
+
+    async fn copy_from_reader(&mut self, reader: &dyn Reader) -> Result<usize>;
+
+    async fn copy_range_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<usize>;
+}
+
+#[async_trait]
+impl<W: Writer + ?Sized> WriteExt for W {
+    async fn write_protobuf(&mut self, msg: &impl Message) -> Result<usize> {
+        let offset = self.tell().await?;
+
+        let len = msg.encoded_len();
+
+        self.write_u32_le(len as u32).await?;
+        self.write_all(&msg.encode_to_vec()).await?;
+
+        Ok(offset)
+    }
+
+    async fn write_magics(
+        &mut self,
+        pos: usize,
+        major_version: i16,
+        minor_version: i16,
+        magic: &[u8],
+    ) -> Result<()> {
+        self.write_i64_le(pos as i64).await?;
+        self.write_i16_le(major_version).await?;
+        self.write_i16_le(minor_version).await?;
+        self.write_all(magic).await?;
+        Ok(())
+    }
+
+    async fn copy_from_reader(&mut self, reader: &dyn Reader) -> Result<usize> {
+        let mut stream = reader.get_stream().await?;
+        let mut copied = 0usize;
+        while let Some(chunk) = stream.next().await {
+            let bytes = chunk?;
+            copied += bytes.len();
+            self.write_all(&bytes).await?;
+        }
+        Ok(copied)
+    }
+
+    async fn copy_range_from_reader(
+        &mut self,
+        reader: &dyn Reader,
+        range: Range<usize>,
+    ) -> Result<usize> {
+        let mut stream = reader.get_range_stream(range).await?;
+        let mut copied = 0usize;
+        while let Some(chunk) = stream.next().await {
+            let bytes = chunk?;
+            copied += bytes.len();
+            self.write_all(&bytes).await?;
+        }
+        Ok(copied)
+    }
+}
+
+pub trait Reader: std::fmt::Debug + Send + Sync + DeepSizeOf {
+    fn path(&self) -> &Path;
+
+    /// Suggest optimal I/O size per storage device.
+    fn block_size(&self) -> usize;
+
+    /// Suggest optimal I/O parallelism per storage device.
+    fn io_parallelism(&self) -> usize;
+
+    /// Object/File Size.
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>>;
+
+    /// Read a range of bytes from the object.
+    ///
+    /// TODO: change to read_at()?
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>>;
+
+    /// Read all bytes from the object.
+    ///
+    /// By default this reads the size in a separate IOP but some implementations
+    /// may not need the size beforehand.
+    fn get_all(&self) -> BoxFuture<'_, object_store::Result<Bytes>>;
+
+    /// Read the entire object as a byte stream.
+    fn get_stream(&self) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let bytes = self.get_all().await?;
+            Ok(futures::stream::once(async move { Ok(bytes) }).boxed())
+        })
+    }
+
+    /// Read a byte range as a byte stream.
+    fn get_range_stream(
+        &self,
+        range: Range<usize>,
+    ) -> BoxFuture<'_, object_store::Result<ByteStream>> {
+        Box::pin(async move {
+            let bytes = self.get_range(range).await?;
+            Ok(futures::stream::once(async move { Ok(bytes) }).boxed())
+        })
+    }
+}

vendor/lance-io/src/uring.rs

@@ -0,0 +1,84 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! io_uring-based I/O for disks with high IOPS capacity (e.g. NVMe)
+//!
+//! This module provides two implementations of the [`Reader`](crate::traits::Reader) trait
+//! using Linux's io_uring interface for asynchronous I/O.
+//!
+//! One of these uses a pool of dedicated background threads which each own an io_uring instance.
+//! Read requests are submitted to a background thread's pool.
+//!
+//! The other implementation uses a thread-local io_uring instance.  This only works if the future
+//! is polled by the same thread that submitted the request.  This means that the runtime must be
+//! a single-threaded runtime.
+//!
+//! # Configuration
+//!
+//! The io_uring reader is enabled by using the `file+uring://` URI scheme instead of `file://`.
+//! Additional tuning parameters are controlled by environment variables:
+//!
+//! - `LANCE_URING_CURRENT_THREAD` - Use thread-local io_uring (default: false)
+//! - `LANCE_URING_BLOCK_SIZE` - Block size in bytes (default: 4KB)
+//! - `LANCE_URING_IO_PARALLELISM` - Max concurrent operations (default: 128)
+//! - `LANCE_URING_QUEUE_DEPTH` - io_uring queue depth (default: 16K)
+//! - `LANCE_URING_THREAD_COUNT` - Number of io_uring threads to use (default: 2)
+//! - `LANCE_URING_SUBMIT_BATCH_SIZE` - Number of requests to batch before submitting (default: 128)
+//! - `LANCE_URING_POLL_TIMEOUT_MS` - Thread poll timeout in milliseconds (default: 10)
+//!
+//! Note: the block size and io parallelism are not actually used by the io_uring implementation.  These
+//! variables just control what the filesystem reports up to Lance.
+//!
+//! # Platform Support
+//!
+//! This module is only available on Linux and requires kernel 5.1 or newer.
+//! On other platforms, the code falls back to [`LocalObjectReader`](crate::local::LocalObjectReader).
+//!
+//! # Example
+//!
+//! ```no_run
+//! # use lance_io::object_store::ObjectStore;
+//! # async fn example() -> lance_core::Result<()> {
+//! // Enable io_uring by using the file+uring:// scheme
+//! let uri = "file+uring:///path/to/file.dat";
+//! let (store, path) = ObjectStore::from_uri(uri).await?;
+//! let reader = store.open(&path).await?;
+//!
+//! // Reader will use io_uring
+//! let data = reader.get_range(0..1024).await?;
+//! # Ok(())
+//! # }
+//! ```
+
+mod future;
+mod reader;
+mod requests;
+mod thread;
+
+// Thread-local io_uring implementation for current-thread runtimes
+pub(crate) mod current_thread;
+pub(crate) mod current_thread_future;
+
+#[cfg(test)]
+mod tests;
+
+use std::sync::LazyLock;
+
+pub(crate) use current_thread::UringCurrentThreadReader;
+pub use reader::UringReader;
+
+/// Default block size for io_uring reads (4KB)
+pub const DEFAULT_URING_BLOCK_SIZE: usize = 4 * 1024;
+
+/// Default I/O parallelism for io_uring (128 concurrent operations)
+pub const DEFAULT_URING_IO_PARALLELISM: usize = 128;
+
+/// Default io_uring queue depth (16K entries)
+pub const DEFAULT_URING_QUEUE_DEPTH: usize = 16 * 1024;
+
+/// Cached `LANCE_URING_BLOCK_SIZE` env var, read once at first access.
+pub(crate) static URING_BLOCK_SIZE: LazyLock<Option<usize>> = LazyLock::new(|| {
+    std::env::var("LANCE_URING_BLOCK_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+});

vendor/lance-io/src/uring/current_thread.rs

@@ -0,0 +1,430 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Thread-local io_uring implementation for current-thread runtimes.
+//!
+//! This implementation creates a thread-local IoUring instance per thread
+//! and directly processes completions during future polling, eliminating
+//! the need for background threads and MPSC channels.
+
+use super::requests::{IoRequest, RequestState};
+use super::{DEFAULT_URING_BLOCK_SIZE, DEFAULT_URING_IO_PARALLELISM, URING_BLOCK_SIZE};
+use crate::local::to_local_path;
+use crate::traits::Reader;
+use crate::uring::DEFAULT_URING_QUEUE_DEPTH;
+use crate::utils::tracking_store::IOTracker;
+use bytes::{Bytes, BytesMut};
+use futures::FutureExt;
+use futures::future::BoxFuture;
+use io_uring::{IoUring, opcode, types};
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result};
+use object_store::path::Path;
+
+use std::cell::{LazyCell, RefCell};
+use std::collections::HashMap;
+use std::fs::File;
+use std::future::Future;
+use std::io::{self, ErrorKind};
+use std::ops::Range;
+use std::pin::Pin;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Mutex};
+use tracing::instrument;
+
+// Re-use file handle types from reader.rs
+use super::reader::{CacheKey, CachedReaderData, HANDLE_CACHE, UringFileHandle};
+
+/// Global counter for generating unique user_data values
+static USER_DATA_COUNTER: AtomicU64 = AtomicU64::new(1);
+
+/// Thread-local io_uring instance with pending requests
+struct ThreadLocalUring {
+    ring: IoUring,
+    pending: HashMap<u64, Arc<IoRequest>>,
+}
+
+thread_local! {
+    static URING: LazyCell<RefCell<ThreadLocalUring>> = LazyCell::new(|| {
+        let queue_depth = std::env::var("LANCE_URING_QUEUE_DEPTH")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_QUEUE_DEPTH);
+
+        let ring = IoUring::builder()
+            // Ensures work is only done in submit_and_wait
+            .setup_defer_taskrun()
+            // Enable perf. optimization when there is only one issuer thread
+            .setup_single_issuer()
+            .build(queue_depth as u32)
+            .expect("Failed to create io_uring");
+
+        log::debug!(
+            "Created thread-local io_uring with queue depth {}",
+            queue_depth
+        );
+
+        RefCell::new(ThreadLocalUring {
+            ring,
+            pending: HashMap::new(),
+        })
+    });
+}
+
+/// Push request to thread-local submission queue
+pub(super) fn push_request(request: Arc<IoRequest>) -> io::Result<()> {
+    URING.with(|cell| {
+        let mut uring = cell.borrow_mut();
+
+        // Generate unique user_data
+        let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        // Get buffer pointer, adjusting for any bytes already read (short read retry)
+        let (buffer_ptr, read_offset, read_length) = {
+            let state = request.state.lock().unwrap();
+            let br = state.bytes_read;
+            (
+                unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+                request.offset + br as u64,
+                (request.length - br) as u32,
+            )
+        };
+
+        // Prepare read operation
+        let read_op =
+            opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length).offset(read_offset);
+
+        // Get submission queue
+        let mut sq = uring.ring.submission();
+
+        // Check if SQ has space
+        if sq.is_full() {
+            drop(sq);
+            return Err(io::Error::new(
+                io::ErrorKind::WouldBlock,
+                "io_uring submission queue full",
+            ));
+        }
+
+        // Push to SQ
+        unsafe {
+            sq.push(&read_op.build().user_data(user_data))
+                .map_err(|_| io::Error::other("Failed to push to SQ"))?;
+        }
+        drop(sq);
+
+        // Track request in pending map
+        uring.pending.insert(user_data, request);
+
+        // Don't submit here - let the future handle submission
+
+        Ok(())
+    })
+}
+
+/// Process completions from thread-local IoUring
+pub(super) fn process_thread_local_completions() -> io::Result<usize> {
+    URING.with(|cell| {
+        let mut uring = cell.borrow_mut();
+        let mut completed = 0;
+        let mut retries: Vec<Arc<IoRequest>> = Vec::new();
+
+        // Collect completions first to avoid borrowing ring and pending simultaneously
+        let cqes: Vec<_> = uring
+            .ring
+            .completion()
+            .map(|cqe| (cqe.user_data(), cqe.result()))
+            .collect();
+
+        for (user_data, result) in cqes {
+            if let Some(request) = uring.pending.remove(&user_data) {
+                let mut state = request.state.lock().unwrap();
+
+                if result < 0 {
+                    // Kernel error
+                    state.err = Some(io::Error::from_raw_os_error(-result));
+                    state.completed = true;
+                } else if result == 0 {
+                    // EOF before full read completed
+                    let br = state.bytes_read;
+                    state.err = Some(io::Error::new(
+                        io::ErrorKind::UnexpectedEof,
+                        format!("unexpected EOF: read {} of {} bytes", br, request.length),
+                    ));
+                    state.buffer.truncate(br);
+                    state.completed = true;
+                } else {
+                    // Positive result: n bytes read
+                    let n = result as usize;
+                    state.bytes_read += n;
+                    let br = state.bytes_read;
+
+                    if br >= request.length {
+                        // Full read complete
+                        state.buffer.truncate(br);
+                        state.completed = true;
+                    } else {
+                        // Short read — need retry; don't mark completed or wake
+                        drop(state);
+                        retries.push(request);
+
+                        continue;
+                    }
+                }
+
+                // Wake waiting future
+                if let Some(waker) = state.waker.take() {
+                    drop(state);
+                    waker.wake();
+                }
+
+                completed += 1;
+            } else {
+                log::warn!("Received completion for unknown user_data: {}", user_data);
+            }
+        }
+
+        // Resubmit short-read retries
+        for request in retries {
+            // Generate unique user_data
+            let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+            let (buffer_ptr, read_offset, read_length) = {
+                let state = request.state.lock().unwrap();
+                let br = state.bytes_read;
+                (
+                    unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+                    request.offset + br as u64,
+                    (request.length - br) as u32,
+                )
+            };
+
+            let read_op = opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length)
+                .offset(read_offset);
+
+            let mut sq = uring.ring.submission();
+            if sq.is_full() {
+                drop(sq);
+                request.fail(io::Error::new(
+                    io::ErrorKind::WouldBlock,
+                    "io_uring submission queue full during retry",
+                ));
+                continue;
+            }
+
+            unsafe {
+                if sq.push(&read_op.build().user_data(user_data)).is_err() {
+                    request.fail(io::Error::other("Failed to push short-read retry to SQ"));
+                    continue;
+                }
+            }
+            drop(sq);
+
+            uring.pending.insert(user_data, request);
+        }
+
+        if completed > 0 {
+            log::trace!("Processed {} completions", completed);
+        }
+
+        Ok(completed)
+    })
+}
+
+/// Submit all pending requests and wait with timeout 0 (non-blocking)
+pub(super) fn submit_and_wait_thread_local() -> io::Result<()> {
+    URING.with(|cell| {
+        let uring = cell.borrow_mut();
+        // Submit with wait=1 (do at least some work)
+        uring.ring.submit_and_wait(1)?;
+        Ok(())
+    })
+}
+
+/// Thread-local io_uring-based reader for current-thread runtimes
+#[derive(Debug)]
+pub struct UringCurrentThreadReader {
+    /// File handle
+    handle: Arc<UringFileHandle>,
+
+    /// Block size for I/O operations
+    block_size: usize,
+
+    /// File size (determined at open time)
+    size: usize,
+
+    /// I/O tracker for monitoring operations
+    io_tracker: Arc<IOTracker>,
+}
+
+impl DeepSizeOf for UringCurrentThreadReader {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // Skip file handle (just a system resource)
+        // Only count the path's deep size
+        self.handle.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl UringCurrentThreadReader {
+    /// Open a file with thread-local io_uring
+    ///
+    /// This reuses the file handle caching infrastructure from UringReader
+    #[instrument(level = "debug")]
+    pub(crate) async fn open(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<Box<dyn Reader>> {
+        // Determine block size with environment variable override
+        let block_size = URING_BLOCK_SIZE.unwrap_or(block_size.max(DEFAULT_URING_BLOCK_SIZE));
+
+        let cache_key = CacheKey::new(path, block_size);
+
+        // Try to get from cache first
+        if let Some(data) = HANDLE_CACHE.get(&cache_key).await {
+            // Use known_size if provided, otherwise use cached size
+            let size = known_size.unwrap_or(data.size);
+            return Ok(Box::new(Self {
+                handle: data.handle,
+                block_size,
+                size,
+                io_tracker,
+            }) as Box<dyn Reader>);
+        }
+
+        // Cache miss - open file and get size
+        let path_clone = path.clone();
+        let local_path = to_local_path(path);
+
+        let data = tokio::task::spawn_blocking(move || {
+            let file = File::open(&local_path).map_err(|e| match e.kind() {
+                ErrorKind::NotFound => Error::not_found(path_clone.to_string()),
+                _ => e.into(),
+            })?;
+
+            // Get size from known_size or file metadata
+            let size = match known_size {
+                Some(s) => s,
+                None => file.metadata()?.len() as usize,
+            };
+
+            Ok::<_, Error>(CachedReaderData {
+                handle: Arc::new(UringFileHandle::new(file, path_clone)),
+                size,
+            })
+        })
+        .await??;
+
+        // Insert into cache
+        HANDLE_CACHE.insert(cache_key, data.clone()).await;
+
+        // Return new reader instance
+        Ok(Box::new(Self {
+            handle: data.handle.clone(),
+            block_size,
+            size: data.size,
+            io_tracker,
+        }) as Box<dyn Reader>)
+    }
+
+    /// Submit a read request and return a future
+    fn submit_read(
+        &self,
+        offset: u64,
+        length: usize,
+    ) -> Pin<Box<dyn Future<Output = object_store::Result<Bytes>> + Send>> {
+        let mut buffer = BytesMut::with_capacity(length);
+        unsafe {
+            buffer.set_len(length);
+        }
+
+        let request = Arc::new(IoRequest {
+            fd: self.handle.fd,
+            offset,
+            length,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer,
+                bytes_read: 0,
+            }),
+        });
+
+        match push_request(request.clone()) {
+            Ok(()) => Box::pin(super::current_thread_future::UringCurrentThreadFuture::new(
+                request,
+            )),
+            Err(e) => Box::pin(async move {
+                Err(object_store::Error::Generic {
+                    store: "io_uring_ct",
+                    source: Box::new(e),
+                })
+            }),
+        }
+    }
+}
+
+impl Reader for UringCurrentThreadReader {
+    fn path(&self) -> &Path {
+        &self.handle.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        std::env::var("LANCE_URING_IO_PARALLELISM")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_IO_PARALLELISM)
+    }
+
+    /// Returns the file size
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move { Ok(self.size) })
+    }
+
+    /// Read a range of bytes using thread-local io_uring
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+        let num_bytes = range.len() as u64;
+        let range_u64 = (range.start as u64)..(range.end as u64);
+
+        let metrics = self.io_tracker.begin_io("get");
+        self.submit_read(range.start as u64, range.len())
+            .map(move |result| {
+                metrics.record(&result, num_bytes);
+                if result.is_ok() {
+                    io_tracker.record_read("get_range", path, num_bytes, Some(range_u64));
+                }
+                result
+            })
+            .boxed()
+    }
+
+    /// Read the entire file using thread-local io_uring
+    #[instrument(level = "debug", skip(self))]
+    fn get_all(&self) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let size = self.size;
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+
+        let metrics = self.io_tracker.begin_io("get");
+        self.submit_read(0, size)
+            .map(move |result| {
+                let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64);
+                metrics.record(&result, num_bytes);
+                if result.is_ok() {
+                    io_tracker.record_read("get_all", path, num_bytes, None);
+                }
+                result
+            })
+            .boxed()
+    }
+}

vendor/lance-io/src/uring/current_thread_future.rs

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Future implementation for thread-local io_uring operations.
+//!
+//! This future actively processes completions during polling instead of
+//! relying on background tasks.
+
+use super::current_thread::{process_thread_local_completions, submit_and_wait_thread_local};
+use super::requests::IoRequest;
+use bytes::Bytes;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+/// Future that awaits completion of a thread-local io_uring read operation
+pub struct UringCurrentThreadFuture {
+    request: Arc<IoRequest>,
+}
+
+impl UringCurrentThreadFuture {
+    pub(super) fn new(request: Arc<IoRequest>) -> Self {
+        Self { request }
+    }
+}
+
+impl Future for UringCurrentThreadFuture {
+    type Output = object_store::Result<Bytes>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        // Check thread safety
+        if self.request.thread_id != std::thread::current().id() {
+            panic!("Request thread ID does not match current thread ID");
+        }
+
+        // First, check if we've been completed by some other future polling for completions.
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Take result and return Ready
+            match state.err.take() {
+                Some(err) => {
+                    return Poll::Ready(Err(object_store::Error::Generic {
+                        store: "io_uring_ct",
+                        source: Box::new(err),
+                    }));
+                }
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    return Poll::Ready(Ok(bytes));
+                }
+            }
+        }
+
+        drop(state);
+
+        // If not, then we should do any available work and then process completions.
+        if let Err(e) = submit_and_wait_thread_local() {
+            log::debug!("Submit and wait error: {:?}", e);
+        }
+
+        if let Err(e) = process_thread_local_completions() {
+            log::warn!("Error processing completions: {:?}", e);
+        }
+
+        // Check if our request completed
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Take result and return Ready
+            match state.err.take() {
+                Some(err) => {
+                    return Poll::Ready(Err(object_store::Error::Generic {
+                        store: "io_uring_ct",
+                        source: Box::new(err),
+                    }));
+                }
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    return Poll::Ready(Ok(bytes));
+                }
+            }
+        }
+
+        // Not done yet - immediately wake and return Pending (don't store waker)
+        // which will force the future to be polled again.  This is intentionally
+        // a busy loop.  io_uring is intended for fast disks where read latency is
+        // so small that the cost of a true context switch (parking and unparking)
+        // would be too high.
+        //
+        // We are effectively doing a "yield" here while we wait for
+        // the io_uring thread to complete the request.
+        drop(state);
+        cx.waker().wake_by_ref();
+        Poll::Pending
+    }
+}

vendor/lance-io/src/uring/future.rs

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Future implementation for io_uring read operations.
+
+use super::requests::IoRequest;
+use bytes::Bytes;
+use std::future::Future;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::task::{Context, Poll};
+
+/// Future that awaits completion of an io_uring read operation.
+///
+/// This future is woken by the io_uring thread when the operation completes.
+pub(super) struct UringReadFuture {
+    pub(super) request: Arc<IoRequest>,
+}
+
+impl Future for UringReadFuture {
+    type Output = object_store::Result<Bytes>;
+
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let mut state = self.request.state.lock().unwrap();
+
+        if state.completed {
+            // Operation completed - take the result
+            match state.err.take() {
+                Some(err) => Poll::Ready(Err(object_store::Error::Generic {
+                    store: "io_uring",
+                    source: Box::new(err),
+                })),
+                None => {
+                    let br = state.bytes_read;
+                    state.buffer.truncate(br);
+                    let bytes = std::mem::take(&mut state.buffer).freeze();
+                    Poll::Ready(Ok(bytes))
+                }
+            }
+        } else {
+            // Operation not yet complete - store waker and return Pending
+            state.waker = Some(cx.waker().clone());
+            Poll::Pending
+        }
+    }
+}

vendor/lance-io/src/uring/reader.rs

@@ -0,0 +1,301 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! UringReader implementation.
+
+use super::future::UringReadFuture;
+use super::requests::IoRequest;
+use super::thread::{SUBMITTED_COUNTER, THREAD_SELECTOR, URING_THREADS};
+use super::{DEFAULT_URING_BLOCK_SIZE, DEFAULT_URING_IO_PARALLELISM, URING_BLOCK_SIZE};
+use crate::local::to_local_path;
+use crate::traits::Reader;
+use crate::uring::requests::RequestState;
+use crate::utils::tracking_store::IOTracker;
+use bytes::{Bytes, BytesMut};
+use futures::FutureExt;
+use futures::future::BoxFuture;
+use lance_core::deepsize::DeepSizeOf;
+use lance_core::{Error, Result};
+use object_store::path::Path;
+use std::fs::File;
+use std::future::Future;
+use std::io::{self, ErrorKind};
+use std::ops::Range;
+use std::os::unix::io::{AsRawFd, RawFd};
+use std::pin::Pin;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, LazyLock, Mutex};
+use std::time::Duration;
+use tracing::instrument;
+
+/// Cache key for UringReader instances.
+/// We cache by (path, block_size) because block_size affects reader behavior.
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
+pub(super) struct CacheKey {
+    path: String,
+    block_size: usize,
+}
+
+impl CacheKey {
+    pub(super) fn new(path: &Path, block_size: usize) -> Self {
+        Self {
+            path: path.to_string(),
+            block_size,
+        }
+    }
+}
+
+/// Data stored in the cache for each opened file.
+#[derive(Clone)]
+pub(super) struct CachedReaderData {
+    pub(super) handle: Arc<UringFileHandle>,
+    pub(super) size: usize,
+}
+
+/// Global cache of open file handles.
+/// Entries expire after 60 seconds to ensure files are eventually closed.
+pub(super) static HANDLE_CACHE: LazyLock<moka::future::Cache<CacheKey, CachedReaderData>> =
+    LazyLock::new(|| {
+        moka::future::Cache::builder()
+            .time_to_live(Duration::from_secs(60))
+            .max_capacity(10_000)
+            .build()
+    });
+
+/// File handle for io_uring operations.
+///
+/// Keeps the file alive and provides the raw file descriptor.
+#[derive(Debug)]
+pub(super) struct UringFileHandle {
+    /// The file (kept alive via Arc)
+    #[allow(unused)]
+    file: Arc<File>,
+
+    /// Raw file descriptor for io_uring
+    pub(super) fd: RawFd,
+
+    /// Object store path
+    pub(super) path: Path,
+}
+
+impl UringFileHandle {
+    pub(super) fn new(file: File, path: Path) -> Self {
+        let fd = file.as_raw_fd();
+        Self {
+            file: Arc::new(file),
+            fd,
+            path,
+        }
+    }
+}
+
+/// io_uring-based reader for local files.
+///
+/// This reader uses a dedicated process-wide thread running an io_uring event loop
+/// for high-performance asynchronous I/O.
+#[derive(Debug)]
+pub struct UringReader {
+    /// File handle
+    handle: Arc<UringFileHandle>,
+
+    /// Block size for I/O operations
+    block_size: usize,
+
+    /// File size (determined at open time)
+    size: usize,
+
+    /// I/O tracker for monitoring operations
+    io_tracker: Arc<IOTracker>,
+}
+
+impl DeepSizeOf for UringReader {
+    fn deep_size_of_children(&self, context: &mut lance_core::deepsize::Context) -> usize {
+        // Skip file handle (just a system resource)
+        // Only count the path's deep size
+        self.handle.path.as_ref().deep_size_of_children(context)
+    }
+}
+
+impl UringReader {
+    /// Open a file with io_uring.
+    ///
+    /// This is the internal constructor used by ObjectStore.
+    #[instrument(level = "debug")]
+    pub(crate) async fn open(
+        path: &Path,
+        block_size: usize,
+        known_size: Option<usize>,
+        io_tracker: Arc<IOTracker>,
+    ) -> Result<Box<dyn Reader>> {
+        // Determine block size with environment variable override
+        let block_size = URING_BLOCK_SIZE.unwrap_or(block_size.max(DEFAULT_URING_BLOCK_SIZE));
+
+        let cache_key = CacheKey::new(path, block_size);
+
+        // Try to get from cache first
+        if let Some(data) = HANDLE_CACHE.get(&cache_key).await {
+            // Use known_size if provided, otherwise use cached size
+            let size = known_size.unwrap_or(data.size);
+            return Ok(Box::new(Self {
+                handle: data.handle,
+                block_size,
+                size,
+                io_tracker,
+            }) as Box<dyn Reader>);
+        }
+
+        // Cache miss - open file and get size
+        let path_clone = path.clone();
+        let local_path = to_local_path(path);
+
+        let data = tokio::task::spawn_blocking(move || {
+            let file = File::open(&local_path).map_err(|e| match e.kind() {
+                ErrorKind::NotFound => Error::not_found(path_clone.to_string()),
+                _ => e.into(),
+            })?;
+
+            // Get size from known_size or file metadata
+            let size = match known_size {
+                Some(s) => s,
+                None => file.metadata()?.len() as usize,
+            };
+
+            Ok::<_, Error>(CachedReaderData {
+                handle: Arc::new(UringFileHandle::new(file, path_clone)),
+                size,
+            })
+        })
+        .await??;
+
+        // Insert into cache
+        HANDLE_CACHE.insert(cache_key, data.clone()).await;
+
+        // Return new reader instance
+        Ok(Box::new(Self {
+            handle: data.handle.clone(),
+            block_size,
+            size: data.size,
+            io_tracker,
+        }) as Box<dyn Reader>)
+    }
+
+    /// Submit a read request to the io_uring thread via channel and return a future.
+    fn submit_read(
+        &self,
+        offset: u64,
+        length: usize,
+    ) -> Pin<Box<dyn Future<Output = object_store::Result<Bytes>> + Send>> {
+        let mut buffer = BytesMut::with_capacity(length);
+        unsafe {
+            buffer.set_len(length);
+        }
+
+        // Create IoRequest with all data
+        let request = Arc::new(IoRequest {
+            fd: self.handle.fd,
+            offset,
+            length,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer,
+                bytes_read: 0,
+            }),
+        });
+
+        // Increment submitted counter before sending to channel
+        SUBMITTED_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        // Select thread in round-robin fashion
+        let thread_idx =
+            (THREAD_SELECTOR.fetch_add(1, Ordering::Relaxed) as usize) % URING_THREADS.len();
+
+        // Send to selected thread via channel
+        match URING_THREADS[thread_idx]
+            .request_tx
+            .send(Arc::clone(&request))
+        {
+            Ok(()) => {
+                // Return future that will be woken when operation completes
+                Box::pin(UringReadFuture { request })
+            }
+            Err(_) => {
+                // Thread died - decrement counter and return error future
+                SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed);
+                Box::pin(async move {
+                    Err(object_store::Error::Generic {
+                        store: "UringReader",
+                        source: Box::new(io::Error::new(
+                            io::ErrorKind::BrokenPipe,
+                            "io_uring thread died",
+                        )),
+                    })
+                })
+            }
+        }
+    }
+}
+
+impl Reader for UringReader {
+    fn path(&self) -> &Path {
+        &self.handle.path
+    }
+
+    fn block_size(&self) -> usize {
+        self.block_size
+    }
+
+    fn io_parallelism(&self) -> usize {
+        std::env::var("LANCE_URING_IO_PARALLELISM")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(DEFAULT_URING_IO_PARALLELISM)
+    }
+
+    /// Returns the file size.
+    fn size(&self) -> BoxFuture<'_, object_store::Result<usize>> {
+        Box::pin(async move { Ok(self.size) })
+    }
+
+    /// Read a range of bytes using io_uring.
+    #[instrument(level = "debug", skip(self))]
+    fn get_range(&self, range: Range<usize>) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+        let num_bytes = range.len() as u64;
+        let range_u64 = (range.start as u64)..(range.end as u64);
+
+        let metrics = self.io_tracker.begin_io("get");
+        self.submit_read(range.start as u64, range.len())
+            .map(move |result| {
+                metrics.record(&result, num_bytes);
+                if result.is_ok() {
+                    io_tracker.record_read("get_range", path, num_bytes, Some(range_u64));
+                }
+                result
+            })
+            .boxed()
+    }
+
+    /// Read the entire file using io_uring.
+    #[instrument(level = "debug", skip(self))]
+    fn get_all(&self) -> BoxFuture<'static, object_store::Result<Bytes>> {
+        let size = self.size;
+        let io_tracker = self.io_tracker.clone();
+        let path = self.handle.path.clone();
+
+        let metrics = self.io_tracker.begin_io("get");
+        self.submit_read(0, size)
+            .map(move |result| {
+                let num_bytes = result.as_ref().map_or(0, |bytes| bytes.len() as u64);
+                metrics.record(&result, num_bytes);
+                if result.is_ok() {
+                    io_tracker.record_read("get_all", path, num_bytes, None);
+                }
+                result
+            })
+            .boxed()
+    }
+}

vendor/lance-io/src/uring/requests.rs

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Protocol types for communication between UringReader and the io_uring thread.
+
+use bytes::BytesMut;
+use std::io;
+use std::os::unix::io::RawFd;
+use std::sync::Mutex;
+use std::task::Waker;
+use std::thread::ThreadId;
+
+pub(super) struct RequestState {
+    pub completed: bool,
+    pub waker: Option<Waker>,
+    pub err: Option<io::Error>,
+    pub buffer: BytesMut,
+    /// Accumulated bytes read across retries (for handling short reads).
+    pub bytes_read: usize,
+}
+
+/// I/O request object that contains all state for a single read operation.
+/// This is shared between the submitter, uring thread, and future via Arc.
+pub(super) struct IoRequest {
+    /// File descriptor to read from.
+    pub fd: RawFd,
+
+    /// Byte offset to start reading from.
+    pub offset: u64,
+
+    /// Number of bytes to read.
+    pub length: usize,
+
+    pub thread_id: ThreadId,
+
+    /// Completion flag - set to true when operation completes.
+    pub state: Mutex<RequestState>,
+}
+
+impl IoRequest {
+    /// Mark this request as failed with the given error.
+    ///
+    /// Sets the error, marks completed, and wakes any waiting future.
+    /// Used when a request cannot be submitted (e.g. SQ full).
+    pub(super) fn fail(&self, err: io::Error) {
+        let mut state = self.state.lock().unwrap();
+        state.err = Some(err);
+        state.completed = true;
+        if let Some(waker) = state.waker.take() {
+            drop(state);
+            waker.wake();
+        }
+    }
+}

vendor/lance-io/src/uring/tests.rs

@@ -0,0 +1,392 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Tests for io_uring reader implementation.
+
+use crate::object_store::ObjectStore;
+use lance_core::Result;
+use std::io::Write;
+use std::time::Duration;
+use tempfile::NamedTempFile;
+
+/// Helper to create a temporary file with test data
+fn create_test_file(size: usize) -> Result<(NamedTempFile, Vec<u8>)> {
+    let mut file = NamedTempFile::new()?;
+    let data: Vec<u8> = (0..size).map(|i| (i % 256) as u8).collect();
+    file.write_all(&data)?;
+    file.flush()?;
+    Ok((file, data))
+}
+
+#[tokio::test]
+async fn test_read_small_file() -> Result<()> {
+    let (file, expected_data) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read entire file
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_range() -> Result<()> {
+    let (file, expected_data) = create_test_file(4096)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read a range in the middle
+    let range = 1000..2000;
+    let data = reader.get_range(range.clone()).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[range]);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_multiple_ranges() -> Result<()> {
+    let (file, expected_data) = create_test_file(8192)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read multiple ranges
+    let ranges = vec![0..100, 500..600, 2000..3000];
+    for range in ranges {
+        let data = reader.get_range(range.clone()).await.unwrap();
+        assert_eq!(data.as_ref(), &expected_data[range]);
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_size() -> Result<()> {
+    let size = 5000;
+    let (file, _) = create_test_file(size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    assert_eq!(reader.size().await.unwrap(), size);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_concurrent_reads() -> Result<()> {
+    let (file, expected_data) = create_test_file(16384)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+
+    // Perform multiple concurrent reads
+    let mut tasks = vec![];
+    for i in 0..10 {
+        let reader_clone = store.open(&path).await?;
+        let expected = expected_data.clone();
+        tasks.push(tokio::spawn(async move {
+            let range = (i * 1000)..((i + 1) * 1000);
+            let data = reader_clone.get_range(range.clone()).await.unwrap();
+            assert_eq!(data.as_ref(), &expected[range]);
+        }));
+    }
+
+    // Wait for all tasks
+    for task in tasks {
+        task.await.unwrap();
+    }
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_large_file_read() -> Result<()> {
+    // Test with a larger file (1MB)
+    let size = 1024 * 1024;
+    let (file, expected_data) = create_test_file(size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read entire file
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.len(), size);
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_read_edge_cases() -> Result<()> {
+    let (file, expected_data) = create_test_file(4096)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Read from start
+    let data = reader.get_range(0..100).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[0..100]);
+
+    // Read to end
+    let data = reader.get_range(4000..4096).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[4000..4096]);
+
+    // Read single byte
+    let data = reader.get_range(2000..2001).await.unwrap();
+    assert_eq!(data.as_ref(), &expected_data[2000..2001]);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_file_not_found() {
+    let uri = "file+uring:///nonexistent/file.dat";
+    let (store, path) = ObjectStore::from_uri(uri).await.unwrap();
+
+    // Should fail to open non-existent file
+    let result = store.open(&path).await;
+    assert!(result.is_err());
+}
+
+#[tokio::test]
+async fn test_block_size_and_parallelism() -> Result<()> {
+    let (file, _) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Check default values (or configured values)
+    assert!(reader.block_size() > 0);
+    assert!(reader.io_parallelism() > 0);
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_path() -> Result<()> {
+    let (file, _) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Verify path is preserved
+    assert_eq!(reader.path(), &path);
+
+    Ok(())
+}
+
+/// Test that reading past EOF returns an error.
+///
+/// This exercises the case where `known_size` passed to `open_with_size` is larger
+/// than the actual file, causing io_uring to hit EOF before the full read completes.
+#[tokio::test]
+async fn test_short_read_get_all() -> Result<()> {
+    let actual_size: usize = 8192;
+    let (file, _expected_data) = create_test_file(actual_size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+
+    // Open with inflated known_size — the reader will think the file is 2x its real size
+    let inflated_size = actual_size * 2;
+    let reader = store.open_with_size(&path, inflated_size).await?;
+
+    // get_all() will submit a read for inflated_size bytes from an actual_size file.
+    // The kernel reads actual_size bytes then returns 0 (EOF) — this should be an error.
+    let result = reader.get_all().await;
+    assert!(result.is_err(), "reading past EOF should return an error");
+
+    Ok(())
+}
+
+/// Test that a range read extending past EOF returns an error.
+#[tokio::test]
+async fn test_short_read_get_range_past_eof() -> Result<()> {
+    let actual_size: usize = 8192;
+    let (file, _expected_data) = create_test_file(actual_size)?;
+    let file_path = file.path().to_str().unwrap();
+    let uri = format!("file+uring://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Request a range that starts inside the file but extends past EOF.
+    // File is 8192 bytes; reading 4096..16384 hits EOF — this should be an error.
+    let range_start = 4096;
+    let range_end = actual_size * 2; // 16384, well past EOF
+    let result = reader.get_range(range_start..range_end).await;
+    assert!(
+        result.is_err(),
+        "range extending past EOF should return an error"
+    );
+
+    Ok(())
+}
+
+/// Test that when push_to_sq fails (SQ full), the request's future returns
+/// an error instead of hanging forever.
+///
+/// This directly tests the thread-path scenario: create an IoUring with
+/// queue_depth=2, fill the SQ, then try to push a 3rd request. The 3rd
+/// request's future should return an error within the timeout.
+///
+/// BUG: currently the failed push silently drops the request, so the
+/// future hangs and the timeout fires.
+#[tokio::test]
+async fn test_retry_sq_full_thread() -> Result<()> {
+    use super::future::UringReadFuture;
+    use super::requests::{IoRequest, RequestState};
+    use super::thread::push_to_sq;
+    use bytes::BytesMut;
+    use io_uring::IoUring;
+    use std::collections::HashMap;
+    use std::os::unix::io::AsRawFd;
+    use std::sync::{Arc, Mutex};
+
+    let (file, _) = create_test_file(4096)?;
+    let fd = file.as_file().as_raw_fd();
+
+    // Create a tiny ring with queue_depth=2
+    let mut ring = IoUring::new(2).unwrap();
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::new();
+
+    // Helper to create a request
+    let make_request = || {
+        Arc::new(IoRequest {
+            fd,
+            offset: 0,
+            length: 4096,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer: BytesMut::zeroed(4096),
+                bytes_read: 0,
+            }),
+        })
+    };
+
+    // Fill the SQ (capacity=2)
+    let _r1 = make_request();
+    let _r2 = make_request();
+    push_to_sq(&mut ring, &mut pending, _r1).unwrap();
+    push_to_sq(&mut ring, &mut pending, _r2).unwrap();
+
+    // 3rd push should fail — SQ is full
+    let r3 = make_request();
+    let push_result = push_to_sq(&mut ring, &mut pending, r3.clone());
+    assert!(push_result.is_err(), "3rd push should fail (SQ full)");
+
+    // r3's future should return an error, not hang forever.
+    // BUG: currently nobody sets completed=true or err on r3, so the future hangs.
+    let future = UringReadFuture { request: r3 };
+    let result = tokio::time::timeout(Duration::from_secs(2), future).await;
+    assert!(
+        result.is_ok(),
+        "future timed out — request was dropped without error on SQ-full push failure"
+    );
+
+    Ok(())
+}
+
+/// Test that when push_to_sq fails (SQ full) on the current-thread path,
+/// the request's future returns an error instead of hanging forever.
+///
+/// Uses UringCurrentThreadFuture (which will be a no-op poller since the
+/// thread-local URING has no knowledge of this request) after push_to_sq
+/// has already completed the request with an error.
+#[tokio::test(flavor = "current_thread")]
+async fn test_retry_sq_full_current_thread() -> Result<()> {
+    use super::current_thread_future::UringCurrentThreadFuture;
+    use super::requests::{IoRequest, RequestState};
+    use super::thread::push_to_sq;
+    use bytes::BytesMut;
+    use io_uring::IoUring;
+    use std::collections::HashMap;
+    use std::os::unix::io::AsRawFd;
+    use std::sync::{Arc, Mutex};
+
+    let (file, _) = create_test_file(4096)?;
+    let fd = file.as_file().as_raw_fd();
+
+    // Create a tiny ring with queue_depth=2
+    let mut ring = IoUring::new(2).unwrap();
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::new();
+
+    let make_request = || {
+        Arc::new(IoRequest {
+            fd,
+            offset: 0,
+            length: 4096,
+            thread_id: std::thread::current().id(),
+            state: Mutex::new(RequestState {
+                completed: false,
+                waker: None,
+                err: None,
+                buffer: BytesMut::zeroed(4096),
+                bytes_read: 0,
+            }),
+        })
+    };
+
+    // Fill the SQ (capacity=2)
+    push_to_sq(&mut ring, &mut pending, make_request()).unwrap();
+    push_to_sq(&mut ring, &mut pending, make_request()).unwrap();
+
+    // 3rd push should fail — SQ is full
+    let r3 = make_request();
+    let push_result = push_to_sq(&mut ring, &mut pending, r3.clone());
+    assert!(push_result.is_err(), "3rd push should fail (SQ full)");
+
+    // r3's future should return an error, not hang forever.
+    let future = UringCurrentThreadFuture::new(r3);
+    let result = tokio::time::timeout(Duration::from_secs(2), future).await;
+    assert!(
+        result.is_ok(),
+        "future timed out — request was dropped without error on SQ-full push failure"
+    );
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn test_uring_not_enabled_with_file_scheme() -> Result<()> {
+    // Verify that files opened with file:// don't use uring
+    let (file, expected_data) = create_test_file(1024)?;
+    let file_path = file.path().to_str().unwrap();
+    // Use regular file:// scheme, should NOT use uring
+    let uri = format!("file://{}", file_path);
+
+    let (store, path) = ObjectStore::from_uri(&uri).await?;
+    let reader = store.open(&path).await?;
+
+    // Should still be able to read, just won't use uring
+    let data = reader.get_all().await.unwrap();
+    assert_eq!(data.as_ref(), expected_data.as_slice());
+
+    Ok(())
+}

vendor/lance-io/src/uring/thread.rs

@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Dedicated thread for io_uring operations.
+//!
+//! This module provides a background thread that owns an io_uring instance
+//! and processes read requests from a channel. Readers send requests via
+//! an MPSC channel, and the thread handles submission and completion processing.
+
+use super::DEFAULT_URING_QUEUE_DEPTH;
+use super::requests::IoRequest;
+use io_uring::{IoUring, opcode, types};
+use std::collections::HashMap;
+use std::io;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender, sync_channel};
+use std::sync::{Arc, LazyLock};
+use std::time::{Duration, Instant};
+
+/// Handle to the io_uring background thread.
+///
+/// This provides a channel sender for submitting read requests to the thread.
+pub(super) struct UringThreadHandle {
+    pub request_tx: SyncSender<Arc<IoRequest>>,
+}
+
+/// Lazy-initialized io_uring thread pool.
+///
+/// Multiple threads are spawned on first access and run until process exit.
+pub(super) static URING_THREADS: LazyLock<Vec<UringThreadHandle>> = LazyLock::new(|| {
+    let queue_depth = get_queue_depth();
+    let thread_count = get_thread_count();
+
+    let mut threads = Vec::with_capacity(thread_count);
+
+    for i in 0..thread_count {
+        let (tx, rx) = sync_channel(queue_depth);
+
+        std::thread::Builder::new()
+            .name(format!("lance-uring-{}", i))
+            .spawn(move || run_uring_thread(rx, queue_depth, i))
+            .expect("Failed to spawn io_uring thread");
+
+        threads.push(UringThreadHandle { request_tx: tx });
+    }
+
+    log::info!(
+        "io_uring thread pool spawned ({} threads, queue_depth={})",
+        thread_count,
+        queue_depth
+    );
+
+    threads
+});
+
+/// Atomic counter for round-robin thread selection.
+pub(super) static THREAD_SELECTOR: AtomicU64 = AtomicU64::new(0);
+
+/// Counter for generating unique user_data values.
+///
+/// Each io_uring operation needs a unique user_data ID to match completions
+/// with their corresponding requests.
+static USER_DATA_COUNTER: AtomicU64 = AtomicU64::new(1);
+
+/// Counter for requests that have been submitted to the thread but not yet received.
+///
+/// This tracks requests sitting in the channel queue waiting to be received by the thread.
+pub(super) static SUBMITTED_COUNTER: AtomicU64 = AtomicU64::new(0);
+
+/// Default batch size for submission - how many requests to batch before calling submit().
+const DEFAULT_SUBMIT_BATCH_SIZE: usize = 128;
+
+/// Default number of io_uring threads.
+const DEFAULT_URING_THREAD_COUNT: usize = 2;
+
+/// Get the configured queue depth from environment variable.
+fn get_queue_depth() -> usize {
+    std::env::var("LANCE_URING_QUEUE_DEPTH")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_URING_QUEUE_DEPTH)
+}
+
+/// Get the configured poll timeout from environment variable.
+fn get_poll_timeout() -> Duration {
+    let timeout_ms = std::env::var("LANCE_URING_POLL_TIMEOUT_MS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(10);
+
+    Duration::from_millis(timeout_ms)
+}
+
+/// Get the configured submit batch size from environment variable.
+fn get_submit_batch_size() -> usize {
+    std::env::var("LANCE_URING_SUBMIT_BATCH_SIZE")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_SUBMIT_BATCH_SIZE)
+}
+
+/// Get the configured number of uring threads from environment variable.
+fn get_thread_count() -> usize {
+    std::env::var("LANCE_URING_THREAD_COUNT")
+        .ok()
+        .and_then(|s| s.parse().ok())
+        .unwrap_or(DEFAULT_URING_THREAD_COUNT)
+}
+
+/// Main loop for the io_uring thread.
+///
+/// This thread:
+/// 1. Receives requests from the channel
+/// 2. Submits them to io_uring
+/// 3. Processes completions
+/// 4. Wakes futures via their wakers
+fn run_uring_thread(request_rx: Receiver<Arc<IoRequest>>, queue_depth: usize, thread_id: usize) {
+    // Create local io_uring instance
+    let mut ring = IoUring::builder()
+        // .setup_sqpoll(100)
+        .build(queue_depth as u32)
+        .expect("Failed to create io_uring");
+
+    let mut pending: HashMap<u64, Arc<IoRequest>> = HashMap::with_capacity(queue_depth);
+    let poll_timeout = get_poll_timeout();
+    let submit_batch_size = get_submit_batch_size();
+    let mut last_log = Instant::now();
+    let log_interval = Duration::from_millis(100);
+    let mut completed_iops = 0usize;
+    let mut completed_sectors = 0usize;
+    let mut min_in_flight = usize::MAX;
+
+    loop {
+        // Track minimum in-flight count
+        let in_flight = pending.len();
+        min_in_flight = min_in_flight.min(in_flight);
+
+        // Log in-flight requests every 100ms
+        let now = Instant::now();
+        if now.duration_since(last_log) >= log_interval {
+            let submitted = SUBMITTED_COUNTER.load(Ordering::Relaxed);
+            log::info!(
+                "io_uring[{}]: {} submitted, {} in flight (min {}), {} iops completed, {} sectors completed",
+                thread_id,
+                submitted,
+                in_flight,
+                min_in_flight,
+                completed_iops,
+                completed_sectors
+            );
+            last_log = now;
+            completed_iops = 0; // Reset counter after logging
+            completed_sectors = 0; // Reset counter after logging
+            min_in_flight = usize::MAX; // Reset min tracker
+        }
+
+        // Process all available completions first
+        let mut needs_submit = false;
+        let completions = process_completions(&mut ring, &mut pending);
+        match completions {
+            Ok(result) => {
+                completed_iops += result.iops;
+                completed_sectors += result.sectors;
+
+                // Resubmit any short-read retries
+                for request in result.retries {
+                    if let Err(e) = push_to_sq(&mut ring, &mut pending, request) {
+                        log::error!("Failed to resubmit short read: {}", e);
+                    } else {
+                        needs_submit = true;
+                    }
+                }
+            }
+            Err(e) => {
+                log::error!("Error processing io_uring completions: {}", e);
+            }
+        }
+
+        min_in_flight = min_in_flight.min(pending.len());
+
+        // Batch submit requests - keep pulling from channel and pushing to SQ
+        // until we hit batch size or channel is empty
+        let mut batch_count = 0;
+        loop {
+            // Try to receive new request
+            // Use recv_timeout only when pending is empty, otherwise use try_recv
+            let recv_result = if pending.is_empty() && batch_count == 0 {
+                // No operations in flight and no batch started - we can afford to wait with timeout
+                request_rx.recv_timeout(poll_timeout).map_err(|e| match e {
+                    RecvTimeoutError::Timeout => std::sync::mpsc::TryRecvError::Empty,
+                    RecvTimeoutError::Disconnected => std::sync::mpsc::TryRecvError::Disconnected,
+                })
+            } else {
+                // Operations in flight or batch in progress - busy loop with try_recv
+                request_rx.try_recv()
+            };
+
+            match recv_result {
+                Ok(request) => {
+                    // Decrement submitted counter when we receive the request from channel
+                    SUBMITTED_COUNTER.fetch_sub(1, Ordering::Relaxed);
+
+                    // Push to submission queue (but don't submit yet)
+                    if let Err(e) = push_to_sq(&mut ring, &mut pending, request) {
+                        log::error!("Failed to push to io_uring SQ: {}", e);
+                    } else {
+                        batch_count += 1;
+                    }
+
+                    // Break if we've hit the batch size limit
+                    if batch_count >= submit_batch_size {
+                        break;
+                    }
+                }
+                Err(std::sync::mpsc::TryRecvError::Empty) => {
+                    // No more requests in channel - break to submit the batch
+                    break;
+                }
+                Err(std::sync::mpsc::TryRecvError::Disconnected) => {
+                    // All senders dropped - submit batch and shutdown
+                    if batch_count > 0
+                        && let Err(e) = ring.submit()
+                    {
+                        log::error!(
+                            "io_uring[{}]: Failed to submit io_uring batch: {}",
+                            thread_id,
+                            e
+                        );
+                    }
+                    log::info!(
+                        "io_uring thread {} shutting down (channel disconnected)",
+                        thread_id
+                    );
+                    return;
+                }
+            }
+        }
+
+        // Submit if we have any requests (from channel or retries)
+        if (batch_count > 0 || needs_submit)
+            && let Err(e) = ring.submit()
+        {
+            log::error!(
+                "Failed to submit io_uring batch of {} requests: {}",
+                batch_count,
+                e
+            );
+        }
+    }
+}
+
+/// Push a read request to the io_uring submission queue (without submitting).
+///
+/// This generates a unique user_data ID, prepares the read operation,
+/// and pushes it to the SQ. The caller is responsible for calling ring.submit().
+pub(super) fn push_to_sq(
+    ring: &mut IoUring,
+    pending: &mut HashMap<u64, Arc<IoRequest>>,
+    request: Arc<IoRequest>,
+) -> io::Result<()> {
+    // Generate unique user_data
+    let user_data = USER_DATA_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+    // Get buffer pointer, adjusting for any bytes already read (short read retry)
+    let (buffer_ptr, read_offset, read_length) = {
+        let state = request.state.lock().unwrap();
+        let br = state.bytes_read;
+        (
+            unsafe { state.buffer.as_ptr().add(br) as *mut u8 },
+            request.offset + br as u64,
+            (request.length - br) as u32,
+        )
+    };
+
+    // Prepare read operation
+    let read_op =
+        opcode::Read::new(types::Fd(request.fd), buffer_ptr, read_length).offset(read_offset);
+
+    // Get submission queue
+    let mut sq = ring.submission();
+
+    // Check if SQ has space
+    if sq.is_full() {
+        drop(sq);
+        request.fail(io::Error::new(
+            io::ErrorKind::WouldBlock,
+            "io_uring submission queue full",
+        ));
+        return Err(io::Error::new(
+            io::ErrorKind::WouldBlock,
+            "io_uring submission queue full",
+        ));
+    }
+
+    // Push to SQ
+    unsafe {
+        if sq.push(&read_op.build().user_data(user_data)).is_err() {
+            drop(sq);
+            request.fail(io::Error::other("Failed to push to SQ"));
+            return Err(io::Error::other("Failed to push to SQ"));
+        }
+    }
+    drop(sq);
+
+    // Track request in pending map
+    pending.insert(user_data, request);
+
+    Ok(())
+}
+
+struct CompletionResult {
+    iops: usize,
+    sectors: usize,
+    retries: Vec<Arc<IoRequest>>,
+}
+
+/// Process all available completions from the io_uring.
+///
+/// This iterates through the completion queue, matches completions to requests,
+/// updates their state, and wakes any waiting futures. Short reads are collected
+/// into `retries` for resubmission; EOF before a full read is an error.
+///
+/// Returns completion stats and a list of requests needing resubmission.
+fn process_completions(
+    ring: &mut IoUring,
+    pending: &mut HashMap<u64, Arc<IoRequest>>,
+) -> io::Result<CompletionResult> {
+    let mut iops = 0;
+    let mut sectors = 0;
+    let mut retries = Vec::new();
+
+    // Process all available completions
+    for cqe in ring.completion() {
+        let user_data = cqe.user_data();
+        let result = cqe.result();
+
+        // Look up request
+        if let Some(request) = pending.remove(&user_data) {
+            let mut state = request.state.lock().unwrap();
+
+            if result < 0 {
+                // Kernel error
+                state.err = Some(io::Error::from_raw_os_error(-result));
+                state.completed = true;
+            } else if result == 0 {
+                // EOF before full read completed
+                let br = state.bytes_read;
+                state.err = Some(io::Error::new(
+                    io::ErrorKind::UnexpectedEof,
+                    format!("unexpected EOF: read {} of {} bytes", br, request.length),
+                ));
+                state.buffer.truncate(br);
+                state.completed = true;
+            } else {
+                // Positive result: n bytes read
+                let n = result as usize;
+                state.bytes_read += n;
+                let br = state.bytes_read;
+
+                if br >= request.length {
+                    // Full read complete
+                    state.buffer.truncate(br);
+                    state.completed = true;
+
+                    if request.length > 0 {
+                        let first_sector = request.offset / 4096;
+                        let last_sector = (request.offset + request.length as u64 - 1) / 4096;
+                        let num_sectors = (last_sector - first_sector + 1) as usize;
+                        sectors += num_sectors;
+                    }
+                } else {
+                    // Short read — need retry; don't mark completed or wake
+                    drop(state);
+                    retries.push(request);
+                    continue;
+                }
+            }
+
+            // Wake the future if it's waiting
+            if let Some(waker) = state.waker.take() {
+                drop(state); // Release lock before waking
+                waker.wake();
+            }
+
+            iops += 1;
+        } else {
+            log::warn!("Received completion for unknown user_data: {}", user_data);
+        }
+    }
+
+    Ok(CompletionResult {
+        iops,
+        sectors,
+        retries,
+    })
+}

vendor/lance-io/src/utils.rs

@@ -0,0 +1,291 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+use std::{cmp::min, num::NonZero, sync::atomic::AtomicU64};
+
+use byteorder::{ByteOrder, LittleEndian};
+use bytes::Bytes;
+use lance_core::deepsize::DeepSizeOf;
+use prost::Message;
+use serde::{Deserialize, Serialize};
+
+use crate::traits::{ProtoStruct, Reader};
+use lance_core::{Error, Result};
+
+pub mod tracking_store;
+
+/// Read a protobuf message at file position 'pos'.
+///
+/// We write protobuf by first writing the length of the message as a u32,
+/// followed by the message itself.
+pub async fn read_message<M: Message + Default>(reader: &dyn Reader, pos: usize) -> Result<M> {
+    let file_size = reader.size().await?;
+    // A message is a u32 length prefix followed by its body; both must lie before
+    // the end. A `pos` too close to the end means the reader size is too small
+    // (e.g. a stale cached size). Reject it rather than slice a short buffer and
+    // panic.
+    if pos + 4 > file_size {
+        return Err(Error::io("file size is too small".to_string()));
+    }
+
+    let range = pos..min(pos + reader.block_size(), file_size);
+    let buf = reader.get_range(range.clone()).await?;
+    let msg_len = LittleEndian::read_u32(&buf) as usize;
+
+    if msg_len + 4 > buf.len() {
+        let remaining_range = range.end..min(4 + pos + msg_len, file_size);
+        let remaining_bytes = reader.get_range(remaining_range).await?;
+        let buf = [buf, remaining_bytes].concat();
+        if buf.len() < msg_len + 4 {
+            return Err(Error::io("file size is too small".to_string()));
+        }
+        Ok(M::decode(&buf[4..4 + msg_len])?)
+    } else {
+        Ok(M::decode(&buf[4..4 + msg_len])?)
+    }
+}
+
+/// Read a Protobuf-backed struct at file position: `pos`.
+// TODO: pub(crate)
+pub async fn read_struct<
+    M: Message + Default + 'static,
+    T: ProtoStruct<Proto = M> + TryFrom<M, Error = Error>,
+>(
+    reader: &dyn Reader,
+    pos: usize,
+) -> Result<T> {
+    let msg = read_message::<M>(reader, pos).await?;
+    T::try_from(msg)
+}
+
+pub async fn read_last_block(reader: &dyn Reader) -> object_store::Result<Bytes> {
+    let file_size = reader.size().await?;
+    let block_size = reader.block_size();
+    let begin = file_size.saturating_sub(block_size);
+    reader.get_range(begin..file_size).await
+}
+
+pub fn read_metadata_offset(bytes: &Bytes) -> Result<usize> {
+    let len = bytes.len();
+    if len < 16 {
+        return Err(Error::io(format!(
+            "does not have sufficient data, len: {}, bytes: {:?}",
+            len, bytes
+        )));
+    }
+    let offset_bytes = bytes.slice(len - 16..len - 8);
+    Ok(LittleEndian::read_u64(offset_bytes.as_ref()) as usize)
+}
+
+/// Read the version from the footer bytes
+pub fn read_version(bytes: &Bytes) -> Result<(u16, u16)> {
+    let len = bytes.len();
+    if len < 8 {
+        return Err(Error::io(format!(
+            "does not have sufficient data, len: {}, bytes: {:?}",
+            len, bytes
+        )));
+    }
+
+    let major_version = LittleEndian::read_u16(bytes.slice(len - 8..len - 6).as_ref());
+    let minor_version = LittleEndian::read_u16(bytes.slice(len - 6..len - 4).as_ref());
+    Ok((major_version, minor_version))
+}
+
+/// Read protobuf from a buffer.
+pub fn read_message_from_buf<M: Message + Default>(buf: &Bytes) -> Result<M> {
+    let msg_len = LittleEndian::read_u32(buf) as usize;
+    Ok(M::decode(&buf[4..4 + msg_len])?)
+}
+
+/// Read a Protobuf-backed struct from a buffer.
+pub fn read_struct_from_buf<
+    M: Message + Default,
+    T: ProtoStruct<Proto = M> + TryFrom<M, Error = Error>,
+>(
+    buf: &Bytes,
+) -> Result<T> {
+    let msg: M = read_message_from_buf(buf)?;
+    T::try_from(msg)
+}
+
+/// A cached file size.
+///
+/// This wraps an atomic u64 to allow setting the cached file size without
+/// needed a mutable reference.
+///
+/// Zero is interpreted as unknown.
+#[derive(Debug, DeepSizeOf)]
+pub struct CachedFileSize(AtomicU64);
+
+impl<'de> Deserialize<'de> for CachedFileSize {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        let size = Option::<u64>::deserialize(deserializer)?.unwrap_or(0);
+        Ok(Self::new(size))
+    }
+}
+
+impl Serialize for CachedFileSize {
+    fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let size = self.0.load(std::sync::atomic::Ordering::Relaxed);
+        if size == 0 {
+            serializer.serialize_none()
+        } else {
+            serializer.serialize_u64(size)
+        }
+    }
+}
+
+impl From<Option<NonZero<u64>>> for CachedFileSize {
+    fn from(size: Option<NonZero<u64>>) -> Self {
+        match size {
+            Some(size) => Self(AtomicU64::new(size.into())),
+            None => Self(AtomicU64::new(0)),
+        }
+    }
+}
+
+impl Default for CachedFileSize {
+    fn default() -> Self {
+        Self(AtomicU64::new(0))
+    }
+}
+
+impl Clone for CachedFileSize {
+    fn clone(&self) -> Self {
+        Self(AtomicU64::new(
+            self.0.load(std::sync::atomic::Ordering::Relaxed),
+        ))
+    }
+}
+
+impl PartialEq for CachedFileSize {
+    fn eq(&self, other: &Self) -> bool {
+        self.0.load(std::sync::atomic::Ordering::Relaxed)
+            == other.0.load(std::sync::atomic::Ordering::Relaxed)
+    }
+}
+
+impl Eq for CachedFileSize {}
+
+impl CachedFileSize {
+    /// Create a `CachedFileSize` from a raw byte count.
+    ///
+    /// Passing `0` is equivalent to calling [`unknown`](Self::unknown): the
+    /// type interprets zero as "size not yet known".
+    pub fn new(size: u64) -> Self {
+        Self(AtomicU64::new(size))
+    }
+
+    pub fn unknown() -> Self {
+        Self(AtomicU64::new(0))
+    }
+
+    pub fn get(&self) -> Option<NonZero<u64>> {
+        NonZero::new(self.0.load(std::sync::atomic::Ordering::Relaxed))
+    }
+
+    pub fn set(&self, size: NonZero<u64>) {
+        self.0
+            .store(size.into(), std::sync::atomic::Ordering::Relaxed);
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use bytes::Bytes;
+    use object_store::path::Path;
+
+    use crate::{
+        Error, Result,
+        object_reader::CloudObjectReader,
+        object_store::{DEFAULT_DOWNLOAD_RETRY_COUNT, ObjectStore},
+        object_writer::ObjectWriter,
+        traits::{ProtoStruct, WriteExt, Writer},
+        utils::read_struct,
+    };
+
+    // Bytes is a prost::Message, since we don't have any .proto files in this crate we
+    // can use it to simulate a real message object.
+    #[derive(Debug, PartialEq)]
+    struct BytesWrapper(Bytes);
+
+    impl ProtoStruct for BytesWrapper {
+        type Proto = Bytes;
+    }
+
+    impl From<&BytesWrapper> for Bytes {
+        fn from(value: &BytesWrapper) -> Self {
+            value.0.clone()
+        }
+    }
+
+    impl TryFrom<Bytes> for BytesWrapper {
+        type Error = Error;
+        fn try_from(value: Bytes) -> Result<Self> {
+            Ok(Self(value))
+        }
+    }
+
+    #[tokio::test]
+    async fn test_write_proto_structs() {
+        let store = ObjectStore::memory();
+        let path = Path::from("/foo");
+
+        let mut object_writer = ObjectWriter::new(&store, &path).await.unwrap();
+        assert_eq!(object_writer.tell().await.unwrap(), 0);
+
+        let some_message = BytesWrapper(Bytes::from(vec![10, 20, 30]));
+
+        let pos = object_writer.write_struct(&some_message).await.unwrap();
+        assert_eq!(pos, 0);
+        object_writer.shutdown().await.unwrap();
+
+        let object_reader =
+            CloudObjectReader::new(store.inner, path, 1024, None, DEFAULT_DOWNLOAD_RETRY_COUNT)
+                .unwrap();
+        let actual: BytesWrapper = read_struct(&object_reader, pos).await.unwrap();
+        assert_eq!(some_message, actual);
+    }
+
+    #[tokio::test]
+    async fn test_copy_reader_to_writer() {
+        let store = ObjectStore::memory();
+        let src = Path::from("/src");
+        let dst = Path::from("/dst");
+        store.put(&src, b"abcdef").await.unwrap();
+
+        let reader = store.open(&src).await.unwrap();
+        let mut writer = store.create(&dst).await.unwrap();
+        let copied = writer.copy_from_reader(reader.as_ref()).await.unwrap();
+        writer.shutdown().await.unwrap();
+
+        assert_eq!(copied, 6);
+        assert_eq!(store.read_one_all(&dst).await.unwrap().as_ref(), b"abcdef");
+    }
+
+    #[tokio::test]
+    async fn test_copy_reader_range_to_writer() {
+        let store = ObjectStore::memory();
+        let src = Path::from("/src-range");
+        let dst = Path::from("/dst-range");
+        store.put(&src, b"abcdef").await.unwrap();
+
+        let reader = store.open(&src).await.unwrap();
+        let mut writer = store.create(&dst).await.unwrap();
+        let copied = writer
+            .copy_range_from_reader(reader.as_ref(), 2..5)
+            .await
+            .unwrap();
+        writer.shutdown().await.unwrap();
+
+        assert_eq!(copied, 3);
+        assert_eq!(store.read_one_all(&dst).await.unwrap().as_ref(), b"cde");
+    }
+}

vendor/lance-io/src/utils/tracking_store.rs

@@ -0,0 +1,583 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The Lance Authors
+
+//! Make assertions about IO operations to an [ObjectStore].
+//!
+//! When testing code that performs IO, you will often want to make assertions
+//! about the number of reads and writes performed, the amount of data read or
+//! written, and the number of disjoint periods where at least one IO is in-flight.
+//!
+//! This modules provides [`IOTracker`] which can be used to wrap any object store.
+use std::fmt::{Display, Formatter};
+use std::ops::Range;
+#[cfg(feature = "test-util")]
+use std::sync::atomic::AtomicU16;
+use std::sync::{Arc, Mutex};
+#[cfg(feature = "metrics")]
+use std::time::Instant;
+
+use bytes::Bytes;
+use futures::StreamExt;
+use futures::TryStreamExt;
+use futures::stream::BoxStream;
+use object_store::path::Path;
+use object_store::{
+    CopyOptions, GetOptions, GetRange, GetResult, ListResult, MultipartUpload, ObjectMeta,
+    ObjectStore, PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions,
+    Result as OSResult, UploadPart,
+};
+
+use crate::object_store::WrappingObjectStore;
+#[cfg(feature = "metrics")]
+use crate::object_store::metrics::{InFlightGuard, record_outcome};
+
+#[derive(Debug, Default, Clone)]
+pub struct IOTracker {
+    stats: Arc<Mutex<IoStats>>,
+    /// The `base` label for the object store metrics published by IO that
+    /// bypasses the `object_store` layer (see [`Self::begin_io`]). `None` when
+    /// the IO cannot be attributed to a store, in which case no metrics are
+    /// published.
+    #[cfg(feature = "metrics")]
+    metrics_base: Option<Arc<str>>,
+}
+
+impl IOTracker {
+    /// Get IO statistics and reset the counters (incremental pattern).
+    ///
+    /// This returns the accumulated statistics since the last call and resets
+    /// the internal counters to zero.
+    pub fn incremental_stats(&self) -> IoStats {
+        std::mem::take(&mut *self.stats.lock().unwrap())
+    }
+
+    /// Get a snapshot of current IO statistics without resetting counters.
+    ///
+    /// This returns a clone of the current statistics without modifying the
+    /// internal state. Use this when you need to check stats without resetting.
+    pub fn stats(&self) -> IoStats {
+        self.stats.lock().unwrap().clone()
+    }
+
+    /// Record a read operation for tracking.
+    ///
+    /// This is used by readers that bypass the ObjectStore layer (like LocalObjectReader)
+    /// to ensure their IO operations are still tracked.
+    pub fn record_read(
+        &self,
+        #[allow(unused_variables)] method: &'static str,
+        #[allow(unused_variables)] path: Path,
+        num_bytes: u64,
+        #[allow(unused_variables)] range: Option<Range<u64>>,
+    ) {
+        let mut stats = self.stats.lock().unwrap();
+        stats.read_iops += 1;
+        stats.read_bytes += num_bytes;
+        #[cfg(feature = "test-util")]
+        stats.requests.push(IoRequestRecord {
+            method,
+            path,
+            range,
+        });
+    }
+
+    /// Record a write operation for tracking.
+    ///
+    /// This is used by writers that bypass the ObjectStore layer (like LocalWriter)
+    /// to ensure their IO operations are still tracked.
+    pub fn record_write(
+        &self,
+        #[allow(unused_variables)] method: &'static str,
+        #[allow(unused_variables)] path: Path,
+        num_bytes: u64,
+    ) {
+        let mut stats = self.stats.lock().unwrap();
+        stats.write_iops += 1;
+        stats.written_bytes += num_bytes;
+        #[cfg(feature = "test-util")]
+        stats.requests.push(IoRequestRecord {
+            method,
+            path,
+            range: None,
+        });
+    }
+
+    /// Label the metrics published through [`Self::begin_io`] with the prefix of
+    /// the store this tracker belongs to, so IO that bypasses the `object_store`
+    /// layer carries the same `base` label as the store's metered operations.
+    ///
+    /// Only `meter_store` should call this, so that labelling the tracker and
+    /// wrapping the store stay inseparable — see the rationale there.
+    #[cfg(feature = "metrics")]
+    pub(crate) fn set_metrics_base(&mut self, base: &str) {
+        self.metrics_base = Some(base.into());
+    }
+
+    /// Begin an operation that talks to storage without going through the
+    /// `object_store` layer, and so is invisible to the `MeteredObjectStore`
+    /// wrapper: the optimized local reads and writes go straight to the
+    /// filesystem. `operation` must be one of the labels that wrapper uses
+    /// (`get`, `put`, `head`, ...) so this IO aggregates with the rest.
+    ///
+    /// The returned guard keeps the in-flight gauge raised until it is dropped.
+    #[cfg(feature = "metrics")]
+    pub fn begin_io(&self, operation: &'static str) -> IoMetricsGuard {
+        IoMetricsGuard {
+            state: self.metrics_base.as_ref().map(|base| IoMetricsState {
+                _in_flight: InFlightGuard::new(base, operation),
+                base: base.clone(),
+                operation,
+                start: Instant::now(),
+            }),
+        }
+    }
+
+    /// Without the `metrics` feature there is nothing to publish.
+    #[cfg(not(feature = "metrics"))]
+    pub fn begin_io(&self, _operation: &'static str) -> IoMetricsGuard {
+        IoMetricsGuard {}
+    }
+}
+
+/// Publishes the object store metrics for a single operation that bypassed the
+/// `object_store` layer (see [`IOTracker::begin_io`]).
+///
+/// The operation is only counted by [`Self::record`]; one dropped before that —
+/// a cancelled read, an abandoned write — counts as neither a success nor a
+/// failure, and only lowers the in-flight gauge.
+#[must_use = "the operation is not recorded until `record` is called"]
+pub struct IoMetricsGuard {
+    #[cfg(feature = "metrics")]
+    state: Option<IoMetricsState>,
+}
+
+#[cfg(feature = "metrics")]
+struct IoMetricsState {
+    base: Arc<str>,
+    operation: &'static str,
+    start: Instant,
+    /// Lowers the in-flight gauge when the guard is dropped.
+    _in_flight: InFlightGuard,
+}
+
+impl IoMetricsGuard {
+    /// Record the operation's count and latency, along with `num_bytes`
+    /// transferred if `result` is `Ok` or an error if it is not.
+    pub fn record<T, E>(self, result: &std::result::Result<T, E>, num_bytes: u64) {
+        #[cfg(feature = "metrics")]
+        if let Some(state) = self.state {
+            record_outcome(
+                &state.base,
+                state.operation,
+                state.start,
+                num_bytes,
+                result.is_err(),
+            );
+        }
+        #[cfg(not(feature = "metrics"))]
+        let _ = (result, num_bytes);
+    }
+}
+
+impl WrappingObjectStore for IOTracker {
+    fn wrap(&self, _store_prefix: &str, target: Arc<dyn ObjectStore>) -> Arc<dyn ObjectStore> {
+        Arc::new(IoTrackingStore::new(target, self.stats.clone()))
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct IoStats {
+    pub read_iops: u64,
+    pub read_bytes: u64,
+    pub write_iops: u64,
+    pub written_bytes: u64,
+    // This is only really meaningful in tests where there isn't any concurrent IO.
+    #[cfg(feature = "test-util")]
+    /// Number of disjoint periods where at least one IO is in-flight.
+    pub num_stages: u64,
+    #[cfg(feature = "test-util")]
+    pub requests: Vec<IoRequestRecord>,
+}
+
+/// Assertions on IO statistics.
+/// assert_io_eq!(io_stats, read_iops, 1);
+/// assert_io_eq!(io_stats, write_iops, 0, "should be no writes");
+/// assert_io_eq!(io_stats, num_hops, 1, "should be just {}", "one hop");
+#[cfg(feature = "test-util")]
+#[macro_export]
+macro_rules! assert_io_eq {
+    ($io_stats:expr, $field:ident, $expected:expr) => {
+        assert_eq!(
+            $io_stats.$field, $expected,
+            "Expected {} to be {}, got {}. Requests: {:#?}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests
+        );
+    };
+    ($io_stats:expr, $field:ident, $expected:expr, $($arg:tt)+) => {
+        assert_eq!(
+            $io_stats.$field, $expected,
+            "Expected {} to be {}, got {}. Requests: {:#?} {}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests,
+            format_args!($($arg)+)
+        );
+    };
+}
+
+#[cfg(feature = "test-util")]
+#[macro_export]
+macro_rules! assert_io_gt {
+    ($io_stats:expr, $field:ident, $expected:expr) => {
+        assert!(
+            $io_stats.$field > $expected,
+            "Expected {} to be > {}, got {}. Requests: {:#?}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests
+        );
+    };
+    ($io_stats:expr, $field:ident, $expected:expr, $($arg:tt)+) => {
+        assert!(
+            $io_stats.$field > $expected,
+            "Expected {} to be > {}, got {}. Requests: {:#?} {}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests,
+            format_args!($($arg)+)
+        );
+    };
+}
+
+#[cfg(feature = "test-util")]
+#[macro_export]
+macro_rules! assert_io_lt {
+    ($io_stats:expr, $field:ident, $expected:expr) => {
+        assert!(
+            $io_stats.$field < $expected,
+            "Expected {} to be < {}, got {}. Requests: {:#?}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests
+        );
+    };
+    ($io_stats:expr, $field:ident, $expected:expr, $($arg:tt)+) => {
+        assert!(
+            $io_stats.$field < $expected,
+            "Expected {} to be < {}, got {}. Requests: {:#?} {}",
+            stringify!($field),
+            $expected,
+            $io_stats.$field,
+            $io_stats.requests,
+            format_args!($($arg)+)
+        );
+    };
+}
+
+// These request records only exist for test-only diagnostics.
+#[cfg(feature = "test-util")]
+#[derive(Clone)]
+pub struct IoRequestRecord {
+    pub method: &'static str,
+    pub path: Path,
+    pub range: Option<Range<u64>>,
+}
+
+#[cfg(feature = "test-util")]
+impl std::fmt::Debug for IoRequestRecord {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        // For example: "put /path/to/file range: 0-100"
+        write!(
+            f,
+            "IORequest(method={}, path=\"{}\"",
+            self.method, self.path
+        )?;
+        if let Some(range) = &self.range {
+            write!(f, ", range={:?}", range)?;
+        }
+        write!(f, ")")?;
+        Ok(())
+    }
+}
+
+impl Display for IoStats {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:#?}", self)
+    }
+}
+
+#[derive(Debug)]
+pub struct IoTrackingStore {
+    target: Arc<dyn ObjectStore>,
+    stats: Arc<Mutex<IoStats>>,
+    #[cfg(feature = "test-util")]
+    active_requests: Arc<AtomicU16>,
+}
+
+impl Display for IoTrackingStore {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:#?}", self)
+    }
+}
+
+impl IoTrackingStore {
+    pub fn new(target: Arc<dyn ObjectStore>, stats: Arc<Mutex<IoStats>>) -> Self {
+        Self {
+            target,
+            stats,
+            #[cfg(feature = "test-util")]
+            active_requests: Arc::new(AtomicU16::new(0)),
+        }
+    }
+
+    fn record_read(
+        &self,
+        method: &'static str,
+        path: Path,
+        num_bytes: u64,
+        range: Option<Range<u64>>,
+    ) {
+        let mut stats = self.stats.lock().unwrap();
+        stats.read_iops += 1;
+        stats.read_bytes += num_bytes;
+        #[cfg(feature = "test-util")]
+        stats.requests.push(IoRequestRecord {
+            method,
+            path,
+            range,
+        });
+        #[cfg(not(feature = "test-util"))]
+        let _ = (method, path, range); // Suppress unused variable warnings
+    }
+
+    fn record_write(&self, method: &'static str, path: Path, num_bytes: u64) {
+        let mut stats = self.stats.lock().unwrap();
+        stats.write_iops += 1;
+        stats.written_bytes += num_bytes;
+        #[cfg(feature = "test-util")]
+        stats.requests.push(IoRequestRecord {
+            method,
+            path,
+            range: None,
+        });
+        #[cfg(not(feature = "test-util"))]
+        let _ = (method, path); // Suppress unused variable warnings
+    }
+
+    #[cfg(feature = "test-util")]
+    fn stage_guard(&self) -> StageGuard {
+        StageGuard::new(self.active_requests.clone(), self.stats.clone())
+    }
+
+    #[cfg(not(feature = "test-util"))]
+    fn stage_guard(&self) -> StageGuard {
+        StageGuard
+    }
+}
+
+#[async_trait::async_trait]
+#[deny(clippy::missing_trait_methods)]
+impl ObjectStore for IoTrackingStore {
+    async fn put_opts(
+        &self,
+        location: &Path,
+        bytes: PutPayload,
+        opts: PutOptions,
+    ) -> OSResult<PutResult> {
+        let _guard = self.stage_guard();
+        self.record_write(
+            "put_opts",
+            location.to_owned(),
+            bytes.content_length() as u64,
+        );
+        self.target.put_opts(location, bytes, opts).await
+    }
+
+    async fn put_multipart_opts(
+        &self,
+        location: &Path,
+        opts: PutMultipartOptions,
+    ) -> OSResult<Box<dyn MultipartUpload>> {
+        let _guard = self.stage_guard();
+        let target = self.target.put_multipart_opts(location, opts).await?;
+        Ok(Box::new(IoTrackingMultipartUpload {
+            target,
+            stats: self.stats.clone(),
+            #[cfg(feature = "test-util")]
+            path: location.to_owned(),
+            #[cfg(feature = "test-util")]
+            _guard,
+        }))
+    }
+
+    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
+        let _guard = self.stage_guard();
+        let range = match &options.range {
+            Some(GetRange::Bounded(range)) => Some(range.clone()),
+            _ => None, // TODO: fill in other options.
+        };
+        let result = self.target.get_opts(location, options).await;
+        if let Ok(result) = &result {
+            let num_bytes = result.range.end - result.range.start;
+
+            self.record_read("get_opts", location.to_owned(), num_bytes, range);
+        }
+        result
+    }
+
+    async fn get_ranges(&self, location: &Path, ranges: &[Range<u64>]) -> OSResult<Vec<Bytes>> {
+        let _guard = self.stage_guard();
+        let result = self.target.get_ranges(location, ranges).await;
+        if let Ok(result) = &result {
+            self.record_read(
+                "get_ranges",
+                location.to_owned(),
+                result.iter().map(|b| b.len() as u64).sum(),
+                None,
+            );
+        }
+        result
+    }
+
+    fn delete_stream(
+        &self,
+        locations: BoxStream<'static, OSResult<Path>>,
+    ) -> BoxStream<'static, OSResult<Path>> {
+        let stats = Arc::clone(&self.stats);
+        let tracked = locations
+            .map_ok(move |path| {
+                let mut stats = stats.lock().unwrap();
+                stats.write_iops += 1;
+                #[cfg(feature = "test-util")]
+                stats.requests.push(IoRequestRecord {
+                    method: "delete",
+                    path: path.clone(),
+                    range: None,
+                });
+                path
+            })
+            .boxed();
+        self.target.delete_stream(tracked)
+    }
+
+    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        let _guard = self.stage_guard();
+        self.record_read("list", prefix.cloned().unwrap_or_default(), 0, None);
+        self.target.list(prefix)
+    }
+
+    fn list_with_offset(
+        &self,
+        prefix: Option<&Path>,
+        offset: &Path,
+    ) -> BoxStream<'static, OSResult<ObjectMeta>> {
+        self.record_read(
+            "list_with_offset",
+            prefix.cloned().unwrap_or_default(),
+            0,
+            None,
+        );
+        self.target.list_with_offset(prefix, offset)
+    }
+
+    async fn list_with_delimiter(&self, prefix: Option<&Path>) -> OSResult<ListResult> {
+        let _guard = self.stage_guard();
+        self.record_read(
+            "list_with_delimiter",
+            prefix.cloned().unwrap_or_default(),
+            0,
+            None,
+        );
+        self.target.list_with_delimiter(prefix).await
+    }
+
+    async fn copy_opts(&self, from: &Path, to: &Path, opts: CopyOptions) -> OSResult<()> {
+        let _guard = self.stage_guard();
+        self.record_write("copy", from.to_owned(), 0);
+        self.target.copy_opts(from, to, opts).await
+    }
+
+    async fn rename_opts(&self, from: &Path, to: &Path, opts: RenameOptions) -> OSResult<()> {
+        let _guard = self.stage_guard();
+        self.record_write("rename", from.to_owned(), 0);
+        self.target.rename_opts(from, to, opts).await
+    }
+}
+
+#[derive(Debug)]
+struct IoTrackingMultipartUpload {
+    target: Box<dyn MultipartUpload>,
+    #[cfg(feature = "test-util")]
+    path: Path,
+    stats: Arc<Mutex<IoStats>>,
+    #[cfg(feature = "test-util")]
+    _guard: StageGuard,
+}
+
+#[async_trait::async_trait]
+impl MultipartUpload for IoTrackingMultipartUpload {
+    async fn abort(&mut self) -> OSResult<()> {
+        self.target.abort().await
+    }
+
+    async fn complete(&mut self) -> OSResult<PutResult> {
+        self.target.complete().await
+    }
+
+    fn put_part(&mut self, payload: PutPayload) -> UploadPart {
+        {
+            let mut stats = self.stats.lock().unwrap();
+            stats.write_iops += 1;
+            stats.written_bytes += payload.content_length() as u64;
+            #[cfg(feature = "test-util")]
+            stats.requests.push(IoRequestRecord {
+                method: "put_part",
+                path: self.path.to_owned(),
+                range: None,
+            });
+        }
+        self.target.put_part(payload)
+    }
+}
+
+#[cfg(feature = "test-util")]
+#[derive(Debug)]
+struct StageGuard {
+    active_requests: Arc<AtomicU16>,
+    stats: Arc<Mutex<IoStats>>,
+}
+
+#[cfg(not(feature = "test-util"))]
+struct StageGuard;
+
+#[cfg(feature = "test-util")]
+impl StageGuard {
+    fn new(active_requests: Arc<AtomicU16>, stats: Arc<Mutex<IoStats>>) -> Self {
+        active_requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        Self {
+            active_requests,
+            stats,
+        }
+    }
+}
+
+#[cfg(feature = "test-util")]
+impl Drop for StageGuard {
+    fn drop(&mut self) {
+        if self
+            .active_requests
+            .fetch_sub(1, std::sync::atomic::Ordering::SeqCst)
+            == 1
+        {
+            let mut stats = self.stats.lock().unwrap();
+            stats.num_stages += 1;
+        }
+    }
+}