Add skip_merge option for FTS indexes

Cargo.toml

@@ -15,37 +15,37 @@ categories = ["database-implementations"]
 rust-version = "1.78.0"
 
 [workspace.dependencies]
-lance = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=1.0.3-rc.1", default-features = false, "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=1.0.3-rc.1", "tag" = "v1.0.3-rc.1", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=2.0.0-beta.8", default-features = false, "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=2.0.0-beta.8", "tag" = "v2.0.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "56.1", optional = false }
-arrow-array = "56.1"
-arrow-data = "56.1"
-arrow-ipc = "56.1"
-arrow-ord = "56.1"
-arrow-schema = "56.1"
-arrow-select = "56.1"
-arrow-cast = "56.1"
+arrow = { version = "57.2", optional = false }
+arrow-array = "57.2"
+arrow-data = "57.2"
+arrow-ipc = "57.2"
+arrow-ord = "57.2"
+arrow-schema = "57.2"
+arrow-select = "57.2"
+arrow-cast = "57.2"
 async-trait = "0"
-datafusion = { version = "50.0.0", default-features = false }
-datafusion-catalog = "50.0.0"
-datafusion-common = { version = "50.0.0", default-features = false }
-datafusion-execution = "50.0.0"
-datafusion-expr = "50.0.0"
-datafusion-physical-plan = "50.0.0"
+datafusion = { version = "51.0", default-features = false }
+datafusion-catalog = "51.0"
+datafusion-common = { version = "51.0", default-features = false }
+datafusion-execution = "51.0"
+datafusion-expr = "51.0"
+datafusion-physical-plan = "51.0"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
     "num-traits",

nodejs/lancedb/indices.ts

@@ -558,6 +558,12 @@ export interface FtsOptions {
    * whether to only index the prefix of the token for ngram tokenizer
    */
   prefixOnly?: boolean;
+
+  /**
+   * Whether to skip the partition merge stage after indexing.
+   * Useful for distributed indexing where merges are handled separately.
+   */
+  skipMerge?: boolean;
 }
 
 export class Index {
@@ -726,6 +732,7 @@ export class Index {
         options?.ngramMinLength,
         options?.ngramMaxLength,
         options?.prefixOnly,
+        options?.skipMerge,
       ),
     );
   }

nodejs/src/index.rs

@@ -157,6 +157,7 @@ impl Index {
         ngram_min_length: Option<u32>,
         ngram_max_length: Option<u32>,
         prefix_only: Option<bool>,
+        skip_merge: Option<bool>,
     ) -> Self {
         let mut opts = FtsIndexBuilder::default();
         if let Some(with_position) = with_position {
@@ -192,6 +193,9 @@ impl Index {
         if let Some(prefix_only) = prefix_only {
             opts = opts.ngram_prefix_only(prefix_only);
         }
+        if let Some(skip_merge) = skip_merge {
+            opts = opts.skip_merge(skip_merge);
+        }
 
         Self {
             inner: Mutex::new(Some(LanceDbIndex::FTS(opts))),

python/Cargo.toml

@@ -14,15 +14,15 @@ name = "_lancedb"
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "56.1", features = ["pyarrow"] }
+arrow = { version = "57.2", features = ["pyarrow"] }
 async-trait = "0.1"
 lancedb = { path = "../rust/lancedb", default-features = false }
 lance-core.workspace = true
 lance-namespace.workspace = true
 lance-io.workspace = true
 env_logger.workspace = true
-pyo3 = { version = "0.25.1", features = ["extension-module", "abi3-py39"] }
-pyo3-async-runtimes = { version = "0.25.0", features = [
+pyo3 = { version = "0.26", features = ["extension-module", "abi3-py39"] }
+pyo3-async-runtimes = { version = "0.26", features = [
     "attributes",
     "tokio-runtime",
 ] }
@@ -32,7 +32,7 @@ snafu.workspace = true
 tokio = { version = "1.40", features = ["sync"] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.25.0", features = [
+pyo3-build-config = { version = "0.26", features = [
     "extension-module",
     "abi3-py39",
 ] }

python/python/lancedb/_lancedb.pyi

@@ -179,7 +179,6 @@ class Table:
         cleanup_since_ms: Optional[int] = None,
         delete_unverified: Optional[bool] = None,
     ) -> OptimizeStats: ...
-    async def uri(self) -> str: ...
     @property
     def tags(self) -> Tags: ...
     def query(self) -> Query: ...

python/python/lancedb/index.py

@@ -127,6 +127,9 @@ class FTS:
     ascii_folding : bool, default True
         Whether to fold ASCII characters. This converts accented characters to
         their ASCII equivalent. For example, "café" would be converted to "cafe".
+    skip_merge : bool, default False
+        Whether to skip the partition merge stage after indexing. This can be
+        useful for distributed indexing where merges are handled separately.
     """
 
     with_position: bool = False
@@ -140,6 +143,7 @@ class FTS:
     ngram_min_length: int = 3
     ngram_max_length: int = 3
     prefix_only: bool = False
+    skip_merge: bool = False
 
 
 @dataclass

python/python/lancedb/remote/table.py

@@ -176,6 +176,7 @@ class RemoteTable(Table):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
+        skip_merge: bool = False,
         name: Optional[str] = None,
     ):
         config = FTS(
@@ -190,6 +191,7 @@ class RemoteTable(Table):
             ngram_min_length=ngram_min_length,
             ngram_max_length=ngram_max_length,
             prefix_only=prefix_only,
+            skip_merge=skip_merge,
         )
         LOOP.run(
             self._table.create_index(
@@ -655,14 +657,6 @@ class RemoteTable(Table):
     def stats(self):
         return LOOP.run(self._table.stats())
 
-    @property
-    def uri(self) -> str:
-        """The table URI (storage location).
-
-        For remote tables, this fetches the location from the server via describe.
-        """
-        return LOOP.run(self._table.uri())
-
     def take_offsets(self, offsets: list[int]) -> LanceTakeQueryBuilder:
         return LanceTakeQueryBuilder(self._table.take_offsets(offsets))
 

python/python/lancedb/table.py

@@ -892,6 +892,7 @@ class Table(ABC):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
+        skip_merge: bool = False,
         wait_timeout: Optional[timedelta] = None,
         name: Optional[str] = None,
     ):
@@ -956,6 +957,9 @@ class Table(ABC):
             The maximum length of an n-gram.
         prefix_only: bool, default False
             Whether to only index the prefix of the token for ngram tokenizer.
+        skip_merge: bool, default False
+            Only available with use_tantivy=False.
+            If True, skip the partition merge stage after indexing.
         wait_timeout: timedelta, optional
             The timeout to wait if indexing is asynchronous.
         name: str, optional
@@ -2218,10 +2222,6 @@ class LanceTable(Table):
     def stats(self) -> TableStatistics:
         return LOOP.run(self._table.stats())
 
-    @property
-    def uri(self) -> str:
-        return LOOP.run(self._table.uri())
-
     def create_scalar_index(
         self,
         column: str,
@@ -2263,6 +2263,7 @@ class LanceTable(Table):
         ngram_min_length: int = 3,
         ngram_max_length: int = 3,
         prefix_only: bool = False,
+        skip_merge: bool = False,
         name: Optional[str] = None,
     ):
         if not use_tantivy:
@@ -2286,6 +2287,8 @@ class LanceTable(Table):
             else:
                 tokenizer_configs = self.infer_tokenizer_configs(tokenizer_name)
 
+            tokenizer_configs["skip_merge"] = skip_merge
+
             config = FTS(
                 **tokenizer_configs,
             )
@@ -3610,20 +3613,6 @@ class AsyncTable:
         """
         return await self._inner.stats()
 
-    async def uri(self) -> str:
-        """
-        Get the table URI (storage location).
-
-        For remote tables, this fetches the location from the server via describe.
-        For local tables, this returns the dataset URI.
-
-        Returns
-        -------
-        str
-            The full storage location of the table (e.g., S3/GCS path).
-        """
-        return await self._inner.uri()
-
     async def add(
         self,
         data: DATA,

python/python/tests/test_remote_db.py

@@ -405,7 +405,10 @@ def test_table_create_indices():
 
         # Test create_fts_index with custom name
         table.create_fts_index(
-            "text", wait_timeout=timedelta(seconds=2), name="custom_fts_idx"
+            "text",
+            wait_timeout=timedelta(seconds=2),
+            name="custom_fts_idx",
+            skip_merge=True,
         )
 
         # Test create_index with custom name
@@ -427,6 +430,7 @@ def test_table_create_indices():
         fts_req = received_requests[1]
         assert "name" in fts_req
         assert fts_req["name"] == "custom_fts_idx"
+        assert fts_req["skip_merge"] is True
 
         # Check vector index request has custom name
         vector_req = received_requests[2]

python/python/tests/test_table.py

@@ -1967,9 +1967,3 @@ def test_add_table_with_empty_embeddings(tmp_path):
         on_bad_vectors="drop",
     )
     assert table.count_rows() == 1
-
-
-def test_table_uri(tmp_path):
-    db = lancedb.connect(tmp_path)
-    table = db.create_table("my_table", data=[{"x": 0}])
-    assert table.uri == str(tmp_path / "my_table.lance")

python/src/arrow.rs

@@ -36,7 +36,10 @@ impl RecordBatchStream {
 impl RecordBatchStream {
     #[getter]
     pub fn schema(&self, py: Python) -> PyResult<Py<PyAny>> {
-        (*self.schema).clone().into_pyarrow(py)
+        (*self.schema)
+            .clone()
+            .into_pyarrow(py)
+            .map(|obj| obj.unbind())
     }
 
     pub fn __aiter__(self_: PyRef<'_, Self>) -> PyRef<'_, Self> {
@@ -54,7 +57,8 @@ impl RecordBatchStream {
                 .ok_or_else(|| PyStopAsyncIteration::new_err(""))?;
             #[allow(deprecated)]
             let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
-                inner_next.infer_error()?.to_pyarrow(py)
+                let bound = inner_next.infer_error()?.to_pyarrow(py)?;
+                Ok(bound.unbind())
             })?;
             Ok(py_obj)
         })

python/src/index.rs

@@ -50,7 +50,8 @@ pub fn extract_index_params(source: &Option<Bound<'_, PyAny>>) -> PyResult<Lance
                     .ascii_folding(params.ascii_folding)
                     .ngram_min_length(params.ngram_min_length)
                     .ngram_max_length(params.ngram_max_length)
-                    .ngram_prefix_only(params.prefix_only);
+                    .ngram_prefix_only(params.prefix_only)
+                    .skip_merge(params.skip_merge);
                 Ok(LanceDbIndex::FTS(inner_opts))
             },
             "IvfFlat" => {
@@ -179,6 +180,7 @@ struct FtsParams {
     ngram_min_length: u32,
     ngram_max_length: u32,
     prefix_only: bool,
+    skip_merge: bool,
 }
 
 #[derive(FromPyObject)]

python/src/permutation.rs

@@ -282,8 +282,10 @@ impl PyPermutationReader {
         future_into_py(slf.py(), async move {
             let schema = reader.output_schema(selection).await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }

python/src/query.rs

@@ -455,8 +455,10 @@ impl Query {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }
@@ -537,8 +539,10 @@ impl TakeQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }
@@ -635,8 +639,10 @@ impl FTSQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }
@@ -817,8 +823,10 @@ impl VectorQuery {
         future_into_py(self_.py(), async move {
             let schema = inner.output_schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }

python/src/table.rs

@@ -288,8 +288,10 @@ impl Table {
         future_into_py(self_.py(), async move {
             let schema = inner.schema().await.infer_error()?;
             #[allow(deprecated)]
-            let py_obj: Py<PyAny> =
-                Python::with_gil(|py| -> PyResult<Py<PyAny>> { schema.to_pyarrow(py) })?;
+            let py_obj: Py<PyAny> = Python::with_gil(|py| -> PyResult<Py<PyAny>> {
+                let bound = schema.to_pyarrow(py)?;
+                Ok(bound.unbind())
+            })?;
             Ok(py_obj)
         })
     }
@@ -502,11 +504,6 @@ impl Table {
         })
     }
 
-    pub fn uri(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
-        let inner = self_.inner_ref()?.clone();
-        future_into_py(self_.py(), async move { inner.uri().await.infer_error() })
-    }
-
     pub fn __repr__(&self) -> String {
         match &self.inner {
             None => format!("ClosedTable({})", self.name),

rust/lancedb/src/connection.rs

@@ -9,7 +9,6 @@ use std::sync::Arc;
 use arrow_array::RecordBatchReader;
 use arrow_schema::{Field, SchemaRef};
 use lance::dataset::ReadParams;
-use lance::io::ObjectStoreParams;
 use lance_namespace::models::{
     CreateNamespaceRequest, CreateNamespaceResponse, DescribeNamespaceRequest,
     DescribeNamespaceResponse, DropNamespaceRequest, DropNamespaceResponse, ListNamespacesRequest,
@@ -40,18 +39,7 @@ use crate::Table;
 pub use lance_encoding::version::LanceFileVersion;
 #[cfg(feature = "remote")]
 use lance_io::object_store::StorageOptions;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
-
-fn update_storage_options<F>(store_params: &mut ObjectStoreParams, update: F)
-where
-    F: FnOnce(&mut HashMap<String, String>),
-{
-    let mut options = store_params.storage_options().cloned().unwrap_or_default();
-    update(&mut options);
-    store_params.storage_options_accessor = Some(Arc::new(
-        StorageOptionsAccessor::with_static_options(options),
-    ));
-}
+use lance_io::object_store::StorageOptionsProvider;
 
 /// A builder for configuring a [`Connection::table_names`] operation
 pub struct TableNamesBuilder {
@@ -258,16 +246,16 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let store_params = self
+        let store_options = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
+            .get_or_insert(Default::default())
+            .storage_options
             .get_or_insert(Default::default());
-        update_storage_options(store_params, |options| {
-            options.insert(key.into(), value.into());
-        });
+        store_options.insert(key.into(), value.into());
         self
     }
 
@@ -281,18 +269,19 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let store_params = self
+        let store_options = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
+            .get_or_insert(Default::default())
+            .storage_options
             .get_or_insert(Default::default());
-        update_storage_options(store_params, |options| {
-            for (key, value) in pairs {
-                options.insert(key.into(), value.into());
-            }
-        });
+
+        for (key, value) in pairs {
+            store_options.insert(key.into(), value.into());
+        }
         self
     }
 
@@ -329,23 +318,24 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This has no effect in LanceDB Cloud.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn enable_v2_manifest_paths(mut self, use_v2_manifest_paths: bool) -> Self {
-        let store_params = self
+        let storage_options = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
+            .get_or_insert_with(Default::default)
+            .storage_options
             .get_or_insert_with(Default::default);
-        update_storage_options(store_params, |options| {
-            options.insert(
-                OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(),
-                if use_v2_manifest_paths {
-                    "true".to_string()
-                } else {
-                    "false".to_string()
-                },
-            );
-        });
+
+        storage_options.insert(
+            OPT_NEW_TABLE_V2_MANIFEST_PATHS.to_string(),
+            if use_v2_manifest_paths {
+                "true".to_string()
+            } else {
+                "false".to_string()
+            },
+        );
         self
     }
 
@@ -354,19 +344,20 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// The default is `LanceFileVersion::Stable`.
     #[deprecated(since = "0.15.1", note = "Use `database_options` instead")]
     pub fn data_storage_version(mut self, data_storage_version: LanceFileVersion) -> Self {
-        let store_params = self
+        let storage_options = self
             .request
             .write_options
             .lance_write_params
             .get_or_insert_with(Default::default)
             .store_params
+            .get_or_insert_with(Default::default)
+            .storage_options
             .get_or_insert_with(Default::default);
-        update_storage_options(store_params, |options| {
-            options.insert(
-                OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
-                data_storage_version.to_string(),
-            );
-        });
+
+        storage_options.insert(
+            OPT_NEW_TABLE_STORAGE_VERSION.to_string(),
+            data_storage_version.to_string(),
+        );
         self
     }
 
@@ -390,17 +381,13 @@ impl<const HAS_DATA: bool> CreateTableBuilder<HAS_DATA> {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        let store_params = self
-            .request
+        self.request
             .write_options
             .lance_write_params
             .get_or_insert(Default::default())
             .store_params
-            .get_or_insert(Default::default());
-        let initial = store_params.storage_options().cloned().unwrap_or_default();
-        store_params.storage_options_accessor = Some(Arc::new(
-            StorageOptionsAccessor::with_initial_and_provider(initial, provider),
-        ));
+            .get_or_insert(Default::default())
+            .storage_options_provider = Some(provider);
         self
     }
 }
@@ -463,15 +450,15 @@ impl OpenTableBuilder {
     ///
     /// See available options at <https://lancedb.com/docs/storage/>
     pub fn storage_option(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
-        let store_params = self
+        let storage_options = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
+            .get_or_insert(Default::default())
+            .storage_options
             .get_or_insert(Default::default());
-        update_storage_options(store_params, |options| {
-            options.insert(key.into(), value.into());
-        });
+        storage_options.insert(key.into(), value.into());
         self
     }
 
@@ -485,17 +472,18 @@ impl OpenTableBuilder {
         mut self,
         pairs: impl IntoIterator<Item = (impl Into<String>, impl Into<String>)>,
     ) -> Self {
-        let store_params = self
+        let storage_options = self
             .request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
+            .get_or_insert(Default::default())
+            .storage_options
             .get_or_insert(Default::default());
-        update_storage_options(store_params, |options| {
-            for (key, value) in pairs {
-                options.insert(key.into(), value.into());
-            }
-        });
+
+        for (key, value) in pairs {
+            storage_options.insert(key.into(), value.into());
+        }
         self
     }
 
@@ -519,16 +507,12 @@ impl OpenTableBuilder {
     /// This allows tables to automatically refresh cloud storage credentials
     /// when they expire, enabling long-running operations on remote storage.
     pub fn storage_options_provider(mut self, provider: Arc<dyn StorageOptionsProvider>) -> Self {
-        let store_params = self
-            .request
+        self.request
             .lance_read_params
             .get_or_insert(Default::default())
             .store_options
-            .get_or_insert(Default::default());
-        let initial = store_params.storage_options().cloned().unwrap_or_default();
-        store_params.storage_options_accessor = Some(Arc::new(
-            StorageOptionsAccessor::with_initial_and_provider(initial, provider),
-        ));
+            .get_or_insert(Default::default())
+            .storage_options_provider = Some(provider);
         self
     }
 

rust/lancedb/src/database/listing.rs

@@ -12,7 +12,7 @@ use lance::dataset::{builder::DatasetBuilder, ReadParams, WriteMode};
 use lance::io::{ObjectStore, ObjectStoreParams, WrappingObjectStore};
 use lance_datafusion::utils::StreamingWriteSource;
 use lance_encoding::version::LanceFileVersion;
-use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider};
+use lance_io::object_store::StorageOptionsProvider;
 use lance_table::io::commit::commit_handler_from_url;
 use object_store::local::LocalFileSystem;
 use snafu::ResultExt;
@@ -356,11 +356,7 @@ impl ListingDatabase {
                     .clone()
                     .unwrap_or_else(|| Arc::new(lance::session::Session::default()));
                 let os_params = ObjectStoreParams {
-                    storage_options_accessor: Some(Arc::new(
-                        StorageOptionsAccessor::with_static_options(
-                            options.storage_options.clone(),
-                        ),
-                    )),
+                    storage_options: Some(options.storage_options.clone()),
                     ..Default::default()
                 };
                 let (object_store, base_path) = ObjectStore::from_uri_and_params(
@@ -467,20 +463,9 @@ impl ListingDatabase {
         validate_table_name(name)?;
 
         let mut uri = self.uri.clone();
-        // If the URI does not end with a path separator, add one
-        // Use forward slash for URIs (http://, s3://, gs://, file://, etc.)
-        // Use platform-specific separator for local paths without scheme
-        let has_scheme = uri.contains("://");
-        let ends_with_separator = uri.ends_with('/') || uri.ends_with('\\');
-
-        if !ends_with_separator {
-            if has_scheme {
-                // URIs always use forward slash
-                uri.push('/');
-            } else {
-                // Local path without scheme - use platform separator
-                uri.push(std::path::MAIN_SEPARATOR);
-            }
+        // If the URI does not end with a slash, add one
+        if !uri.ends_with('/') {
+            uri.push('/');
         }
         // Append the table name with the lance file extension
         uri.push_str(&format!("{}.{}", name, LANCE_FILE_EXTENSION));
@@ -496,9 +481,7 @@ impl ListingDatabase {
 
     async fn drop_tables(&self, names: Vec<String>) -> Result<()> {
         let object_store_params = ObjectStoreParams {
-            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
-                self.storage_options.clone(),
-            ))),
+            storage_options: Some(self.storage_options.clone()),
             ..Default::default()
         };
         let mut uri = self.uri.clone();
@@ -547,7 +530,7 @@ impl ListingDatabase {
             .lance_write_params
             .as_ref()
             .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options());
+            .and_then(|sp| sp.storage_options.as_ref());
 
         let storage_version_override = storage_options
             .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
@@ -599,25 +582,20 @@ impl ListingDatabase {
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
         if !self.storage_options.is_empty() {
-            let store_params = write_params
+            let storage_options = write_params
                 .store_params
+                .get_or_insert_with(Default::default)
+                .storage_options
                 .get_or_insert_with(Default::default);
-            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
-            self.inherit_storage_options(&mut storage_options);
-            store_params.storage_options_accessor = Some(Arc::new(
-                StorageOptionsAccessor::with_static_options(storage_options),
-            ));
+            self.inherit_storage_options(storage_options);
         }
 
         // Set storage options provider if available
-        if let Some(provider) = self.storage_options_provider.clone() {
-            let store_params = write_params
+        if self.storage_options_provider.is_some() {
+            write_params
                 .store_params
-                .get_or_insert_with(Default::default);
-            let initial = store_params.storage_options().cloned().unwrap_or_default();
-            store_params.storage_options_accessor = Some(Arc::new(
-                StorageOptionsAccessor::with_initial_and_provider(initial, provider),
-            ));
+                .get_or_insert_with(Default::default)
+                .storage_options_provider = self.storage_options_provider.clone();
         }
 
         write_params.data_storage_version = self
@@ -903,9 +881,7 @@ impl Database for ListingDatabase {
         validate_table_name(&request.target_table_name)?;
 
         let storage_params = ObjectStoreParams {
-            storage_options_accessor: Some(Arc::new(StorageOptionsAccessor::with_static_options(
-                self.storage_options.clone(),
-            ))),
+            storage_options: Some(self.storage_options.clone()),
             ..Default::default()
         };
         let read_params = ReadParams {
@@ -970,29 +946,24 @@ impl Database for ListingDatabase {
         // be dropped from the cache when python GCs the table object, which
         // confounds reuse across tables.
         if !self.storage_options.is_empty() {
-            let store_params = request
+            let storage_options = request
                 .lance_read_params
                 .get_or_insert_with(Default::default)
                 .store_options
+                .get_or_insert_with(Default::default)
+                .storage_options
                 .get_or_insert_with(Default::default);
-            let mut storage_options = store_params.storage_options().cloned().unwrap_or_default();
-            self.inherit_storage_options(&mut storage_options);
-            store_params.storage_options_accessor = Some(Arc::new(
-                StorageOptionsAccessor::with_static_options(storage_options),
-            ));
+            self.inherit_storage_options(storage_options);
         }
 
         // Set storage options provider if available
-        if let Some(provider) = self.storage_options_provider.clone() {
-            let store_params = request
+        if self.storage_options_provider.is_some() {
+            request
                 .lance_read_params
                 .get_or_insert_with(Default::default)
                 .store_options
-                .get_or_insert_with(Default::default);
-            let initial = store_params.storage_options().cloned().unwrap_or_default();
-            store_params.storage_options_accessor = Some(Arc::new(
-                StorageOptionsAccessor::with_initial_and_provider(initial, provider),
-            ));
+                .get_or_insert_with(Default::default)
+                .storage_options_provider = self.storage_options_provider.clone();
         }
 
         // Some ReadParams are exposed in the OpenTableBuilder, but we also
@@ -1100,7 +1071,6 @@ mod tests {
     use crate::table::{Table, TableDefinition};
     use arrow_array::{Int32Array, RecordBatch, StringArray};
     use arrow_schema::{DataType, Field, Schema};
-    use std::path::PathBuf;
     use tempfile::tempdir;
 
     async fn setup_database() -> (tempfile::TempDir, ListingDatabase) {
@@ -1899,9 +1869,7 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options_accessor: Some(Arc::new(
-                        StorageOptionsAccessor::with_static_options(storage_options),
-                    )),
+                    storage_options: Some(storage_options),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -1975,9 +1943,7 @@ mod tests {
         let write_options = WriteOptions {
             lance_write_params: Some(lance::dataset::WriteParams {
                 store_params: Some(lance::io::ObjectStoreParams {
-                    storage_options_accessor: Some(Arc::new(
-                        StorageOptionsAccessor::with_static_options(storage_options),
-                    )),
+                    storage_options: Some(storage_options),
                     ..Default::default()
                 }),
                 ..Default::default()
@@ -2080,19 +2046,6 @@ mod tests {
         assert_eq!(db_options.new_table_config.enable_stable_row_ids, None);
     }
 
-    #[tokio::test]
-    async fn test_table_uri() {
-        let (_tempdir, db) = setup_database().await;
-
-        let mut pb = PathBuf::new();
-        pb.push(db.uri.clone());
-        pb.push("test.lance");
-
-        let expected = pb.to_str().unwrap();
-        let uri = db.table_uri("test").ok().unwrap();
-        assert_eq!(uri, expected);
-    }
-
     #[tokio::test]
     async fn test_namespace_client() {
         let (_tempdir, db) = setup_database().await;

rust/lancedb/src/index/scalar.rs

@@ -53,5 +53,8 @@ pub struct LabelListIndexBuilder {}
 
 pub use lance_index::scalar::inverted::query::*;
 pub use lance_index::scalar::FullTextSearchQuery;
+/// Builder for full text search (FTS) index parameters.
+///
+/// Use [`FtsIndexBuilder::skip_merge`] to skip the partition merge stage after indexing.
 pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder;
 pub use lance_index::scalar::InvertedIndexParams;

rust/lancedb/src/remote/table.rs

@@ -204,7 +204,6 @@ pub struct RemoteTable<S: HttpSend = Sender> {
     server_version: ServerVersion,
 
     version: RwLock<Option<u64>>,
-    location: RwLock<Option<String>>,
 }
 
 impl<S: HttpSend> RemoteTable<S> {
@@ -222,7 +221,6 @@ impl<S: HttpSend> RemoteTable<S> {
             identifier,
             server_version,
             version: RwLock::new(None),
-            location: RwLock::new(None),
         }
     }
 
@@ -641,7 +639,6 @@ impl<S: HttpSend> RemoteTable<S> {
 struct TableDescription {
     version: u64,
     schema: JsonSchema,
-    location: Option<String>,
 }
 
 impl<S: HttpSend> std::fmt::Display for RemoteTable<S> {
@@ -670,7 +667,6 @@ mod test_utils {
                 identifier: name,
                 server_version: version.map(ServerVersion).unwrap_or_default(),
                 version: RwLock::new(None),
-                location: RwLock::new(None),
             }
         }
     }
@@ -1465,28 +1461,8 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
             message: "table_definition is not supported on LanceDB cloud.".into(),
         })
     }
-    async fn uri(&self) -> Result<String> {
-        // Check if we already have the location cached
-        {
-            let location = self.location.read().await;
-            if let Some(ref loc) = *location {
-                return Ok(loc.clone());
-            }
-        }
-
-        // Fetch from server via describe
-        let description = self.describe().await?;
-        let location = description.location.ok_or_else(|| Error::NotSupported {
-            message: "Table URI not supported by the server".into(),
-        })?;
-
-        // Cache the location for future use
-        {
-            let mut cached_location = self.location.write().await;
-            *cached_location = Some(location.clone());
-        }
-
-        Ok(location)
+    fn dataset_uri(&self) -> &str {
+        "NOT_SUPPORTED"
     }
 
     async fn storage_options(&self) -> Option<HashMap<String, String>> {
@@ -2648,6 +2624,11 @@ mod tests {
                 serde_json::to_value(InvertedIndexParams::default()).unwrap(),
                 Index::FTS(Default::default()),
             ),
+            (
+                "FTS",
+                serde_json::to_value(InvertedIndexParams::default().skip_merge(true)).unwrap(),
+                Index::FTS(InvertedIndexParams::default().skip_merge(true)),
+            ),
         ];
 
         for (index_type, expected_body, index) in cases {
@@ -3356,69 +3337,4 @@ mod tests {
         let result = table.drop_columns(&["old_col1", "old_col2"]).await.unwrap();
         assert_eq!(result.version, 5);
     }
-
-    #[tokio::test]
-    async fn test_uri() {
-        let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.method(), "POST");
-            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
-
-            http::Response::builder()
-                .status(200)
-                .body(r#"{"version": 1, "schema": {"fields": []}, "location": "s3://bucket/path/to/table"}"#)
-                .unwrap()
-        });
-
-        let uri = table.uri().await.unwrap();
-        assert_eq!(uri, "s3://bucket/path/to/table");
-    }
-
-    #[tokio::test]
-    async fn test_uri_missing_location() {
-        let table = Table::new_with_handler("my_table", |request| {
-            assert_eq!(request.method(), "POST");
-            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
-
-            // Server returns response without location field
-            http::Response::builder()
-                .status(200)
-                .body(r#"{"version": 1, "schema": {"fields": []}}"#)
-                .unwrap()
-        });
-
-        let result = table.uri().await;
-        assert!(result.is_err());
-        assert!(matches!(&result, Err(Error::NotSupported { .. })));
-    }
-
-    #[tokio::test]
-    async fn test_uri_caching() {
-        use std::sync::atomic::{AtomicUsize, Ordering};
-        use std::sync::Arc;
-
-        let call_count = Arc::new(AtomicUsize::new(0));
-        let call_count_clone = call_count.clone();
-
-        let table = Table::new_with_handler("my_table", move |request| {
-            assert_eq!(request.url().path(), "/v1/table/my_table/describe/");
-            call_count_clone.fetch_add(1, Ordering::SeqCst);
-
-            http::Response::builder()
-                .status(200)
-                .body(
-                    r#"{"version": 1, "schema": {"fields": []}, "location": "gs://bucket/table"}"#,
-                )
-                .unwrap()
-        });
-
-        // First call should fetch from server
-        let uri1 = table.uri().await.unwrap();
-        assert_eq!(uri1, "gs://bucket/table");
-        assert_eq!(call_count.load(Ordering::SeqCst), 1);
-
-        // Second call should use cached value
-        let uri2 = table.uri().await.unwrap();
-        assert_eq!(uri2, "gs://bucket/table");
-        assert_eq!(call_count.load(Ordering::SeqCst), 1); // Still 1, no new call
-    }
 }

rust/lancedb/src/table.rs

@@ -40,7 +40,7 @@ use lance_index::vector::pq::PQBuildParams;
 use lance_index::vector::sq::builder::SQBuildParams;
 use lance_index::DatasetIndexExt;
 use lance_index::IndexType;
-use lance_io::object_store::{LanceNamespaceStorageOptionsProvider, StorageOptionsAccessor};
+use lance_io::object_store::LanceNamespaceStorageOptionsProvider;
 use lance_namespace::models::{
     QueryTableRequest as NsQueryTableRequest, QueryTableRequestColumns,
     QueryTableRequestFullTextQuery, QueryTableRequestVector, StringFtsQuery,
@@ -608,8 +608,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
     async fn list_versions(&self) -> Result<Vec<Version>>;
     /// Get the table definition.
     async fn table_definition(&self) -> Result<TableDefinition>;
-    /// Get the table URI (storage location)
-    async fn uri(&self) -> Result<String>;
+    /// Get the table URI
+    fn dataset_uri(&self) -> &str;
     /// Get the storage options used when opening this table, if any.
     async fn storage_options(&self) -> Option<HashMap<String, String>>;
     /// Poll until the columns are fully indexed. Will return Error::Timeout if the columns
@@ -1317,12 +1317,11 @@ impl Table {
         self.inner.list_indices().await
     }
 
-    /// Get the table URI (storage location)
+    /// Get the underlying dataset URI
     ///
-    /// Returns the full storage location of the table (e.g., S3/GCS path).
-    /// For remote tables, this fetches the location from the server via describe.
-    pub async fn uri(&self) -> Result<String> {
-        self.inner.uri().await
+    /// Warning: This is an internal API and the return value is subject to change.
+    pub fn dataset_uri(&self) -> &str {
+        self.inner.dataset_uri()
     }
 
     /// Get the storage options used when opening this table, if any.
@@ -1425,7 +1424,9 @@ impl Table {
             })
             .collect::<Vec<_>>();
 
-        let unioned = Arc::new(UnionExec::new(projected_plans));
+        let unioned = UnionExec::try_new(projected_plans).map_err(|e| Error::Runtime {
+            message: format!("Failed to build union plan: {e}"),
+        })?;
         // We require 1 partition in the final output
         let repartitioned = RepartitionExec::try_new(
             unioned,
@@ -1666,14 +1667,18 @@ impl NativeTable {
 
         // Use DatasetBuilder::from_namespace which automatically fetches location
         // and storage options from the namespace
-        let builder = DatasetBuilder::from_namespace(namespace_client.clone(), table_id)
-            .await
-            .map_err(|e| match e {
-                lance::Error::Namespace { source, .. } => Error::Runtime {
-                    message: format!("Failed to get table info from namespace: {:?}", source),
-                },
-                source => Error::Lance { source },
-            })?;
+        let builder = DatasetBuilder::from_namespace(
+            namespace_client.clone(),
+            table_id,
+            false, // Don't ignore namespace storage options
+        )
+        .await
+        .map_err(|e| match e {
+            lance::Error::Namespace { source, .. } => Error::Runtime {
+                message: format!("Failed to get table info from namespace: {:?}", source),
+            },
+            source => Error::Lance { source },
+        })?;
 
         let dataset = builder
             .with_read_params(params)
@@ -1877,10 +1882,7 @@ impl NativeTable {
         let store_params = params
             .store_params
             .get_or_insert_with(ObjectStoreParams::default);
-        let initial = store_params.storage_options().cloned().unwrap_or_default();
-        store_params.storage_options_accessor = Some(Arc::new(
-            StorageOptionsAccessor::with_initial_and_provider(initial, storage_options_provider),
-        ));
+        store_params.storage_options_provider = Some(storage_options_provider);
 
         // Patch the params if we have a write store wrapper
         let params = match write_store_wrapper.clone() {
@@ -3232,8 +3234,8 @@ impl BaseTable for NativeTable {
         Ok(results.into_iter().flatten().collect())
     }
 
-    async fn uri(&self) -> Result<String> {
-        Ok(self.uri.clone())
+    fn dataset_uri(&self) -> &str {
+        self.uri.as_str()
     }
 
     async fn storage_options(&self) -> Option<HashMap<String, String>> {
@@ -3241,7 +3243,7 @@ impl BaseTable for NativeTable {
             .get()
             .await
             .ok()
-            .and_then(|dataset| dataset.initial_storage_options().cloned())
+            .and_then(|dataset| dataset.storage_options().cloned())
     }
 
     async fn index_stats(&self, index_name: &str) -> Result<Option<IndexStatistics>> {

rust/lancedb/src/table/dataset.rs

@@ -100,7 +100,8 @@ impl DatasetRef {
                 let should_checkout = match &target_ref {
                     refs::Ref::Version(_, Some(target_ver)) => version != target_ver,
                     refs::Ref::Version(_, None) => true, // No specific version, always checkout
-                    refs::Ref::Tag(_) => true,           // Always checkout for tags
+                    refs::Ref::VersionNumber(target_ver) => version != target_ver,
+                    refs::Ref::Tag(_) => true, // Always checkout for tags
                 };
 
                 if should_checkout {