chore: update lance dependency to v6.0.0-beta.6

2026-05-23 06:50:40 +00:00 · 2026-04-29 07:27:24 +00:00
48 changed files with 932 additions and 2897 deletions
--- a/.bumpversion.toml
+++ b/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.28.0-beta.11"
+current_version = "0.28.0-beta.10"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,40 +13,40 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=7.0.0-beta.5", default-features = false, "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=7.0.0-beta.5", "tag" = "v7.0.0-beta.5", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=6.0.0-beta.6", default-features = false, "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=6.0.0-beta.6", default-features = false, "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=6.0.0-beta.6", default-features = false, "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=6.0.0-beta.6", "tag" = "v6.0.0-beta.6", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
-arrow = { version = "58.0.0", optional = false }
-arrow-array = "58.0.0"
-arrow-data = "58.0.0"
-arrow-ipc = "58.0.0"
-arrow-ord = "58.0.0"
-arrow-schema = "58.0.0"
-arrow-select = "58.0.0"
-arrow-cast = "58.0.0"
+arrow = { version = "58.1", optional = false }
+arrow-array = "58.1"
+arrow-data = "58.1"
+arrow-ipc = "58.1"
+arrow-ord = "58.1"
+arrow-schema = "58.1"
+arrow-select = "58.1"
+arrow-cast = "58.1"
 async-trait = "0"
-datafusion = { version = "53.0.0", default-features = false }
-datafusion-catalog = "53.0.0"
-datafusion-common = { version = "53.0.0", default-features = false }
-datafusion-execution = "53.0.0"
-datafusion-expr = "53.0.0"
-datafusion-functions = "53.0.0"
-datafusion-physical-plan = "53.0.0"
-datafusion-physical-expr = "53.0.0"
-datafusion-sql = "53.0.0"
+datafusion = { version = "53.1", default-features = false }
+datafusion-catalog = "53.1"
+datafusion-common = { version = "53.1", default-features = false }
+datafusion-execution = "53.1"
+datafusion-expr = "53.1"
+datafusion-functions = "53.1"
+datafusion-physical-plan = "53.1"
+datafusion-physical-expr = "53.1"
+datafusion-sql = "53.1"
 env_logger = "0.11"
 half = { "version" = "2.7.1", default-features = false, features = [
    "num-traits",
@@ -54,7 +54,7 @@ half = { "version" = "2.7.1", default-features = false, features = [
 futures = "0"
 log = "0.4"
 moka = { version = "0.12", features = ["future"] }
-object_store = "0.13.2"
+object_store = "0.12.0"
 pin-project = "1.0.7"
 rand = "0.9"
 snafu = "0.8"
--- a/docs/src/java/java.md
+++ b/docs/src/java/java.md
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
 <dependency>
    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-core</artifactId>
-    <version>0.28.0-beta.11</version>
+    <version>0.28.0-beta.10</version>
 </dependency>
 ```

--- a/docs/src/js/interfaces/ConnectionOptions.md
+++ b/docs/src/js/interfaces/ConnectionOptions.md
@@ -41,29 +41,6 @@ for testing purposes.

 ***

-### manifestEnabled?
-
-```ts
-optional manifestEnabled: boolean;
-```
-
-(For LanceDB OSS only): use directory namespace manifests as the source
-of truth for table metadata. Existing directory-listed root tables are
-migrated into the manifest on access.
-
-***
-
-### namespaceClientProperties?
-
-```ts
-optional namespaceClientProperties: Record<string, string>;
-```
-
-(For LanceDB OSS only): extra properties for the backing namespace
-client used by manifest-enabled native connections.
-
-***
-
 ### readConsistencyInterval?

 ```ts
--- a/java/lancedb-core/pom.xml
+++ b/java/lancedb-core/pom.xml
@@ -8,7 +8,7 @@
    <parent>
      <groupId>com.lancedb</groupId>
      <artifactId>lancedb-parent</artifactId>
-      <version>0.28.0-beta.11</version>
+      <version>0.28.0-beta.10</version>
      <relativePath>../pom.xml</relativePath>
    </parent>

--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@

    <groupId>com.lancedb</groupId>
    <artifactId>lancedb-parent</artifactId>
-    <version>0.28.0-beta.11</version>
+    <version>0.28.0-beta.10</version>
    <packaging>pom</packaging>
    <name>${project.artifactId}</name>
    <description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>7.0.0-beta.5</lance-core.version>
+        <lance-core.version>6.0.0-beta.6</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
--- a/nodejs/Cargo.toml
+++ b/nodejs/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "lancedb-nodejs"
 edition.workspace = true
-version = "0.28.0-beta.11"
+version = "0.28.0-beta.10"
 publish = false
 license.workspace = true
 description.workspace = true
@@ -16,7 +16,7 @@ crate-type = ["cdylib"]
 async-trait.workspace = true
 arrow-ipc.workspace = true
 arrow-array.workspace = true
-arrow-buffer = "58.0.0"
+arrow-buffer = "58.1"
 half.workspace = true
 arrow-schema.workspace = true
 env_logger.workspace = true
--- a/nodejs/npm/darwin-arm64/package.json
+++ b/nodejs/npm/darwin-arm64/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-darwin-arm64",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["darwin"],
 	"cpu": ["arm64"],
 	"main": "lancedb.darwin-arm64.node",
--- a/nodejs/npm/linux-arm64-gnu/package.json
+++ b/nodejs/npm/linux-arm64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-gnu",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-gnu.node",
--- a/nodejs/npm/linux-arm64-musl/package.json
+++ b/nodejs/npm/linux-arm64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-arm64-musl",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["arm64"],
 	"main": "lancedb.linux-arm64-musl.node",
--- a/nodejs/npm/linux-x64-gnu/package.json
+++ b/nodejs/npm/linux-x64-gnu/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-gnu",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-gnu.node",
--- a/nodejs/npm/linux-x64-musl/package.json
+++ b/nodejs/npm/linux-x64-musl/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-linux-x64-musl",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["linux"],
 	"cpu": ["x64"],
 	"main": "lancedb.linux-x64-musl.node",
--- a/nodejs/npm/win32-arm64-msvc/package.json
+++ b/nodejs/npm/win32-arm64-msvc/package.json
@@ -1,6 +1,6 @@
 {
  "name": "@lancedb/lancedb-win32-arm64-msvc",
-  "version": "0.28.0-beta.11",
+  "version": "0.28.0-beta.10",
  "os": [
    "win32"
  ],
--- a/nodejs/npm/win32-x64-msvc/package.json
+++ b/nodejs/npm/win32-x64-msvc/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "@lancedb/lancedb-win32-x64-msvc",
-	"version": "0.28.0-beta.11",
+	"version": "0.28.0-beta.10",
 	"os": ["win32"],
 	"cpu": ["x64"],
 	"main": "lancedb.win32-x64-msvc.node",
--- a/nodejs/package-lock.json
+++ b/nodejs/package-lock.json
@@ -1,12 +1,12 @@
 {
  "name": "@lancedb/lancedb",
-  "version": "0.28.0-beta.11",
+  "version": "0.28.0-beta.10",
  "lockfileVersion": 3,
  "requires": true,
  "packages": {
    "": {
      "name": "@lancedb/lancedb",
-      "version": "0.28.0-beta.11",
+      "version": "0.28.0-beta.10",
      "cpu": [
        "x64",
        "arm64"
--- a/nodejs/package.json
+++ b/nodejs/package.json
@@ -11,7 +11,7 @@
    "ann"
  ],
  "private": false,
-  "version": "0.28.0-beta.11",
+  "version": "0.28.0-beta.10",
  "main": "dist/index.js",
  "exports": {
    ".": "./dist/index.js",
@@ -75,6 +75,7 @@
    "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
    "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
+    "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
    "build": "npm run build:debug && npm run tsc",
    "build-release": "npm run build:release && npm run tsc",
    "tsc": "tsc -b",
--- a/nodejs/src/connection.rs
+++ b/nodejs/src/connection.rs
@@ -67,12 +67,6 @@ impl Connection {
                builder = builder.storage_option(key, value);
            }
        }
-        if let Some(manifest_enabled) = options.manifest_enabled {
-            builder = builder.manifest_enabled(manifest_enabled);
-        }
-        if let Some(namespace_client_properties) = options.namespace_client_properties {
-            builder = builder.namespace_client_properties(namespace_client_properties);
-        }

        // Create client config, optionally with header provider
        let client_config = options.client_config.unwrap_or_default();
--- a/nodejs/src/lib.rs
+++ b/nodejs/src/lib.rs
@@ -37,13 +37,6 @@ pub struct ConnectionOptions {
    ///
    /// The available options are described at https://docs.lancedb.com/storage/
    pub storage_options: Option<HashMap<String, String>>,
-    /// (For LanceDB OSS only): use directory namespace manifests as the source
-    /// of truth for table metadata. Existing directory-listed root tables are
-    /// migrated into the manifest on access.
-    pub manifest_enabled: Option<bool>,
-    /// (For LanceDB OSS only): extra properties for the backing namespace
-    /// client used by manifest-enabled native connections.
-    pub namespace_client_properties: Option<HashMap<String, String>>,
    /// (For LanceDB OSS only): the session to use for this connection. Holds
    /// shared caches and other session-specific state.
    pub session: Option<session::Session>,
--- a/python/.bumpversion.toml
+++ b/python/.bumpversion.toml
@@ -1,5 +1,5 @@
 [tool.bumpversion]
-current_version = "0.31.0-beta.11"
+current_version = "0.31.0-beta.10"
 parse = """(?x)
    (?P<major>0|[1-9]\\d*)\\.
    (?P<minor>0|[1-9]\\d*)\\.
--- a/python/Cargo.toml
+++ b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb-python"
-version = "0.31.0-beta.11"
+version = "0.31.0-beta.10"
 publish = false
 edition.workspace = true
 description = "Python bindings for LanceDB"
@@ -15,7 +15,7 @@ name = "_lancedb"
 crate-type = ["cdylib"]

 [dependencies]
-arrow = { version = "58.0.0", features = ["pyarrow"] }
+arrow = { workspace = true, features = ["pyarrow"] }
 async-trait = "0.1"
 bytes = "1"
 lancedb = { path = "../rust/lancedb", default-features = false }
@@ -35,8 +35,7 @@ futures.workspace = true
 serde = "1"
 serde_json = "1"
 snafu.workspace = true
-tokio = { version = "1.40", features = ["sync", "rt-multi-thread"] }
-libc = "0.2"
+tokio = { version = "1.40", features = ["sync"] }

 [build-dependencies]
 pyo3-build-config = { version = "0.28", features = [
--- a/python/python/lancedb/init.py
+++ b/python/python/lancedb/init.py
@@ -7,6 +7,7 @@ import os
 from concurrent.futures import ThreadPoolExecutor
 from datetime import timedelta
 from typing import Dict, Optional, Union, Any, List
+import warnings

 __version__ = importlib.metadata.version("lancedb")

@@ -72,7 +73,6 @@ def connect(
    client_config: Union[ClientConfig, Dict[str, Any], None] = None,
    storage_options: Optional[Dict[str, str]] = None,
    session: Optional[Session] = None,
-    manifest_enabled: bool = False,
    namespace_client_impl: Optional[str] = None,
    namespace_client_properties: Optional[Dict[str, str]] = None,
    namespace_client_pushdown_operations: Optional[List[str]] = None,
@@ -111,10 +111,6 @@ def connect(
    storage_options: dict, optional
        Additional options for the storage backend. See available options at
        <https://docs.lancedb.com/storage/>
-    manifest_enabled : bool, default False
-        When true for local/native connections, use directory namespace
-        manifests as the source of truth for table metadata. Existing
-        directory-listed root tables are migrated into the manifest on access.
    session: Session, optional
        (For LanceDB OSS only)
        A session to use for this connection. Sessions allow you to configure
@@ -162,11 +158,11 @@ def connect(
    conn : DBConnection
        A connection to a LanceDB database.
    """
-    if namespace_client_impl is not None:
-        if namespace_client_properties is None:
+    if namespace_client_impl is not None or namespace_client_properties is not None:
+        if namespace_client_impl is None or namespace_client_properties is None:
            raise ValueError(
-                "namespace_client_properties must be provided when "
-                "namespace_client_impl is set"
+                "Both namespace_client_impl and "
+                "namespace_client_properties must be provided"
            )
        if kwargs:
            raise ValueError(f"Unknown keyword arguments: {kwargs}")
@@ -179,12 +175,6 @@ def connect(
            namespace_client_pushdown_operations=namespace_client_pushdown_operations,
        )

-    if namespace_client_properties is not None and not manifest_enabled:
-        raise ValueError(
-            "namespace_client_impl must be provided when using "
-            "namespace_client_properties unless manifest_enabled=True"
-        )
-
    if namespace_client_pushdown_operations is not None:
        raise ValueError(
            "namespace_client_pushdown_operations is only valid when "
@@ -222,8 +212,6 @@ def connect(
        read_consistency_interval=read_consistency_interval,
        storage_options=storage_options,
        session=session,
-        manifest_enabled=manifest_enabled,
-        namespace_client_properties=namespace_client_properties,
    )


@@ -301,8 +289,6 @@ def deserialize_conn(
            parsed["uri"],
            read_consistency_interval=rci,
            storage_options=storage_options,
-            manifest_enabled=parsed.get("manifest_enabled", False),
-            namespace_client_properties=parsed.get("namespace_client_properties"),
        )
    else:
        raise ValueError(f"Unknown connection_type: {connection_type}")
@@ -318,8 +304,6 @@ async def connect_async(
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]] = None,
    storage_options: Optional[Dict[str, str]] = None,
    session: Optional[Session] = None,
-    manifest_enabled: bool = False,
-    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> AsyncConnection:
    """Connect to a LanceDB database.

@@ -359,13 +343,6 @@ async def connect_async(
        cache sizes for index and metadata caches, which can significantly
        impact memory use and performance. They can also be re-used across
        multiple connections to share the same cache state.
-    manifest_enabled : bool, default False
-        When true for local/native connections, use directory namespace
-        manifests as the source of truth for table metadata. Existing
-        directory-listed root tables are migrated into the manifest on access.
-    namespace_client_properties : dict, optional
-        Additional directory namespace client properties to use with
-        ``manifest_enabled=True``.

    Examples
    --------
@@ -408,8 +385,6 @@ async def connect_async(
            client_config,
            storage_options,
            session,
-            manifest_enabled,
-            namespace_client_properties,
        )
    )

@@ -437,3 +412,13 @@ __all__ = [
    "Table",
    "__version__",
 ]
+
+
+def __warn_on_fork():
+    warnings.warn(
+        "lance is not fork-safe. If you are using multiprocessing, use spawn instead.",
+    )
+
+
+if hasattr(os, "register_at_fork"):
+    os.register_at_fork(before=__warn_on_fork)  # type: ignore[attr-defined]
--- a/python/python/lancedb/_lancedb.pyi
+++ b/python/python/lancedb/_lancedb.pyi
@@ -242,8 +242,6 @@ async def connect(
    client_config: Optional[Union[ClientConfig, Dict[str, Any]]],
    storage_options: Optional[Dict[str, str]],
    session: Optional[Session],
-    manifest_enabled: bool = False,
-    namespace_client_properties: Optional[Dict[str, str]] = None,
 ) -> Connection: ...

 class RecordBatchStream:
@@ -442,7 +440,7 @@ class AsyncPermutationBuilder:
    async def execute(self) -> Table: ...

 def async_permutation_builder(
-    table: Table,
+    table: Table, dest_table_name: str
 ) -> AsyncPermutationBuilder: ...
 def fts_query_to_json(query: Any) -> str: ...

--- a/python/python/lancedb/background_loop.py
+++ b/python/python/lancedb/background_loop.py
@@ -2,9 +2,7 @@
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

 import asyncio
-import os
 import threading
-import warnings


 class BackgroundEventLoop:
@@ -15,9 +13,6 @@ class BackgroundEventLoop:
    """

    def __init__(self):
-        self._start()
-
-    def _start(self):
        self.loop = asyncio.new_event_loop()
        self.thread = threading.Thread(
            target=self.loop.run_forever,
@@ -36,30 +31,3 @@ class BackgroundEventLoop:


 LOOP = BackgroundEventLoop()
-
-_FORK_WARNED = False
-
-
-def _reset_after_fork():
-    # Threads do not survive fork(), so the asyncio loop in LOOP.thread is
-    # dead in the child. Re-initialize the singleton in place so existing
-    # `from .background_loop import LOOP` references in other modules see
-    # the new state. The Rust-side tokio runtime is reset analogously by a
-    # pthread_atfork hook installed in the _lancedb extension.
-    LOOP._start()
-    global _FORK_WARNED
-    if not _FORK_WARNED:
-        _FORK_WARNED = True
-        warnings.warn(
-            "lancedb fork support is experimental: the internal async "
-            "runtime has been reset in the forked child, but a small chance "
-            "of deadlock remains if other state was mid-operation at fork "
-            "time. The 'forkserver' or 'spawn' multiprocessing start method "
-            "is likely a safer alternative.",
-            RuntimeWarning,
-            stacklevel=2,
-        )
-
-
-if hasattr(os, "register_at_fork"):
-    os.register_at_fork(after_in_child=_reset_after_fork)
--- a/python/python/lancedb/db.py
+++ b/python/python/lancedb/db.py
@@ -590,13 +590,8 @@ class LanceDBConnection(DBConnection):
        read_consistency_interval: Optional[timedelta] = None,
        storage_options: Optional[Dict[str, str]] = None,
        session: Optional[Session] = None,
-        manifest_enabled: bool = False,
-        namespace_client_properties: Optional[Dict[str, str]] = None,
        _inner: Optional[LanceDbConnection] = None,
    ):
-        self.storage_options = storage_options
-        self._manifest_enabled = manifest_enabled
-        self._namespace_client_properties = namespace_client_properties
        if _inner is not None:
            self._conn = _inner
            self._cached_namespace_client = None
@@ -638,8 +633,6 @@ class LanceDBConnection(DBConnection):
                None,
                storage_options,
                session,
-                manifest_enabled,
-                namespace_client_properties,
            )

        # TODO: It would be nice if we didn't store self.storage_options but it is
@@ -647,6 +640,7 @@ class LanceDBConnection(DBConnection):
        # work because some paths like LanceDBConnection.from_inner will lose the
        # storage_options.  Also, this class really shouldn't be holding any state
        # beyond _conn.
+        self.storage_options = storage_options
        self._conn = AsyncConnection(LOOP.run(do_connect()))
        self._cached_namespace_client: Optional[LanceNamespace] = None

@@ -683,8 +677,6 @@ class LanceDBConnection(DBConnection):
                "connection_type": "local",
                "uri": self.uri,
                "storage_options": self.storage_options,
-                "manifest_enabled": self._manifest_enabled,
-                "namespace_client_properties": self._namespace_client_properties,
                "read_consistency_interval_seconds": (
                    rci.total_seconds() if rci else None
                ),
--- a/python/python/lancedb/permutation.py
+++ b/python/python/lancedb/permutation.py
@@ -1,11 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import copy
-import json
-
 from deprecation import deprecated
+from lancedb import AsyncConnection, DBConnection
 import pyarrow as pa
+import json

 from ._lancedb import async_permutation_builder, PermutationReader
 from .table import LanceTable
@@ -37,7 +36,10 @@ class PermutationBuilder:
    be referenced by name in the future.  If names are not provided then they can only
    be referenced by their ordinal index.  There is no requirement to name every split.

-    The permutation is stored in memory and will be lost when the program exits.
+    By default, the permutation will be stored in memory and will be lost when the
+    program exits.  To persist the permutation (for very large datasets or to share
+    the permutation across multiple workers) use the [persist](#persist) method to
+    create a permanent table.
    """

    def __init__(self, table: LanceTable):
@@ -49,6 +51,15 @@ class PermutationBuilder:
        """
        self._async = async_permutation_builder(table)

+    def persist(
+        self, database: Union[DBConnection, AsyncConnection], table_name: str
+    ) -> "PermutationBuilder":
+        """
+        Persist the permutation to the given database.
+        """
+        self._async.persist(database, table_name)
+        return self
+
    def split_random(
        self,
        *,
@@ -369,44 +380,20 @@ class Permutation:

    def __init__(
        self,
-        base_table: LanceTable,
-        permutation_table: Optional[LanceTable],
-        split: int,
+        reader: PermutationReader,
        selection: dict[str, str],
        batch_size: int,
        transform_fn: Callable[pa.RecordBatch, Any],
-        offset: Optional[int] = None,
-        limit: Optional[int] = None,
-        connection_factory: Optional[Callable[[str], LanceTable]] = None,
-        _reader: Optional[PermutationReader] = None,
    ):
        """
        Internal constructor.  Use [from_tables](#from_tables) instead.
        """
-        assert base_table is not None, "base_table is required"
+        assert reader is not None, "reader is required"
        assert selection is not None, "selection is required"
-        self.base_table = base_table
-        self.permutation_table = permutation_table
-        self.split = split
+        self.reader = reader
        self.selection = selection
        self.transform_fn = transform_fn
        self.batch_size = batch_size
-        self.offset = offset
-        self.limit = limit
-        self.connection_factory = connection_factory
-        if _reader is None:
-            _reader = LOOP.run(self._build_reader())
-        self.reader: PermutationReader = _reader
-
-    async def _build_reader(self) -> PermutationReader:
-        reader = await PermutationReader.from_tables(
-            self.base_table, self.permutation_table, self.split
-        )
-        if self.offset is not None:
-            reader = await reader.with_offset(self.offset)
-        if self.limit is not None:
-            reader = await reader.with_limit(self.limit)
-        return reader

    def _with_selection(self, selection: dict[str, str]) -> "Permutation":
        """
@@ -415,97 +402,21 @@ class Permutation:
        Does not validation of the selection and it replaces it entirely.  This is not
        intended for public use.
        """
-        new = copy.copy(self)
-        new.selection = selection
-        return new
+        return Permutation(self.reader, selection, self.batch_size, self.transform_fn)
+
+    def _with_reader(self, reader: PermutationReader) -> "Permutation":
+        """
+        Creates a new permutation with the given reader
+
+        This is an internal method and should not be used directly.
+        """
+        return Permutation(reader, self.selection, self.batch_size, self.transform_fn)

    def with_batch_size(self, batch_size: int) -> "Permutation":
        """
        Creates a new permutation with the given batch size
        """
-        new = copy.copy(self)
-        new.batch_size = batch_size
-        return new
-
-    def with_connection_factory(
-        self, connection_factory: Callable[[str], LanceTable]
-    ) -> "Permutation":
-        """
-        Creates a new permutation that will use ``connection_factory`` to reopen
-        the base table when this permutation is unpickled in a worker process.
-
-        The factory is a callable that takes a single argument — the base table
-        name — and returns a [LanceTable]. It must be picklable; the worker
-        will pickle it via standard ``pickle`` and call it to recover the base
-        table. Picklable callables in practice means top-level (module-level)
-        functions, ``functools.partial`` of such functions, or instances of
-        picklable classes implementing ``__call__``. Lambdas and closures over
-        local variables don't pickle with the default protocol.
-
-        Setting a factory is necessary when the URI alone is not enough to
-        re-open the connection — most importantly for LanceDB Cloud (``db://``)
-        connections, where ``api_key`` and ``region`` aren't recoverable from
-        the connection object after construction.
-
-        For local file or cloud-storage paths the factory is optional: if not
-        set, ``__getstate__`` falls back to capturing
-        ``(uri, storage_options, namespace_path)`` and re-opening via
-        ``lancedb.connect(uri, storage_options=...)``.
-
-        Examples
-        --------
-        Basic native (file-system path), parameterized via ``functools.partial``::
-
-            import functools, lancedb
-            from lancedb.permutation import Permutation
-
-            def open_native_table(uri: str, table_name: str):
-                return lancedb.connect(uri).open_table(table_name)
-
-            factory = functools.partial(open_native_table, "/data/lance_db")
-            permutation = Permutation.identity(
-                factory("training")
-            ).with_connection_factory(factory)
-
-        Native via :func:`lancedb.connect_namespace` (e.g. a directory- or
-        REST-backed namespace client). The factory takes the
-        implementation name and properties dict as partial-bound args so
-        the worker can rebuild the same namespace connection::
-
-            def open_via_namespace(
-                impl: str, properties: dict[str, str], table_name: str,
-            ):
-                return lancedb.connect_namespace(impl, properties).open_table(
-                    table_name,
-                )
-
-            factory = functools.partial(
-                open_via_namespace,
-                "dir",
-                {"root": "/data/lance_db"},
-            )
-
-        LanceDB Cloud, reading credentials from env vars at worker startup
-        so secrets aren't pickled into the dataset::
-
-            import os, lancedb
-
-            def open_remote_table(table_name: str):
-                db = lancedb.connect(
-                    "db://my-database",
-                    api_key=os.environ["LANCEDB_API_KEY"],
-                    region=os.environ.get("LANCEDB_REGION", "us-east-1"),
-                )
-                return db.open_table(table_name)
-
-            permutation = Permutation.identity(
-                open_remote_table("training")
-            ).with_connection_factory(open_remote_table)
-        """
-        assert connection_factory is not None, "connection_factory is required"
-        new = copy.copy(self)
-        new.connection_factory = connection_factory
-        return new
+        return Permutation(self.reader, self.selection, batch_size, self.transform_fn)

    @classmethod
    def identity(cls, table: LanceTable) -> "Permutation":
@@ -578,126 +489,11 @@ class Permutation:
            schema = await reader.output_schema(None)
            initial_selection = {name: name for name in schema.names}
            return cls(
-                base_table,
-                permutation_table,
-                split,
-                initial_selection,
-                DEFAULT_BATCH_SIZE,
-                Transforms.arrow2python,
-                _reader=reader,
+                reader, initial_selection, DEFAULT_BATCH_SIZE, Transforms.arrow2python
            )

        return LOOP.run(do_from_tables())

-    def __getstate__(self) -> dict[str, Any]:
-        """Build a picklable state dict for this permutation.
-
-        The base table is captured either via a user-supplied
-        ``connection_factory`` (see [with_connection_factory]) or, as a
-        fallback, by introspecting ``(uri, storage_options, namespace_path)``
-        on the connection. The permutation table — always an in-memory
-        LanceDB table — is captured as a pyarrow Table (which pickles via
-        Arrow IPC natively). The reader is dropped from the wire format;
-        ``__setstate__`` rebuilds it from the restored tables.
-        """
-        permutation_data: Optional[pa.Table] = None
-        if self.permutation_table is not None:
-            permutation_data = self.permutation_table.to_arrow()
-
-        common = {
-            "base_table_name": self.base_table.name,
-            "permutation_data": permutation_data,
-            "split": self.split,
-            "selection": self.selection,
-            "batch_size": self.batch_size,
-            "transform_fn": self.transform_fn,
-            "offset": self.offset,
-            "limit": self.limit,
-            "connection_factory": self.connection_factory,
-        }
-
-        if self.connection_factory is not None:
-            # The factory carries enough state to recover the base table on
-            # its own; we don't need to capture the URI / storage options /
-            # namespace from the existing connection.
-            return common
-
-        # URI-introspection fallback: only viable for native (OSS) connections
-        # where (uri, storage_options) is enough to reopen. Remote / cloud
-        # connections don't expose recoverable api_key / region — those users
-        # must call with_connection_factory().
-        try:
-            base_uri = self.base_table._conn.uri
-            storage_options = self.base_table._conn.storage_options
-        except AttributeError as e:
-            raise ValueError(
-                "Cannot pickle this Permutation: the base table's connection "
-                "does not expose a uri/storage_options, which usually means it "
-                "is a remote (LanceDB Cloud) connection. Call "
-                "Permutation.with_connection_factory(...) first to provide a "
-                "picklable callable that re-opens the base table from a worker "
-                "process."
-            ) from e
-
-        if base_uri.startswith("memory://"):
-            # In-memory base tables don't exist in any worker process by
-            # default, so dump the entire base table into the pickle. This
-            # can be expensive for large datasets — users with large
-            # in-memory base tables should either persist them or set a
-            # connection_factory.
-            return {
-                **common,
-                "base_table_data": self.base_table.to_arrow(),
-            }
-
-        return {
-            **common,
-            "base_table_uri": base_uri,
-            "base_table_namespace": self.base_table._namespace_path,
-            "base_table_storage_options": storage_options,
-        }
-
-    def __setstate__(self, state: dict[str, Any]) -> None:
-        from . import connect
-
-        connection_factory = state["connection_factory"]
-        if connection_factory is not None:
-            base_table = connection_factory(state["base_table_name"])
-        elif "base_table_data" in state:
-            # In-memory base table inlined into the pickle; rebuild the same
-            # way we rebuild the in-memory permutation table.
-            mem_db = connect("memory://")
-            base_table = mem_db.create_table(
-                state["base_table_name"], state["base_table_data"]
-            )
-        else:
-            base_db = connect(
-                state["base_table_uri"],
-                storage_options=state["base_table_storage_options"],
-            )
-            base_table = base_db.open_table(
-                state["base_table_name"],
-                namespace_path=state["base_table_namespace"] or None,
-            )
-
-        permutation_table: Optional[LanceTable] = None
-        if state["permutation_data"] is not None:
-            mem_db = connect("memory://")
-            permutation_table = mem_db.create_table(
-                "permutation", state["permutation_data"]
-            )
-
-        self.base_table = base_table
-        self.permutation_table = permutation_table
-        self.split = state["split"]
-        self.selection = state["selection"]
-        self.batch_size = state["batch_size"]
-        self.transform_fn = state["transform_fn"]
-        self.offset = state["offset"]
-        self.limit = state["limit"]
-        self.connection_factory = connection_factory
-        self.reader = LOOP.run(self._build_reader())
-
    @property
    def schema(self) -> pa.Schema:
        async def do_output_schema():
@@ -964,9 +760,7 @@ class Permutation:
        for expensive operations such as image decoding.
        """
        assert transform is not None, "transform is required"
-        new = copy.copy(self)
-        new.transform_fn = transform
-        return new
+        return Permutation(self.reader, self.selection, self.batch_size, transform)

    def __getitem__(self, index: int) -> Any:
        """
@@ -1001,10 +795,12 @@ class Permutation:
        """
        Skip the first `skip` rows of the permutation
        """
-        new = copy.copy(self)
-        new.offset = skip
-        new.reader = LOOP.run(new._build_reader())
-        return new
+
+        async def do_with_skip():
+            reader = await self.reader.with_offset(skip)
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_skip())

    @deprecated(details="Use with_take instead")
    def take(self, limit: int) -> "Permutation":
@@ -1022,10 +818,12 @@ class Permutation:
        """
        Limit the permutation to `limit` rows (following any `skip`)
        """
-        new = copy.copy(self)
-        new.limit = limit
-        new.reader = LOOP.run(new._build_reader())
-        return new
+
+        async def do_with_take():
+            reader = await self.reader.with_limit(limit)
+            return self._with_reader(reader)
+
+        return LOOP.run(do_with_take())

    @deprecated(details="Use with_repeat instead")
    def repeat(self, times: int) -> "Permutation":
--- a/python/python/lancedb/query.py
+++ b/python/python/lancedb/query.py
@@ -1643,7 +1643,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
    def _validate_query(self, query, vector=None, text=None):
        if query is not None and (vector is not None or text is not None):
            raise ValueError(
-                "You can either provide a string query in search() method "
+                "You can either provide a string query in search() method"
                "or set `vector()` and `text()` explicitly for hybrid search."
                "But not both."
            )
--- a/python/python/tests/test_permutation.py
+++ b/python/python/tests/test_permutation.py
@@ -9,6 +9,21 @@ from lancedb import DBConnection, Table, connect
 from lancedb.permutation import Permutation, Permutations, permutation_builder


+def test_permutation_persistence(tmp_path):
+    db = connect(tmp_path)
+    tbl = db.create_table("test_table", pa.table({"x": range(100), "y": range(100)}))
+
+    permutation_tbl = (
+        permutation_builder(tbl).shuffle().persist(db, "test_permutation").execute()
+    )
+    assert permutation_tbl.count_rows() == 100
+
+    re_open = db.open_table("test_permutation")
+    assert re_open.count_rows() == 100
+
+    assert permutation_tbl.to_arrow() == re_open.to_arrow()
+
+
 def test_split_random_ratios(mem_db):
    """Test random splitting with ratios."""
    tbl = mem_db.create_table(
--- a/python/python/tests/test_remote_db.py
+++ b/python/python/tests/test_remote_db.py
@@ -6,8 +6,6 @@ import contextlib
 from datetime import timedelta
 import http.server
 import json
-import multiprocessing as mp
-import sys
 import threading
 import time
 from unittest.mock import MagicMock, patch
@@ -1232,82 +1230,3 @@ def test_background_loop_cancellation(exception):
        with pytest.raises(exception):
            loop.run(None)
        mock_future.cancel.assert_called_once()
-
-
-def _remote_fork_child(port: int, queue) -> None:
-    # Build a fresh Connection in the child so we exercise the at-fork-child
-    # tokio runtime reset rather than relying on an inherited reqwest client.
-    db = lancedb.connect(
-        "db://dev",
-        api_key="fake",
-        host_override=f"http://localhost:{port}",
-        client_config={
-            "retry_config": {"retries": 0},
-            "timeout_config": {"connect_timeout": 2, "read_timeout": 2},
-        },
-    )
-    queue.put(db.table_names())
-
-
-@pytest.mark.skipif(
-    sys.platform != "linux",
-    reason=(
-        "fork() is unavailable on Windows and unsafe on macOS "
-        "(Apple frameworks/TLS are not fork-safe)"
-    ),
-)
-def test_remote_connection_after_fork():
-    """A freshly-built remote Connection in a forked child should not hang.
-
-    The pyo3-async-runtimes tokio runtime would otherwise be inherited from
-    the parent with dead worker threads; the at-fork-child handler in our
-    runtime module rebuilds it on first use in the child.
-    """
-
-    def handler(request):
-        request.send_response(200)
-        request.send_header("Content-Type", "application/json")
-        request.end_headers()
-        request.wfile.write(b'{"tables": []}')
-
-    server = http.server.HTTPServer(("localhost", 0), make_mock_http_handler(handler))
-    port = server.server_address[1]
-    server_thread = threading.Thread(target=server.serve_forever)
-    server_thread.start()
-    try:
-        # Hit the server in the parent first so the runtime + LOOP are warm
-        # before fork; a fresh child must still succeed.
-        parent_db = lancedb.connect(
-            "db://dev",
-            api_key="fake",
-            host_override=f"http://localhost:{port}",
-            client_config={
-                "retry_config": {"retries": 0},
-                "timeout_config": {"connect_timeout": 2, "read_timeout": 2},
-            },
-        )
-        assert parent_db.table_names() == []
-
-        ctx = mp.get_context("fork")
-        queue = ctx.Queue()
-        proc = ctx.Process(target=_remote_fork_child, args=(port, queue))
-        proc.start()
-        proc.join(timeout=15)
-
-        if proc.is_alive():
-            proc.terminate()
-            proc.join(timeout=5)
-            if proc.is_alive():
-                proc.kill()
-                proc.join()
-            pytest.fail("Remote connection hung after fork")
-
-        assert proc.exitcode == 0, f"child exited with code {proc.exitcode}"
-        assert not queue.empty(), "child produced no result"
-        assert queue.get() == []
-
-        # Parent connection must still be usable after the child returned.
-        assert parent_db.table_names() == []
-    finally:
-        server.shutdown()
-        server_thread.join()
--- a/python/python/tests/test_torch.py
+++ b/python/python/tests/test_torch.py
@@ -1,29 +1,14 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright The LanceDB Authors

-import functools
-import multiprocessing as mp
-import pickle
-import sys
-
-import lancedb
 import pyarrow as pa
 import pytest
-from lancedb.permutation import Permutation, Permutations, permutation_builder
 from lancedb.util import tbl_to_tensor
+from lancedb.permutation import Permutation

 torch = pytest.importorskip("torch")


-def _open_native_table(uri: str, table_name: str):
-    """Top-level connection factory used by the explicit-factory pickle test.
-
-    Defined at module scope so that pickle can resolve it by name in the
-    worker / unpickling process.
-    """
-    return lancedb.connect(uri).open_table(table_name)
-
-
 def test_table_dataloader(mem_db):
    table = mem_db.create_table("test_table", pa.table({"a": range(1000)}))
    dataloader = torch.utils.data.DataLoader(
@@ -55,156 +40,3 @@ def test_permutation_dataloader(mem_db):
    for batch in dataloader:
        assert batch.size(0) == 1
        assert batch.size(1) == 10
-
-
-def test_permutation_is_picklable(tmp_db):
-    """A Permutation must be picklable so it can be used with PyTorch's
-    DataLoader when num_workers > 0 (which uses multiprocessing and pickles
-    the dataset to pass it to worker processes)."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(1000)}))
-    permutation = Permutation.identity(table)
-
-    pickled = pickle.dumps(permutation)
-    restored = pickle.loads(pickled)
-
-    assert len(restored) == 1000
-    rows = restored.__getitems__([0, 1, 2])
-    assert rows == [{"a": 0}, {"a": 1}, {"a": 2}]
-
-
-def test_permutation_with_memory_base_is_picklable(mem_db):
-    """An in-memory base table is inlined into the pickle as Arrow IPC bytes
-    and rebuilt on the other side as an in-memory LanceTable, so the
-    Permutation round-trips even though the original database can't be
-    reopened across processes."""
-    table = mem_db.create_table("test_table", pa.table({"a": range(50)}))
-    permutation = Permutation.identity(table)
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == 50
-    assert restored.__getitems__([0, 10, 49]) == [{"a": 0}, {"a": 10}, {"a": 49}]
-
-
-def test_permutation_dataloader_multiprocessing(tmp_db):
-    """Using a Permutation with a PyTorch DataLoader that has num_workers > 0
-    must work end-to-end. Each worker process gets a pickled copy of the
-    dataset and reads batches from it."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(1000)}))
-    permutation = Permutation.identity(table)
-
-    dataloader = torch.utils.data.DataLoader(
-        permutation,
-        batch_size=10,
-        shuffle=True,
-        num_workers=2,
-        multiprocessing_context="spawn",
-    )
-    seen = 0
-    for batch in dataloader:
-        assert batch["a"].size(0) == 10
-        seen += batch["a"].size(0)
-    assert seen == 1000
-
-
-def test_permutation_pickle_with_connection_factory(tmp_path):
-    """When the user provides a connection_factory, pickling should round-trip
-    through that factory rather than introspecting the connection URI. Useful
-    for remote / cloud connections where the URI alone isn't reopenable."""
-    db = lancedb.connect(tmp_path)
-    db.create_table("test_table", pa.table({"a": range(50)}))
-
-    factory = functools.partial(_open_native_table, str(tmp_path))
-    permutation = Permutation.identity(factory("test_table")).with_connection_factory(
-        factory
-    )
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == 50
-    # The factory survives pickling and is what powered base-table reopen.
-    assert restored.connection_factory is not None
-    assert restored.connection_factory.func is _open_native_table
-    assert restored.__getitems__([0, 1, 2]) == [{"a": 0}, {"a": 1}, {"a": 2}]
-
-
-def test_permutation_with_builder_is_picklable(tmp_db):
-    """A Permutation built from a non-identity permutation table must round-trip
-    through pickle while preserving the row order defined by the permutation."""
-    table = tmp_db.create_table("test_table", pa.table({"a": range(100)}))
-    perm_tbl = (
-        permutation_builder(table)
-        .split_random(ratios=[0.8, 0.2], seed=42, split_names=["train", "test"])
-        .shuffle(seed=42)
-        .execute()
-    )
-    permutations = Permutations(table, perm_tbl)
-    permutation = permutations["train"]
-
-    indices = list(range(len(permutation)))
-    expected = permutation.__getitems__(indices)
-
-    restored = pickle.loads(pickle.dumps(permutation))
-
-    assert len(restored) == len(permutation)
-    assert restored.__getitems__(indices) == expected
-
-
-def _multiworker_dataloader_target(db_uri: str, result_queue):
-    import lancedb
-    from lancedb.permutation import Permutation
-
-    db = lancedb.connect(db_uri)
-    table = db.open_table("test_table")
-    permutation = Permutation.identity(table)
-
-    dataloader = torch.utils.data.DataLoader(
-        permutation,
-        batch_size=10,
-        num_workers=2,
-        multiprocessing_context="fork",
-    )
-    count = 0
-    for batch in dataloader:
-        assert batch["a"].size(0) == 10
-        count += 1
-    result_queue.put(count)
-
-
-@pytest.mark.skipif(
-    sys.platform != "linux",
-    reason=(
-        "fork() is unavailable on Windows and unsafe on macOS "
-        "(Apple frameworks/TLS are not fork-safe)"
-    ),
-)
-def test_permutation_dataloader_fork_workers(tmp_path):
-    """A Permutation used by a fork-based DataLoader should not hang.
-
-    PyTorch's DataLoader uses fork-based multiprocessing by default on Linux.
-    LanceDB drives async work through a background asyncio thread that does
-    not survive a fork, so any LOOP.run() in a worker blocks forever.
-    """
-    import lancedb
-
-    db_uri = str(tmp_path / "db")
-    db = lancedb.connect(db_uri)
-    db.create_table("test_table", pa.table({"a": list(range(1000))}))
-
-    ctx = mp.get_context("spawn")
-    queue = ctx.Queue()
-    proc = ctx.Process(target=_multiworker_dataloader_target, args=(db_uri, queue))
-    proc.start()
-    proc.join(timeout=30)
-
-    if proc.is_alive():
-        proc.terminate()
-        proc.join(timeout=5)
-        if proc.is_alive():
-            proc.kill()
-            proc.join()
-        pytest.fail("Permutation hung when iterated in a fork-based DataLoader worker")
-
-    assert proc.exitcode == 0, f"child exited with code {proc.exitcode}"
-    assert not queue.empty(), "child produced no batches"
-    assert queue.get() == 100
--- a/python/src/arrow.rs
+++ b/python/src/arrow.rs
@@ -3,8 +3,6 @@

 use std::sync::Arc;

-use crate::error::PythonErrorExt;
-use crate::runtime::future_into_py;
 use arrow::{
    datatypes::SchemaRef,
    pyarrow::{IntoPyArrow, ToPyArrow},
@@ -14,6 +12,9 @@ use lancedb::arrow::SendableRecordBatchStream;
 use pyo3::{
    Bound, Py, PyAny, PyRef, PyResult, Python, exceptions::PyStopAsyncIteration, pyclass, pymethods,
 };
+use pyo3_async_runtimes::tokio::future_into_py;
+
+use crate::error::PythonErrorExt;

 #[pyclass]
 pub struct RecordBatchStream {
--- a/python/src/connection.rs
+++ b/python/src/connection.rs
@@ -7,12 +7,6 @@ use std::{
    time::Duration,
 };

-use crate::{
-    error::PythonErrorExt,
-    namespace::{create_namespace_storage_options_provider, extract_namespace_arc},
-    runtime::future_into_py,
-    table::Table,
-};
 use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
 use lancedb::{
    connection::Connection as LanceConnection,
@@ -26,6 +20,13 @@ use pyo3::{
    pyclass, pyfunction, pymethods,
    types::{PyDict, PyDictMethods},
 };
+use pyo3_async_runtimes::tokio::future_into_py;
+
+use crate::{
+    error::PythonErrorExt,
+    namespace::{create_namespace_storage_options_provider, extract_namespace_arc},
+    table::Table,
+};

 #[pyclass]
 pub struct Connection {
@@ -524,7 +525,7 @@ impl Connection {
 }

 #[pyfunction]
-#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None, manifest_enabled=false, namespace_client_properties=None))]
+#[pyo3(signature = (uri, api_key=None, region=None, host_override=None, read_consistency_interval=None, client_config=None, storage_options=None, session=None))]
 #[allow(clippy::too_many_arguments)]
 pub fn connect(
    py: Python<'_>,
@@ -536,8 +537,6 @@ pub fn connect(
    client_config: Option<PyClientConfig>,
    storage_options: Option<HashMap<String, String>>,
    session: Option<crate::session::Session>,
-    manifest_enabled: bool,
-    namespace_client_properties: Option<HashMap<String, String>>,
 ) -> PyResult<Bound<'_, PyAny>> {
    future_into_py(py, async move {
        let mut builder = lancedb::connect(&uri);
@@ -557,12 +556,6 @@ pub fn connect(
        if let Some(storage_options) = storage_options {
            builder = builder.storage_options(storage_options);
        }
-        if manifest_enabled {
-            builder = builder.manifest_enabled(true);
-        }
-        if let Some(namespace_client_properties) = namespace_client_properties {
-            builder = builder.namespace_client_properties(namespace_client_properties);
-        }
        #[cfg(feature = "remote")]
        if let Some(client_config) = client_config {
            builder = builder.client_config(client_config.into());
--- a/python/src/index.rs
+++ b/python/src/index.rs
@@ -13,7 +13,7 @@ use pyo3::{
    Bound, FromPyObject, PyAny, PyResult, Python,
    exceptions::{PyKeyError, PyValueError},
    intern, pyclass, pymethods,
-    types::{PyAnyMethods, PyString},
+    types::PyAnyMethods,
 };

 use crate::util::parse_distance_type;
@@ -22,7 +22,7 @@ pub fn class_name(ob: &'_ Bound<'_, PyAny>) -> PyResult<String> {
    let full_name = ob
        .getattr(intern!(ob.py(), "__class__"))?
        .getattr(intern!(ob.py(), "__name__"))?;
-    let full_name = full_name.cast::<PyString>()?.to_string_lossy();
+    let full_name = full_name.cast()?.to_string_lossy();

    match full_name.rsplit_once('.') {
        Some((_, name)) => Ok(name.to_string()),
--- a/python/src/lib.rs
+++ b/python/src/lib.rs
@@ -28,7 +28,6 @@ pub mod index;
 pub mod namespace;
 pub mod permutation;
 pub mod query;
-pub mod runtime;
 pub mod session;
 pub mod table;
 pub mod util;
--- a/python/src/permutation.rs
+++ b/python/src/permutation.rs
@@ -4,7 +4,7 @@
 use std::sync::{Arc, Mutex};

 use crate::{
-    arrow::RecordBatchStream, error::PythonErrorExt, runtime::future_into_py, table::Table,
+    arrow::RecordBatchStream, connection::Connection, error::PythonErrorExt, table::Table,
 };
 use arrow::pyarrow::{PyArrowType, ToPyArrow};
 use lancedb::{
@@ -21,6 +21,7 @@ use pyo3::{
    pyclass, pymethods,
    types::{PyAnyMethods, PyDict, PyDictMethods, PyType},
 };
+use pyo3_async_runtimes::tokio::future_into_py;

 fn table_from_py<'a>(table: Bound<'a, PyAny>) -> PyResult<Bound<'a, Table>> {
    if table.hasattr("_inner")? {
@@ -79,6 +80,24 @@ impl PyAsyncPermutationBuilder {

 #[pymethods]
 impl PyAsyncPermutationBuilder {
+    #[pyo3(signature = (database, table_name))]
+    pub fn persist(
+        slf: PyRefMut<'_, Self>,
+        database: Bound<'_, PyAny>,
+        table_name: String,
+    ) -> PyResult<Self> {
+        let conn = if database.hasattr("_conn")? {
+            database
+                .getattr("_conn")?
+                .getattr("_inner")?
+                .cast_into::<Connection>()?
+        } else {
+            database.getattr("_inner")?.cast_into::<Connection>()?
+        };
+        let database = conn.borrow().database()?;
+        slf.modify(|builder| builder.persist(database, table_name))
+    }
+
    #[pyo3(signature = (*, ratios=None, counts=None, fixed=None, seed=None, split_names=None))]
    pub fn split_random(
        slf: PyRefMut<'_, Self>,
--- a/python/src/query.rs
+++ b/python/src/query.rs
@@ -4,11 +4,6 @@
 use std::sync::Arc;
 use std::time::Duration;

-use crate::expr::PyExpr;
-use crate::runtime::future_into_py;
-use crate::util::parse_distance_type;
-use crate::{arrow::RecordBatchStream, util::PyLanceDB};
-use crate::{error::PythonErrorExt, index::class_name};
 use arrow::array::Array;
 use arrow::array::ArrayData;
 use arrow::array::make_array;
@@ -27,6 +22,7 @@ use lancedb::query::{
    VectorQuery as LanceDbVectorQuery,
 };
 use lancedb::table::AnyQuery;
+use pyo3::Borrowed;
 use pyo3::Bound;
 use pyo3::IntoPyObject;
 use pyo3::PyAny;
@@ -38,15 +34,20 @@ use pyo3::pyfunction;
 use pyo3::pymethods;
 use pyo3::types::PyList;
 use pyo3::types::{PyDict, PyString};
-use pyo3::{Borrowed, FromPyObject, exceptions::PyRuntimeError};
+use pyo3::{FromPyObject, exceptions::PyRuntimeError};
 use pyo3::{PyErr, pyclass};
 use pyo3::{exceptions::PyValueError, intern};
+use pyo3_async_runtimes::tokio::future_into_py;

-impl<'a, 'py> FromPyObject<'a, 'py> for PyLanceDB<FtsQuery> {
+use crate::expr::PyExpr;
+use crate::util::parse_distance_type;
+use crate::{arrow::RecordBatchStream, util::PyLanceDB};
+use crate::{error::PythonErrorExt, index::class_name};
+
+impl FromPyObject<'_, '_> for PyLanceDB<FtsQuery> {
    type Error = PyErr;

-    fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult<Self> {
-        let ob = ob.to_owned();
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> PyResult<Self> {
        match class_name(&ob)?.as_str() {
            "MatchQuery" => {
                let query = ob.getattr("query")?.extract()?;
--- a/python/src/runtime.rs
+++ b/python/src/runtime.rs
@@ -1,142 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright The LanceDB Authors
-
-//! Fork-safe wrapper around tokio + pyo3-async-runtimes.
-//!
-//! `pyo3_async_runtimes::tokio` keeps its multi-threaded runtime in a
-//! `OnceLock` that can never be replaced.  Tokio's worker threads do not
-//! survive `fork()`, so once a child inherits a "frozen" runtime, every
-//! `future_into_py` call hangs forever.
-//!
-//! We sidestep the global by routing every future through our own
-//! [`LanceRuntime`] (a [`pyo3_async_runtimes::generic::Runtime`] impl) backed
-//! by an [`AtomicPtr`] to a tokio runtime that we own.  A `pthread_atfork`
-//! child handler nulls the pointer; the next `spawn` rebuilds the runtime in
-//! the child.  This mirrors the pattern used in the Lance Python bindings.
-
-use std::future::Future;
-use std::pin::Pin;
-use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
-
-use pyo3::{Bound, PyAny, PyResult, Python, conversion::IntoPyObject};
-use pyo3_async_runtimes::{
-    TaskLocals,
-    generic::{ContextExt, JoinError, Runtime},
-};
-use tokio::{runtime, task};
-
-static RUNTIME: AtomicPtr<runtime::Runtime> = AtomicPtr::new(std::ptr::null_mut());
-static RUNTIME_INSTALLING: AtomicBool = AtomicBool::new(false);
-static ATFORK_INSTALLED: AtomicBool = AtomicBool::new(false);
-
-fn create_runtime() -> runtime::Runtime {
-    runtime::Builder::new_multi_thread()
-        .enable_all()
-        .thread_name("lancedb-tokio-worker")
-        .build()
-        .expect("Failed to build tokio runtime")
-}
-
-fn get_runtime() -> &'static runtime::Runtime {
-    loop {
-        let ptr = RUNTIME.load(Ordering::SeqCst);
-        if !ptr.is_null() {
-            return unsafe { &*ptr };
-        }
-        if !RUNTIME_INSTALLING.fetch_or(true, Ordering::SeqCst) {
-            break;
-        }
-        std::thread::yield_now();
-    }
-    if !ATFORK_INSTALLED.fetch_or(true, Ordering::SeqCst) {
-        install_atfork();
-    }
-    let new_ptr = Box::into_raw(Box::new(create_runtime()));
-    RUNTIME.store(new_ptr, Ordering::SeqCst);
-    unsafe { &*new_ptr }
-}
-
-/// Runs in async-signal context after `fork()` in the child.  We can only
-/// touch atomics here; we deliberately leak the previous runtime because
-/// dropping a tokio `Runtime` would try to join its (now-dead) worker
-/// threads and hang.
-extern "C" fn atfork_child() {
-    RUNTIME.store(std::ptr::null_mut(), Ordering::SeqCst);
-    RUNTIME_INSTALLING.store(false, Ordering::SeqCst);
-}
-
-#[cfg(not(windows))]
-fn install_atfork() {
-    unsafe { libc::pthread_atfork(None, None, Some(atfork_child)) };
-}
-
-#[cfg(windows)]
-fn install_atfork() {}
-
-/// Marker type implementing [`Runtime`] over our fork-safe runtime slot.
-pub struct LanceRuntime;
-
-/// Newtype wrapper around `tokio::task::JoinError` so we can implement the
-/// foreign [`JoinError`] trait without violating orphan rules.
-pub struct LanceJoinError(task::JoinError);
-
-impl JoinError for LanceJoinError {
-    fn is_panic(&self) -> bool {
-        self.0.is_panic()
-    }
-    fn into_panic(self) -> Box<dyn std::any::Any + Send + 'static> {
-        self.0.into_panic()
-    }
-}
-
-impl Runtime for LanceRuntime {
-    type JoinError = LanceJoinError;
-    type JoinHandle = Pin<Box<dyn Future<Output = Result<(), Self::JoinError>> + Send>>;
-
-    fn spawn<F>(fut: F) -> Self::JoinHandle
-    where
-        F: Future<Output = ()> + Send + 'static,
-    {
-        let handle = get_runtime().spawn(fut);
-        Box::pin(async move { handle.await.map_err(LanceJoinError) })
-    }
-
-    fn spawn_blocking<F>(f: F) -> Self::JoinHandle
-    where
-        F: FnOnce() + Send + 'static,
-    {
-        let handle = get_runtime().spawn_blocking(f);
-        Box::pin(async move { handle.await.map_err(LanceJoinError) })
-    }
-}
-
-tokio::task_local! {
-    static TASK_LOCALS: std::cell::OnceCell<TaskLocals>;
-}
-
-impl ContextExt for LanceRuntime {
-    fn scope<F, R>(locals: TaskLocals, fut: F) -> Pin<Box<dyn Future<Output = R> + Send>>
-    where
-        F: Future<Output = R> + Send + 'static,
-    {
-        let cell = std::cell::OnceCell::new();
-        cell.set(locals).unwrap();
-        Box::pin(TASK_LOCALS.scope(cell, fut))
-    }
-
-    fn get_task_locals() -> Option<TaskLocals> {
-        TASK_LOCALS
-            .try_with(|c| c.get().cloned())
-            .unwrap_or_default()
-    }
-}
-
-/// Drop-in replacement for `pyo3_async_runtimes::tokio::future_into_py` that
-/// uses our fork-safe runtime.
-pub fn future_into_py<F, T>(py: Python<'_>, fut: F) -> PyResult<Bound<'_, PyAny>>
-where
-    F: Future<Output = PyResult<T>> + Send + 'static,
-    T: for<'py> IntoPyObject<'py> + Send + 'static,
-{
-    pyo3_async_runtimes::generic::future_into_py::<LanceRuntime, _, T>(py, fut)
-}
--- a/python/src/table.rs
+++ b/python/src/table.rs
@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 use std::{collections::HashMap, sync::Arc};

-use crate::runtime::future_into_py;
 use crate::{
    connection::Connection,
    error::PythonErrorExt,
@@ -25,6 +24,7 @@ use pyo3::{
    pyclass, pymethods,
    types::{IntoPyDict, PyAnyMethods, PyDict, PyDictMethods},
 };
+use pyo3_async_runtimes::tokio::future_into_py;

 mod scannable;

--- a/python/src/table/scannable.rs
+++ b/python/src/table/scannable.rs
@@ -14,7 +14,7 @@ use lancedb::{
    arrow::{SendableRecordBatchStream, SimpleRecordBatchStream},
    data::scannable::Scannable,
 };
-use pyo3::{FromPyObject, Py, PyAny, Python, types::PyAnyMethods};
+use pyo3::{Borrowed, FromPyObject, Py, PyAny, Python, types::PyAnyMethods};

 /// Adapter that implements Scannable for a Python reader factory callable.
 ///
@@ -126,11 +126,10 @@ impl Scannable for PyScannable {
    }
 }

-impl<'a, 'py> FromPyObject<'a, 'py> for PyScannable {
+impl FromPyObject<'_, '_> for PyScannable {
    type Error = pyo3::PyErr;

-    fn extract(ob: pyo3::Borrowed<'a, 'py, PyAny>) -> pyo3::PyResult<Self> {
-        let ob = ob.to_owned();
+    fn extract(ob: Borrowed<'_, '_, PyAny>) -> pyo3::PyResult<Self> {
        // Convert from Scannable dataclass.
        let schema: PyArrowType<Schema> = ob.getattr("schema")?.extract()?;
        let schema = Arc::new(schema.0);
--- a/rust/lancedb/Cargo.toml
+++ b/rust/lancedb/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "lancedb"
-version = "0.28.0-beta.11"
+version = "0.28.0-beta.10"
 edition.workspace = true
 description = "LanceDB: A serverless, low-latency vector database for AI applications"
 license.workspace = true
@@ -108,20 +108,10 @@ test-log = "0.2"

 [features]
 default = []
-aws = [
-    "lance/aws",
-    "lance-io/aws",
-    "lance-namespace-impls/dir-aws",
-    "object_store/aws",
-]
+aws = ["lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws"]
 oss = ["lance/oss", "lance-io/oss", "lance-namespace-impls/dir-oss"]
 gcs = ["lance/gcp", "lance-io/gcp", "lance-namespace-impls/dir-gcp"]
-azure = [
-    "lance/azure",
-    "lance-io/azure",
-    "lance-namespace-impls/dir-azure",
-    "lance-namespace-impls/credential-vendor-azure",
-]
+azure = ["lance/azure", "lance-io/azure", "lance-namespace-impls/dir-azure"]
 huggingface = [
    "lance/huggingface",
    "lance-io/huggingface",
--- a/rust/lancedb/src/connection.rs
+++ b/rust/lancedb/src/connection.rs
@@ -590,15 +590,6 @@ pub struct ConnectRequest {
    /// storage options.
    pub namespace_client_properties: HashMap<String, String>,

-    /// Use directory namespace manifests as the source of truth for native
-    /// LanceDB table metadata.
-    ///
-    /// When enabled for a local/native connection, LanceDB returns a
-    /// namespace-backed database directly. Directory listing fallback remains
-    /// enabled for migration, and directory-listing-to-manifest migration is
-    /// forced on.
-    pub manifest_enabled: bool,
-
    /// The interval at which to check for updates from other processes.
    ///
    /// If None, then consistency is not checked. For performance
@@ -639,7 +630,6 @@ impl ConnectBuilder {
                read_consistency_interval: None,
                options: HashMap::new(),
                namespace_client_properties: HashMap::new(),
-                manifest_enabled: false,
                session: None,
            },
            embedding_registry: None,
@@ -801,17 +791,6 @@ impl ConnectBuilder {
        self
    }

-    /// Enable or disable manifest-backed directory namespace mode for local
-    /// native connections.
-    ///
-    /// When enabled, the connection uses the directory namespace database
-    /// directly for all table operations and forces
-    /// `dir_listing_to_manifest_migration_enabled=true`.
-    pub fn manifest_enabled(mut self, enabled: bool) -> Self {
-        self.request.manifest_enabled = enabled;
-        self
-    }
-
    /// The interval at which to check for updates from other processes. This
    /// only affects LanceDB OSS.
    ///
@@ -907,16 +886,6 @@ impl ConnectBuilder {
    pub async fn execute(self) -> Result<Connection> {
        if self.request.uri.starts_with("db") {
            self.execute_remote()
-        } else if self.request.manifest_enabled {
-            let internal = Arc::new(
-                ListingDatabase::connect_manifest_enabled_namespace_database(&self.request).await?,
-            );
-            Ok(Connection {
-                internal,
-                embedding_registry: self
-                    .embedding_registry
-                    .unwrap_or_else(|| Arc::new(MemoryRegistry::new())),
-            })
        } else {
            let internal = Arc::new(ListingDatabase::connect_with_options(&self.request).await?);
            Ok(Connection {
@@ -1163,9 +1132,6 @@ mod tests {
    use lance_testing::datagen::{BatchGenerator, IncrementingInt32};
    use tempfile::tempdir;

-    use crate::database::listing::{ListingDatabaseOptions, OPT_NEW_TABLE_V2_MANIFEST_PATHS};
-    use crate::database::namespace::LanceNamespaceDatabase;
-    use crate::table::NativeTable;
    use crate::test_utils::connection::new_test_connection;

    use super::*;
@@ -1238,147 +1204,6 @@ mod tests {
        );
    }

-    #[tokio::test]
-    async fn test_connect_with_manifest_enabled_uses_directory_namespace() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-
-        let db = connect(uri)
-            .manifest_enabled(true)
-            .storage_option("timeout", "30s")
-            .namespace_client_property("manifest_enabled", "false")
-            .namespace_client_property("dir_listing_to_manifest_migration_enabled", "false")
-            .execute()
-            .await
-            .unwrap();
-
-        assert!(
-            db.database()
-                .as_any()
-                .downcast_ref::<LanceNamespaceDatabase>()
-                .is_some()
-        );
-        assert_eq!(db.uri(), uri);
-
-        let (ns_impl, properties) = db.namespace_client_config().await.unwrap();
-        assert_eq!(ns_impl, "dir");
-        assert_eq!(properties.get("root"), Some(&uri.to_string()));
-        assert_eq!(
-            properties.get("manifest_enabled"),
-            Some(&"true".to_string())
-        );
-        assert_eq!(
-            properties.get("dir_listing_to_manifest_migration_enabled"),
-            Some(&"true".to_string())
-        );
-        assert_eq!(properties.get("storage.timeout"), Some(&"30s".to_string()));
-    }
-
-    #[tokio::test]
-    async fn test_manifest_enabled_rejects_commit_engine_uri() {
-        let Err(err) = connect("s3+ddb://bucket/db?ddbTableName=manifest")
-            .manifest_enabled(true)
-            .execute()
-            .await
-        else {
-            panic!("expected manifest-enabled s3+ddb connection to fail");
-        };
-        assert!(
-            matches!(err, Error::NotSupported { message } if message.contains("commit engine URI schemes"))
-        );
-
-        let Err(err) = connect("s3://bucket/db?engine=ddb&ddbTableName=manifest")
-            .manifest_enabled(true)
-            .execute()
-            .await
-        else {
-            panic!("expected manifest-enabled engine query connection to fail");
-        };
-        assert!(
-            matches!(err, Error::NotSupported { message } if message.contains("commit engine"))
-        );
-    }
-
-    #[tokio::test]
-    async fn test_manifest_enabled_connection_migrates_root_listing_table() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
-
-        connect(uri)
-            .execute()
-            .await
-            .unwrap()
-            .create_empty_table("legacy", schema)
-            .execute()
-            .await
-            .unwrap();
-
-        let db = connect(uri).manifest_enabled(true).execute().await.unwrap();
-        let tables = db.table_names().execute().await.unwrap();
-        assert_eq!(tables, vec!["legacy".to_string()]);
-        db.open_table("legacy").execute().await.unwrap();
-    }
-
-    #[tokio::test]
-    async fn test_manifest_enabled_preserves_new_table_options() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let options = ListingDatabaseOptions::builder()
-            .enable_v2_manifest_paths(true)
-            .build();
-        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
-
-        let table = connect(uri)
-            .manifest_enabled(true)
-            .database_options(&options)
-            .execute()
-            .await
-            .unwrap()
-            .create_empty_table("v1_manifest", schema)
-            .storage_option(OPT_NEW_TABLE_V2_MANIFEST_PATHS, "false")
-            .execute()
-            .await
-            .unwrap();
-
-        let native_table = table
-            .base_table()
-            .as_any()
-            .downcast_ref::<NativeTable>()
-            .unwrap();
-        assert!(!native_table.uses_v2_manifest_paths().await.unwrap());
-    }
-
-    #[tokio::test]
-    async fn test_manifest_enabled_vend_input_storage_options() {
-        let tmp_dir = tempdir().unwrap();
-        let uri = tmp_dir.path().to_str().unwrap();
-        let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)]));
-
-        let table = connect(uri)
-            .manifest_enabled(true)
-            .storage_option("test_storage_option", "test_value")
-            .namespace_client_property("vend_input_storage_options", "true")
-            .namespace_client_property(
-                "vend_input_storage_options_refresh_interval_millis",
-                "60000",
-            )
-            .execute()
-            .await
-            .unwrap()
-            .create_empty_table("vended", schema)
-            .execute()
-            .await
-            .unwrap();
-
-        let storage_options = table.latest_storage_options().await.unwrap().unwrap();
-        assert_eq!(
-            storage_options.get("test_storage_option"),
-            Some(&"test_value".to_string())
-        );
-        assert!(storage_options.contains_key("expires_at_millis"));
-    }
-
    #[tokio::test]
    async fn test_table_names() {
        let tc = new_test_connection().await.unwrap();
--- a/rust/lancedb/src/database/listing.rs
+++ b/rust/lancedb/src/database/listing.rs
@@ -285,7 +285,7 @@ const MIRRORED_STORE: &str = "mirroredStore";

 /// A connection to LanceDB
 impl ListingDatabase {
-    pub(crate) fn build_namespace_client_properties(
+    fn build_namespace_client_properties(
        uri: &str,
        storage_options: &HashMap<String, String>,
        namespace_client_properties: HashMap<String, String>,
@@ -298,24 +298,6 @@ impl ListingDatabase {
        properties
    }

-    pub(crate) fn build_manifest_enabled_namespace_client_properties(
-        uri: &str,
-        storage_options: &HashMap<String, String>,
-        namespace_client_properties: HashMap<String, String>,
-    ) -> HashMap<String, String> {
-        let mut properties = Self::build_namespace_client_properties(
-            uri,
-            storage_options,
-            namespace_client_properties,
-        );
-        properties.insert("manifest_enabled".to_string(), "true".to_string());
-        properties.insert(
-            "dir_listing_to_manifest_migration_enabled".to_string(),
-            "true".to_string(),
-        );
-        properties
-    }
-
    async fn connect_namespace_database(
        uri: &str,
        storage_options: HashMap<String, String>,
@@ -341,119 +323,6 @@ impl ListingDatabase {
        ))
    }

-    async fn prepare_namespace_root(
-        uri: &str,
-        storage_options: &HashMap<String, String>,
-        session: Arc<lance::session::Session>,
-    ) -> Result<String> {
-        match url::Url::parse(uri) {
-            Ok(url) if url.scheme().len() == 1 && cfg!(windows) => {
-                let (object_store, _) = ObjectStore::from_uri_and_params(
-                    session.store_registry(),
-                    uri,
-                    &ObjectStoreParams::default(),
-                )
-                .await?;
-                if object_store.is_local() {
-                    Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
-                }
-                Ok(uri.to_string())
-            }
-            Ok(mut url) => {
-                if url.scheme().contains('+') {
-                    return Err(Error::NotSupported {
-                        message: "commit engine URI schemes are not supported for manifest-enabled namespace connections".to_string(),
-                    });
-                }
-
-                for (key, value) in url.query_pairs() {
-                    if key == ENGINE {
-                        return Err(Error::NotSupported {
-                            message: format!(
-                                "commit engine '{}' is not supported for manifest-enabled namespace connections",
-                                value
-                            ),
-                        });
-                    } else if key == MIRRORED_STORE {
-                        return Err(Error::NotSupported {
-                            message: "mirrored store is not supported for manifest-enabled namespace connections"
-                                .to_string(),
-                        });
-                    }
-                }
-
-                url.set_query(None);
-                let plain_uri = url.to_string();
-
-                let os_params = ObjectStoreParams {
-                    storage_options_accessor: if storage_options.is_empty() {
-                        None
-                    } else {
-                        Some(Arc::new(StorageOptionsAccessor::with_static_options(
-                            storage_options.clone(),
-                        )))
-                    },
-                    ..Default::default()
-                };
-                let (object_store, _) = ObjectStore::from_uri_and_params(
-                    session.store_registry(),
-                    &plain_uri,
-                    &os_params,
-                )
-                .await?;
-                if object_store.is_local() {
-                    Self::try_create_dir(&plain_uri).context(CreateDirSnafu {
-                        path: plain_uri.clone(),
-                    })?;
-                }
-
-                Ok(plain_uri)
-            }
-            Err(_) => {
-                let (object_store, _) = ObjectStore::from_uri_and_params(
-                    session.store_registry(),
-                    uri,
-                    &ObjectStoreParams::default(),
-                )
-                .await?;
-                if object_store.is_local() {
-                    Self::try_create_dir(uri).context(CreateDirSnafu { path: uri })?;
-                }
-                Ok(uri.to_string())
-            }
-        }
-    }
-
-    pub(crate) async fn connect_manifest_enabled_namespace_database(
-        request: &ConnectRequest,
-    ) -> Result<LanceNamespaceDatabase> {
-        let options = ListingDatabaseOptions::parse_from_map(&request.options)?;
-        let session = request
-            .session
-            .clone()
-            .unwrap_or_else(|| Arc::new(lance::session::Session::default()));
-        let namespace_root =
-            Self::prepare_namespace_root(&request.uri, &options.storage_options, session.clone())
-                .await?;
-        let ns_properties = Self::build_manifest_enabled_namespace_client_properties(
-            &namespace_root,
-            &options.storage_options,
-            request.namespace_client_properties.clone(),
-        );
-
-        LanceNamespaceDatabase::connect_with_new_table_config(
-            "dir",
-            ns_properties,
-            options.storage_options,
-            request.read_consistency_interval,
-            Some(session),
-            HashSet::new(),
-            options.new_table_config,
-        )
-        .await
-        .map(|db| db.with_uri(request.uri.clone()))
-    }
-
    /// Connect to a listing database
    ///
    /// The URI should be a path to a directory where the tables are stored.
@@ -715,7 +584,7 @@ impl ListingDatabase {
        let commit_handler = commit_handler_from_url(&uri, &Some(object_store_params)).await?;
        for name in names {
            let dir_name = format!("{}.{}", name, LANCE_EXTENSION);
-            let full_path = self.base_path.clone().join(dir_name.clone());
+            let full_path = self.base_path.child(dir_name.clone());

            commit_handler.delete(&full_path).await?;

@@ -821,12 +690,15 @@ impl ListingDatabase {
            store_params.storage_options_accessor = Some(Arc::new(accessor));
        }

-        write_params.data_storage_version = storage_version_override
-            .or(write_params.data_storage_version)
-            .or(self.new_table_config.data_storage_version);
+        write_params.data_storage_version = self
+            .new_table_config
+            .data_storage_version
+            .or(storage_version_override);

-        if let Some(enable_v2_manifest_paths) =
-            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
+        if let Some(enable_v2_manifest_paths) = self
+            .new_table_config
+            .enable_v2_manifest_paths
+            .or(v2_manifest_override)
        {
            write_params.enable_v2_manifest_paths = enable_v2_manifest_paths;
        }
@@ -1286,7 +1158,6 @@ mod tests {
            client_config: Default::default(),
            options: Default::default(),
            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
@@ -1421,7 +1292,6 @@ mod tests {
            client_config: Default::default(),
            options: options.clone(),
            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
@@ -1957,7 +1827,6 @@ mod tests {
            client_config: Default::default(),
            options,
            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
@@ -2064,7 +1933,6 @@ mod tests {
            client_config: Default::default(),
            options,
            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
@@ -2137,7 +2005,6 @@ mod tests {
            client_config: Default::default(),
            options,
            namespace_client_properties: Default::default(),
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
@@ -2335,7 +2202,6 @@ mod tests {
            client_config: Default::default(),
            options: Default::default(),
            namespace_client_properties,
-            manifest_enabled: false,
            read_consistency_interval: None,
            session: None,
        };
--- a/rust/lancedb/src/database/namespace.rs
+++ b/rust/lancedb/src/database/namespace.rs
@@ -24,10 +24,6 @@ use lance_table::io::commit::external_manifest::ExternalManifestCommitHandler;

 use crate::connection::NamespaceClientPushdownOperation;
 use crate::database::ReadConsistency;
-use crate::database::listing::{
-    NewTableConfig, OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS, OPT_NEW_TABLE_STORAGE_VERSION,
-    OPT_NEW_TABLE_V2_MANIFEST_PATHS,
-};
 use crate::error::{Error, Result};
 use crate::table::NativeTable;
 use lance::dataset::WriteMode;
@@ -54,8 +50,6 @@ pub struct LanceNamespaceDatabase {
    ns_impl: String,
    // Namespace properties used to construct the namespace client
    ns_properties: HashMap<String, String>,
-    // Options for tables created by this connection
-    new_table_config: NewTableConfig,
 }

 impl LanceNamespaceDatabase {
@@ -77,15 +71,9 @@ impl LanceNamespaceDatabase {
            pushdown_operations: namespace_client_pushdown_operations,
            ns_impl: namespace_client_impl,
            ns_properties: namespace_client_properties,
-            new_table_config: NewTableConfig::default(),
        }
    }

-    pub(crate) fn with_uri(mut self, uri: impl Into<String>) -> Self {
-        self.uri = uri.into();
-        self
-    }
-
    pub async fn connect(
        ns_impl: &str,
        ns_properties: HashMap<String, String>,
@@ -93,27 +81,6 @@ impl LanceNamespaceDatabase {
        read_consistency_interval: Option<std::time::Duration>,
        session: Option<Arc<lance::session::Session>>,
        pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
-    ) -> Result<Self> {
-        Self::connect_with_new_table_config(
-            ns_impl,
-            ns_properties,
-            storage_options,
-            read_consistency_interval,
-            session,
-            pushdown_operations,
-            NewTableConfig::default(),
-        )
-        .await
-    }
-
-    pub(crate) async fn connect_with_new_table_config(
-        ns_impl: &str,
-        ns_properties: HashMap<String, String>,
-        storage_options: HashMap<String, String>,
-        read_consistency_interval: Option<std::time::Duration>,
-        session: Option<Arc<lance::session::Session>>,
-        pushdown_operations: HashSet<NamespaceClientPushdownOperation>,
-        new_table_config: NewTableConfig,
    ) -> Result<Self> {
        let mut builder = ConnectBuilder::new(ns_impl);
        for (key, value) in ns_properties.clone() {
@@ -135,79 +102,8 @@ impl LanceNamespaceDatabase {
            pushdown_operations,
            ns_impl: ns_impl.to_string(),
            ns_properties,
-            new_table_config,
        })
    }
-
-    fn extract_storage_overrides(
-        &self,
-        request: &DbCreateTableRequest,
-    ) -> Result<(
-        Option<lance_encoding::version::LanceFileVersion>,
-        Option<bool>,
-        Option<bool>,
-    )> {
-        let storage_options = request
-            .write_options
-            .lance_write_params
-            .as_ref()
-            .and_then(|p| p.store_params.as_ref())
-            .and_then(|sp| sp.storage_options());
-
-        let storage_version_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_STORAGE_VERSION))
-            .map(|s| s.parse::<lance_encoding::version::LanceFileVersion>())
-            .transpose()?;
-
-        let v2_manifest_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_V2_MANIFEST_PATHS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_v2_manifest_paths must be a boolean".to_string(),
-            })?;
-
-        let stable_row_ids_override = storage_options
-            .and_then(|opts| opts.get(OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS))
-            .map(|s| s.parse::<bool>())
-            .transpose()
-            .map_err(|_| Error::InvalidInput {
-                message: "enable_stable_row_ids must be a boolean".to_string(),
-            })?;
-
-        Ok((
-            storage_version_override,
-            v2_manifest_override,
-            stable_row_ids_override,
-        ))
-    }
-
-    fn apply_new_table_config(
-        &self,
-        params: &mut lance::dataset::WriteParams,
-        request: &DbCreateTableRequest,
-    ) -> Result<()> {
-        let (storage_version_override, v2_manifest_override, stable_row_ids_override) =
-            self.extract_storage_overrides(request)?;
-
-        params.data_storage_version = storage_version_override
-            .or(params.data_storage_version)
-            .or(self.new_table_config.data_storage_version);
-
-        if let Some(enable_v2_manifest_paths) =
-            v2_manifest_override.or(self.new_table_config.enable_v2_manifest_paths)
-        {
-            params.enable_v2_manifest_paths = enable_v2_manifest_paths;
-        }
-
-        if let Some(enable_stable_row_ids) =
-            stable_row_ids_override.or(self.new_table_config.enable_stable_row_ids)
-        {
-            params.enable_stable_row_ids = enable_stable_row_ids;
-        }
-
-        Ok(())
-    }
 }

 impl std::fmt::Debug for LanceNamespaceDatabase {
@@ -403,12 +299,7 @@ impl Database for LanceNamespaceDatabase {
        };

        // Build write params with storage options and commit handler
-        let mut params = request
-            .write_options
-            .lance_write_params
-            .clone()
-            .unwrap_or_default();
-        self.apply_new_table_config(&mut params, &request)?;
+        let mut params = request.write_options.lance_write_params.unwrap_or_default();

        if matches!(request.mode, CreateTableMode::Overwrite) {
            params.mode = WriteMode::Overwrite;
--- a/rust/lancedb/src/io/object_store.rs
+++ b/rust/lancedb/src/io/object_store.rs
@@ -5,12 +5,11 @@

 use std::{fmt::Formatter, sync::Arc};

-use futures::{StreamExt, TryFutureExt, stream::BoxStream};
+use futures::{TryFutureExt, stream::BoxStream};
 use lance::io::WrappingObjectStore;
 use object_store::{
-    CopyOptions, Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta,
-    ObjectStore, ObjectStoreExt, PutMultipartOptions, PutOptions, PutPayload, PutResult, Result,
-    UploadPart, path::Path,
+    Error, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, Result, UploadPart, path::Path,
 };

 use async_trait::async_trait;
@@ -94,6 +93,20 @@ impl ObjectStore for MirroringObjectStore {
        self.primary.get_opts(location, options).await
    }

+    async fn head(&self, location: &Path) -> Result<ObjectMeta> {
+        self.primary.head(location).await
+    }
+
+    async fn delete(&self, location: &Path) -> Result<()> {
+        if !location.primary_only() {
+            match self.secondary.delete(location).await {
+                Err(Error::NotFound { .. }) | Ok(_) => {}
+                Err(e) => return Err(e),
+            }
+        }
+        self.primary.delete(location).await
+    }
+
    fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, Result<ObjectMeta>> {
        self.primary.list(prefix)
    }
@@ -102,41 +115,22 @@ impl ObjectStore for MirroringObjectStore {
        self.primary.list_with_delimiter(prefix).await
    }

-    fn delete_stream(
-        &self,
-        locations: BoxStream<'static, Result<Path>>,
-    ) -> BoxStream<'static, Result<Path>> {
-        let primary = self.primary.clone();
-        let secondary = self.secondary.clone();
-        locations
-            .map(move |location| {
-                let primary = primary.clone();
-                let secondary = secondary.clone();
-                async move {
-                    let location = location?;
-                    if !location.primary_only() {
-                        match secondary.delete(&location).await {
-                            Err(Error::NotFound { .. }) | Ok(_) => {}
-                            Err(e) => return Err(e),
-                        }
-                    }
-                    primary.delete(&location).await?;
-                    Ok(location)
-                }
-            })
-            .buffered(10)
-            .boxed()
-    }
-
-    async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> Result<()> {
+    async fn copy(&self, from: &Path, to: &Path) -> Result<()> {
        if to.primary_only() {
-            self.primary.copy_opts(from, to, options).await
+            self.primary.copy(from, to).await
        } else {
-            self.secondary.copy_opts(from, to, options.clone()).await?;
-            self.primary.copy_opts(from, to, options).await?;
+            self.secondary.copy(from, to).await?;
+            self.primary.copy(from, to).await?;
            Ok(())
        }
    }
+
+    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> Result<()> {
+        if !to.primary_only() {
+            self.secondary.copy(from, to).await?;
+        }
+        self.primary.copy_if_not_exists(from, to).await
+    }
 }

 #[derive(Debug)]
--- a/rust/lancedb/src/io/object_store/io_tracking.rs
+++ b/rust/lancedb/src/io/object_store/io_tracking.rs
@@ -10,9 +10,9 @@ use bytes::Bytes;
 use futures::stream::BoxStream;
 use lance::io::WrappingObjectStore;
 use object_store::{
-    CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
-    PutMultipartOptions, PutOptions, PutPayload, PutResult, RenameOptions, Result as OSResult,
-    UploadPart, path::Path,
+    GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore,
+    PutMultipartOptions, PutOptions, PutPayload, PutResult, Result as OSResult, UploadPart,
+    path::Path,
 };

 #[derive(Debug, Default)]
@@ -81,6 +81,11 @@ impl IoTrackingStore {
 #[async_trait::async_trait]
 #[deny(clippy::missing_trait_methods)]
 impl ObjectStore for IoTrackingStore {
+    async fn put(&self, location: &Path, bytes: PutPayload) -> OSResult<PutResult> {
+        self.record_write(bytes.content_length() as u64);
+        self.target.put(location, bytes).await
+    }
+
    async fn put_opts(
        &self,
        location: &Path,
@@ -91,6 +96,14 @@ impl ObjectStore for IoTrackingStore {
        self.target.put_opts(location, bytes, opts).await
    }

+    async fn put_multipart(&self, location: &Path) -> OSResult<Box<dyn MultipartUpload>> {
+        let target = self.target.put_multipart(location).await?;
+        Ok(Box::new(IoTrackingMultipartUpload {
+            target,
+            stats: self.stats.clone(),
+        }))
+    }
+
    async fn put_multipart_opts(
        &self,
        location: &Path,
@@ -103,6 +116,15 @@ impl ObjectStore for IoTrackingStore {
        }))
    }

+    async fn get(&self, location: &Path) -> OSResult<GetResult> {
+        let result = self.target.get(location).await;
+        if let Ok(result) = &result {
+            let num_bytes = result.range.end - result.range.start;
+            self.record_read(num_bytes);
+        }
+        result
+    }
+
    async fn get_opts(&self, location: &Path, options: GetOptions) -> OSResult<GetResult> {
        let result = self.target.get_opts(location, options).await;
        if let Ok(result) = &result {
@@ -112,6 +134,14 @@ impl ObjectStore for IoTrackingStore {
        result
    }

+    async fn get_range(&self, location: &Path, range: std::ops::Range<u64>) -> OSResult<Bytes> {
+        let result = self.target.get_range(location, range).await;
+        if let Ok(result) = &result {
+            self.record_read(result.len() as u64);
+        }
+        result
+    }
+
    async fn get_ranges(
        &self,
        location: &Path,
@@ -124,11 +154,20 @@ impl ObjectStore for IoTrackingStore {
        result
    }

-    fn delete_stream(
-        &self,
-        locations: BoxStream<'static, OSResult<Path>>,
-    ) -> BoxStream<'static, OSResult<Path>> {
+    async fn head(&self, location: &Path) -> OSResult<ObjectMeta> {
+        self.record_read(0);
+        self.target.head(location).await
+    }
+
+    async fn delete(&self, location: &Path) -> OSResult<()> {
        self.record_write(0);
+        self.target.delete(location).await
+    }
+
+    fn delete_stream<'a>(
+        &'a self,
+        locations: BoxStream<'a, OSResult<Path>>,
+    ) -> BoxStream<'a, OSResult<Path>> {
        self.target.delete_stream(locations)
    }

@@ -151,14 +190,24 @@ impl ObjectStore for IoTrackingStore {
        self.target.list_with_delimiter(prefix).await
    }

-    async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> OSResult<()> {
+    async fn copy(&self, from: &Path, to: &Path) -> OSResult<()> {
        self.record_write(0);
-        self.target.copy_opts(from, to, options).await
+        self.target.copy(from, to).await
    }

-    async fn rename_opts(&self, from: &Path, to: &Path, options: RenameOptions) -> OSResult<()> {
+    async fn rename(&self, from: &Path, to: &Path) -> OSResult<()> {
        self.record_write(0);
-        self.target.rename_opts(from, to, options).await
+        self.target.rename(from, to).await
+    }
+
+    async fn rename_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.record_write(0);
+        self.target.rename_if_not_exists(from, to).await
+    }
+
+    async fn copy_if_not_exists(&self, from: &Path, to: &Path) -> OSResult<()> {
+        self.record_write(0);
+        self.target.copy_if_not_exists(from, to).await
    }
 }

--- a/rust/lancedb/src/remote/table/insert.rs
+++ b/rust/lancedb/src/remote/table/insert.rs
@@ -105,12 +105,12 @@ impl<S: HttpSend + 'static> RemoteInsertExec<S> {
            1
        };
        let schema = COUNT_SCHEMA.clone();
-        let properties = PlanProperties::new(
+        let properties = Arc::new(PlanProperties::new(
            EquivalenceProperties::new(schema),
            datafusion_physical_plan::Partitioning::UnknownPartitioning(num_partitions),
            datafusion_physical_plan::execution_plan::EmissionType::Final,
            datafusion_physical_plan::execution_plan::Boundedness::Bounded,
-        );
+        ));

        Self {
            table_name,
@@ -118,7 +118,7 @@ impl<S: HttpSend + 'static> RemoteInsertExec<S> {
            client,
            input,
            overwrite,
-            properties: Arc::new(properties),
+            properties,
            add_result: Arc::new(Mutex::new(None)),
            metrics: ExecutionPlanMetricsSet::new(),
            upload_id,
--- a/rust/lancedb/src/table/datafusion/insert.rs
+++ b/rust/lancedb/src/table/datafusion/insert.rs
@@ -95,19 +95,19 @@ impl InsertExec {
    ) -> Self {
        let schema = COUNT_SCHEMA.clone();
        let num_partitions = input.output_partitioning().partition_count();
-        let properties = PlanProperties::new(
+        let properties = Arc::new(PlanProperties::new(
            EquivalenceProperties::new(schema),
            Partitioning::UnknownPartitioning(num_partitions),
            EmissionType::Final,
            Boundedness::Bounded,
-        );
+        ));

        Self {
            ds_wrapper,
            dataset,
            input,
            write_params,
-            properties: Arc::new(properties),
+            properties,
            partial_transactions: Arc::new(Mutex::new(Vec::with_capacity(num_partitions))),
            metrics: ExecutionPlanMetricsSet::new(),
        }
--- a/rust/lancedb/src/table/datafusion/scannable_exec.rs
+++ b/rust/lancedb/src/table/datafusion/scannable_exec.rs
@@ -37,19 +37,19 @@ impl ScannableExec {
    pub fn new(source: Box<dyn Scannable>, tracker: Option<Arc<WriteProgressTracker>>) -> Self {
        let schema = source.schema();
        let eq_properties = EquivalenceProperties::new(schema);
-        let properties = PlanProperties::new(
+        let properties = Arc::new(PlanProperties::new(
            eq_properties,
            Partitioning::UnknownPartitioning(1),
            EmissionType::Incremental,
            datafusion_physical_plan::execution_plan::Boundedness::Bounded,
-        );
+        ));

        let num_rows = source.num_rows();
        let source = Mutex::new(source);
        Self {
            source,
            num_rows,
-            properties: Arc::new(properties),
+            properties,
            tracker,
        }
    }