fix: disable manifest for commit-engine URIs on default connect

Connecting to an external commit-engine URI (e.g. `s3+ddb://...`) went
through the directory namespace with manifest listing enabled by default.
That opens a `__manifest` table under the root, whose URI has no
`ddbTableName` query, so building its commit handler fails with
"`s3+ddb://` scheme and expects exactly one query `ddbTableName`".

Lance v8 silently swallowed that error and fell back to directory
listing, so the connect succeeded. Lance v9's directory namespace only
falls back on not-found errors and propagates everything else, turning
the pre-existing incompatibility into a hard failure (surfaced by
`test_concurrent_dynamodb_commit`).

Disable the manifest namespace when a commit engine is in use, matching
the explicit `manifest_enabled(true)` path which already rejects these
URIs.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Cargo.lock

@@ -3423,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
 
 [[package]]
 name = "fsst"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "rand 0.9.4",
@@ -4774,8 +4774,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
 
 [[package]]
 name = "lance"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -4849,8 +4849,8 @@ dependencies = [
 
 [[package]]
 name = "lance-arrow"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4872,7 +4872,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4886,7 +4886,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -4895,8 +4895,8 @@ dependencies = [
 
 [[package]]
 name = "lance-bitpacking"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrayref",
  "crunchy",
@@ -4906,8 +4906,8 @@ dependencies = [
 
 [[package]]
 name = "lance-core"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -4945,8 +4945,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datafusion"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4976,8 +4976,8 @@ dependencies = [
 
 [[package]]
 name = "lance-datagen"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -4994,8 +4994,8 @@ dependencies = [
 
 [[package]]
 name = "lance-derive"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -5004,8 +5004,8 @@ dependencies = [
 
 [[package]]
 name = "lance-encoding"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5040,8 +5040,8 @@ dependencies = [
 
 [[package]]
 name = "lance-file"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-arith",
  "arrow-array",
@@ -5071,8 +5071,8 @@ dependencies = [
 
 [[package]]
 name = "lance-index"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arc-swap",
  "arrow",
@@ -5137,8 +5137,8 @@ dependencies = [
 
 [[package]]
 name = "lance-io"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "arrow-arith",
@@ -5179,8 +5179,8 @@ dependencies = [
 
 [[package]]
 name = "lance-linalg"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5196,8 +5196,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "async-trait",
@@ -5209,8 +5209,8 @@ dependencies = [
 
 [[package]]
 name = "lance-namespace-impls"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "arrow-ipc",
@@ -5233,7 +5233,7 @@ dependencies = [
  "lance-table",
  "log",
  "object_store",
- "quick-xml 0.38.4",
+ "quick-xml 0.40.1",
  "rand 0.9.4",
  "reqwest 0.12.28",
  "roaring",
@@ -5264,8 +5264,8 @@ dependencies = [
 
 [[package]]
 name = "lance-select"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -5280,8 +5280,8 @@ dependencies = [
 
 [[package]]
 name = "lance-table"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -5320,8 +5320,8 @@ dependencies = [
 
 [[package]]
 name = "lance-testing"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "arrow-array",
  "arrow-schema",
@@ -5334,8 +5334,8 @@ dependencies = [
 
 [[package]]
 name = "lance-tokenizer"
-version = "9.0.0-beta.18"
+version = "9.0.0-beta.19"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
  "icu_segmenter",
  "jieba-rs",
@@ -7688,15 +7688,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -7707,6 +7698,15 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "quick-xml"
+version = "0.40.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"

Cargo.toml

@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"
 
 [workspace.dependencies]
-lance = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }

java/pom.xml

@@ -28,7 +28,7 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>9.0.0-beta.18</lance-core.version>
+        <lance-core.version>9.0.0-beta.19</lance-core.version>
         <spotless.skip>false</spotless.skip>
         <spotless.version>2.30.0</spotless.version>
         <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>

rust/lancedb/src/connection.rs

@@ -1472,6 +1472,33 @@ mod tests {
         );
     }
 
+    // A default connection to a commit-engine URI must disable the manifest
+    // namespace: the `__manifest` table under the root has no `ddbTableName`
+    // query, so building its commit handler would fail. See the DynamoDB
+    // regression fixed alongside the Lance v9 bump.
+    #[cfg(feature = "dynamodb")]
+    #[tokio::test]
+    async fn test_default_connect_disables_manifest_for_commit_engine() {
+        for uri in [
+            "s3+ddb://bucket/db?ddbTableName=manifest",
+            "s3://bucket/db?engine=ddb&ddbTableName=manifest",
+        ] {
+            let db = connect(uri)
+                .storage_option("region", "us-east-1")
+                .execute()
+                .await
+                .unwrap_or_else(|e| panic!("expected {uri} to connect, got {e:?}"));
+
+            let (ns_impl, properties) = db.namespace_client_config().await.unwrap();
+            assert_eq!(ns_impl, "dir");
+            assert_eq!(
+                properties.get("manifest_enabled"),
+                Some(&"false".to_string()),
+                "manifest should be disabled for commit-engine URI {uri}"
+            );
+        }
+    }
+
     #[tokio::test]
     async fn test_manifest_enabled_connection_migrates_root_listing_table() {
         let tmp_dir = tempdir().unwrap();

rust/lancedb/src/database/listing.rs

@@ -323,12 +323,21 @@ impl ListingDatabase {
         namespace_client_properties: HashMap<String, String>,
         read_consistency_interval: Option<std::time::Duration>,
         session: Arc<lance::session::Session>,
+        uses_commit_engine: bool,
     ) -> Result<Arc<LanceNamespaceDatabase>> {
-        let ns_properties = Self::build_namespace_client_properties(
+        let mut ns_properties = Self::build_namespace_client_properties(
             uri,
             &storage_options,
             namespace_client_properties,
         );
+        // The directory namespace enables manifest-based listing by default, which
+        // opens a `__manifest` table under the root. That is incompatible with an
+        // external commit engine (e.g. `s3+ddb://`): the manifest table URI has no
+        // `ddbTableName` query, so building its commit handler fails. Disable the
+        // manifest so these connections fall back to directory listing.
+        if uses_commit_engine {
+            ns_properties.insert("manifest_enabled".to_string(), "false".to_string());
+        }
         Ok(Arc::new(
             LanceNamespaceDatabase::connect(
                 "dir",
@@ -503,6 +512,12 @@ impl ListingDatabase {
                     }
                 }
 
+                // A commit engine is in use either via an explicit `engine=` query
+                // param or a `<scheme>+<engine>` URI scheme (e.g. `s3+ddb`). The
+                // manifest namespace is incompatible with these, so track it to
+                // disable the manifest below.
+                let uses_commit_engine = engine.is_some() || url.scheme().contains('+');
+
                 // Filter out the commit store query param -- it's a lancedb param
                 url.query_pairs_mut().clear();
                 url.query_pairs_mut().extend_pairs(filtered_querys);
@@ -575,6 +590,7 @@ impl ListingDatabase {
                     request.namespace_client_properties.clone(),
                     request.read_consistency_interval,
                     session.clone(),
+                    uses_commit_engine,
                 )
                 .await?;
 
@@ -629,6 +645,7 @@ impl ListingDatabase {
             namespace_client_properties,
             read_consistency_interval,
             session.clone(),
+            false,
         )
         .await?;