fix: disable manifest for commit-engine URIs on default connect

Connecting to an external commit-engine URI (e.g. `s3+ddb://...`) went through the directory namespace with manifest listing enabled by default. That opens a `__manifest` table under the root, whose URI has no `ddbTableName` query, so building its commit handler fails with "`s3+ddb://` scheme and expects exactly one query `ddbTableName`". Lance v8 silently swallowed that error and fell back to directory listing, so the connect succeeded. Lance v9's directory namespace only falls back on not-found errors and propagates everything else, turning the pre-existing incompatibility into a hard failure (surfaced by `test_concurrent_dynamodb_commit`). Disable the manifest namespace when a commit engine is in use, matching the explicit `manifest_enabled(true)` path which already rejects these URIs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
chore: update lance dependency to v9.0.0-beta.19
2026-08-26 07:58:31 +00:00 · 2026-07-09 12:19:18 -07:00 · 2026-07-09 18:42:53 +00:00
5 changed files with 110 additions and 66 deletions
@@ -3423,8 +3423,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"

 [[package]]
 name = "fsst"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "rand 0.9.4",
@@ -4774,8 +4774,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"

 [[package]]
 name = "lance"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -4849,8 +4849,8 @@ dependencies = [

 [[package]]
 name = "lance-arrow"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4872,7 +4872,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-scalar"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4886,7 +4886,7 @@ dependencies = [
 [[package]]
 name = "lance-arrow-stats"
 version = "58.0.0"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -4895,8 +4895,8 @@ dependencies = [

 [[package]]
 name = "lance-bitpacking"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrayref",
 "crunchy",
@@ -4906,8 +4906,8 @@ dependencies = [

 [[package]]
 name = "lance-core"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -4945,8 +4945,8 @@ dependencies = [

 [[package]]
 name = "lance-datafusion"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4976,8 +4976,8 @@ dependencies = [

 [[package]]
 name = "lance-datagen"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -4994,8 +4994,8 @@ dependencies = [

 [[package]]
 name = "lance-derive"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -5004,8 +5004,8 @@ dependencies = [

 [[package]]
 name = "lance-encoding"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5040,8 +5040,8 @@ dependencies = [

 [[package]]
 name = "lance-file"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-arith",
 "arrow-array",
@@ -5071,8 +5071,8 @@ dependencies = [

 [[package]]
 name = "lance-index"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arc-swap",
 "arrow",
@@ -5137,8 +5137,8 @@ dependencies = [

 [[package]]
 name = "lance-io"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "arrow-arith",
@@ -5179,8 +5179,8 @@ dependencies = [

 [[package]]
 name = "lance-linalg"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5196,8 +5196,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "async-trait",
@@ -5209,8 +5209,8 @@ dependencies = [

 [[package]]
 name = "lance-namespace-impls"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "arrow-ipc",
@@ -5233,7 +5233,7 @@ dependencies = [
 "lance-table",
 "log",
 "object_store",
- "quick-xml 0.38.4",
+ "quick-xml 0.40.1",
 "rand 0.9.4",
 "reqwest 0.12.28",
 "roaring",
@@ -5264,8 +5264,8 @@ dependencies = [

 [[package]]
 name = "lance-select"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-buffer",
@@ -5280,8 +5280,8 @@ dependencies = [

 [[package]]
 name = "lance-table"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow",
 "arrow-array",
@@ -5320,8 +5320,8 @@ dependencies = [

 [[package]]
 name = "lance-testing"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "arrow-array",
 "arrow-schema",
@@ -5334,8 +5334,8 @@ dependencies = [

 [[package]]
 name = "lance-tokenizer"
-version = "9.0.0-beta.18"
-source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.18#d581bb9d068c4c3bba22c8a9972b2103d028dbcd"
+version = "9.0.0-beta.19"
+source = "git+https://github.com/lance-format/lance.git?tag=v9.0.0-beta.19#8f0e6d3a7c53438275b134c0ac1afbc80600616e"
 dependencies = [
 "icu_segmenter",
 "jieba-rs",
@@ -7688,15 +7688,6 @@ dependencies = [
 "memchr",
 ]

-[[package]]
-name = "quick-xml"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.39.4"
@@ -7707,6 +7698,15 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "quick-xml"
+version = "0.40.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "quinn"
 version = "0.11.9"
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
 rust-version = "1.91.0"

 [workspace.dependencies]
-lance = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-core = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-datagen = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-file = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-io = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-index = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-linalg = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-namespace-impls = { "version" = "=9.0.0-beta.18", default-features = false, "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-table = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-testing = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-datafusion = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-encoding = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
-lance-arrow = { "version" = "=9.0.0-beta.18", "tag" = "v9.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
+lance = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-core = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-datagen = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-file = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-io = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-index = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-linalg = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-namespace-impls = { "version" = "=9.0.0-beta.19", default-features = false, "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-table = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-testing = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-datafusion = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-encoding = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
+lance-arrow = { "version" = "=9.0.0-beta.19", "tag" = "v9.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
 ahash = "0.8"
 # Note that this one does not include pyarrow
 arrow = { version = "58.0.0", optional = false }
@@ -28,7 +28,7 @@
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <arrow.version>15.0.0</arrow.version>
-        <lance-core.version>9.0.0-beta.18</lance-core.version>
+        <lance-core.version>9.0.0-beta.19</lance-core.version>
        <spotless.skip>false</spotless.skip>
        <spotless.version>2.30.0</spotless.version>
        <spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
@@ -1472,6 +1472,33 @@ mod tests {
        );
    }

+    // A default connection to a commit-engine URI must disable the manifest
+    // namespace: the `__manifest` table under the root has no `ddbTableName`
+    // query, so building its commit handler would fail. See the DynamoDB
+    // regression fixed alongside the Lance v9 bump.
+    #[cfg(feature = "dynamodb")]
+    #[tokio::test]
+    async fn test_default_connect_disables_manifest_for_commit_engine() {
+        for uri in [
+            "s3+ddb://bucket/db?ddbTableName=manifest",
+            "s3://bucket/db?engine=ddb&ddbTableName=manifest",
+        ] {
+            let db = connect(uri)
+                .storage_option("region", "us-east-1")
+                .execute()
+                .await
+                .unwrap_or_else(|e| panic!("expected {uri} to connect, got {e:?}"));
+
+            let (ns_impl, properties) = db.namespace_client_config().await.unwrap();
+            assert_eq!(ns_impl, "dir");
+            assert_eq!(
+                properties.get("manifest_enabled"),
+                Some(&"false".to_string()),
+                "manifest should be disabled for commit-engine URI {uri}"
+            );
+        }
+    }
+
    #[tokio::test]
    async fn test_manifest_enabled_connection_migrates_root_listing_table() {
        let tmp_dir = tempdir().unwrap();
@@ -323,12 +323,21 @@ impl ListingDatabase {
        namespace_client_properties: HashMap<String, String>,
        read_consistency_interval: Option<std::time::Duration>,
        session: Arc<lance::session::Session>,
+        uses_commit_engine: bool,
    ) -> Result<Arc<LanceNamespaceDatabase>> {
-        let ns_properties = Self::build_namespace_client_properties(
+        let mut ns_properties = Self::build_namespace_client_properties(
            uri,
            &storage_options,
            namespace_client_properties,
        );
+        // The directory namespace enables manifest-based listing by default, which
+        // opens a `__manifest` table under the root. That is incompatible with an
+        // external commit engine (e.g. `s3+ddb://`): the manifest table URI has no
+        // `ddbTableName` query, so building its commit handler fails. Disable the
+        // manifest so these connections fall back to directory listing.
+        if uses_commit_engine {
+            ns_properties.insert("manifest_enabled".to_string(), "false".to_string());
+        }
        Ok(Arc::new(
            LanceNamespaceDatabase::connect(
                "dir",
@@ -503,6 +512,12 @@ impl ListingDatabase {
                    }
                }

+                // A commit engine is in use either via an explicit `engine=` query
+                // param or a `<scheme>+<engine>` URI scheme (e.g. `s3+ddb`). The
+                // manifest namespace is incompatible with these, so track it to
+                // disable the manifest below.
+                let uses_commit_engine = engine.is_some() || url.scheme().contains('+');
+
                // Filter out the commit store query param -- it's a lancedb param
                url.query_pairs_mut().clear();
                url.query_pairs_mut().extend_pairs(filtered_querys);
@@ -575,6 +590,7 @@ impl ListingDatabase {
                    request.namespace_client_properties.clone(),
                    request.read_consistency_interval,
                    session.clone(),
+                    uses_commit_engine,
                )
                .await?;

@@ -629,6 +645,7 @@ impl ListingDatabase {
            namespace_client_properties,
            read_consistency_interval,
            session.clone(),
+            false,
        )
        .await?;