fix: honor downstream Lance dependency overrides (#4671)

Consumers that provide a local Lance checkout cannot resolve the
dependency graph because the SDK directly selects a vendored
`lance-arrow`, producing two path packages with the same name and
version. The Rust core now declares its upstream Git dependency
separately so the consuming workspace can override that source
consistently. Python continues inheriting the vendored workspace
dependency, keeping the sliced-struct validity fix in Maturin source
distributions. Standalone SDK builds still select that same fix through
the existing root patch.

Adds a downstream consumer CI regression that resolves a local Lance
override and checks that exactly one canonical `lance-arrow` package is
selected. The Lance version updater also updates the core declaration in
stable, preview and local modes. Public APIs and query behavior are
unchanged.

The workspace declaration lint excludes path dependencies because the
Python binding intentionally uses the bundled source. Duplicate and
unused external declarations remain errors; advisory, license,
banned-package and source checks continue covering the full dependency
graph.

---------

Co-authored-by: Lance Release <lance-dev@lancedb.com>

.github/workflows/pypi-publish.yml

@@ -15,6 +15,9 @@ on:
       - ci/check_cargo_lock.py
       - ci/check_sdist.py
       - python/pyproject.toml
+      - python/Cargo.toml
+      - python/vendor/**
+      - rust/lancedb/Cargo.toml
       - Cargo.toml # Change in dependency frequently breaks builds
       - Cargo.lock
 

.github/workflows/rust.yml

@@ -29,6 +29,63 @@ env:
   RUST_BACKTRACE: "1"
 
 jobs:
+  dependency-overrides:
+    name: Downstream Lance override
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Read the pinned Lance tag
+        id: lance
+        run: |
+          python3 - <<'PYTHON'
+          import os
+          import tomllib
+          from pathlib import Path
+
+          manifest = tomllib.loads(Path("Cargo.toml").read_text())
+          tag = manifest["workspace"]["dependencies"]["lance"]["tag"]
+          with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+              output.write(f"tag={tag}\n")
+          PYTHON
+      - uses: actions/checkout@v6
+        with:
+          repository: lance-format/lance
+          ref: ${{ steps.lance.outputs.tag }}
+          path: local-lance
+      - name: Resolve a downstream workspace with a local Lance override
+        run: |
+          mkdir -p downstream/src
+          touch downstream/src/lib.rs
+          cat > downstream/Cargo.toml <<'TOML'
+          [package]
+          name = "lancedb-local-lance-consumer"
+          version = "0.1.0"
+          edition = "2024"
+
+          [workspace]
+
+          [dependencies]
+          lancedb = { path = "../rust/lancedb", default-features = false }
+          lance-arrow = { path = "../local-lance/rust/lance-arrow" }
+
+          [patch."https://github.com/lance-format/lance.git"]
+          lance-arrow = { path = "../local-lance/rust/lance-arrow" }
+          TOML
+          cargo metadata --manifest-path downstream/Cargo.toml --format-version 1 > downstream/graph.json
+          python3 - <<'PYTHON'
+          import json
+          from pathlib import Path
+
+          graph = json.loads(Path("downstream/graph.json").read_text())
+          packages = [p for p in graph["packages"] if p["name"] == "lance-arrow"]
+          assert len(packages) == 1, [p["id"] for p in packages]
+          assert Path(packages[0]["manifest_path"]).resolve() == Path(
+              "local-lance/rust/lance-arrow/Cargo.toml"
+          ).resolve()
+          PYTHON
+
   lint:
     timeout-minutes: 30
     runs-on: ubuntu-24.04

ci/set_lance_version.py

@@ -2,6 +2,8 @@ import argparse
 import re
 import sys
 import json
+import tomllib
+from pathlib import Path
 
 LANCE_GIT_URL = "https://github.com/lance-format/lance.git"
 
@@ -148,6 +150,18 @@ def update_cargo_toml(line_updater):
     with open("Cargo.toml", "w") as f:
         f.writelines(new_lines)
 
+    core_manifest = Path("rust/lancedb/Cargo.toml")
+    if core_manifest.exists():
+        lines = core_manifest.read_text().splitlines(keepends=True)
+        for index, line in enumerate(lines):
+            if line.startswith("lance-arrow ="):
+                updated = tomllib.loads(line_updater(line))["lance-arrow"]
+                config = {"version": updated} if isinstance(updated, str) else updated
+                if "path" in config:
+                    config["path"] = "../../" + config["path"]
+                lines[index] = dict_to_toml_line("lance-arrow", config)
+        core_manifest.write_text("".join(lines))
+
 
 def set_stable_version(version: str):
     """

ci/tests/test_set_lance_version.py

@@ -4,6 +4,7 @@ import subprocess
 import sys
 import tempfile
 import textwrap
+import tomllib
 import unittest
 from pathlib import Path
 
@@ -83,6 +84,13 @@ lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], "
             with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp:
                 workdir = Path(tmp)
                 (workdir / "Cargo.toml").write_text(CARGO_TOML)
+                core_manifest = workdir / "rust/lancedb/Cargo.toml"
+                core_manifest.parent.mkdir(parents=True)
+                core_manifest.write_text(
+                    "[dependencies]\nlance-arrow = { "
+                    'version = "=1.0.0", git = "https://github.com/lance-format/lance.git", '
+                    'tag = "v1.0.0" }\narrow.workspace = true\n'
+                )
                 command_log = workdir / "commands.log"
                 fake_bin = workdir / "bin"
                 fake_bin.mkdir()
@@ -111,6 +119,26 @@ lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], "
                     + updated_dependencies
                     + UNTOUCHED_DEPENDENCIES,
                 )
+                core = tomllib.loads(core_manifest.read_text())["dependencies"]
+                dependency = core["lance-arrow"]
+                if isinstance(dependency, str):
+                    dependency = {"version": dependency}
+                if version == "local":
+                    self.assertEqual(
+                        dependency["path"], "../../../lance/rust/lance-arrow"
+                    )
+                else:
+                    expected_version = {
+                        "stable": "9.9.9",
+                        "preview": "10.0.0-beta.3",
+                    }.get(version, version.removeprefix("v"))
+                    self.assertEqual(dependency["version"], f"={expected_version}")
+                    if "beta" in expected_version:
+                        self.assertEqual(dependency["git"], LANCE_GIT_URL)
+                        self.assertEqual(dependency["tag"], f"v{expected_version}")
+                    else:
+                        self.assertNotIn("git", dependency)
+                self.assertEqual(core["arrow"], {"workspace": True})
                 commands = command_log.read_text().splitlines()
                 for command in expected_commands:
                     self.assertTrue(

deny.toml

@@ -177,7 +177,9 @@ wildcards = "deny"
 # `[workspace.dependencies]` table: any crate used by more than one member must
 # go through `workspace = true`, and entries nothing uses are an error. This
 # keeps versions from drifting between the core crate and the bindings.
-workspace-dependencies = { duplicates = "deny", unused = "deny" }
+# Python bundles a patched path crate while the Rust SDK uses the overridable
+# upstream source. Check shared external declarations, not these path bindings.
+workspace-dependencies = { duplicates = "deny", unused = "deny", include-path-dependencies = false }
 # Internal workspace crates reference each other via `path = "..."`, which
 # cargo-deny sees as a wildcard version. That's fine for private workspace
 # members (not published to crates.io), so allow it specifically for paths.

python/Cargo.toml

@@ -22,7 +22,6 @@ bytes.workspace = true
 lancedb.workspace = true
 datafusion-common.workspace = true
 lance-core.workspace = true
-# The workspace path dependency includes the patched crate in Maturin sources.
 lance-arrow.workspace = true
 lance-namespace.workspace = true
 lance-namespace-impls.workspace = true

rust/lancedb/Cargo.toml

@@ -46,7 +46,7 @@ lance-index = { workspace = true, features = ["tokenizer-jieba", "tokenizer-lind
 lance-table = { workspace = true }
 lance-linalg = { workspace = true }
 lance-encoding = { workspace = true }
-lance-arrow = { workspace = true }
+lance-arrow = { version = "=15.0.0-beta.1", tag = "v15.0.0-beta.1", git = "https://github.com/lance-format/lance.git" }
 lance-namespace = { workspace = true }
 lance-namespace-impls = { workspace = true }
 metrics = { workspace = true, optional = true }