diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index e71564473..1784c4c8a 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -15,6 +15,9 @@ on: - ci/check_cargo_lock.py - ci/check_sdist.py - python/pyproject.toml + - python/Cargo.toml + - python/vendor/** + - rust/lancedb/Cargo.toml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 930ae67e2..17fe3a7b4 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -29,6 +29,63 @@ env: RUST_BACKTRACE: "1" jobs: + dependency-overrides: + name: Downstream Lance override + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v6 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Read the pinned Lance tag + id: lance + run: | + python3 - <<'PYTHON' + import os + import tomllib + from pathlib import Path + + manifest = tomllib.loads(Path("Cargo.toml").read_text()) + tag = manifest["workspace"]["dependencies"]["lance"]["tag"] + with open(os.environ["GITHUB_OUTPUT"], "a") as output: + output.write(f"tag={tag}\n") + PYTHON + - uses: actions/checkout@v6 + with: + repository: lance-format/lance + ref: ${{ steps.lance.outputs.tag }} + path: local-lance + - name: Resolve a downstream workspace with a local Lance override + run: | + mkdir -p downstream/src + touch downstream/src/lib.rs + cat > downstream/Cargo.toml <<'TOML' + [package] + name = "lancedb-local-lance-consumer" + version = "0.1.0" + edition = "2024" + + [workspace] + + [dependencies] + lancedb = { path = "../rust/lancedb", default-features = false } + lance-arrow = { path = "../local-lance/rust/lance-arrow" } + + [patch."https://github.com/lance-format/lance.git"] + lance-arrow = { path = "../local-lance/rust/lance-arrow" } + TOML + cargo metadata --manifest-path downstream/Cargo.toml --format-version 1 > downstream/graph.json + python3 - <<'PYTHON' + import json + from pathlib import Path + + graph = json.loads(Path("downstream/graph.json").read_text()) + packages = [p for p in graph["packages"] if p["name"] == "lance-arrow"] + assert len(packages) == 1, [p["id"] for p in packages] + assert Path(packages[0]["manifest_path"]).resolve() == Path( + "local-lance/rust/lance-arrow/Cargo.toml" + ).resolve() + PYTHON + lint: timeout-minutes: 30 runs-on: ubuntu-24.04 diff --git a/ci/set_lance_version.py b/ci/set_lance_version.py index c7bf41f55..bef97f8ad 100644 --- a/ci/set_lance_version.py +++ b/ci/set_lance_version.py @@ -2,6 +2,8 @@ import argparse import re import sys import json +import tomllib +from pathlib import Path LANCE_GIT_URL = "https://github.com/lance-format/lance.git" @@ -148,6 +150,18 @@ def update_cargo_toml(line_updater): with open("Cargo.toml", "w") as f: f.writelines(new_lines) + core_manifest = Path("rust/lancedb/Cargo.toml") + if core_manifest.exists(): + lines = core_manifest.read_text().splitlines(keepends=True) + for index, line in enumerate(lines): + if line.startswith("lance-arrow ="): + updated = tomllib.loads(line_updater(line))["lance-arrow"] + config = {"version": updated} if isinstance(updated, str) else updated + if "path" in config: + config["path"] = "../../" + config["path"] + lines[index] = dict_to_toml_line("lance-arrow", config) + core_manifest.write_text("".join(lines)) + def set_stable_version(version: str): """ diff --git a/ci/tests/test_set_lance_version.py b/ci/tests/test_set_lance_version.py index 1493fc59d..1173eab84 100644 --- a/ci/tests/test_set_lance_version.py +++ b/ci/tests/test_set_lance_version.py @@ -4,6 +4,7 @@ import subprocess import sys import tempfile import textwrap +import tomllib import unittest from pathlib import Path @@ -83,6 +84,13 @@ lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], " with self.subTest(version=version), tempfile.TemporaryDirectory() as tmp: workdir = Path(tmp) (workdir / "Cargo.toml").write_text(CARGO_TOML) + core_manifest = workdir / "rust/lancedb/Cargo.toml" + core_manifest.parent.mkdir(parents=True) + core_manifest.write_text( + "[dependencies]\nlance-arrow = { " + 'version = "=1.0.0", git = "https://github.com/lance-format/lance.git", ' + 'tag = "v1.0.0" }\narrow.workspace = true\n' + ) command_log = workdir / "commands.log" fake_bin = workdir / "bin" fake_bin.mkdir() @@ -111,6 +119,26 @@ lance_datafusion = {{ "version" = "=8.2.0-beta.4", "features" = ["substrait"], " + updated_dependencies + UNTOUCHED_DEPENDENCIES, ) + core = tomllib.loads(core_manifest.read_text())["dependencies"] + dependency = core["lance-arrow"] + if isinstance(dependency, str): + dependency = {"version": dependency} + if version == "local": + self.assertEqual( + dependency["path"], "../../../lance/rust/lance-arrow" + ) + else: + expected_version = { + "stable": "9.9.9", + "preview": "10.0.0-beta.3", + }.get(version, version.removeprefix("v")) + self.assertEqual(dependency["version"], f"={expected_version}") + if "beta" in expected_version: + self.assertEqual(dependency["git"], LANCE_GIT_URL) + self.assertEqual(dependency["tag"], f"v{expected_version}") + else: + self.assertNotIn("git", dependency) + self.assertEqual(core["arrow"], {"workspace": True}) commands = command_log.read_text().splitlines() for command in expected_commands: self.assertTrue( diff --git a/deny.toml b/deny.toml index 77412709f..e4982c149 100644 --- a/deny.toml +++ b/deny.toml @@ -177,7 +177,9 @@ wildcards = "deny" # `[workspace.dependencies]` table: any crate used by more than one member must # go through `workspace = true`, and entries nothing uses are an error. This # keeps versions from drifting between the core crate and the bindings. -workspace-dependencies = { duplicates = "deny", unused = "deny" } +# Python bundles a patched path crate while the Rust SDK uses the overridable +# upstream source. Check shared external declarations, not these path bindings. +workspace-dependencies = { duplicates = "deny", unused = "deny", include-path-dependencies = false } # Internal workspace crates reference each other via `path = "..."`, which # cargo-deny sees as a wildcard version. That's fine for private workspace # members (not published to crates.io), so allow it specifically for paths. diff --git a/python/Cargo.toml b/python/Cargo.toml index 82fb7efae..265e4d270 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -22,7 +22,6 @@ bytes.workspace = true lancedb.workspace = true datafusion-common.workspace = true lance-core.workspace = true -# The workspace path dependency includes the patched crate in Maturin sources. lance-arrow.workspace = true lance-namespace.workspace = true lance-namespace-impls.workspace = true diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 006886647..c18db11fe 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -46,7 +46,7 @@ lance-index = { workspace = true, features = ["tokenizer-jieba", "tokenizer-lind lance-table = { workspace = true } lance-linalg = { workspace = true } lance-encoding = { workspace = true } -lance-arrow = { workspace = true } +lance-arrow = { version = "=15.0.0-beta.1", tag = "v15.0.0-beta.1", git = "https://github.com/lance-format/lance.git" } lance-namespace = { workspace = true } lance-namespace-impls = { workspace = true } metrics = { workspace = true, optional = true }