diff --git a/.bumpversion.toml b/.bumpversion.toml index af2f32902..d4c3d13a6 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -70,6 +70,11 @@ filename = "rust/lancedb/Cargo.toml" replace = "\nversion = \"{new_version}\"" search = "\nversion = \"{current_version}\"" +[[tool.bumpversion.files]] +filename = "rust/lancedb/cli/Cargo.toml" +replace = "\nversion = \"{new_version}\"" +search = "\nversion = \"{current_version}\"" + [[tool.bumpversion.files]] filename = "nodejs/Cargo.toml" replace = "\nversion = \"{new_version}\"" diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml index a99cf3320..d61af5632 100644 --- a/.github/actions/setup-protoc/action.yml +++ b/.github/actions/setup-protoc/action.yml @@ -52,7 +52,8 @@ runs: Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;; Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;; macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;; - Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;; + # Windows 11 Arm runners can execute the upstream x64 protoc binary. + Windows/X64|Windows/ARM64) platform="win64"; sha256="$WIN64_SHA256" ;; *) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;; esac zip="$RUNNER_TEMP/protoc.zip" diff --git a/.github/workflows/ci-scripts.yml b/.github/workflows/ci-scripts.yml index d45289c2d..098b097d4 100644 --- a/.github/workflows/ci-scripts.yml +++ b/.github/workflows/ci-scripts.yml @@ -9,6 +9,10 @@ on: - python/pyproject.toml - ci/check_cargo_lock.py - ci/check_sdist.py + - ci/build_cli_linux.sh + - ci/package_cli.py + - .bumpversion.toml + - rust/lancedb/cli/Cargo.toml - ci/set_lance_version.py - ci/tests/** - .github/workflows/ci-scripts.yml @@ -18,6 +22,10 @@ on: - python/pyproject.toml - ci/check_cargo_lock.py - ci/check_sdist.py + - ci/build_cli_linux.sh + - ci/package_cli.py + - .bumpversion.toml + - rust/lancedb/cli/Cargo.toml - ci/set_lance_version.py - ci/tests/** - .github/workflows/ci-scripts.yml diff --git a/.github/workflows/cli-release.yml b/.github/workflows/cli-release.yml new file mode 100644 index 000000000..2ec74e5f3 --- /dev/null +++ b/.github/workflows/cli-release.yml @@ -0,0 +1,167 @@ +name: CLI release packages + +on: + workflow_call: + workflow_dispatch: + pull_request: + paths: + - 'rust/lancedb/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - '.cargo/config.toml' + - '.bumpversion.toml' + - 'LICENSE' + - 'RUST_THIRD_PARTY_LICENSES.html' + - 'ci/build_cli_linux.sh' + - 'ci/package_cli.py' + - 'ci/tests/test_package_cli.py' + - '.github/actions/setup-protoc/**' + - '.github/workflows/cli-release.yml' + - '.github/workflows/gh-release.yml' + +permissions: + contents: read + +concurrency: + group: cli-packages-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + version: + runs-on: ubuntu-24.04 + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - name: Validate release version before building + id: version + run: | + version=$(python ci/package_cli.py version) + echo "version=$version" >> "$GITHUB_OUTPUT" + + build: + needs: version + timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu + runner: ubuntu-2404-8x-x64 + image: quay.io/pypa/manylinux_2_28_x86_64 + protoc: linux-x86_64 + - target: aarch64-unknown-linux-gnu + runner: ubuntu-2404-8x-arm64 + image: quay.io/pypa/manylinux_2_28_aarch64 + protoc: linux-aarch_64 + - target: aarch64-apple-darwin + runner: warp-macos-14-arm64-6x + runs-on: ${{ matrix.runner }} + env: + CLI_TARGET: ${{ matrix.target }} + CLI_VERSION: ${{ needs.version.outputs.version }} + MACOSX_DEPLOYMENT_TARGET: '14.0' + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 + with: + # Leave .cargo/config.toml's CPU/feature flags in effect. + rustflags: '' + cache: false + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2 + with: + key: cli-release-${{ matrix.target }} + save-if: ${{ github.event_name == 'pull_request' || github.ref == 'refs/heads/main' }} + - uses: ./.github/actions/setup-protoc + id: protoc + with: + install: ${{ runner.os == 'macOS' }} + - name: Add swap for Arm fat LTO + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + swap_file="$RUNNER_TEMP/lancedb-cli-swap" + sudo fallocate --length 16G "$swap_file" + sudo chmod 600 "$swap_file" + sudo mkswap "$swap_file" + sudo swapon "$swap_file" + - name: Build Linux CLI against glibc 2.28 + if: runner.os == 'Linux' + env: + MANYLINUX_IMAGE: ${{ matrix.image }} + PROTOC_PLATFORM: ${{ matrix.protoc }} + PROTOC_VERSION: ${{ steps.protoc.outputs.version }} + PROTOC_SHA256: ${{ matrix.protoc == 'linux-x86_64' && steps.protoc.outputs.linux-x86_64-sha256 || steps.protoc.outputs.linux-aarch_64-sha256 }} + run: | + docker run --rm \ + --volume "$PWD:/workspace" --workdir /workspace \ + --volume "$HOME/.cargo:/cargo" --env CARGO_HOME=/cargo \ + --volume "$HOME/.rustup:/rustup" --env RUSTUP_HOME=/rustup \ + --env CLI_TARGET --env PROTOC_PLATFORM --env PROTOC_VERSION --env PROTOC_SHA256 \ + "$MANYLINUX_IMAGE" bash ci/build_cli_linux.sh + - name: Build macOS CLI + if: runner.os == 'macOS' + run: | + cargo build --locked --release -p lancedb-cli --bin lancedb --target "$CLI_TARGET" + strip -S "target/$CLI_TARGET/release/lancedb" + # Stripping can invalidate the linker's ad-hoc signature on Apple Silicon. + codesign --force --sign - "target/$CLI_TARGET/release/lancedb" + - name: Package, inspect runtime dependencies, and smoke-test archive + run: | + python ci/package_cli.py package \ + --binary "target/$CLI_TARGET/release/lancedb" --target "$CLI_TARGET" --output dist + - name: Smoke-test Linux archive on a clean glibc 2.28 system + if: runner.os == 'Linux' + run: | + docker run --rm --volume "$PWD/dist:/artifacts:ro" \ + --env CLI_TARGET --env CLI_VERSION almalinux:8 bash -euo pipefail -c ' + dnf install -y libstdc++ + mkdir -p /tmp/check /tmp/cli-home + export HOME=/tmp/cli-home XDG_CONFIG_HOME=/tmp/cli-home/.config + stem="lancedb-v${CLI_VERSION}-${CLI_TARGET}" + tar -xzf "/artifacts/${stem}.tar.gz" -C /tmp/check + cli="/tmp/check/${stem}/lancedb" + "$cli" --help + test "$("$cli" --version)" = "lancedb ${CLI_VERSION}" + "$cli" profile list + ' + - uses: actions/upload-artifact@v7 + with: + name: cli-archive-${{ matrix.target }} + path: dist/*.tar.gz + if-no-files-found: error + + assemble: + needs: [version, build] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: actions/download-artifact@v8 + with: + pattern: cli-archive-* + merge-multiple: true + path: dist + - name: Require all three archives and generate checksums + run: | + python ci/package_cli.py checksums --output dist + python ci/package_cli.py verify --output dist + - uses: actions/upload-artifact@v7 + with: + name: cli-release + path: dist/* + if-no-files-found: error diff --git a/.github/workflows/gh-release.yml b/.github/workflows/gh-release.yml index 683d55d71..bbaade34c 100644 --- a/.github/workflows/gh-release.yml +++ b/.github/workflows/gh-release.yml @@ -1,8 +1,8 @@ name: GitHub Release # All SDKs share one version, so a single `vX.Y.Z` tag produces a single GitHub -# release covering all of them. The per-package publish workflows (PyPI, NPM, -# Cargo, Maven) trigger off the same tag independently. +# release covering all of them, including CLI binaries. The per-package publish +# workflows (PyPI, NPM, Cargo, Maven) trigger off the same tag independently. on: push: @@ -13,7 +13,11 @@ permissions: contents: read jobs: + cli: + uses: ./.github/workflows/cli-release.yml + gh-release: + needs: cli runs-on: ubuntu-latest permissions: contents: write @@ -22,6 +26,15 @@ jobs: with: fetch-depth: 0 lfs: true + - uses: actions/setup-python@v6 + with: + python-version: '3.13' + - uses: actions/download-artifact@v8 + with: + name: cli-release + path: dist + - name: Verify CLI release assets + run: python ci/package_cli.py verify --output dist - name: Extract version id: extract_version env: @@ -83,3 +96,22 @@ jobs: generate_release_notes: false name: LanceDB v${{ steps.extract_version.outputs.version }} body: ${{ steps.release_notes.outputs.changelog }} + files: | + dist/lancedb-*.tar.gz + dist/SHA256SUMS + fail_on_unmatched_files: true + overwrite_files: true + + report-failure: + runs-on: ubuntu-24.04 + needs: [cli, gh-release] + if: always() && failure() + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v6 + - uses: ./.github/actions/create-failure-issue + with: + job-results: ${{ toJSON(needs) }} + workflow-name: ${{ github.workflow }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f65123f78..491275340 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -14,6 +14,7 @@ on: - docs/web-tests/blob-v2/rs/** - nodejs/Cargo.toml - python/Cargo.toml + - .github/actions/setup-protoc/** - .github/workflows/rust.yml permissions: @@ -324,8 +325,8 @@ jobs: # unreadable outside their own branch anyway, since GitHub scopes # caches to the creating ref. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install Protoc v21.12 - run: choco install --no-progress protoc + - name: Install protoc + uses: ./.github/actions/setup-protoc - name: Build run: | $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT @@ -361,8 +362,8 @@ jobs: # unreadable outside their own branch anyway, since GitHub scopes # caches to the creating ref. save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Install Protoc v21.12 - run: choco install --no-progress protoc + - name: Install protoc + uses: ./.github/actions/setup-protoc - name: Build run: | $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT diff --git a/Cargo.lock b/Cargo.lock index 253eb4cbb..63b4ad423 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "lancedb-cli" -version = "0.1.0" +version = "0.41.0-beta.17" dependencies = [ "anyhow", "arrow-array", diff --git a/ci/build_cli_linux.sh b/ci/build_cli_linux.sh new file mode 100644 index 000000000..dac630222 --- /dev/null +++ b/ci/build_cli_linux.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Run inside a native-architecture manylinux_2_28 container. Cargo and rustup +# are mounted from the runner; keep the image's compiler/toolchain PATH intact. +set -euo pipefail + +export PATH="/cargo/bin:$PATH" +# The manylinux images provide GCC, but do not guarantee Clang or LLD. Native +# dependencies use Clang, and the Arm linker wrapper below also requires LLD. +dnf install -y clang lld +export CC=clang CXX=clang++ +"$CC" --version +"$CXX" --version +ld.lld --version + +# Fail before Cargo downloads or compiles dependencies if either compiler or +# the linker cannot produce a runnable binary in this container. +printf 'int main(void) { return 0; }\n' > /tmp/cli-toolchain-probe.c +"$CC" -fuse-ld=lld /tmp/cli-toolchain-probe.c -o /tmp/cli-toolchain-probe +/tmp/cli-toolchain-probe +"$CXX" -fuse-ld=lld -x c++ /tmp/cli-toolchain-probe.c -o /tmp/cli-toolchain-probe +/tmp/cli-toolchain-probe + +git config --global --add safe.directory /workspace + +curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_PLATFORM}.zip" -o /tmp/protoc.zip +echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - +unzip -q /tmp/protoc.zip -d /tmp/protoc +export PATH="/tmp/protoc/bin:$PATH" + +# GNU ld on Arm has a much higher memory peak for Lance's final link. Use a +# linker wrapper without replacing the repository's CPU and feature rustflags. +if [[ "$CLI_TARGET" == aarch64-unknown-linux-gnu ]]; then + cat > /tmp/cli-linker <<'LINKER' +#!/bin/sh +exec clang -fuse-ld=lld "$@" +LINKER + chmod 755 /tmp/cli-linker + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=/tmp/cli-linker +fi + +cargo build --locked --release -p lancedb-cli --bin lancedb --target "$CLI_TARGET" +strip "target/$CLI_TARGET/release/lancedb" diff --git a/ci/package_cli.py b/ci/package_cli.py new file mode 100644 index 000000000..83f244090 --- /dev/null +++ b/ci/package_cli.py @@ -0,0 +1,230 @@ +"""Package and validate standalone CLI release assets (Python 3.12+). + +Run `version` before building, `package` on each native runner, then `checksums` +after gathering the archives. Run `verify` again immediately before publication. +""" + +import argparse +import hashlib +import os +import re +import subprocess +import tarfile +import tempfile +import tomllib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TARGETS = ( + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", +) +NOTICES = ("LICENSE", "RUST_THIRD_PARTY_LICENSES.html") + + +def release_version(root: Path, ref: str) -> str: + config = tomllib.loads((root / ".bumpversion.toml").read_text())["tool"][ + "bumpversion" + ] + version = config["current_version"] + if not re.fullmatch(r"\d+\.\d+\.\d+(?:-beta\.\d+)?", version): + raise ValueError(f"Unsupported release version: {version}") + for manifest in ("rust/lancedb/Cargo.toml", "rust/lancedb/cli/Cargo.toml"): + package = tomllib.loads((root / manifest).read_text())["package"] + if package["version"] != version: + raise ValueError(f"{manifest} version does not match {version}") + if ref.startswith("refs/tags/") and ref != f"refs/tags/v{version}": + raise ValueError(f"Tag {ref} does not match v{version}") + return version + + +def archive_stem(version: str, target: str) -> str: + return f"lancedb-v{version}-{target}" + + +def command(*args: str) -> str: + return subprocess.check_output(args, text=True) + + +def check_linux_metadata(header: str, dynamic: str, symbols: str, target: str): + machine = ( + "AArch64" if target.startswith("aarch64") else "Advanced Micro Devices X86-64" + ) + if not re.search(rf"Machine:\s+{re.escape(machine)}\s*$", header, re.MULTILINE): + raise ValueError(f"ELF architecture does not match {target}") + allowed = { + "libc.so.6", + "libdl.so.2", + "libm.so.6", + "libpthread.so.0", + "librt.so.1", + "libgcc_s.so.1", + "libstdc++.so.6", + "ld-linux-x86-64.so.2", + "ld-linux-aarch64.so.1", + } + needed = set(re.findall(r"\(NEEDED\).*\[([^\]]+)\]", dynamic)) + if not needed or needed - allowed: + raise ValueError(f"Unexpected runtime libraries: {sorted(needed - allowed)}") + if re.search(r"\((?:RPATH|RUNPATH)\)", dynamic): + raise ValueError("CLI must not depend on a build-container runtime path") + # AlmaLinux 8 provides glibc 2.28, GCC 8's libstdc++, and libgcc. + limits = { + "GLIBC": (2, 28), + "GLIBCXX": (3, 4, 25), + "CXXABI": (1, 3, 11), + "GCC": (7, 0, 0), + } + for namespace, limit in limits.items(): + for value in re.findall(rf"\b{namespace}_([\d.]+)\b", symbols): + if tuple(map(int, value.split("."))) > limit: + raise ValueError( + f"Runtime symbol {namespace}_{value} exceeds Linux baseline" + ) + if "GLIBC_PRIVATE" in symbols: + raise ValueError("CLI must not require private glibc symbols") + + +def check_macos_metadata(arches: str, libraries: str, build: str): + if arches.strip() != "arm64": + raise ValueError("CLI must contain exactly the arm64 architecture") + for library in re.findall(r"^\s+(\S+) \(", libraries, re.MULTILINE): + if not library.startswith(("/usr/lib/", "/System/Library/")): + raise ValueError(f"Non-system macOS runtime library: {library}") + minimum = re.findall(r"^\s*minos ([\d.]+)\s*$", build, re.MULTILINE) + if len(minimum) != 1 or tuple(map(int, minimum[0].split("."))) != (14, 0): + raise ValueError("CLI must target macOS 14.0") + + +def check_runtime(binary: Path, target: str): + path = str(binary.resolve()) + if target.endswith("linux-gnu"): + check_linux_metadata( + command("readelf", "-h", path), + command("readelf", "-d", path), + command("readelf", "--version-info", path), + target, + ) + else: + command("codesign", "--verify", "--strict", path) + check_macos_metadata( + command("lipo", "-archs", path), + command("otool", "-L", path), + command("vtool", "-show-build", path), + ) + + +def verify_archive(archive: Path, version: str, target: str): + stem = archive_stem(version, target) + expected = {f"{stem}/{name}" for name in ("lancedb", *NOTICES)} + with tarfile.open(archive, "r:gz") as tar: + members = tar.getmembers() + if len(members) != len(expected) or {m.name for m in members} != expected: + raise ValueError(f"Unexpected archive contents: {archive.name}") + for member in members: + mode = 0o755 if member.name.endswith("/lancedb") else 0o644 + if not member.isfile() or member.mode != mode or member.size == 0: + raise ValueError(f"Invalid archive member: {member.name}") + + +def smoke_test(archive: Path, version: str, target: str): + verify_archive(archive, version, target) + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(directory, filter="data") + binary = directory / archive_stem(version, target) / "lancedb" + home = directory / "home" + home.mkdir() + env = {k: v for k, v in os.environ.items() if not k.startswith("LANCEDB_")} + env.update(HOME=str(home), XDG_CONFIG_HOME=str(home / ".config")) + for args in (("--help",), ("--version",), ("profile", "list")): + output = subprocess.run( + [str(binary), *args], + cwd=home, + env=env, + check=True, + capture_output=True, + text=True, + timeout=30, + ) + if args == ("--version",) and output.stdout.strip() != f"lancedb {version}": + raise ValueError( + f"Packaged CLI reports the wrong version: {output.stdout!r}" + ) + if args == ("--help",) and "Usage:" not in output.stdout: + raise ValueError("Packaged CLI did not print help") + if ( + args == ("profile", "list") + and "Configured profiles:" not in output.stdout + ): + raise ValueError("Packaged CLI did not list profiles") + + +def package(root: Path, binary: Path, target: str, output: Path, version: str) -> Path: + check_runtime(binary, target) + output.mkdir(parents=True, exist_ok=True) + stem = archive_stem(version, target) + archive = output / f"{stem}.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + for name, source in [ + ("lancedb", binary), + *((name, root / name) for name in NOTICES), + ]: + member = tar.gettarinfo(str(source), arcname=f"{stem}/{name}") + member.mode = 0o755 if name == "lancedb" else 0o644 + member.uid = member.gid = 0 + member.uname = member.gname = "" + with source.open("rb") as file: + tar.addfile(member, file) + smoke_test(archive, version, target) + return archive + + +def checksum_lines(output: Path, version: str) -> str: + expected = {f"{archive_stem(version, target)}.tar.gz": target for target in TARGETS} + found = {path.name for path in output.glob("*.tar.gz")} + if found != expected.keys(): + raise ValueError( + f"Expected exactly these CLI archives: {sorted(expected)}; found {sorted(found)}" + ) + lines = [] + for name, target in sorted(expected.items()): + archive = output / name + verify_archive(archive, version, target) + with archive.open("rb") as file: + digest = hashlib.file_digest(file, "sha256").hexdigest() + lines.append(f"{digest} {name}\n") + return "".join(lines) + + +def verify_checksums(output: Path, version: str): + if (output / "SHA256SUMS").read_text() != checksum_lines(output, version): + raise ValueError("SHA256SUMS does not match the CLI release archives") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("version") + build = commands.add_parser("package") + build.add_argument("--binary", type=Path, required=True) + build.add_argument("--target", choices=TARGETS, required=True) + build.add_argument("--output", type=Path, required=True) + for name in ("checksums", "verify"): + commands.add_parser(name).add_argument("--output", type=Path, required=True) + args = parser.parse_args() + version = release_version(ROOT, os.environ.get("GITHUB_REF", "")) + if args.command == "version": + print(version) + elif args.command == "package": + print(package(ROOT, args.binary, args.target, args.output, version)) + elif args.command == "checksums": + (args.output / "SHA256SUMS").write_text(checksum_lines(args.output, version)) + else: + verify_checksums(args.output, version) + + +if __name__ == "__main__": + main() diff --git a/ci/tests/test_build_cli_linux.py b/ci/tests/test_build_cli_linux.py new file mode 100644 index 000000000..2ac318ad9 --- /dev/null +++ b/ci/tests/test_build_cli_linux.py @@ -0,0 +1,65 @@ +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "build_cli_linux.sh" + + +class BuildCliLinuxTest(unittest.TestCase): + def test_toolchain_failures_stop_before_downloads_and_cargo(self): + # Start with controlled tools so the host's compilers cannot mask a + # missing container dependency. No command can download or build code. + for failed_tool, exit_code in ( + ("dnf", 42), + ("clang", 127), + ("clang++", 127), + ("ld.lld", 127), + ): + with ( + self.subTest(failed_tool=failed_tool), + tempfile.TemporaryDirectory() as tmp, + ): + workdir = Path(tmp) + bin_dir = workdir / "bin" + bin_dir.mkdir() + log = workdir / "commands.log" + for name in ( + "dnf", + "clang", + "clang++", + "ld.lld", + "git", + "curl", + "cargo", + ): + status = exit_code if name == failed_tool else 0 + if name in ("git", "curl", "cargo"): + status = 99 + command = bin_dir / name + command.write_text( + f'#!/bin/sh\necho "{name}" >> "$CLI_TEST_LOG"\nexit {status}\n' + ) + command.chmod(0o755) + env = dict(os.environ, PATH=str(bin_dir), CLI_TEST_LOG=str(log)) + result = subprocess.run( + ["/bin/bash", str(SCRIPT)], + check=False, + cwd=workdir, + env=env, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, exit_code, result.stderr) + commands = log.read_text().splitlines() + self.assertEqual(commands[0], "dnf") + self.assertEqual(commands[-1], failed_tool) + self.assertNotIn("git", commands) + self.assertNotIn("curl", commands) + self.assertNotIn("cargo", commands) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/tests/test_package_cli.py b/ci/tests/test_package_cli.py new file mode 100644 index 000000000..b56a3c8dc --- /dev/null +++ b/ci/tests/test_package_cli.py @@ -0,0 +1,228 @@ +import io +import os +import tarfile +import tempfile +import tomllib +import unittest +from pathlib import Path +from unittest.mock import patch + +from ci import package_cli as cli + +VERSION = "1.2.3-beta.4" + + +class PackageCliTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.output = self.root / "dist" + self.write_versions(VERSION) + for name in cli.NOTICES: + (self.root / name).write_text(f"Test {name}\n") + self.binary = self.root / "lancedb" + self.binary.write_text( + "#!/usr/bin/env python3\n" + "import os, sys\n" + "assert not any(k.startswith('LANCEDB_') for k in os.environ)\n" + "assert os.environ['HOME'] != os.environ.get('ORIGINAL_HOME')\n" + "assert not os.listdir(os.environ['HOME'])\n" + f"print({{('--version',): 'lancedb {VERSION}', " + "('--help',): 'Usage: lancedb', " + "('profile', 'list'): 'Configured profiles:'}[tuple(sys.argv[1:])])\n" + ) + # Packaging must restore executable permissions even if the input lacks them. + self.binary.chmod(0o644) + + def write_versions(self, version): + (self.root / ".bumpversion.toml").write_text( + f'[tool.bumpversion]\ncurrent_version = "{version}"\n' + ) + for manifest in ("rust/lancedb/Cargo.toml", "rust/lancedb/cli/Cargo.toml"): + path = self.root / manifest + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f'[package]\nversion = "{version}"\n') + + def make_package(self, target=cli.TARGETS[0]): + with patch.object(cli, "check_runtime"): + return cli.package(self.root, self.binary, target, self.output, VERSION) + + def test_stable_and_beta_versions_and_tags(self): + for version in ("1.2.3", VERSION): + self.write_versions(version) + for ref in ( + f"refs/tags/v{version}", + "refs/heads/main", + "refs/pull/1/merge", + ): + with self.subTest(version=version, ref=ref): + self.assertEqual(cli.release_version(self.root, ref), version) + + def test_mismatched_tag_and_manifests_fail(self): + with self.assertRaisesRegex(ValueError, "Tag"): + cli.release_version(self.root, "refs/tags/v1.2.3") + for manifest in ("rust/lancedb/Cargo.toml", "rust/lancedb/cli/Cargo.toml"): + self.write_versions(VERSION) + (self.root / manifest).write_text('[package]\nversion = "0.1.0"\n') + with ( + self.subTest(manifest=manifest), + self.assertRaisesRegex(ValueError, "version does not match"), + ): + cli.release_version(self.root, "refs/heads/main") + + def test_cli_participates_in_repository_version_bumps(self): + config = tomllib.loads((cli.ROOT / ".bumpversion.toml").read_text())["tool"][ + "bumpversion" + ] + entry = next( + f + for f in config["files"] + if f.get("filename") == "rust/lancedb/cli/Cargo.toml" + ) + manifest = (cli.ROOT / entry["filename"]).read_text() + version = cli.release_version(cli.ROOT, "") + self.assertIn(entry["search"].format(current_version=version), manifest) + lock = tomllib.loads((cli.ROOT / "Cargo.lock").read_text()) + package = next(p for p in lock["package"] if p["name"] == "lancedb-cli") + self.assertEqual(package["version"], version) + + def test_archive_layout_permissions_and_isolated_smoke_test(self): + with patch.dict( + os.environ, + {"LANCEDB_API_KEY": "unused", "ORIGINAL_HOME": os.environ["HOME"]}, + ): + archive = self.make_package() + self.assertEqual(archive.name, f"lancedb-v{VERSION}-{cli.TARGETS[0]}.tar.gz") + cli.verify_archive(archive, VERSION, cli.TARGETS[0]) + with tarfile.open(archive) as tar: + for member in tar.getmembers(): + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + + def test_wrong_binary_version_fails_smoke_test(self): + self.binary.write_text( + self.binary.read_text().replace(f"lancedb {VERSION}", "lancedb 0.1.0") + ) + with self.assertRaisesRegex(ValueError, "wrong version"): + self.make_package() + + def test_bad_archive_members_are_rejected(self): + archive = self.make_package() + stem = cli.archive_stem(VERSION, cli.TARGETS[0]) + for bad in ( + "missing", + "extra", + "duplicate", + "symlink", + "permissions", + "empty", + "traversal", + ): + with self.subTest(bad=bad): + with tarfile.open(archive, "w:gz") as tar: + for name in ("lancedb", *cli.NOTICES): + if bad == "missing" and name == "LICENSE": + continue + member = tarfile.TarInfo(f"{stem}/{name}") + member.mode = 0o755 if name == "lancedb" else 0o644 + member.size = 1 + if name == "lancedb": + if bad == "symlink": + member.type = tarfile.SYMTYPE + member.linkname = "/usr/bin/true" + member.size = 0 + elif bad == "permissions": + member.mode = 0o644 + elif bad == "empty": + member.size = 0 + elif bad == "traversal": + member.name = "../lancedb" + tar.addfile(member, io.BytesIO(b"x")) + if bad in ("extra", "duplicate"): + member = tarfile.TarInfo( + f"{stem}/{'LICENSE' if bad == 'duplicate' else 'extra'}" + ) + tar.addfile(member) + with self.assertRaises(ValueError): + cli.verify_archive(archive, VERSION, cli.TARGETS[0]) + + def test_checksums_require_exact_target_set_and_detect_tampering(self): + self.make_package() + with self.assertRaisesRegex(ValueError, "exactly"): + cli.checksum_lines(self.output, VERSION) + for target in cli.TARGETS[1:]: + self.make_package(target) + manifest = self.output / "SHA256SUMS" + lines = cli.checksum_lines(self.output, VERSION) + manifest.write_text(lines) + self.assertEqual(len(lines.splitlines()), 3) + cli.verify_checksums(self.output, VERSION) + # Still a valid archive, but a different payload from the published digest. + self.binary.write_text(self.binary.read_text() + "# changed\n") + self.make_package() + with self.assertRaisesRegex(ValueError, "SHA256SUMS"): + cli.verify_checksums(self.output, VERSION) + (self.output / "unexpected.tar.gz").write_bytes(b"extra") + with self.assertRaisesRegex(ValueError, "exactly"): + cli.checksum_lines(self.output, VERSION) + + +class RuntimeMetadataTest(unittest.TestCase): + def test_linux_baseline_and_architectures(self): + for target, machine in ( + (cli.TARGETS[0], "Advanced Micro Devices X86-64"), + (cli.TARGETS[1], "AArch64"), + ): + cli.check_linux_metadata( + f" Machine: {machine}\n", + " (NEEDED) Shared library: [libc.so.6]\n (NEEDED) Shared library: [libgcc_s.so.1]", + "GLIBC_2.28 GLIBCXX_3.4.25 CXXABI_1.3.11 GCC_7.0.0", + target, + ) + + def test_linux_rejects_wrong_arch_new_symbols_and_non_system_libraries(self): + header = " Machine: Advanced Micro Devices X86-64\n" + dynamic = " (NEEDED) Shared library: [libc.so.6]" + cases = [(" Machine: AArch64\n", dynamic, "GLIBC_2.28")] + for symbol in ( + "GLIBC_2.29", + "GLIBCXX_3.4.26", + "CXXABI_1.3.12", + "GCC_12.0.0", + "GLIBC_PRIVATE", + ): + cases.append((header, dynamic, symbol)) + for extra in ( + " (NEEDED) Shared library: [libssl.so.3]", + " (RUNPATH) Library runpath: [/opt/build/lib]", + ): + cases.append((header, dynamic + extra, "GLIBC_2.28")) + for args in cases: + with self.subTest(args=args), self.assertRaises(ValueError): + cli.check_linux_metadata(*args, cli.TARGETS[0]) + + def test_macos_baseline_and_system_libraries(self): + cli.check_macos_metadata( + "arm64\n", + "cli:\n\t/usr/lib/libSystem.B.dylib (compatibility version 1.0.0)", + " minos 14.0\n", + ) + + def test_macos_rejects_wrong_arch_version_and_homebrew_dependencies(self): + for args in ( + ("x86_64", "", " minos 14.0\n"), + ("arm64", "", " minos 15.0\n"), + ("arm64", "", ""), + ( + "arm64", + "\t/opt/homebrew/lib/libssl.dylib (compatibility version 3.0.0)", + " minos 14.0\n", + ), + ): + with self.subTest(args=args), self.assertRaises(ValueError): + cli.check_macos_metadata(*args) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/web/tools/cli.mdx b/docs/web/tools/cli.mdx index 97b058bd9..bfa40c92d 100644 --- a/docs/web/tools/cli.mdx +++ b/docs/web/tools/cli.mdx @@ -12,11 +12,62 @@ statement, or execute a SQL file. SQL requires a running deployment with an ## Install -With Rust and Git installed, build the CLI from the LanceDB repository: +Download a CLI archive and `SHA256SUMS` from the +[LanceDB releases page](https://github.com/lancedb/lancedb/releases). The CLI +shares the SDK release version, including beta versions. + +| Platform | Target | Requirements | +| --- | --- | --- | +| Linux x86_64 | `x86_64-unknown-linux-gnu` | glibc 2.28 or newer; Haswell-compatible CPU with AVX2, FMA, and F16C | +| Linux ARM64 | `aarch64-unknown-linux-gnu` | glibc 2.28 or newer | +| macOS Apple Silicon | `aarch64-apple-darwin` | macOS 14 or newer | + +Set the version and target to match your chosen release and platform: + +```sh +CLI_VERSION=X.Y.Z # Replace with the release version, for example X.Y.Z-beta.N +CLI_TARGET=aarch64-apple-darwin +CLI_STEM="lancedb-v${CLI_VERSION}-${CLI_TARGET}" +CLI_RELEASE_URL="https://github.com/lancedb/lancedb/releases/download/v${CLI_VERSION}" +curl -fLO "${CLI_RELEASE_URL}/${CLI_STEM}.tar.gz" +curl -fLO "${CLI_RELEASE_URL}/SHA256SUMS" +awk -v archive="${CLI_STEM}.tar.gz" '$2 == archive' SHA256SUMS > "${CLI_STEM}.sha256" +``` + +Verify the downloaded archive before extracting it. On Linux: + +```sh +sha256sum --check "${CLI_STEM}.sha256" +``` + +On macOS: + +```sh +shasum -a 256 --check "${CLI_STEM}.sha256" +``` + +After the checksum passes, install the executable into a directory on your `PATH`: + +```sh +tar -xzf "${CLI_STEM}.tar.gz" +mkdir -p "$HOME/.local/bin" +install -m 755 "${CLI_STEM}/lancedb" "$HOME/.local/bin/lancedb" +export PATH="$HOME/.local/bin:$PATH" +lancedb --version +``` + +Add `$HOME/.local/bin` to your shell's `PATH` configuration to use it in future +sessions. Each archive also contains the license and Rust third-party notices. + +### Build from source + +With Rust and Git installed, build the CLI from the LanceDB repository. Check +out a release tag to build that version: ```sh git clone https://github.com/lancedb/lancedb.git cd lancedb +git checkout vX.Y.Z # Replace with the desired release tag cargo install --locked --path rust/lancedb/cli ``` diff --git a/release_process.md b/release_process.md index a1130fdf7..55521774e 100644 --- a/release_process.md +++ b/release_process.md @@ -1,8 +1,9 @@ # Release process -We release four `lancedb` packages: Python, Rust, Java, and Node.js. +We release four `lancedb` SDK packages (Python, Rust, Java, and Node.js) and +standalone `lancedb` command-line binaries. -All four share a single version number, defined by `current_version` in +All SDKs and the CLI share a single version number, defined by `current_version` in `.bumpversion.toml`. One `vX.Y.Z` tag releases all of them, so a breaking change in any SDK bumps the minor version for every SDK. @@ -84,7 +85,10 @@ The workflow creates a new version commit and tag. Compare that tag with the validated candidate and check that only the expected mechanical version and lockfile updates were introduced. Code or dependency changes require renewed validation. Verify all four SDK publishing workflows and the exact registry -artifacts before reporting the stable release as complete. +artifacts, plus the three CLI archives and their `SHA256SUMS` on the GitHub +release, before reporting the stable release as complete. Download the CLI +archives, verify their checksums, and confirm their reported versions on the +corresponding platforms. ## Making releases @@ -95,7 +99,7 @@ The release process uses a handful of GitHub actions to automate the process. │Create Release Commit│ └─┬───────────────────┘ │ ┌──────────────┐ - └──►(tag) vX.Y.Z ─┬─►│GitHub Release├───►GH Release + └──►(tag) vX.Y.Z ─┬─►│GitHub Release├───►CLI packages + GH Release │ └──────────────┘ │ ┌────────────┐ ├─►│PyPI Publish├─────►Python Wheels @@ -134,6 +138,38 @@ Select `bump-minor` according to the version and breaking-change check below. > is needed and fail the `Create Release Commit` job. To fix, select the > "bump minor version" option. +### CLI release packages + +The GitHub Release workflow calls **CLI release packages** before creating the +release. Both stable and beta tags produce these archives: + +* `lancedb-v-x86_64-unknown-linux-gnu.tar.gz` +* `lancedb-v-aarch64-unknown-linux-gnu.tar.gz` +* `lancedb-v-aarch64-apple-darwin.tar.gz` + +Each archive contains a matching top-level directory with the `lancedb` +executable, `LICENSE`, and `RUST_THIRD_PARTY_LICENSES.html`. Linux builds target +glibc 2.28; x86_64 retains the repository's Haswell/AVX2 CPU requirements. macOS +builds target Apple Silicon on macOS 14 or newer. + +Packaging runs on relevant PRs without publishing. To dry-run packaging on a +candidate branch before merging or tagging, run: + +```bash +gh workflow run cli-release.yml --repo lancedb/lancedb --ref +``` + +Inspect the `cli-release` workflow artifact: it must contain all three archives +and a matching `SHA256SUMS`. The workflow checks runtime library compatibility +and runs the extracted executables; Linux also runs on a clean AlmaLinux 8 +system. A tag must match both the CLI and SDK manifests before any build starts. + +A CLI packaging failure blocks GitHub release creation. SDK registry workflows +run independently. Fix the failure and rerun the failed GitHub Release jobs; +the publishing job replaces assets with matching names on an existing release. +Verify beta releases remain marked as prereleases and do not take the latest +release designation. + ## Breaking changes We try to avoid breaking changes, but sometimes they are necessary. When there diff --git a/rust/lancedb/cli/Cargo.toml b/rust/lancedb/cli/Cargo.toml index df9291647..8d0097abb 100644 --- a/rust/lancedb/cli/Cargo.toml +++ b/rust/lancedb/cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-cli" -version = "0.1.0" +version = "0.41.0-beta.17" edition = "2021" license.workspace = true repository.workspace = true diff --git a/rust/lancedb/cli/tests/command_surface.rs b/rust/lancedb/cli/tests/command_surface.rs index 9d2f9c50e..474aadb09 100644 --- a/rust/lancedb/cli/tests/command_surface.rs +++ b/rust/lancedb/cli/tests/command_surface.rs @@ -3,6 +3,21 @@ use std::process::Command; +#[test] +fn version_matches_the_release() { + let output = Command::new(env!("CARGO_BIN_EXE_lancedb")) + .arg("--version") + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!( + String::from_utf8(output.stdout).unwrap().trim(), + format!("lancedb {}", env!("CARGO_PKG_VERSION")) + ); + assert_eq!(lancedb_cli::build_version(), env!("CARGO_PKG_VERSION")); + assert!(output.stderr.is_empty()); +} + #[test] fn an_unconfigured_default_sql_invocation_prints_usage() { use lancedb_cli::server_config::{Config, Profile};