mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-13 14:16:54 +00:00
Colin Patrick McCabe
2044179a8b
Package the standalone lancedb CLI with every stable and beta GitHub release. Publish archives for Linux x86_64, Linux ARM64, and Apple Silicon macOS, along with SHA-256 checksums, licenses, and third-party notices. Synchronize the CLI version with the SDKs and validate that release tags match both manifests. Add PR and manual packaging dry runs, runtime compatibility checks, and smoke tests of extracted executables. GitHub release creation now waits for all three archives to pass validation; SDK registry publishing remains independent. Update installation instructions and the release checklist. Linux binaries target glibc 2.28, retaining existing x86_64 CPU requirements; macOS binaries target macOS 14 or newer.
231 lines
8.9 KiB
Python
231 lines
8.9 KiB
Python
"""Package and validate standalone CLI release assets (Python 3.12+).
|
|
|
|
Run `version` before building, `package` on each native runner, then `checksums`
|
|
after gathering the archives. Run `verify` again immediately before publication.
|
|
"""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import tarfile
|
|
import tempfile
|
|
import tomllib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
TARGETS = (
|
|
"x86_64-unknown-linux-gnu",
|
|
"aarch64-unknown-linux-gnu",
|
|
"aarch64-apple-darwin",
|
|
)
|
|
NOTICES = ("LICENSE", "RUST_THIRD_PARTY_LICENSES.html")
|
|
|
|
|
|
def release_version(root: Path, ref: str) -> str:
|
|
config = tomllib.loads((root / ".bumpversion.toml").read_text())["tool"][
|
|
"bumpversion"
|
|
]
|
|
version = config["current_version"]
|
|
if not re.fullmatch(r"\d+\.\d+\.\d+(?:-beta\.\d+)?", version):
|
|
raise ValueError(f"Unsupported release version: {version}")
|
|
for manifest in ("rust/lancedb/Cargo.toml", "rust/lancedb/cli/Cargo.toml"):
|
|
package = tomllib.loads((root / manifest).read_text())["package"]
|
|
if package["version"] != version:
|
|
raise ValueError(f"{manifest} version does not match {version}")
|
|
if ref.startswith("refs/tags/") and ref != f"refs/tags/v{version}":
|
|
raise ValueError(f"Tag {ref} does not match v{version}")
|
|
return version
|
|
|
|
|
|
def archive_stem(version: str, target: str) -> str:
|
|
return f"lancedb-v{version}-{target}"
|
|
|
|
|
|
def command(*args: str) -> str:
|
|
return subprocess.check_output(args, text=True)
|
|
|
|
|
|
def check_linux_metadata(header: str, dynamic: str, symbols: str, target: str):
|
|
machine = (
|
|
"AArch64" if target.startswith("aarch64") else "Advanced Micro Devices X86-64"
|
|
)
|
|
if not re.search(rf"Machine:\s+{re.escape(machine)}\s*$", header, re.MULTILINE):
|
|
raise ValueError(f"ELF architecture does not match {target}")
|
|
allowed = {
|
|
"libc.so.6",
|
|
"libdl.so.2",
|
|
"libm.so.6",
|
|
"libpthread.so.0",
|
|
"librt.so.1",
|
|
"libgcc_s.so.1",
|
|
"libstdc++.so.6",
|
|
"ld-linux-x86-64.so.2",
|
|
"ld-linux-aarch64.so.1",
|
|
}
|
|
needed = set(re.findall(r"\(NEEDED\).*\[([^\]]+)\]", dynamic))
|
|
if not needed or needed - allowed:
|
|
raise ValueError(f"Unexpected runtime libraries: {sorted(needed - allowed)}")
|
|
if re.search(r"\((?:RPATH|RUNPATH)\)", dynamic):
|
|
raise ValueError("CLI must not depend on a build-container runtime path")
|
|
# AlmaLinux 8 provides glibc 2.28, GCC 8's libstdc++, and libgcc.
|
|
limits = {
|
|
"GLIBC": (2, 28),
|
|
"GLIBCXX": (3, 4, 25),
|
|
"CXXABI": (1, 3, 11),
|
|
"GCC": (7, 0, 0),
|
|
}
|
|
for namespace, limit in limits.items():
|
|
for value in re.findall(rf"\b{namespace}_([\d.]+)\b", symbols):
|
|
if tuple(map(int, value.split("."))) > limit:
|
|
raise ValueError(
|
|
f"Runtime symbol {namespace}_{value} exceeds Linux baseline"
|
|
)
|
|
if "GLIBC_PRIVATE" in symbols:
|
|
raise ValueError("CLI must not require private glibc symbols")
|
|
|
|
|
|
def check_macos_metadata(arches: str, libraries: str, build: str):
|
|
if arches.strip() != "arm64":
|
|
raise ValueError("CLI must contain exactly the arm64 architecture")
|
|
for library in re.findall(r"^\s+(\S+) \(", libraries, re.MULTILINE):
|
|
if not library.startswith(("/usr/lib/", "/System/Library/")):
|
|
raise ValueError(f"Non-system macOS runtime library: {library}")
|
|
minimum = re.findall(r"^\s*minos ([\d.]+)\s*$", build, re.MULTILINE)
|
|
if len(minimum) != 1 or tuple(map(int, minimum[0].split("."))) != (14, 0):
|
|
raise ValueError("CLI must target macOS 14.0")
|
|
|
|
|
|
def check_runtime(binary: Path, target: str):
|
|
path = str(binary.resolve())
|
|
if target.endswith("linux-gnu"):
|
|
check_linux_metadata(
|
|
command("readelf", "-h", path),
|
|
command("readelf", "-d", path),
|
|
command("readelf", "--version-info", path),
|
|
target,
|
|
)
|
|
else:
|
|
command("codesign", "--verify", "--strict", path)
|
|
check_macos_metadata(
|
|
command("lipo", "-archs", path),
|
|
command("otool", "-L", path),
|
|
command("vtool", "-show-build", path),
|
|
)
|
|
|
|
|
|
def verify_archive(archive: Path, version: str, target: str):
|
|
stem = archive_stem(version, target)
|
|
expected = {f"{stem}/{name}" for name in ("lancedb", *NOTICES)}
|
|
with tarfile.open(archive, "r:gz") as tar:
|
|
members = tar.getmembers()
|
|
if len(members) != len(expected) or {m.name for m in members} != expected:
|
|
raise ValueError(f"Unexpected archive contents: {archive.name}")
|
|
for member in members:
|
|
mode = 0o755 if member.name.endswith("/lancedb") else 0o644
|
|
if not member.isfile() or member.mode != mode or member.size == 0:
|
|
raise ValueError(f"Invalid archive member: {member.name}")
|
|
|
|
|
|
def smoke_test(archive: Path, version: str, target: str):
|
|
verify_archive(archive, version, target)
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
directory = Path(tmp)
|
|
with tarfile.open(archive, "r:gz") as tar:
|
|
tar.extractall(directory, filter="data")
|
|
binary = directory / archive_stem(version, target) / "lancedb"
|
|
home = directory / "home"
|
|
home.mkdir()
|
|
env = {k: v for k, v in os.environ.items() if not k.startswith("LANCEDB_")}
|
|
env.update(HOME=str(home), XDG_CONFIG_HOME=str(home / ".config"))
|
|
for args in (("--help",), ("--version",), ("profile", "list")):
|
|
output = subprocess.run(
|
|
[str(binary), *args],
|
|
cwd=home,
|
|
env=env,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=30,
|
|
)
|
|
if args == ("--version",) and output.stdout.strip() != f"lancedb {version}":
|
|
raise ValueError(
|
|
f"Packaged CLI reports the wrong version: {output.stdout!r}"
|
|
)
|
|
if args == ("--help",) and "Usage:" not in output.stdout:
|
|
raise ValueError("Packaged CLI did not print help")
|
|
if (
|
|
args == ("profile", "list")
|
|
and "Configured profiles:" not in output.stdout
|
|
):
|
|
raise ValueError("Packaged CLI did not list profiles")
|
|
|
|
|
|
def package(root: Path, binary: Path, target: str, output: Path, version: str) -> Path:
|
|
check_runtime(binary, target)
|
|
output.mkdir(parents=True, exist_ok=True)
|
|
stem = archive_stem(version, target)
|
|
archive = output / f"{stem}.tar.gz"
|
|
with tarfile.open(archive, "w:gz") as tar:
|
|
for name, source in [
|
|
("lancedb", binary),
|
|
*((name, root / name) for name in NOTICES),
|
|
]:
|
|
member = tar.gettarinfo(str(source), arcname=f"{stem}/{name}")
|
|
member.mode = 0o755 if name == "lancedb" else 0o644
|
|
member.uid = member.gid = 0
|
|
member.uname = member.gname = ""
|
|
with source.open("rb") as file:
|
|
tar.addfile(member, file)
|
|
smoke_test(archive, version, target)
|
|
return archive
|
|
|
|
|
|
def checksum_lines(output: Path, version: str) -> str:
|
|
expected = {f"{archive_stem(version, target)}.tar.gz": target for target in TARGETS}
|
|
found = {path.name for path in output.glob("*.tar.gz")}
|
|
if found != expected.keys():
|
|
raise ValueError(
|
|
f"Expected exactly these CLI archives: {sorted(expected)}; found {sorted(found)}"
|
|
)
|
|
lines = []
|
|
for name, target in sorted(expected.items()):
|
|
archive = output / name
|
|
verify_archive(archive, version, target)
|
|
with archive.open("rb") as file:
|
|
digest = hashlib.file_digest(file, "sha256").hexdigest()
|
|
lines.append(f"{digest} {name}\n")
|
|
return "".join(lines)
|
|
|
|
|
|
def verify_checksums(output: Path, version: str):
|
|
if (output / "SHA256SUMS").read_text() != checksum_lines(output, version):
|
|
raise ValueError("SHA256SUMS does not match the CLI release archives")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
commands = parser.add_subparsers(dest="command", required=True)
|
|
commands.add_parser("version")
|
|
build = commands.add_parser("package")
|
|
build.add_argument("--binary", type=Path, required=True)
|
|
build.add_argument("--target", choices=TARGETS, required=True)
|
|
build.add_argument("--output", type=Path, required=True)
|
|
for name in ("checksums", "verify"):
|
|
commands.add_parser(name).add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
version = release_version(ROOT, os.environ.get("GITHUB_REF", ""))
|
|
if args.command == "version":
|
|
print(version)
|
|
elif args.command == "package":
|
|
print(package(ROOT, args.binary, args.target, args.output, version))
|
|
elif args.command == "checksums":
|
|
(args.output / "SHA256SUMS").write_text(checksum_lines(args.output, version))
|
|
else:
|
|
verify_checksums(args.output, version)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|