diff --git a/.github/workflows/make-release-commit.yml b/.github/workflows/make-release-commit.yml index 21b359df..c267c396 100644 --- a/.github/workflows/make-release-commit.yml +++ b/.github/workflows/make-release-commit.yml @@ -84,7 +84,7 @@ jobs: run: | pip install bump-my-version PyGithub packaging bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP - bash ci/update_lockfiles.sh + bash ci/update_lockfiles.sh --amend - name: Push new version tag if: ${{ !inputs.dry_run }} uses: ad-m/github-push-action@master diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8ef96f2e..175a92e4 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -505,6 +505,8 @@ jobs: name: vectordb NPM Publish needs: [node, node-macos, node-linux-gnu, node-windows] runs-on: ubuntu-latest + permissions: + contents: write # Only runs on tags that matches the make-release action if: startsWith(github.ref, 'refs/tags/v') steps: @@ -537,6 +539,10 @@ jobs: # We need to deprecate the old package to avoid confusion. # Each time we publish a new version, it gets undeprecated. run: npm deprecate vectordb "Use @lancedb/lancedb instead." + - name: Update package-lock.json + run: bash ci/update_lockfiles.sh + - name: Push new commit + uses: ad-m/github-push-action@master - name: Notify Slack Action uses: ravsamhq/notify-slack-action@2.3.0 if: ${{ always() }} diff --git a/Cargo.lock b/Cargo.lock index 54fecdac..bafd2929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2836,7 +2836,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b456eec9a7cd44268f958006308a05d703c578ab5c86fa119f3bb5bc97d01899" dependencies = [ "rand 0.8.5", ] @@ -3929,7 +3930,8 @@ dependencies = [ [[package]] name = "lance" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6acee7cab66a2fbb68238ba9dd83a8b4fea4bedad02e2ef2ebb8f35f0cdd09" dependencies = [ "arrow", "arrow-arith", @@ -3993,7 +3995,8 @@ dependencies = [ [[package]] name = "lance-arrow" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "866ae620f88cbbbebcd24cee23c41d0ac847652bbf38d467b8fb95302007723c" dependencies = [ "arrow-array", "arrow-buffer", @@ -4011,7 +4014,8 @@ dependencies = [ [[package]] name = "lance-core" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a41540cd34c16371d2e72f270cfd755f7677cdeb597925f151f9512a70673e6" dependencies = [ "arrow-array", "arrow-buffer", @@ -4048,7 +4052,8 @@ dependencies = [ [[package]] name = "lance-datafusion" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cac3480432121d87ea77e50a08bd3a2d81bbe202844fbb0fd2b260216990e1" dependencies = [ "arrow", "arrow-array", @@ -4078,7 +4083,8 @@ dependencies = [ [[package]] name = "lance-datagen" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d498251ffd9da2eabb6b75e8bcd16f74c9b42a647c9388e6b00882ca4acf343" dependencies = [ "arrow", "arrow-array", @@ -4094,7 +4100,8 @@ dependencies = [ [[package]] name = "lance-encoding" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904f17e867748ead2d2a7def04b88039ee4881159812f76e542899d28effe21f" dependencies = [ "arrayref", "arrow", @@ -4134,7 +4141,8 @@ dependencies = [ [[package]] name = "lance-file" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff100a7a4b8e13c59f5bf462df6d28e486f7c372269f70df2c0c2cc26371b365" dependencies = [ "arrow-arith", "arrow-array", @@ -4169,7 +4177,8 @@ dependencies = [ [[package]] name = "lance-index" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f715e3c1ed90a4d3d5d16eac94173dfdbffe328902dada01826bf1a289e1765c" dependencies = [ "arrow", "arrow-array", @@ -4225,7 +4234,8 @@ dependencies = [ [[package]] name = "lance-io" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36646da2c6db604971cda4e1a1932d67c2b7c3b1572a218f454f4df2502348d" dependencies = [ "arrow", "arrow-arith", @@ -4265,7 +4275,8 @@ dependencies = [ [[package]] name = "lance-linalg" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b020e1433361ddf73d82415065e4d1cff4d351ecc0dd57613e3fa91f909e395b" dependencies = [ "arrow-array", "arrow-ord", @@ -4289,7 +4300,8 @@ dependencies = [ [[package]] name = "lance-table" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7fb06ea96820c15e606bbf9bd9c85b97aecad3a8de85d1b36c64a38b990f31" dependencies = [ "arrow", "arrow-array", @@ -4329,7 +4341,8 @@ dependencies = [ [[package]] name = "lance-testing" version = "0.29.0" -source = "git+https://github.com/lancedb/lance.git?tag=v0.29.0-beta.2#0cfaf95bb914d589fde2f01c6f5ef0ef0beddbca" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b869111ee91597dbc3f140f3d1c6213c984d190b9103a151c72d166868bcfa" dependencies = [ "arrow-array", "arrow-schema", @@ -4340,7 +4353,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.20.0-beta.1" +version = "0.20.0-beta.2" dependencies = [ "arrow", "arrow-array", @@ -4427,7 +4440,7 @@ dependencies = [ [[package]] name = "lancedb-node" -version = "0.20.0-beta.1" +version = "0.20.0-beta.2" dependencies = [ "arrow-array", "arrow-ipc", @@ -4452,7 +4465,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.20.0-beta.1" +version = "0.20.0-beta.2" dependencies = [ "arrow-array", "arrow-ipc", @@ -4472,7 +4485,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.23.0-beta.1" +version = "0.23.0-beta.2" dependencies = [ "arrow", "env_logger", diff --git a/Cargo.toml b/Cargo.toml index e51407db..c5634706 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,14 +21,14 @@ categories = ["database-implementations"] rust-version = "1.78.0" [workspace.dependencies] -lance = { "version" = "=0.29.0", "features" = ["dynamodb"], tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-io = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-index = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-linalg = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-table = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-testing = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-datafusion = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } -lance-encoding = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } +lance = { "version" = "=0.29.0", "features" = ["dynamodb"] } +lance-io = "=0.29.0" +lance-index = "=0.29.0" +lance-linalg = "=0.29.0" +lance-table = "=0.29.0" +lance-testing = "=0.29.0" +lance-datafusion = "=0.29.0" +lance-encoding = "=0.29.0" # Note that this one does not include pyarrow arrow = { version = "55.1", optional = false } arrow-array = "55.1" diff --git a/ci/set_lance_version.py b/ci/set_lance_version.py new file mode 100644 index 00000000..f3a02093 --- /dev/null +++ b/ci/set_lance_version.py @@ -0,0 +1,174 @@ +import argparse +import sys +import json + + +def run_command(command: str) -> str: + """ + Run a shell command and return stdout as a string. + If exit code is not 0, raise an exception with the stderr output. + """ + import subprocess + + result = subprocess.run(command, shell=True, capture_output=True, text=True) + if result.returncode != 0: + raise Exception(f"Command failed with error: {result.stderr.strip()}") + return result.stdout.strip() + + +def get_latest_stable_version() -> str: + version_line = run_command("cargo info lance | grep '^version:'") + version = version_line.split(" ")[1].strip() + return version + + +def get_latest_preview_version() -> str: + lance_tags = run_command( + "git ls-remote --tags https://github.com/lancedb/lance.git | grep 'refs/tags/v[0-9beta.-]\\+$'" + ).splitlines() + lance_tags = ( + tag.split("refs/tags/")[1] + for tag in lance_tags + if "refs/tags/" in tag and "beta" in tag + ) + from packaging.version import Version + + latest = max( + (tag[1:] for tag in lance_tags if tag.startswith("v")), key=lambda t: Version(t) + ) + return str(latest) + + +def extract_features(line: str) -> list: + """ + Extracts the features from a line in Cargo.toml. + Example: 'lance = { "version" = "=0.29.0", "features" = ["dynamodb"] }' + Returns: ['dynamodb'] + """ + import re + + match = re.search(r'"features"\s*=\s*\[(.*?)\]', line) + if match: + features_str = match.group(1) + return [f.strip('"') for f in features_str.split(",")] + return [] + + +def update_cargo_toml(line_updater): + """ + Updates the Cargo.toml file by applying the line_updater function to each line. + The line_updater function should take a line as input and return the updated line. + """ + with open("Cargo.toml", "r") as f: + lines = f.readlines() + + new_lines = [] + for line in lines: + if line.startswith("lance"): + # Update the line using the provided function + new_lines.append(line_updater(line)) + else: + # Keep the line unchanged + new_lines.append(line) + + with open("Cargo.toml", "w") as f: + f.writelines(new_lines) + + +def set_stable_version(version: str): + """ + Sets lines to + lance = { "version" = "=0.29.0", "features" = ["dynamodb"] } + lance-io = "=0.29.0" + ... + """ + + def line_updater(line: str) -> str: + package_name = line.split("=", maxsplit=1)[0].strip() + features = extract_features(line) + if features: + return f'{package_name} = {{ "version" = "={version}", "features" = {json.dumps(features)} }}\n' + else: + return f'{package_name} = "={version}"\n' + + update_cargo_toml(line_updater) + + +def set_preview_version(version: str): + """ + Sets lines to + lance = { "version" = "=0.29.0", "features" = ["dynamodb"], tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } + lance-io = { version = "=0.29.0", tag = "v0.29.0-beta.2", git="https://github.com/lancedb/lance.git" } + ... + """ + + def line_updater(line: str) -> str: + package_name = line.split("=", maxsplit=1)[0].strip() + features = extract_features(line) + base_version = version.split("-")[0] # Get the base version without beta suffix + if features: + return f'{package_name} = {{ "version" = "={base_version}", "features" = {json.dumps(features)}, "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n' + else: + return f'{package_name} = {{ "version" = "={base_version}", "tag" = "v{version}", "git" = "https://github.com/lancedb/lance.git" }}\n' + + update_cargo_toml(line_updater) + + +def set_local_version(): + """ + Sets lines to + lance = { path = "../lance/rust/lance", features = ["dynamodb"] } + lance-io = { path = "../lance/rust/lance-io" } + ... + """ + + def line_updater(line: str) -> str: + package_name = line.split("=", maxsplit=1)[0].strip() + features = extract_features(line) + if features: + return f'{package_name} = {{ "path" = "../lance/rust/{package_name}", "features" = {json.dumps(features)} }}\n' + else: + return f'{package_name} = {{ "path" = "../lance/rust/{package_name}" }}\n' + + update_cargo_toml(line_updater) + + +parser = argparse.ArgumentParser(description="Set the version of the Lance package.") +parser.add_argument( + "version", + type=str, + help="The version to set for the Lance package. Use 'stable' for the latest stable version, 'preview' for latest preview version, or a specific version number (e.g., '0.1.0'). You can also specify 'local' to use a local path.", +) +args = parser.parse_args() + +if args.version == "stable": + latest_stable_version = get_latest_stable_version() + print( + f"Found latest stable version: \033[1mv{latest_stable_version}\033[0m", + file=sys.stderr, + ) + set_stable_version(latest_stable_version) +elif args.version == "preview": + latest_preview_version = get_latest_preview_version() + print( + f"Found latest preview version: \033[1mv{latest_preview_version}\033[0m", + file=sys.stderr, + ) + set_preview_version(latest_preview_version) +elif args.version == "local": + set_local_version() +else: + # Parse the version number. + version = args.version + # Ignore initial v if present. + if version.startswith("v"): + version = version[1:] + + if "beta" in version: + set_preview_version(version) + else: + set_stable_version(version) + +print("Updating lockfiles...", file=sys.stderr, end="") +run_command("cargo metadata > /dev/null") +print(" done.", file=sys.stderr) diff --git a/ci/update_lockfiles.sh b/ci/update_lockfiles.sh index b91e0b95..7a375c83 100755 --- a/ci/update_lockfiles.sh +++ b/ci/update_lockfiles.sh @@ -1,19 +1,30 @@ #!/usr/bin/env bash set -euo pipefail +AMEND=false + +for arg in "$@"; do + if [[ "$arg" == "--amend" ]]; then + AMEND=true + fi +done + # This updates the lockfile without building -cargo metadata > /dev/null +cargo metadata --quiet > /dev/null pushd nodejs || exit 1 -npm install --package-lock-only +npm install --package-lock-only --silent popd pushd node || exit 1 -npm install --package-lock-only +npm install --package-lock-only --silent popd if git diff --quiet --exit-code; then echo "No lockfile changes to commit; skipping amend." -else +elif $AMEND; then git add Cargo.lock nodejs/package-lock.json node/package-lock.json git commit --amend --no-edit +else + git add Cargo.lock nodejs/package-lock.json node/package-lock.json + git commit -m "Update lockfiles" fi diff --git a/node/package-lock.json b/node/package-lock.json index f516df02..1e7e340b 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -326,6 +326,66 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@lancedb/vectordb-darwin-arm64": { + "version": "0.20.0-beta.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.20.0-beta.2.tgz", + "integrity": "sha512-H9PmJ/5KSvstVzR8Q7T22+eHRjJZ2ef3aA3gdFxXvoMi3xQ0MGIxz23HuKHGTRT4tfl1nNnpOPb2W7Na8etK9w==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-darwin-x64": { + "version": "0.20.0-beta.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.20.0-beta.2.tgz", + "integrity": "sha512-9AQkv4tIys+vg0cplZtSE48o61jd7EnmuMkUht+vLORL5/HAma84eAoU9lXHT7zAtPAQmL+98Bfvcsx7fJ6mVw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-linux-arm64-gnu": { + "version": "0.20.0-beta.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.20.0-beta.2.tgz", + "integrity": "sha512-eQWoJz2ePml7NyEInTBeakWx56+5c6r2p3F+iHC5tsLuznn6eFX90koXJunRxH1WXHDN48ECUlEmKypgfEmn4w==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-linux-x64-gnu": { + "version": "0.20.0-beta.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.20.0-beta.2.tgz", + "integrity": "sha512-/+84U+Dt07m8Jk0b8h+SvOzlrynITPP3SDBOlB+OonwmGSxirXhc8gkfNZctgXOJYKMyRIRSsMHP/QNjOp2ajA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-win32-x64-msvc": { + "version": "0.20.0-beta.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.20.0-beta.2.tgz", + "integrity": "sha512-bgdunAPnknBh/5oO+vr6RXMr6wb3hHugNPXcIidxYMQvgFa8uhaAKtgYkAKuoyUReOYo8DGtVkZxNUUpZbF7/A==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@neon-rs/cli": { "version": "0.0.160", "resolved": "https://registry.npmjs.org/@neon-rs/cli/-/cli-0.0.160.tgz",