From 5349e8b1db57dda7ec7c61d6ee55b81625bf3acc Mon Sep 17 00:00:00 2001 From: Will Jones Date: Fri, 17 May 2024 11:24:38 -0700 Subject: [PATCH] ci: make preview releases (#1302) This PR changes the release process. Some parts are more complex, and other parts I've simplified. ## Simplifications * Combined `Create Release Commit` and `Create Python Release Commit` into a single workflow. By default, it does a release of all packages, but you can still choose to make just a Python or just Node/Rust release through the arguments. This will make it rarer that we create a Node release but forget about Python or vice-versa. * Releases are automatically generated once a tag is pushed. This eliminates the manual step of creating the release. * Release notes are automatically generated and changes are categorized based on the PR labels. * Removed the use of `LANCEDB_RELEASE_TOKEN` in favor of just using `GITHUB_TOKEN` where it wasn't necessary. In the one place it is necessary, I left a comment as to why it is. * Reused the version in `python/Cargo.toml` so we don't have two different versions in Python LanceDB. ## New changes * We now can create `preview` / `beta` releases. By default `Create Release Commit` will create a preview release, but you can select a "stable" release type and it will create a full stable release. * For Python, pre-releases go to fury.io instead of PyPI * `bump2version` was deprecated, so upgraded to `bump-my-version`. This also seems to better support semantic versioning with pre-releases. * `ci` changes will now be shown in the changelog, allowing changes like this to be visible to users. `chore` is still hidden. ## Versioning **NOTE**: unlike how it is in lance repo right now, the version in main is the last one released, including beta versions. --------- Co-authored-by: Lance Release Co-authored-by: Weston Pace --- .bumpversion.cfg | 22 ---- .bumpversion.toml | 57 ++++++++++ .github/release.yml | 25 ---- .github/release_notes.json | 41 +++++++ .github/workflows/cargo-publish.yml | 8 +- .github/workflows/make-release-commit.yml | 86 ++++++++++---- .github/workflows/npm-publish.yml | 99 +++++++++++++++- .github/workflows/pypi-publish.yml | 107 +++++++++++++----- .../workflows/python-make-release-commit.yml | 56 --------- .github/workflows/upload_wheel/action.yml | 54 +++++---- ci/bump_version.sh | 49 ++++++++ ci/check_breaking_changes.py | 35 ++++++ ci/semver_sort.py | 35 ++++++ docs/src/basic.md | 30 +++++ nodejs/npm/win32-x64-msvc/package.json | 2 +- python/.bumpversion.cfg | 8 -- python/.bumpversion.toml | 34 ++++++ python/Cargo.toml | 2 +- python/pyproject.toml | 2 +- release_process.md | 55 ++++++++- 20 files changed, 609 insertions(+), 198 deletions(-) delete mode 100644 .bumpversion.cfg create mode 100644 .bumpversion.toml delete mode 100644 .github/release.yml create mode 100644 .github/release_notes.json delete mode 100644 .github/workflows/python-make-release-commit.yml create mode 100644 ci/bump_version.sh create mode 100644 ci/check_breaking_changes.py create mode 100644 ci/semver_sort.py delete mode 100644 python/.bumpversion.cfg create mode 100644 python/.bumpversion.toml diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index 6f7e779e..00000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,22 +0,0 @@ -[bumpversion] -current_version = 0.4.20 -commit = True -message = Bump version: {current_version} β†’ {new_version} -tag = True -tag_name = v{new_version} - -[bumpversion:file:node/package.json] - -[bumpversion:file:nodejs/package.json] - -[bumpversion:file:nodejs/npm/darwin-x64/package.json] - -[bumpversion:file:nodejs/npm/darwin-arm64/package.json] - -[bumpversion:file:nodejs/npm/linux-x64-gnu/package.json] - -[bumpversion:file:nodejs/npm/linux-arm64-gnu/package.json] - -[bumpversion:file:rust/ffi/node/Cargo.toml] - -[bumpversion:file:rust/lancedb/Cargo.toml] diff --git a/.bumpversion.toml b/.bumpversion.toml new file mode 100644 index 00000000..c2ebbb26 --- /dev/null +++ b/.bumpversion.toml @@ -0,0 +1,57 @@ +[tool.bumpversion] +current_version = "0.4.21-beta.0" +parse = """(?x) + (?P0|[1-9]\\d*)\\. + (?P0|[1-9]\\d*)\\. + (?P0|[1-9]\\d*) + (?:-(?P[a-zA-Z-]+)\\.(?P0|[1-9]\\d*))? +""" +serialize = [ + "{major}.{minor}.{patch}-{pre_l}.{pre_n}", + "{major}.{minor}.{patch}", +] +search = "{current_version}" +replace = "{new_version}" +regex = false +ignore_missing_version = false +ignore_missing_files = false +tag = true +sign_tags = false +tag_name = "v{new_version}" +tag_message = "Bump version: {current_version} β†’ {new_version}" +allow_dirty = true +commit = true +message = "Bump version: {current_version} β†’ {new_version}" +commit_args = "" + +[tool.bumpversion.parts.pre_l] +values = ["beta", "final"] +optional_value = "final" + +[[tool.bumpversion.files]] +filename = "node/package.json" +search = "\"version\": \"{current_version}\"," +replace = "\"version\": \"{new_version}\"," + +[[tool.bumpversion.files]] +filename = "nodejs/package.json" +search = "\"version\": \"{current_version}\"," +replace = "\"version\": \"{new_version}\"," + +# nodejs binary packages +[[tool.bumpversion.files]] +glob = "nodejs/npm/*/package.json" +search = "\"version\": \"{current_version}\"," +replace = "\"version\": \"{new_version}\"," + +# Cargo files +# ------------ +[[tool.bumpversion.files]] +filename = "rust/ffi/node/Cargo.toml" +search = "\nversion = \"{current_version}\"" +replace = "\nversion = \"{new_version}\"" + +[[tool.bumpversion.files]] +filename = "rust/lancedb/Cargo.toml" +search = "\nversion = \"{current_version}\"" +replace = "\nversion = \"{new_version}\"" \ No newline at end of file diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 99bf5c9b..00000000 --- a/.github/release.yml +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: create separate templates for Python and other releases. -changelog: - exclude: - labels: - - ci - - chore - categories: - - title: Breaking Changes πŸ›  - labels: - - breaking-change - - title: New Features πŸŽ‰ - labels: - - enhancement - - title: Bug Fixes πŸ› - labels: - - bug - - title: Documentation πŸ“š - labels: - - documentation - - title: Performance Improvements πŸš€ - labels: - - performance - - title: Other Changes - labels: - - "*" diff --git a/.github/release_notes.json b/.github/release_notes.json new file mode 100644 index 00000000..a9786069 --- /dev/null +++ b/.github/release_notes.json @@ -0,0 +1,41 @@ +{ + "ignore_labels": ["chore"], + "pr_template": "- ${{TITLE}} by @${{AUTHOR}} in ${{URL}}", + "categories": [ + { + "title": "## πŸ† Highlights", + "labels": ["highlight"] + }, + { + "title": "## πŸ›  Breaking Changes", + "labels": ["breaking-change"] + }, + { + "title": "## ⚠️ Deprecations ", + "labels": ["deprecation"] + }, + { + "title": "## πŸŽ‰ New Features", + "labels": ["enhancement"] + }, + { + "title": "## πŸ› Bug Fixes", + "labels": ["bug"] + }, + { + "title": "## πŸ“š Documentation", + "labels": ["documentation"] + }, + { + "title": "## πŸš€ Performance Improvements", + "labels": ["performance"] + }, + { + "title": "## Other Changes" + }, + { + "title": "## πŸ”§ Build and CI", + "labels": ["ci"] + } + ] +} \ No newline at end of file diff --git a/.github/workflows/cargo-publish.yml b/.github/workflows/cargo-publish.yml index 50cdd125..8bb085b0 100644 --- a/.github/workflows/cargo-publish.yml +++ b/.github/workflows/cargo-publish.yml @@ -1,8 +1,12 @@ name: Cargo Publish on: - release: - types: [ published ] + push: + tags-ignore: + # We don't publish pre-releases for Rust. Crates.io is just a source + # distribution, so we don't need to publish pre-releases. + - 'v*-beta*' + - '*-v*' # for example, python-vX.Y.Z env: # This env var is used by Swatinem/rust-cache@v2 for the cache diff --git a/.github/workflows/make-release-commit.yml b/.github/workflows/make-release-commit.yml index b2fa261e..6747f11e 100644 --- a/.github/workflows/make-release-commit.yml +++ b/.github/workflows/make-release-commit.yml @@ -1,37 +1,62 @@ name: Create release commit +# This workflow increments versions, tags the version, and pushes it. +# When a tag is pushed, another workflow is triggered that creates a GH release +# and uploads the binaries. This workflow is only for creating the tag. + +# This script will enforce that a minor version is incremented if there are any +# breaking changes since the last minor increment. However, it isn't able to +# differentiate between breaking changes in Node versus Python. If you wish to +# bypass this check, you can manually increment the version and push the tag. on: workflow_dispatch: inputs: dry_run: description: 'Dry run (create the local commit/tags but do not push it)' required: true - default: "false" - type: choice - options: - - "true" - - "false" - part: + default: false + type: boolean + type: description: 'What kind of release is this?' required: true - default: 'patch' + default: 'preview' type: choice options: - - patch - - minor - - major + - preview + - stable + python: + description: 'Make a Python release' + required: true + default: true + type: boolean + other: + description: 'Make a Node/Rust release' + required: true + default: true + type: boolean + bump-minor: + description: 'Bump minor version' + required: true + default: false + type: boolean jobs: - bump-version: + make-release: + # Creates tag and GH release. The GH release will trigger the build and release jobs. runs-on: ubuntu-latest + permissions: + contents: write steps: - - name: Check out main - uses: actions/checkout@v4 + - name: Output Inputs + run: echo "${{ toJSON(github.event.inputs) }}" + - uses: actions/checkout@v4 with: - ref: main - persist-credentials: false fetch-depth: 0 lfs: true + # It's important we use our token here, as the default token will NOT + # trigger any workflows watching for new tags. See: + # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow + token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - name: Set git configs for bumpversion shell: bash run: | @@ -41,19 +66,34 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Bump version, create tag and commit + - name: Bump Python version + if: ${{ inputs.python }} + working-directory: python + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - pip install bump2version - bumpversion --verbose ${{ inputs.part }} - - name: Push new version and tag - if: ${{ inputs.dry_run }} == "false" + # Need to get the commit before bumping the version, so we can + # determine if there are breaking changes in the next step as well. + echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV + + pip install bump-my-version PyGithub packaging + bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v + - name: Bump Node/Rust version + if: ${{ inputs.other }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + pip install bump-my-version PyGithub packaging + bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP + - name: Push new version tag + if: ${{ !inputs.dry_run }} uses: ad-m/github-push-action@master with: + # Need to use PAT here too to trigger next workflow. See comment above. github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - branch: main + branch: ${{ github.ref }} tags: true - uses: ./.github/workflows/update_package_lock if: ${{ inputs.dry_run }} == "false" with: - github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index f6a64e9e..c629795e 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -1,8 +1,9 @@ name: NPM Publish on: - release: - types: [published] + push: + tags: + - 'v*' jobs: node: @@ -274,9 +275,15 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} run: | + # Tag beta as "preview" instead of default "latest". See lancedb + # npm publish step for more info. + if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then + PUBLISH_ARGS="--tag preview" + fi + mv */*.tgz . for filename in *.tgz; do - npm publish $filename + npm publish $PUBLISH_ARGS $filename done release-nodejs: @@ -316,11 +323,23 @@ jobs: - name: Publish to NPM env: NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }} - run: npm publish --access public + # By default, things are published to the latest tag. This is what is + # installed by default if the user does not specify a version. This is + # good for stable releases, but for pre-releases, we want to publish to + # the "preview" tag so they can install with `npm install lancedb@preview`. + # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420 + run: | + if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then + npm publish --access public --tag preview + else + npm publish --access public + fi update-package-lock: needs: [release] runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout uses: actions/checkout@v4 @@ -331,11 +350,13 @@ jobs: lfs: true - uses: ./.github/workflows/update_package_lock with: - github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} update-package-lock-nodejs: needs: [release-nodejs] runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Checkout uses: actions/checkout@v4 @@ -346,4 +367,70 @@ jobs: lfs: true - uses: ./.github/workflows/update_package_lock_nodejs with: - github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + gh-release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + - name: Extract version + id: extract_version + env: + GITHUB_REF: ${{ github.ref }} + run: | + set -e + echo "Extracting tag and version from $GITHUB_REF" + if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then + VERSION=${BASH_REMATCH[1]} + TAG=v$VERSION + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + else + echo "Failed to extract version from $GITHUB_REF" + exit 1 + fi + echo "Extracted version $VERSION from $GITHUB_REF" + if [[ $VERSION =~ beta ]]; then + echo "This is a beta release" + + # Get last release (that is not this one) + FROM_TAG=$(git tag --sort='version:refname' \ + | grep ^v \ + | grep -vF "$TAG" \ + | python ci/semver_sort.py v \ + | tail -n 1) + else + echo "This is a stable release" + # Get last stable tag (ignore betas) + FROM_TAG=$(git tag --sort='version:refname' \ + | grep ^v \ + | grep -vF "$TAG" \ + | grep -v beta \ + | python ci/semver_sort.py v \ + | tail -n 1) + fi + echo "Found from tag $FROM_TAG" + echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT + - name: Create Release Notes + id: release_notes + uses: mikepenz/release-changelog-builder-action@v4 + with: + configuration: .github/release_notes.json + toTag: ${{ steps.extract_version.outputs.tag }} + fromTag: ${{ steps.extract_version.outputs.from_tag }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create GH release + uses: softprops/action-gh-release@v2 + with: + prerelease: ${{ contains('beta', github.ref) }} + tag_name: ${{ steps.extract_version.outputs.tag }} + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: false + name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }} + body: ${{ steps.release_notes.outputs.changelog }} diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index c913003d..98e39675 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -1,18 +1,16 @@ name: PyPI Publish on: - release: - types: [published] + push: + tags: + - 'python-v*' jobs: linux: - # Only runs on tags that matches the python-make-release action - if: startsWith(github.ref, 'refs/tags/python-v') name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} timeout-minutes: 60 strategy: matrix: - python-minor-version: ["8"] config: - platform: x86_64 manylinux: "2_17" @@ -34,25 +32,22 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.${{ matrix.python-minor-version }} + python-version: 3.8 - uses: ./.github/workflows/build_linux_wheel with: - python-minor-version: ${{ matrix.python-minor-version }} + python-minor-version: 8 args: "--release --strip ${{ matrix.config.extra_args }}" arm-build: ${{ matrix.config.platform == 'aarch64' }} manylinux: ${{ matrix.config.manylinux }} - uses: ./.github/workflows/upload_wheel with: - token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} - repo: "pypi" + pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} + fury_token: ${{ secrets.FURY_TOKEN }} mac: - # Only runs on tags that matches the python-make-release action - if: startsWith(github.ref, 'refs/tags/python-v') timeout-minutes: 60 runs-on: ${{ matrix.config.runner }} strategy: matrix: - python-minor-version: ["8"] config: - target: x86_64-apple-darwin runner: macos-13 @@ -63,7 +58,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.ref }} fetch-depth: 0 lfs: true - name: Set up Python @@ -72,38 +66,95 @@ jobs: python-version: 3.12 - uses: ./.github/workflows/build_mac_wheel with: - python-minor-version: ${{ matrix.python-minor-version }} + python-minor-version: 8 args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels" - uses: ./.github/workflows/upload_wheel with: - python-minor-version: ${{ matrix.python-minor-version }} - token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} - repo: "pypi" + pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} + fury_token: ${{ secrets.FURY_TOKEN }} windows: - # Only runs on tags that matches the python-make-release action - if: startsWith(github.ref, 'refs/tags/python-v') timeout-minutes: 60 runs-on: windows-latest - strategy: - matrix: - python-minor-version: ["8"] steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.ref }} fetch-depth: 0 lfs: true - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.${{ matrix.python-minor-version }} + python-version: 3.8 - uses: ./.github/workflows/build_windows_wheel with: - python-minor-version: ${{ matrix.python-minor-version }} + python-minor-version: 8 args: "--release --strip" vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }} - uses: ./.github/workflows/upload_wheel with: - python-minor-version: ${{ matrix.python-minor-version }} - token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} - repo: "pypi" + pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }} + fury_token: ${{ secrets.FURY_TOKEN }} + gh-release: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + lfs: true + - name: Extract version + id: extract_version + env: + GITHUB_REF: ${{ github.ref }} + run: | + set -e + echo "Extracting tag and version from $GITHUB_REF" + if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then + VERSION=${BASH_REMATCH[1]} + TAG=python-v$VERSION + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + else + echo "Failed to extract version from $GITHUB_REF" + exit 1 + fi + echo "Extracted version $VERSION from $GITHUB_REF" + if [[ $VERSION =~ beta ]]; then + echo "This is a beta release" + + # Get last release (that is not this one) + FROM_TAG=$(git tag --sort='version:refname' \ + | grep ^python-v \ + | grep -vF "$TAG" \ + | python ci/semver_sort.py python-v \ + | tail -n 1) + else + echo "This is a stable release" + # Get last stable tag (ignore betas) + FROM_TAG=$(git tag --sort='version:refname' \ + | grep ^python-v \ + | grep -vF "$TAG" \ + | grep -v beta \ + | python ci/semver_sort.py python-v \ + | tail -n 1) + fi + echo "Found from tag $FROM_TAG" + echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT + - name: Create Python Release Notes + id: python_release_notes + uses: mikepenz/release-changelog-builder-action@v4 + with: + configuration: .github/release_notes.json + toTag: ${{ steps.extract_version.outputs.tag }} + fromTag: ${{ steps.extract_version.outputs.from_tag }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Create Python GH release + uses: softprops/action-gh-release@v2 + with: + prerelease: ${{ contains('beta', github.ref) }} + tag_name: ${{ steps.extract_version.outputs.tag }} + token: ${{ secrets.GITHUB_TOKEN }} + generate_release_notes: false + name: Python LanceDB v${{ steps.extract_version.outputs.version }} + body: ${{ steps.python_release_notes.outputs.changelog }} diff --git a/.github/workflows/python-make-release-commit.yml b/.github/workflows/python-make-release-commit.yml deleted file mode 100644 index ebd48506..00000000 --- a/.github/workflows/python-make-release-commit.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Python - Create release commit - -on: - workflow_dispatch: - inputs: - dry_run: - description: 'Dry run (create the local commit/tags but do not push it)' - required: true - default: "false" - type: choice - options: - - "true" - - "false" - part: - description: 'What kind of release is this?' - required: true - default: 'patch' - type: choice - options: - - patch - - minor - - major - -jobs: - bump-version: - runs-on: ubuntu-latest - steps: - - name: Check out main - uses: actions/checkout@v4 - with: - ref: main - persist-credentials: false - fetch-depth: 0 - lfs: true - - name: Set git configs for bumpversion - shell: bash - run: | - git config user.name 'Lance Release' - git config user.email 'lance-dev@lancedb.com' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Bump version, create tag and commit - working-directory: python - run: | - pip install bump2version - bumpversion --verbose ${{ inputs.part }} - - name: Push new version and tag - if: ${{ inputs.dry_run }} == "false" - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }} - branch: main - tags: true - diff --git a/.github/workflows/upload_wheel/action.yml b/.github/workflows/upload_wheel/action.yml index 8494e01e..c7597bcb 100644 --- a/.github/workflows/upload_wheel/action.yml +++ b/.github/workflows/upload_wheel/action.yml @@ -2,28 +2,44 @@ name: upload-wheel description: "Upload wheels to Pypi" inputs: - os: - required: true - description: "ubuntu-22.04 or macos-13" - repo: - required: false - description: "pypi or testpypi" - default: "pypi" - token: + pypi_token: required: true description: "release token for the repo" + fury_token: + required: true + description: "release token for the fury repo" runs: using: "composite" steps: - - name: Install dependencies - shell: bash - run: | - python -m pip install --upgrade pip - pip install twine - - name: Publish wheel - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ inputs.token }} - shell: bash - run: twine upload --repository ${{ inputs.repo }} target/wheels/lancedb-*.whl + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install twine + - name: Choose repo + shell: bash + id: choose_repo + run: | + if [ ${{ github.ref }} == "*beta*" ]; then + echo "repo=fury" >> $GITHUB_OUTPUT + else + echo "repo=pypi" >> $GITHUB_OUTPUT + fi + - name: Publish to PyPI + working-directory: python + shell: bash + env: + FURY_TOKEN: ${{ inputs.fury_token }} + PYPI_TOKEN: ${{ inputs.pypi_token }} + run: | + if [ ${{ steps.choose_repo.outputs.repo }} == "fury" ]; then + WHEEL=$(ls target/wheels/lancedb-*.whl 2> /dev/null | head -n 1) + echo "Uploading $WHEEL to Fury" + curl -f -F package=@$WHEEL https://$FURY_TOKEN@push.fury.io/lancedb/ + else + twine upload --repository ${{ steps.choose_repo.outputs.repo }} \ + --username __token__ \ + --password $PYPI_TOKEN \ + target/wheels/lancedb-*.whl + fi diff --git a/ci/bump_version.sh b/ci/bump_version.sh new file mode 100644 index 00000000..20d792d1 --- /dev/null +++ b/ci/bump_version.sh @@ -0,0 +1,49 @@ +set -e + +RELEASE_TYPE=${1:-"stable"} +BUMP_MINOR=${2:-false} +TAG_PREFIX=${3:-"v"} # Such as "python-v" +HEAD_SHA=${4:-$(git rev-parse HEAD)} + +readonly SELF_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) + +PREV_TAG=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1) +echo "Found previous tag $PREV_TAG" + +# Initially, we don't want to tag if we are doing stable, because we will bump +# again later. See comment at end for why. +if [[ "$RELEASE_TYPE" == 'stable' ]]; then + BUMP_ARGS="--no-tag" +fi + +# If last is stable and not bumping minor +if [[ $PREV_TAG != *beta* ]]; then + if [[ "$BUMP_MINOR" != "false" ]]; then + # X.Y.Z -> X.(Y+1).0-beta.0 + bump-my-version bump -vv $BUMP_ARGS minor + else + # X.Y.Z -> X.Y.(Z+1)-beta.0 + bump-my-version bump -vv $BUMP_ARGS patch + fi +else + if [[ "$BUMP_MINOR" != "false" ]]; then + # X.Y.Z-beta.N -> X.(Y+1).0-beta.0 + bump-my-version bump -vv $BUMP_ARGS minor + else + # X.Y.Z-beta.N -> X.Y.Z-beta.(N+1) + bump-my-version bump -vv $BUMP_ARGS pre_n + fi +fi + +# The above bump will always bump to a pre-release version. If we are releasing +# a stable version, bump the pre-release level ("pre_l") to make it stable. +if [[ $RELEASE_TYPE == 'stable' ]]; then + # X.Y.Z-beta.N -> X.Y.Z + bump-my-version bump -vv pre_l +fi + +# Validate that we have incremented version appropriately for breaking changes +LAST_STABLE_RELEASE=$(git tag --sort='version:refname' | grep ^$TAG_PREFIX | grep -v beta | python $SELF_DIR/semver_sort.py $TAG_PREFIX | tail -n 1) +LAST_STABLE_VERSION=$(echo $LAST_STABLE_RELEASE | sed "s/^$TAG_PREFIX//") +NEW_VERSION=$(git describe --tags --exact-match HEAD | sed "s/^$TAG_PREFIX//") +python $SELF_DIR/check_breaking_changes.py $LAST_STABLE_RELEASE $HEAD_SHA $LAST_STABLE_VERSION $NEW_VERSION diff --git a/ci/check_breaking_changes.py b/ci/check_breaking_changes.py new file mode 100644 index 00000000..bc7a562b --- /dev/null +++ b/ci/check_breaking_changes.py @@ -0,0 +1,35 @@ +""" +Check whether there are any breaking changes in the PRs between the base and head commits. +If there are, assert that we have incremented the minor version. +""" +import argparse +import os +from packaging.version import parse + +from github import Github + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("base") + parser.add_argument("head") + parser.add_argument("last_stable_version") + parser.add_argument("current_version") + args = parser.parse_args() + + repo = Github(os.environ["GITHUB_TOKEN"]).get_repo(os.environ["GITHUB_REPOSITORY"]) + commits = repo.compare(args.base, args.head).commits + prs = (pr for commit in commits for pr in commit.get_pulls()) + + for pr in prs: + if any(label.name == "breaking-change" for label in pr.labels): + print(f"Breaking change in PR: {pr.html_url}") + break + else: + print("No breaking changes found.") + exit(0) + + last_stable_version = parse(args.last_stable_version) + current_version = parse(args.current_version) + if current_version.minor <= last_stable_version.minor: + print("Minor version is not greater than the last stable version.") + exit(1) diff --git a/ci/semver_sort.py b/ci/semver_sort.py new file mode 100644 index 00000000..b90ba331 --- /dev/null +++ b/ci/semver_sort.py @@ -0,0 +1,35 @@ +""" +Takes a list of semver strings and sorts them in ascending order. +""" + +import sys +from packaging.version import parse, InvalidVersion + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser() + parser.add_argument("prefix", default="v") + args = parser.parse_args() + + # Read the input from stdin + lines = sys.stdin.readlines() + + # Parse the versions + versions = [] + for line in lines: + line = line.strip() + try: + version_str = line.removeprefix(args.prefix) + version = parse(version_str) + except InvalidVersion: + # There are old tags that don't follow the semver format + print(f"Invalid version: {line}", file=sys.stderr) + continue + versions.append((line, version)) + + # Sort the versions + versions.sort(key=lambda x: x[1]) + + # Print the sorted versions as original strings + for line, _ in versions: + print(line) diff --git a/docs/src/basic.md b/docs/src/basic.md index 62f487cb..2897d375 100644 --- a/docs/src/basic.md +++ b/docs/src/basic.md @@ -44,6 +44,36 @@ !!! info "Please also make sure you're using the same version of Arrow as in the [lancedb crate](https://github.com/lancedb/lancedb/blob/main/Cargo.toml)" +### Preview releases + +Stable releases are created about every 2 weeks. For the latest features and bug +fixes, you can install the preview release. These releases receive the same +level of testing as stable releases, but are not guaranteed to be available for +more than 6 months after they are released. Once your application is stable, we +recommend switching to stable releases. + +=== "Python" + + ```shell + pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb + ``` + +=== "Typescript" + + ```shell + npm install vectordb@preview + ``` + +=== "Rust" + + We don't push preview releases to crates.io, but you can referent the tag + in GitHub within your Cargo dependencies: + + ```toml + [dependencies] + lancedb = { git = "https://github.com/lancedb/lancedb.git", tag = "vX.Y.Z-beta.N" } + ``` + ## Connect to a database === "Python" diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 2450e238..accdd228 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.4.14", + "version": "0.4.20", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/python/.bumpversion.cfg b/python/.bumpversion.cfg deleted file mode 100644 index 2749d862..00000000 --- a/python/.bumpversion.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[bumpversion] -current_version = 0.6.13 -commit = True -message = [python] Bump version: {current_version} β†’ {new_version} -tag = True -tag_name = python-v{new_version} - -[bumpversion:file:pyproject.toml] diff --git a/python/.bumpversion.toml b/python/.bumpversion.toml new file mode 100644 index 00000000..74b29615 --- /dev/null +++ b/python/.bumpversion.toml @@ -0,0 +1,34 @@ +[tool.bumpversion] +current_version = "0.6.13" +parse = """(?x) + (?P0|[1-9]\\d*)\\. + (?P0|[1-9]\\d*)\\. + (?P0|[1-9]\\d*) + (?:-(?P[a-zA-Z-]+)\\.(?P0|[1-9]\\d*))? +""" +serialize = [ + "{major}.{minor}.{patch}-{pre_l}.{pre_n}", + "{major}.{minor}.{patch}", +] +search = "{current_version}" +replace = "{new_version}" +regex = false +ignore_missing_version = false +ignore_missing_files = false +tag = true +sign_tags = false +tag_name = "python-v{new_version}" +tag_message = "Bump version: {current_version} β†’ {new_version}" +allow_dirty = true +commit = true +message = "Bump version: {current_version} β†’ {new_version}" +commit_args = "" + +[tool.bumpversion.parts.pre_l] +values = ["beta", "final"] +optional_value = "final" + +[[tool.bumpversion.files]] +filename = "Cargo.toml" +search = "\nversion = \"{current_version}\"" +replace = "\nversion = \"{new_version}\"" diff --git a/python/Cargo.toml b/python/Cargo.toml index 89403a2f..0e4c5ea3 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.4.10" +version = "0.6.13" edition.workspace = true description = "Python bindings for LanceDB" license.workspace = true diff --git a/python/pyproject.toml b/python/pyproject.toml index 6ad974c2..e28336ef 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lancedb" -version = "0.6.13" +# version in Cargo.toml dependencies = [ "deprecation", "pylance==0.10.12", diff --git a/release_process.md b/release_process.md index dbc0467e..979cb8db 100644 --- a/release_process.md +++ b/release_process.md @@ -8,6 +8,51 @@ The Python package is versioned and released separately from the Rust and Node.j ones. For Rust and Node.js, the release process is shared between `lancedb` and `vectordb` for now. +## Preview releases + +LanceDB has full releases about every 2 weeks, but in between we make frequent +preview releases. These are released as `0.x.y.betaN` versions. They receive the +same level of testing as normal releases and let you get access to the latest +features. However, we do not guarantee that preview releases will be available +more than 6 months after they are released. We may delete the preview releases +from the packaging index after a while. Once your application is stable, we +recommend switching to full releases, which will never be removed from package +indexes. + +## Making releases + +The release process uses a handful of GitHub actions to automate the process. + +```text + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚Create Release Commitβ”‚ + β””β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β–ΊPython GH Release + β”œβ”€β”€β–Ί(tag) python-vX.Y.Z ───►│PyPI Publishβ”œβ”€β”€ + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ └──►Python Wheels + β”‚ + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + └──►(tag) vX.Y.Z ───┬──────►│NPM Publishβ”œβ”€β”€β”¬β”€β”€β–ΊRust/Node GH Release + β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ └──►NPM Packages + β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + └──────►│Cargo Publishβ”œβ”€β”€β”€β–ΊCargo Release + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +To start a release, trigger a `Create Release Commit` action from +[the workflows page](https://github.com/lancedb/lancedb/actions/workflows/make-release-commit.yml) +(Click on "Run workflow"). + +* **For a preview release**, leave the default parameters. +* **For a stable release**, set the `release_type` input to `stable`. + +> [!IMPORTANT] +> If there was a breaking change since the last stable release, and we haven't +> done so yet, we should increment the minor version. The CI will detect if this +> is needed and fail the `Create Release Commit` job. To fix, select the +> "bump minor version" option. + ## Breaking changes We try to avoid breaking changes, but sometimes they are necessary. When there @@ -21,12 +66,10 @@ body of the PR. A CI job will add a `breaking-change` label to the PR, which is what will ultimately be used to CI to determine if the minor version should be incremented. -A CI job will validate that if a `breaking-change` label is added, the minor -version is incremented in the `Cargo.toml` and `pyproject.toml` files. The only -exception is if it has already been incremented since the last stable release. - -**It is the responsibility of the PR author to increment the minor version when -appropriate.** +> [!IMPORTANT] +> Reviewers should check that PRs with breaking changes receive the `breaking-change` +> label. If a PR is missing the label, please add it, even if after it was merged. +> This label is used in the release process. Some things that are considered breaking changes: