diff --git a/.github/workflows/make_release_commit.yml b/.github/workflows/make_release_commit.yml new file mode 100644 index 00000000..860689c1 --- /dev/null +++ b/.github/workflows/make_release_commit.yml @@ -0,0 +1,70 @@ +name: Create release commit + +on: + workflow_dispatch: + inputs: + dry_run: + description: 'Just create the local commit/tags but do not push it' + required: true + default: "false" + type: choice + options: + - "true" + - "false" + part: + description: 'What kind of release is this?' + required: true + default: 'patch' + type: choice + options: + - patch + - minor + - major + +jobs: + bump-version: + runs-on: ubuntu-latest + steps: + - name: Check out main + uses: actions/checkout@v3 + with: + ref: main + persist-credentials: false + fetch-depth: 0 + lfs: true + - name: Install cargo utils + run: cargo install cargo-bump cargo-get + - name: Bump vectordb + working-directory: rust/vectordb + run: | + cargo bump ${{ inputs.part }} + echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV + - name: Bump rust/ffi/node + working-directory: rust/ffi/node + run: | + cargo bump ${{ inputs.part }} + echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV + - name: Bump node + working-directory: node + run: | + npm version ${{ inputs.part }} + echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV + - name: Create tag + run: | + if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then + echo "Version mismatch between rust/vectordb and rust/ffi/node" + exit 1 + fi + if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then + echo "Version mismatch between rust/vectordb and node" + exit 1 + fi + export TAG="v$CRATE_VERSION'" + git tag $TAG + - name: Push new version and tag + if: ${{ inputs.dry_run }} == "false" + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.RELEASE_TOKEN }} + branch: main + tags: true \ No newline at end of file diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 49780ee2..5a6c81e8 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -103,74 +103,3 @@ jobs: - name: Test run: | npm run test - node-macos: - runs-on: macos-12 - # needs: draft-release - strategy: - fail-fast: false - matrix: - target: [x86_64-apple-darwin, aarch64-apple-darwin] - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Install system dependencies - run: brew install protobuf - - name: Install npm dependencies - run: | - cd node - npm ci - - name: Install rustup target - if: ${{ matrix.target == 'aarch64-apple-darwin' }} - run: rustup target add aarch64-apple-darwin - - name: Build MacOS native node modules - run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} - # - uses: softprops/action-gh-release@v1 - # with: - # draft: true - # files: node/dist/vectordb-darwin*.tgz - # fail_on_unmatched_files: true - - node-linux: - name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }}) - runs-on: ubuntu-latest - # needs: draft-release - strategy: - fail-fast: false - matrix: - libc: - - gnu - # TODO: re-enable musl once we have refactored to pre-built containers - # Right now we have to build node from source which is too expensive. - # - musl - arch: - - x86_64 - # - aarch64 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up QEMU - if: ${{ matrix.arch == 'aarch64' }} - uses: docker/setup-qemu-action@v2 - with: - platforms: arm64 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Build Linux GNU native node modules - if: ${{ matrix.libc == 'gnu' }} - run: | - docker run \ - -v $(pwd):/io -w /io \ - quay.io/pypa/manylinux2014_${{ matrix.arch }} \ - bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu - - name: Build musl Linux native node modules - if: ${{ matrix.libc == 'musl' }} - run: | - docker run --platform linux/arm64/v8 \ - -v $(pwd):/io -w /io \ - quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \ - bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl - # - uses: softprops/action-gh-release@v1 - # with: - # draft: true - # files: node/dist/vectordb-linux*.tgz - # fail_on_unmatched_files: true \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d8f0253..aaa7c80e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: Prepare Release # TODO: bump versions in CI -# TODO: keep python release separate for now. +# NOTE: Python is a separate release for now. on: push: @@ -41,34 +41,6 @@ jobs: draft: true files: target/package/vectordb-*.crate fail_on_unmatched_files: true - - python: - runs-on: ubuntu-latest - needs: draft-release - defaults: - run: - shell: bash - working-directory: python - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - lfs: true - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - name: Build wheel - run: | - pip install wheel - python setup.py sdist bdist_wheel - - uses: softprops/action-gh-release@v1 - with: - draft: true - files: | - python/dist/lancedb-*.tar.gz - python/dist/lancedb-*.whl - fail_on_unmatched_files: true node: runs-on: ubuntu-latest @@ -116,6 +88,9 @@ jobs: run: | cd node npm ci + - name: Install rustup target + if: ${{ matrix.target == 'aarch64-apple-darwin' }} + run: rustup target add aarch64-apple-darwin - name: Build MacOS native node modules run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} - uses: softprops/action-gh-release@v1 @@ -138,17 +113,19 @@ jobs: # - musl arch: - x86_64 - # - aarch64 + - aarch64 steps: - name: Checkout uses: actions/checkout@v2 + - name: Change owner to root (for npm) + # The docker container is run as root, so we need the files to be owned by root + # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773 + run: sudo chown -R root:root . - name: Set up QEMU if: ${{ matrix.arch == 'aarch64' }} uses: docker/setup-qemu-action@v2 with: platforms: arm64 - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - name: Build Linux GNU native node modules if: ${{ matrix.libc == 'gnu' }} run: | @@ -170,7 +147,7 @@ jobs: fail_on_unmatched_files: true release: - needs: [python, node, node-macos, node-linux, rust] + needs: [rust, node, node-macos, node-linux] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 diff --git a/Cross.toml b/Cross.toml deleted file mode 100644 index e9396965..00000000 --- a/Cross.toml +++ /dev/null @@ -1,39 +0,0 @@ -# These make sure our builds are compatible with old glibc versions. -[target.x86_64-unknown-linux-gnu] -pre-build = [ - # Install newer gfortran - "yum install -y openssl-devel unzip gcc-gfortran", - "scl enable devtoolset-11 bash", - # protobuf is too old, so we directly download binaries - "PB_REL=https://github.com/protocolbuffers/protobuf/releases", - "PB_VERSION=23.1", - "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local", -] -image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos" - -[target.aarch64-unknown-linux-gnu] -pre-build = [ - "yum install -y openssl-devel unzip", - # protobuf is too old, so we directly download binaries - "PB_REL=https://github.com/protocolbuffers/protobuf/releases", - "PB_VERSION=23.1", - "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", - "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local", -] -# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos -image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos" - -[target.x86_64-unknown-linux-musl] -# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl -pre-build = [ - "dpkg --add-architecture $CROSS_DEB_ARCH", - "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH", -] - -[target.aarch64-unknown-linux-musl] -# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl -pre-build = [ - "dpkg --add-architecture $CROSS_DEB_ARCH", - "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH", -] diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh index 328358ef..6cc0b291 100644 --- a/ci/build_linux_artifacts.sh +++ b/ci/build_linux_artifacts.sh @@ -18,7 +18,7 @@ setup_dependencies() { apk add openssl-dev else # manylinux2014 - yum install -y openssl-devel unzip + yum install -y openssl-devel unzip fi if [[ $1 == x86_64* ]]; then @@ -48,28 +48,19 @@ install_node() { else nvm install --no-progress 17 # latest that supports glibc 2.17 fi - - printenv - echo "Node version:" - npm --version - which npm - which node } install_rust() { echo "Installing rust..." curl https://sh.rustup.rs -sSf | bash -s -- -y - - printenv - export PATH="$PATH:/root/.cargo/bin" - - printenv } build_node_binary() { echo "Building node library for $1..." pushd node + + npm ci if [[ $1 == *musl ]]; then # This is needed for cargo to allow build cdylibs with musl @@ -77,10 +68,8 @@ build_node_binary() { fi # We don't pass in target, since the native target here already matches # and openblas-src doesn't do well with cross-compilation. - source ~/.bashrc - npm run check-npm --script-shell bash - npm run build-release --script-shell bash - npm run pack-build --script-shell bash + npm run build-release + npm run pack-build popd } diff --git a/ci/release_process.md b/ci/release_process.md index f2728a5e..d90df33b 100644 --- a/ci/release_process.md +++ b/ci/release_process.md @@ -1,38 +1,10 @@ +# How to release -How to release the node module +This is for the Rust crate and Node module. For now, the Python module is +released separately. -### 1. Bump the versions - - - -```shell -pushd rust/vectordb -cargo bump minor -popd - -pushd rust/ffi/node -cargo bump minor -popd - -pushd python -cargo bump minor -popd - -pushd node -npm version minor -popd - -git add -u -git commit -m "Bump versions" -git push -``` - -### 2. Push a new tag - -```shell -git tag vX.X.X -git push --tag vX.X.X -``` +The release is started by bumping the versions and pushing a new tag. To do this +automatically, use the `make_release_commit` GitHub action. When the tag is pushed, GitHub actions will start building the libraries and will upload them to a draft release. @@ -46,7 +18,7 @@ artifacts will be released to crates.io, NPM, and PyPI. ## Manual process -You can build the artifacts locally on a MacOS machine. +You can also build the artifacts locally on a MacOS machine. ### Build the MacOS release libraries @@ -76,11 +48,12 @@ docker run \ You can change `ARCH` to `x86_64`. -Similar script for musl binaries: +Similar script for musl binaries (not yet working): ```shell ARCH=aarch64 docker run \ + --user $(id -u) \ -v $(pwd):/io -w /io \ quay.io/pypa/musllinux_1_1_$ARCH \ bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl diff --git a/node/README.md b/node/README.md index 662ce848..272dd9a1 100644 --- a/node/README.md +++ b/node/README.md @@ -9,7 +9,8 @@ npm install vectordb ``` This will download the appropriate native library for your platform. We currently -support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. +support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not +yet support Windows or musl-based Linux (such as Alpine Linux). ## Usage