make node available to all users

.github/workflows/make_release_commit.yml

@@ -0,0 +1,70 @@
+name: Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Just create the local commit/tags but do not push it'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Install cargo utils
+      run: cargo install cargo-bump cargo-get
+    - name: Bump vectordb
+      working-directory: rust/vectordb
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
+    - name: Bump rust/ffi/node
+      working-directory: rust/ffi/node
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV 
+    - name: Bump node
+      working-directory: node
+      run: |
+        npm version ${{ inputs.part }}
+        echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV 
+    - name: Create tag
+      run: |
+        if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and rust/ffi/node"
+          exit 1
+        fi
+        if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and node"
+          exit 1
+        fi
+        export TAG="v$CRATE_VERSION'"
+        git tag $TAG
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.RELEASE_TOKEN }}
+        branch: main
+        tags: true

.github/workflows/node.yml

@@ -103,74 +103,3 @@ jobs:
     - name: Test
       run: |
         npm run test
-  node-macos:
-    runs-on: macos-12
-    # needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        target: [x86_64-apple-darwin, aarch64-apple-darwin]
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Install system dependencies
-      run: brew install protobuf
-    - name: Install npm dependencies
-      run: |
-        cd node
-        npm ci
-    - name: Install rustup target
-      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
-      run: rustup target add aarch64-apple-darwin
-    - name: Build MacOS native node modules
-      run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
-    # - uses: softprops/action-gh-release@v1
-    #   with:
-    #     draft: true
-    #     files: node/dist/vectordb-darwin*.tgz
-    #     fail_on_unmatched_files: true
-  
-  node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
-    runs-on: ubuntu-latest
-    # needs: draft-release
-    strategy:
-      fail-fast: false
-      matrix:
-        libc:
-          - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
-          # Right now we have to build node from source which is too expensive.
-          # - musl
-        arch:
-          - x86_64
-          # - aarch64
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v2
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2  
-      with:
-        platforms: arm64
-    - name: Setup tmate session
-      uses: mxschmitt/action-tmate@v3
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
-      run: |
-        docker run \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build musl Linux native node modules
-      if: ${{ matrix.libc == 'musl' }}
-      run: |
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-    # - uses: softprops/action-gh-release@v1
-    #   with:
-    #     draft: true
-    #     files: node/dist/vectordb-linux*.tgz
-    #     fail_on_unmatched_files: true

.github/workflows/release.yml

@@ -1,7 +1,7 @@
 name: Prepare Release
 
 # TODO: bump versions in CI
-# TODO: keep python release separate for now.
+# NOTE: Python is a separate release for now.
 
 on:
   push:
@@ -41,34 +41,6 @@ jobs:
           draft: true
           files: target/package/vectordb-*.crate
           fail_on_unmatched_files: true
-    
-  python:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Build wheel
-        run: |
-          pip install wheel
-          python setup.py sdist bdist_wheel
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: |
-            python/dist/lancedb-*.tar.gz 
-            python/dist/lancedb-*.whl
-          fail_on_unmatched_files: true
 
   node:
     runs-on: ubuntu-latest
@@ -116,6 +88,9 @@ jobs:
       run: |
         cd node
         npm ci
+    - name: Install rustup target
+      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+      run: rustup target add aarch64-apple-darwin
     - name: Build MacOS native node modules
       run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
     - uses: softprops/action-gh-release@v1
@@ -138,17 +113,19 @@ jobs:
           # - musl
         arch:
           - x86_64
-          # - aarch64
+          - aarch64
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - name: Change owner to root (for npm)
+      # The docker container is run as root, so we need the files to be owned by root
+      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+      run: sudo chown -R root:root .
     - name: Set up QEMU
       if: ${{ matrix.arch == 'aarch64' }}
       uses: docker/setup-qemu-action@v2  
       with:
         platforms: arm64
-    - name: Setup tmate session
-      uses: mxschmitt/action-tmate@v3
     - name: Build Linux GNU native node modules
       if: ${{ matrix.libc == 'gnu' }}
       run: |
@@ -170,7 +147,7 @@ jobs:
         fail_on_unmatched_files: true
 
   release:
-    needs: [python, node, node-macos, node-linux, rust]
+    needs: [rust, node, node-macos, node-linux]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v3

Cross.toml

@@ -1,39 +0,0 @@
-# These make sure our builds are compatible with old glibc versions.
-[target.x86_64-unknown-linux-gnu]
-pre-build = [
-    # Install newer gfortran
-    "yum install -y openssl-devel unzip gcc-gfortran",
-    "scl enable devtoolset-11 bash",
-    # protobuf is too old, so we directly download binaries
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
-]
-image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
-
-[target.aarch64-unknown-linux-gnu]
-pre-build = [
-    "yum install -y openssl-devel unzip",
-    # protobuf is too old, so we directly download binaries
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
-]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
-image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
-
-[target.x86_64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]
-
-[target.aarch64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]

ci/build_linux_artifacts.sh

@@ -18,7 +18,7 @@ setup_dependencies() {
         apk add openssl-dev
     else
         # manylinux2014
-        yum install -y openssl-devel unzip   
+        yum install -y openssl-devel unzip
     fi
 
     if [[ $1 == x86_64* ]]; then
@@ -48,39 +48,33 @@ install_node() {
     else
         nvm install --no-progress 17 # latest that supports glibc 2.17
     fi
-
-    printenv
-    echo "Node version:"
-    npm --version
-    which npm
-    which node
 }
 
 install_rust() {
     echo "Installing rust..."
     curl https://sh.rustup.rs -sSf | bash -s -- -y
-
-    printenv
-
     export PATH="$PATH:/root/.cargo/bin"
-
-    printenv
 }
 
 build_node_binary() {
     echo "Building node library for $1..."
     pushd node
+
+    npm ci
     
     if [[ $1 == *musl ]]; then
         # This is needed for cargo to allow build cdylibs with musl
         export RUSTFLAGS="-C target-feature=-crt-static"
     fi
+
+    # Cargo can run out of memory while pulling dependencies, espcially when running
+    # in QEMU. This is a workaround for that.
+    export CARGO_NET_GIT_FETCH_WITH_CLI=true
+
     # We don't pass in target, since the native target here already matches
     # and openblas-src doesn't do well with cross-compilation.
-    source ~/.bashrc
+    npm run build-release
-    npm run check-npm --script-shell bash
+    npm run pack-build
-    npm run build-release --script-shell bash
-    npm run pack-build --script-shell bash
 
     popd
 }

ci/release_process.md

@@ -1,38 +1,10 @@
+# How to release
 
-How to release the node module
+This is for the Rust crate and Node module. For now, the Python module is
+released separately.
 
-### 1. Bump the versions
+The release is started by bumping the versions and pushing a new tag. To do this
-
+automatically, use the `make_release_commit` GitHub action.
-<!-- TODO: we also need to bump the optional dependencies for node! -->
-
-```shell
-pushd rust/vectordb
-cargo bump minor
-popd
-
-pushd rust/ffi/node
-cargo bump minor
-popd
-
-pushd python
-cargo bump minor
-popd
-
-pushd node
-npm version minor
-popd
-
-git add -u
-git commit -m "Bump versions"
-git push
-```
-
-### 2. Push a new tag
-
-```shell
-git tag vX.X.X
-git push --tag vX.X.X
-```
 
 When the tag is pushed, GitHub actions will start building the libraries and
 will upload them to a draft release.
@@ -46,7 +18,7 @@ artifacts will be released to crates.io, NPM, and PyPI.
 
 ## Manual process
 
-You can build the artifacts locally on a MacOS machine.
+You can also build the artifacts locally on a MacOS machine.
 
 ### Build the MacOS release libraries
 
@@ -76,11 +48,12 @@ docker run \
 
 You can change `ARCH` to `x86_64`.
 
-Similar script for musl binaries:
+Similar script for musl binaries (not yet working):
 
 ```shell
 ARCH=aarch64
 docker run \
+    --user $(id -u) \
     -v $(pwd):/io -w /io \
     quay.io/pypa/musllinux_1_1_$ARCH \
     bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl

node/README.md

@@ -9,7 +9,8 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
 
 ## Usage