make node available to all users

.github/workflows/make_release_commit.yml

@@ -0,0 +1,70 @@
+name: Create release commit
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Just create the local commit/tags but do not push it'
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "true"
+          - "false"
+      part:
+        description: 'What kind of release is this?'
+        required: true
+        default: 'patch'
+        type: choice
+        options:
+          - patch
+          - minor
+          - major
+
+jobs:
+  bump-version:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out main
+      uses: actions/checkout@v3
+      with:
+        ref: main
+        persist-credentials: false
+        fetch-depth: 0
+        lfs: true
+    - name: Install cargo utils
+      run: cargo install cargo-bump cargo-get
+    - name: Bump vectordb
+      working-directory: rust/vectordb
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV
+    - name: Bump rust/ffi/node
+      working-directory: rust/ffi/node
+      run: |
+        cargo bump ${{ inputs.part }}
+        echo "FFI_CRATE_VERSION=$(cargo get version)" >> $GITHUB_ENV 
+    - name: Bump node
+      working-directory: node
+      run: |
+        npm version ${{ inputs.part }}
+        echo "NPM_PACKAGE_VERSION=$(cat package.json | jq -r '.version')" >> $GITHUB_ENV 
+    - name: Create tag
+      run: |
+        if [ "$CRATE_VERSION" != "$FFI_CRATE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and rust/ffi/node"
+          exit 1
+        fi
+        if [ "$CRATE_VERSION" != "$NPM_PACKAGE_VERSION" ]; then
+          echo "Version mismatch between rust/vectordb and node"
+          exit 1
+        fi
+        export TAG="v$CRATE_VERSION'"
+        git tag $TAG
+    - name: Push new version and tag
+      if: ${{ inputs.dry_run }} == "false"
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.RELEASE_TOKEN }}
+        branch: main
+        tags: true

.github/workflows/release.yml

@@ -1,6 +1,7 @@
 name: Prepare Release
 
-# Based on https://github.com/dherman/neon-prebuild-example/blob/eaa4d33d682e5eb7abbc3da7aed153a1b1acb1b3/.github/workflows/publish.yml
+# TODO: bump versions in CI
+# NOTE: Python is a separate release for now.
 
 on:
   push:
@@ -38,35 +39,7 @@ jobs:
       - uses: softprops/action-gh-release@v1
         with:
           draft: true
-          files: target/vectordb-*.crate
-          fail_on_unmatched_files: true
-    
-  python:
-    runs-on: ubuntu-latest
-    needs: draft-release
-    defaults:
-      run:
-        shell: bash
-        working-directory: python
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          lfs: true
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Build wheel
-        run: |
-          pip install wheel
-          python setup.py sdist bdist_wheel
-      - uses: softprops/action-gh-release@v1
-        with:
-          draft: true
-          files: |
-            python/dist/lancedb-*.tar.gz 
-            python/dist/lancedb-*.whl
+          files: target/package/vectordb-*.crate
           fail_on_unmatched_files: true
 
   node:
@@ -115,6 +88,9 @@ jobs:
       run: |
         cd node
         npm ci
+    - name: Install rustup target
+      if: ${{ matrix.target == 'aarch64-apple-darwin' }}
+      run: rustup target add aarch64-apple-darwin
     - name: Build MacOS native node modules
       run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
     - uses: softprops/action-gh-release@v1
@@ -141,6 +117,10 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v2
+    - name: Change owner to root (for npm)
+      # The docker container is run as root, so we need the files to be owned by root
+      # Otherwise npm is a nightmare: https://github.com/npm/cli/issues/3773
+      run: sudo chown -R root:root .
     - name: Set up QEMU
       if: ${{ matrix.arch == 'aarch64' }}
       uses: docker/setup-qemu-action@v2  
@@ -167,7 +147,7 @@ jobs:
         fail_on_unmatched_files: true
 
   release:
-    needs: [python, node, node-macos, node-linux, rust]
+    needs: [rust, node, node-macos, node-linux]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v3

Cross.toml

@@ -1,39 +0,0 @@
-# These make sure our builds are compatible with old glibc versions.
-[target.x86_64-unknown-linux-gnu]
-pre-build = [
-    # Install newer gfortran
-    "yum install -y openssl-devel unzip gcc-gfortran",
-    "scl enable devtoolset-11 bash",
-    # protobuf is too old, so we directly download binaries
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
-]
-image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
-
-[target.aarch64-unknown-linux-gnu]
-pre-build = [
-    "yum install -y openssl-devel unzip",
-    # protobuf is too old, so we directly download binaries
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
-]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
-image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
-
-[target.x86_64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]
-
-[target.aarch64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]

ci/build_linux_artifacts.sh

@@ -18,7 +18,7 @@ setup_dependencies() {
         apk add openssl-dev
     else
         # manylinux2014
-        yum install -y openssl-devel unzip   
+        yum install -y openssl-devel unzip
     fi
 
     if [[ $1 == x86_64* ]]; then
@@ -44,9 +44,9 @@ install_node() {
         # This node version is 15, we need 16 or higher:
         # apk add nodejs-current npm 
         # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s 17
+        nvm install -s --no-progress 17
     else
-        nvm install 17 # latest that supports glibc 2.17
+        nvm install --no-progress 17 # latest that supports glibc 2.17
     fi
 }
 
@@ -59,11 +59,18 @@ install_rust() {
 build_node_binary() {
     echo "Building node library for $1..."
     pushd node
+
+    npm ci
     
     if [[ $1 == *musl ]]; then
         # This is needed for cargo to allow build cdylibs with musl
         export RUSTFLAGS="-C target-feature=-crt-static"
     fi
+
+    # Cargo can run out of memory while pulling dependencies, espcially when running
+    # in QEMU. This is a workaround for that.
+    export CARGO_NET_GIT_FETCH_WITH_CLI=true
+
     # We don't pass in target, since the native target here already matches
     # and openblas-src doesn't do well with cross-compilation.
     npm run build-release

ci/build_macos_artifacts.sh

@@ -2,6 +2,20 @@
 # Usage: ./build_macos_artifacts.sh [target]
 # Targets supported: x86_64-apple-darwin aarch64-apple-darwin
 
+prebuild_rust() {
+    # Building here for the sake of easier debugging.
+    pushd rust/ffi/node
+
+    for target in $1
+    do
+        echo "Building rust library for $target"
+        export RUST_BACKTRACE=1
+        cargo build --release --target $target
+    done
+
+    popd
+}
+
 build_node_binaries() {
     pushd node
     
@@ -19,4 +33,6 @@ if [ -n "$1" ]; then
 else
     targets="x86_64-apple-darwin aarch64-apple-darwin"
 fi
+
+prebuild_rust $targets
 build_node_binaries $targets

ci/release_process.md

@@ -1,56 +1,24 @@
+# How to release
 
-How to release the node module
+This is for the Rust crate and Node module. For now, the Python module is
+released separately.
 
-### 1. Bump the versions
-
-<!-- TODO: we also need to bump the optional dependencies for node! -->
-
-```shell
-pushd rust/vectordb
-cargo bump minor
-popd
-
-pushd rust/ffi/node
-cargo bump minor
-popd
-
-pushd python
-cargo bump minor
-popd
-
-pushd node
-npm version minor
-popd
-
-git add -u
-git commit -m "Bump versions"
-git push
-```
-
-### 2. Push a new tag
-
-```shell
-git tag vX.X.X
-git push --tag vX.X.X
-```
+The release is started by bumping the versions and pushing a new tag. To do this
+automatically, use the `make_release_commit` GitHub action.
 
 When the tag is pushed, GitHub actions will start building the libraries and
-will upload them to a draft release. Wait for those jobs to complete.
+will upload them to a draft release.
 
-### 3. Publish the release
+While those jobs are running, edit the release notes as needed. For example, 
+bring relevant new features and bugfixes to the top of the notes and the testing
+and CI changes to the bottom.
 
-Once the jobs are complete, you can edit the 
-
-2. Push a tag, such as vX.X.X. Once the tag is pushrf, GitHub actions will start
-   building the native libraries and uploading them to a draft release. Wait for
-   those jobs to complete.
-3. If the libraries are successful, edit the changelog and then publish the
-   release. Once you publish, a new action will start and upload all the 
-   release artifacts to npm.
+Once the jobs have finished, the release will be marked as not draft and the
+artifacts will be released to crates.io, NPM, and PyPI.
 
 ## Manual process
 
-You can build the artifacts locally on a MacOS machine.
+You can also build the artifacts locally on a MacOS machine.
 
 ### Build the MacOS release libraries
 
@@ -80,11 +48,12 @@ docker run \
 
 You can change `ARCH` to `x86_64`.
 
-Similar script for musl binaries:
+Similar script for musl binaries (not yet working):
 
 ```shell
 ARCH=aarch64
 docker run \
+    --user $(id -u) \
     -v $(pwd):/io -w /io \
     quay.io/pypa/musllinux_1_1_$ARCH \
     bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
@@ -92,7 +61,15 @@ docker run \
 
 <!--
 
-For debugging, use this snippet:
+For debugging, use these snippets:
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/manylinux2014_$ARCH \
+    bash
+```
 
 ```shell
 ARCH=aarch64
@@ -101,6 +78,8 @@ docker run -it \
     quay.io/pypa/musllinux_1_1_$ARCH \
     bash
 ```
+
+Note: musllinux_1_1 is Alpine Linux 3.12
 -->
 
 ```

node/README.md

@@ -9,7 +9,8 @@ npm install vectordb
 ```
 
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, Intel MacOS, and ARM (M1/M2) MacOS.
+support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
+yet support Windows or musl-based Linux (such as Alpine Linux).
 
 ## Usage
 

node/package.json

@@ -11,7 +11,8 @@
     "cross-release": "cargo-cp-artifact --artifact cdylib vectordb-node index.node -- cross build --message-format=json --release -p vectordb-node",
     "test": "mocha -recursive dist/test",
     "lint": "eslint src --ext .js,.ts",
-    "pack-build": "neon pack-build"
+    "pack-build": "neon pack-build",
+    "check-npm": "printenv && which node && which npm && npm --version"
   },
   "repository": {
     "type": "git",