use manylinux containers locally

.github/workflows/release.yml

@@ -37,8 +37,8 @@ jobs:
         run: cargo package --all-features
       - uses: softprops/action-gh-release@v1
         with:
-          files: |
+          draft: true
-            rust/target/vectordb-*.crate
+          files: target/vectordb-*.crate
     
   python:
     runs-on: ubuntu-latest
@@ -55,11 +55,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.10
+          python-version: "3.10"
       - name: Build wheel
-        run: python setup.py sdist bdist_wheel
+        run: |
+          pip install wheel
+          python setup.py sdist bdist_wheel
       - uses: softprops/action-gh-release@v1
         with:
+          draft: true
           files: |
             python/dist/lancedb-*.tar.gz 
             python/dist/lancedb-*.whl
@@ -79,7 +82,6 @@ jobs:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
-      - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: |
           sudo apt update
@@ -91,12 +93,14 @@ jobs:
           npm pack
       - uses: softprops/action-gh-release@v1
         with:
+          draft: true
           files: node/vectordb-*.tgz
 
   node-macos:
     runs-on: macos-12
     needs: draft-release
     strategy:
+      fail-fast: false
       matrix:
         target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
@@ -104,35 +108,54 @@ jobs:
       uses: actions/checkout@v2
     - name: Install system dependencies
       run: brew install protobuf
+    - name: Install npm dependencies
+      run: |
+        cd node
+        npm ci
     - name: Build MacOS native node modules
       run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
     - uses: softprops/action-gh-release@v1
       with:
+        draft: true
         files: node/dist/vectordb-darwin*.tgz
 
   node-linux:
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
     runs-on: ubuntu-latest
     needs: draft-release
     strategy:
+      fail-fast: false
       matrix:
-        target:
+        libc:
-          - x86_64-unknown-linux-gnu:centos
+          - gnu
-          - aarch64-unknown-linux-gnu:centos
+          - musl
-          - aarch64-unknown-linux-musl
+        arch:
-          - x86_64-unknown-linux-musl
+          - x86_64
+          - aarch64
     steps:
-    - name: Install system dependencies
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
       run: |
-        sudo apt update
+        docker run \
-        sudo apt install -y protobuf-compiler libssl-dev
+          -v $(pwd):/io -w /io \
-    - name: Build Linux native node modules
+          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
-      run: bash ci/build_linux_artifacts.sh ${{ matrix.target }}
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
+      run: |
+        docker run \
+          -v $(pwd):/io -w /io \
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
     - uses: softprops/action-gh-release@v1
       with:
+        draft: true
         files: node/dist/vectordb-linux*.tgz
 
   release:
-    needs: [python, node, node-macos, node-linux]
+    needs: [python, node, node-macos, node-linux, rust]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/download-artifact@v3

Cross.toml

@@ -1,14 +1,39 @@
 # These make sure our builds are compatible with old glibc versions.
 [target.x86_64-unknown-linux-gnu]
 pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    # Install newer gfortran
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+    "yum install -y openssl-devel unzip gcc-gfortran",
+    "scl enable devtoolset-11 bash",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
 ]
 image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
 
 [target.aarch64-unknown-linux-gnu]
 pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "yum install -y openssl-devel unzip",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
+    # protobuf is too old, so we directly download binaries
+    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
+    "PB_VERSION=23.1",
+    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
+    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
 ]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
 image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
+
+[target.x86_64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]
+
+[target.aarch64-unknown-linux-musl]
+# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
+]

ci/build_linux_artifacts.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -13,28 +14,69 @@
 
 set -e
 
-build_node_binaries() {
+setup_dependencies() {
+    echo "Installing system dependencies..."
+    if [[ $1 == *musl ]]; then
+        # musllinux
+        apk add openssl-dev
+    else
+        # manylinux2014
+        yum install -y openssl-devel unzip   
+    fi
+
+    if [[ $1 == x86_64* ]]; then
+        ARCH=x86_64
+    else
+        # gnu target
+        ARCH=aarch_64
+    fi
+    
+    # Install new enough protobuf (yum-provided is old)
+    PB_REL=https://github.com/protocolbuffers/protobuf/releases
+    PB_VERSION=23.1
+    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
+}
+
+install_node() {
+    echo "Installing node..."
+    if [[ $1 == *musl ]]; then
+        apk add nodejs-current npm
+    else
+        curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+        source "$HOME"/.nvm/nvm.sh
+        nvm install 17 # latest that supports glibc 2.17
+    fi
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+    rustup target add $1
+}
+
+build_node_binary() {
+    echo "Building node library for $1..."
     pushd node
     
-    for target in $1
+    if [[ $1 == *musl ]]; then
-    do
+        # This is needed for cargo to allow build cdylibs with musl
-        echo "Building node library for $target"
+        export RUSTFLAGS="-C target-feature=-crt-static"
-        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+    fi
-        if [[ $target == x86_64* ]]; then
+    npm run build-release -- --target $1
-            export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+    npm run pack-build -- --target $1
-        else
+
-            export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8"
-        fi
-        npm run cross-release -- --target $target
-        npm run pack-build -- --target $target
-    done
     popd
 }
 
-if [ -n "$1" ]; then
+TARGET=${1:-x86_64-unknown-linux-gnu}
-    targets=$1
+# Others:
-else
+# aarch64-unknown-linux-gnu
-    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+# x86_64-unknown-linux-musl
-    targets="aarch64-unknown-linux-gnu"
+# aarch64-unknown-linux-musl
-fi
+
-build_node_binaries $targets
+setup_dependencies $TARGET
+install_node
+install_rust $TARGET
+build_node_binary $TARGET

rust/ffi/node/release_process.md

@@ -68,18 +68,37 @@ bash ci/build_macos_artifacts.sh
 
 ### Build the Linux release libraries
 
-One-time setup, building the Docker container
+To build a Linux library, we need to use docker with a different build script:
-
-```shell
-cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
-```
-
-To build:
 
 ```shell
+ARCH=aarch64
 docker run \
-    -v /var/run/docker.sock:/var/run/docker.sock \
     -v $(pwd):/io -w /io \
-    lancedb-node-build \
+    quay.io/pypa/manylinux2014_$ARCH \
-    bash ci/build_linux_artifacts.sh
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
 ```
+
+You can change `ARCH` to `x86_64`.
+
+Similar script for musl binaries:
+
+```shell
+ARCH=aarch64
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
+```
+
+<!--
+
+For debugging, use this snippet:
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash
+```
+-->