try manylinux again

.github/workflows/release.yml

@@ -120,7 +120,7 @@ jobs:
         files: node/dist/vectordb-darwin*.tgz
 
   node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    name: Linux ${{ matrix.settings.arch}} native node module
     runs-on: ubuntu-latest
     needs: draft-release
     strategy:
@@ -129,26 +129,49 @@ jobs:
         libc:
           - gnu
           - musl
-        arch:
+        settings:
-          - x86_64
+          - container: quay.io/pypa/manylinux2014_x86_64
-          - aarch64
+            arch: x86_64
+            protoc_arch: x86_64
+          - container:  quay.io/pypa/manylinux2014_aarch64
+            arch: aarch64
+            protoc_arch: aarch_64
+    container:
+      image: ${{ matrix.settings.container }}
+      options: --platform linux/amd64
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Build Linux GNU native node modules
+    - name: Install system dependencies
-      if: ${{ matrix.libc == 'gnu' }}
       run: |
-        docker run \
+        yum install -y openssl-devel unzip
-          -v $(pwd):/io -w /io \
+        # Install new enough protobuf (yum-provided is old)
-          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
+        PB_REL=https://github.com/protocolbuffers/protobuf/releases
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+        PB_VERSION=23.1
-    - name: Build musl Linux native node modules
+        curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip
-      if: ${{ matrix.libc == 'musl' }}
+        unzip protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip -d /usr/local
+    - name: Install Node
       run: |
-        docker run \
+        curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-          -v $(pwd):/io -w /io \
+        source "$HOME"/.nvm/nvm.sh
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
+        nvm install 17 # latest that supports glibc 2.17
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
+    - name: Setup Rust
+      run: |
+        curl https://sh.rustup.rs -sSf | bash -s -- -y
+        echo "/root/.cargo/bin" >> $GITHUB_PATH
+    - name: Install npm dependencies
+      run: npm ci
+    - name: Build ${{ matrix.settings.libc }} library
+      run: |
+        BUILD_TARGET=${{ matrix.settings.arch }}-unknown-linux-${{ matrix.settings.libc }}
+        rustup target add $BUILD_TARGET
+        export RUSTFLAGS="-C target-feature=-crt-static" # for musl
+        npm run build-release -- --target $BUILD_TARGET
+        npm run pack-build -- --target $BUILD_TARGET
     - uses: softprops/action-gh-release@v1
       with:
         draft: true

ci/build_linux_artifacts.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -14,69 +13,28 @@
 
 set -e
 
-setup_dependencies() {
+build_node_binaries() {
-    echo "Installing system dependencies..."
-    if [[ $1 == *musl ]]; then
-        # musllinux
-        apk add openssl-dev
-    else
-        # manylinux2014
-        yum install -y openssl-devel unzip   
-    fi
-
-    if [[ $1 == x86_64* ]]; then
-        ARCH=x86_64
-    else
-        # gnu target
-        ARCH=aarch_64
-    fi
-    
-    # Install new enough protobuf (yum-provided is old)
-    PB_REL=https://github.com/protocolbuffers/protobuf/releases
-    PB_VERSION=23.1
-    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
-}
-
-install_node() {
-    echo "Installing node..."
-    if [[ $1 == *musl ]]; then
-        apk add nodejs-current npm
-    else
-        curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-        source "$HOME"/.nvm/nvm.sh
-        nvm install 17 # latest that supports glibc 2.17
-    fi
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-    rustup target add $1
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
     pushd node
     
-    if [[ $1 == *musl ]]; then
+    for target in $1
-        # This is needed for cargo to allow build cdylibs with musl
+    do
-        export RUSTFLAGS="-C target-feature=-crt-static"
+        echo "Building node library for $target"
-    fi
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
-    npm run build-release -- --target $1
+        export CROSS_CONTAINER_OPTS="--platform linux/amd64"
-    npm run pack-build -- --target $1
+        if [[ $target == *musl ]]; then
-
+            # This is needed for cargo to allow build cdylibs with musl
+            RUSTFLAGS="-C target-feature=-crt-static"
+        fi
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
     popd
 }
 
-TARGET=${1:-x86_64-unknown-linux-gnu}
+if [ -n "$1" ]; then
-# Others:
+    targets=$1
-# aarch64-unknown-linux-gnu
+else
-# x86_64-unknown-linux-musl
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
-# aarch64-unknown-linux-musl
+    targets="aarch64-unknown-linux-gnu"
-
+fi
-setup_dependencies $TARGET
+build_node_binaries $targets
-install_node
-install_rust $TARGET
-build_node_binary $TARGET

rust/ffi/node/release_process.md

@@ -68,37 +68,18 @@ bash ci/build_macos_artifacts.sh
 
 ### Build the Linux release libraries
 
-To build a Linux library, we need to use docker with a different build script:
+One-time setup, building the Docker container
+
+```shell
+cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+```
+
+To build:
 
 ```shell
-ARCH=aarch64
 docker run \
+    -v /var/run/docker.sock:/var/run/docker.sock \
     -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
+    lancedb-node-build \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
+    bash ci/build_linux_artifacts.sh
 ```
-
-You can change `ARCH` to `x86_64`.
-
-Similar script for musl binaries:
-
-```shell
-ARCH=aarch64
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
-```
-
-<!--
-
-For debugging, use this snippet:
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash
-```
--->