give up on musl for now

.github/workflows/release.yml

@@ -39,6 +39,7 @@ jobs:
         with:
           draft: true
           files: target/vectordb-*.crate
+          fail_on_unmatched_files: true
     
   python:
     runs-on: ubuntu-latest
@@ -66,6 +67,7 @@ jobs:
           files: |
             python/dist/lancedb-*.tar.gz 
             python/dist/lancedb-*.whl
+          fail_on_unmatched_files: true
 
   node:
     runs-on: ubuntu-latest
@@ -95,6 +97,7 @@ jobs:
         with:
           draft: true
           files: node/vectordb-*.tgz
+          fail_on_unmatched_files: true
 
   node-macos:
     runs-on: macos-12
@@ -118,9 +121,10 @@ jobs:
       with:
         draft: true
         files: node/dist/vectordb-darwin*.tgz
+        fail_on_unmatched_files: true
 
   node-linux:
-    name: Linux ${{ matrix.settings.arch}} native node module
+    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
     runs-on: ubuntu-latest
     needs: draft-release
     strategy:
@@ -128,54 +132,39 @@ jobs:
       matrix:
         libc:
           - gnu
-          - musl
+          # TODO: re-enable musl once we have refactored to pre-built containers
-        settings:
+          # Right now we have to build node from source which is too expensive.
-          - container: quay.io/pypa/manylinux2014_x86_64
+          # - musl
-            arch: x86_64
+        arch:
-            protoc_arch: x86_64
+          - x86_64
-          # TODO: find a way to build aarch64 on GA
+          - aarch64
-          # - container:  quay.io/pypa/manylinux2014_aarch64
-          #   arch: aarch64
-          #   protoc_arch: aarch_64
-    container:
-      image: ${{ matrix.settings.container }}
-      options: --platform linux/amd64
-    defaults:
-      run:
-        shell: bash
-        working-directory: node
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Install system dependencies
+    - name: Set up QEMU
-      run: |
+      if: ${{ matrix.arch == 'aarch64' }}
-        yum install -y openssl-devel unzip
+      uses: docker/setup-qemu-action@v2  
-        # Install new enough protobuf (yum-provided is old)
-        PB_REL=https://github.com/protocolbuffers/protobuf/releases
-        PB_VERSION=23.1
-        curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip
-        unzip protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip -d /usr/local
-    - name: Setup Node
-      uses: actions/setup-node@v3
       with:
-        node-version: 17 # Last that support glibc 2.17
+        platforms: arm64
-    - name: Setup Rust
+    - name: Build Linux GNU native node modules
+      if: ${{ matrix.libc == 'gnu' }}
       run: |
-        curl https://sh.rustup.rs -sSf | bash -s -- -y
+        docker run \
-        echo "/root/.cargo/bin" >> $GITHUB_PATH
+          -v $(pwd):/io -w /io \
-    - name: Install npm dependencies
+          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
-      run: npm ci
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
-    - name: Build ${{ matrix.settings.libc }} library
+    - name: Build musl Linux native node modules
+      if: ${{ matrix.libc == 'musl' }}
       run: |
-        BUILD_TARGET=${{ matrix.settings.arch }}-unknown-linux-${{ matrix.settings.libc }}
+        docker run --platform linux/arm64/v8 \
-        rustup target add $BUILD_TARGET
+          -v $(pwd):/io -w /io \
-        export RUSTFLAGS="-C target-feature=-crt-static" # for musl
+          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-        npm run build-release -- --target $BUILD_TARGET
+          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
-        npm run pack-build -- --target $BUILD_TARGET
     - uses: softprops/action-gh-release@v1
       with:
         draft: true
         files: node/dist/vectordb-linux*.tgz
+        fail_on_unmatched_files: true
 
   release:
     needs: [python, node, node-macos, node-linux, rust]

ci/build_linux_artifacts.sh

@@ -1,3 +1,4 @@
+#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -6,35 +7,78 @@
 # - aarch64-unknown-linux-musl
 # - x86_64-unknown-linux-musl
 
-# On MacOS, need to run in a linux container:
+# TODO: refactor this into a Docker container we can pull
-# docker run -v $(pwd):/io -w /io 
-
-# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
 
 set -e
 
-build_node_binaries() {
+setup_dependencies() {
+    echo "Installing system dependencies..."
+    if [[ $1 == *musl ]]; then
+        # musllinux
+        apk add openssl-dev
+    else
+        # manylinux2014
+        yum install -y openssl-devel unzip   
+    fi
+
+    if [[ $1 == x86_64* ]]; then
+        ARCH=x86_64
+    else
+        # gnu target
+        ARCH=aarch_64
+    fi
+    
+    # Install new enough protobuf (yum-provided is old)
+    PB_REL=https://github.com/protocolbuffers/protobuf/releases
+    PB_VERSION=23.1
+    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
+    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
+}
+
+install_node() {
+    echo "Installing node..."
+    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+    source "$HOME"/.bashrc
+
+    if [[ $1 == *musl ]]; then
+        # This node version is 15, we need 16 or higher:
+        # apk add nodejs-current npm 
+        # So instead we install from source (nvm doesn't provide binaries for musl):
+        nvm install -s 17
+    else
+        nvm install 17 # latest that supports glibc 2.17
+    fi
+}
+
+install_rust() {
+    echo "Installing rust..."
+    curl https://sh.rustup.rs -sSf | bash -s -- -y
+    export PATH="$PATH:/root/.cargo/bin"
+}
+
+build_node_binary() {
+    echo "Building node library for $1..."
     pushd node
     
-    for target in $1
+    if [[ $1 == *musl ]]; then
-    do
+        # This is needed for cargo to allow build cdylibs with musl
-        echo "Building node library for $target"
+        export RUSTFLAGS="-C target-feature=-crt-static"
-        # cross doesn't yet pass this down to Docker, so we do it ourselves.
+    fi
-        export CROSS_CONTAINER_OPTS="--platform linux/amd64"
+    # We don't pass in target, since the native target here already matches
-        if [[ $target == *musl ]]; then
+    # and openblas-src doesn't do well with cross-compilation.
-            # This is needed for cargo to allow build cdylibs with musl
+    npm run build-release
-            RUSTFLAGS="-C target-feature=-crt-static"
+    npm run pack-build
-        fi
+
-        npm run cross-release -- --target $target
-        npm run pack-build -- --target $target
-    done
     popd
 }
 
-if [ -n "$1" ]; then
+TARGET=${1:-x86_64-unknown-linux-gnu}
-    targets=$1
+# Others:
-else
+# aarch64-unknown-linux-gnu
-    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
+# x86_64-unknown-linux-musl
-    targets="aarch64-unknown-linux-gnu"
+# aarch64-unknown-linux-musl
-fi
+
-build_node_binaries $targets
+setup_dependencies $TARGET
+install_node $TARGET
+install_rust
+build_node_binary $TARGET

ci/release_process.md

@@ -68,18 +68,44 @@ bash ci/build_macos_artifacts.sh
 
 ### Build the Linux release libraries
 
-One-time setup, building the Docker container
+To build a Linux library, we need to use docker with a different build script:
-
-```shell
-cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
-```
-
-To build:
 
 ```shell
+ARCH=aarch64
 docker run \
-    -v /var/run/docker.sock:/var/run/docker.sock \
     -v $(pwd):/io -w /io \
-    lancedb-node-build \
+    quay.io/pypa/manylinux2014_$ARCH \
-    bash ci/build_linux_artifacts.sh
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
 ```
+
+You can change `ARCH` to `x86_64`.
+
+Similar script for musl binaries:
+
+```shell
+ARCH=aarch64
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
+```
+
+<!--
+
+For debugging, use this snippet:
+
+```shell
+ARCH=aarch64
+docker run -it \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_$ARCH \
+    bash
+```
+-->
+
+```
+docker run \
+    -v $(pwd):/io -w /io \
+    quay.io/pypa/musllinux_1_1_aarch64 \
+    bash alpine_repro.sh
+```

ci/ubuntu_build.dockerfile

@@ -1,31 +0,0 @@
-# On MacOS, need to run in a linux container:
-# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
-# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
-FROM ubuntu:20.04
-
-ARG DEBIAN_FRONTEND=noninteractive
-ENV TZ=Europe/Moscow
-
-RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
-    software-properties-common npm docker.io
-
-# Install rust
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
-
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-# Install cross
-# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
-RUN cargo install cross --git https://github.com/cross-rs/cross
-
-# Install additional build targets
-RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
-
-# Install node
-RUN npm install npm@latest -g && \
-    npm install n -g && \
-    n latest
-
-# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
-ENV CROSS_CONTAINER_IN_CONTAINER=true
-ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1