use manylinux containers locally

.github/workflows/release.yml

@@ -39,7 +39,6 @@ jobs:
         with:
           draft: true
           files: target/vectordb-*.crate
-          fail_on_unmatched_files: true
     
   python:
     runs-on: ubuntu-latest
@@ -67,7 +66,6 @@ jobs:
           files: |
             python/dist/lancedb-*.tar.gz 
             python/dist/lancedb-*.whl
-          fail_on_unmatched_files: true
 
   node:
     runs-on: ubuntu-latest
@@ -97,7 +95,6 @@ jobs:
         with:
           draft: true
           files: node/vectordb-*.tgz
-          fail_on_unmatched_files: true
 
   node-macos:
     runs-on: macos-12
@@ -121,10 +118,9 @@ jobs:
       with:
         draft: true
         files: node/dist/vectordb-darwin*.tgz
-        fail_on_unmatched_files: true
 
   node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
+    name: Linux ${{ matrix.settings.arch}} native node module
     runs-on: ubuntu-latest
     needs: draft-release
     strategy:
@@ -132,20 +128,17 @@ jobs:
       matrix:
         libc:
           - gnu
-          # TODO: re-enable musl once we have refactored to pre-built containers
+          - musl
-          # Right now we have to build node from source which is too expensive.
-          # - musl
         arch:
           - x86_64
           - aarch64
+    defaults:
+      run:
+        shell: bash
+        working-directory: node
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Set up QEMU
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2  
-      with:
-        platforms: arm64
     - name: Build Linux GNU native node modules
       if: ${{ matrix.libc == 'gnu' }}
       run: |
@@ -156,7 +149,7 @@ jobs:
     - name: Build musl Linux native node modules
       if: ${{ matrix.libc == 'musl' }}
       run: |
-        docker run --platform linux/arm64/v8 \
+        docker run \
           -v $(pwd):/io -w /io \
           quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
           bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
@@ -164,7 +157,6 @@ jobs:
       with:
         draft: true
         files: node/dist/vectordb-linux*.tgz
-        fail_on_unmatched_files: true
 
   release:
     needs: [python, node, node-macos, node-linux, rust]

ci/build_linux_artifacts.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -7,7 +6,10 @@
 # - aarch64-unknown-linux-musl
 # - x86_64-unknown-linux-musl
 
-# TODO: refactor this into a Docker container we can pull
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
 
 set -e
 
@@ -37,15 +39,11 @@ setup_dependencies() {
 
 install_node() {
     echo "Installing node..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-    source "$HOME"/.bashrc
-
     if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
+        apk add nodejs-current npm
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s 17
     else
+        curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+        source "$HOME"/.nvm/nvm.sh
         nvm install 17 # latest that supports glibc 2.17
     fi
 }
@@ -54,6 +52,7 @@ install_rust() {
     echo "Installing rust..."
     curl https://sh.rustup.rs -sSf | bash -s -- -y
     export PATH="$PATH:/root/.cargo/bin"
+    rustup target add $1
 }
 
 build_node_binary() {
@@ -64,10 +63,8 @@ build_node_binary() {
         # This is needed for cargo to allow build cdylibs with musl
         export RUSTFLAGS="-C target-feature=-crt-static"
     fi
-    # We don't pass in target, since the native target here already matches
+    npm run build-release -- --target $1
-    # and openblas-src doesn't do well with cross-compilation.
+    npm run pack-build -- --target $1
-    npm run build-release
-    npm run pack-build
 
     popd
 }
@@ -79,6 +76,6 @@ TARGET=${1:-x86_64-unknown-linux-gnu}
 # aarch64-unknown-linux-musl
 
 setup_dependencies $TARGET
-install_node $TARGET
+install_node
-install_rust
+install_rust $TARGET
 build_node_binary $TARGET

ci/ubuntu_build.dockerfile

@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1

rust/ffi/node/release_process.md

@@ -102,10 +102,3 @@ docker run -it \
     bash
 ```
 -->
-
-```
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_aarch64 \
-    bash alpine_repro.sh
-```