fixes for action

.github/workflows/release.yml

@@ -38,8 +38,8 @@ jobs:
       - uses: softprops/action-gh-release@v1
         with:
           draft: true
-          files: target/vectordb-*.crate
+          files: |
-          fail_on_unmatched_files: true
+            rust/target/vectordb-*.crate
     
   python:
     runs-on: ubuntu-latest
@@ -58,16 +58,13 @@ jobs:
         with:
           python-version: "3.10"
       - name: Build wheel
-        run: |
+        run: python setup.py sdist bdist_wheel
-          pip install wheel
-          python setup.py sdist bdist_wheel
       - uses: softprops/action-gh-release@v1
         with:
           draft: true
           files: |
             python/dist/lancedb-*.tar.gz 
             python/dist/lancedb-*.whl
-          fail_on_unmatched_files: true
 
   node:
     runs-on: ubuntu-latest
@@ -84,6 +81,7 @@ jobs:
           node-version: 20
           cache: 'npm'
           cache-dependency-path: node/package-lock.json
+      - uses: Swatinem/rust-cache@v2
       - name: Install dependencies
         run: |
           sudo apt update
@@ -97,13 +95,11 @@ jobs:
         with:
           draft: true
           files: node/vectordb-*.tgz
-          fail_on_unmatched_files: true
 
   node-macos:
     runs-on: macos-12
     needs: draft-release
     strategy:
-      fail-fast: false
       matrix:
         target: [x86_64-apple-darwin, aarch64-apple-darwin]
     steps:
@@ -112,59 +108,40 @@ jobs:
     - name: Install system dependencies
       run: brew install protobuf
     - name: Install npm dependencies
-      run: |
+      run: npm ci
-        cd node
-        npm ci
     - name: Build MacOS native node modules
       run: bash ci/build_macos_artifacts.sh ${{ matrix.target }}
     - uses: softprops/action-gh-release@v1
       with:
         draft: true
         files: node/dist/vectordb-darwin*.tgz
-        fail_on_unmatched_files: true
 
   node-linux:
-    name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }})
     runs-on: ubuntu-latest
     needs: draft-release
     strategy:
-      fail-fast: false
       matrix:
-        libc:
+        target:
-          - gnu
+          - x86_64-unknown-linux-gnu:centos
-          # TODO: re-enable musl once we have refactored to pre-built containers
+          - aarch64-unknown-linux-gnu:centos
-          # Right now we have to build node from source which is too expensive.
+          - aarch64-unknown-linux-musl
-          # - musl
+          - x86_64-unknown-linux-musl
-        arch:
-          - x86_64
-          - aarch64
     steps:
     - name: Checkout
       uses: actions/checkout@v2
-    - name: Set up QEMU
+    - name: Install system dependencies
-      if: ${{ matrix.arch == 'aarch64' }}
-      uses: docker/setup-qemu-action@v2  
-      with:
-        platforms: arm64
-    - name: Build Linux GNU native node modules
-      if: ${{ matrix.libc == 'gnu' }}
       run: |
-        docker run \
+        sudo apt update
-          -v $(pwd):/io -w /io \
+        sudo apt install -y protobuf-compiler libssl-dev
-          quay.io/pypa/manylinux2014_${{ matrix.arch }} \
+        cargo install cross
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu
+    - name: Install npm dependencies
-    - name: Build musl Linux native node modules
+      run: npm ci
-      if: ${{ matrix.libc == 'musl' }}
+    - name: Build Linux native node modules
-      run: |
+      run: bash ci/build_linux_artifacts.sh ${{ matrix.target }}
-        docker run --platform linux/arm64/v8 \
-          -v $(pwd):/io -w /io \
-          quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \
-          bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl
     - uses: softprops/action-gh-release@v1
       with:
         draft: true
         files: node/dist/vectordb-linux*.tgz
-        fail_on_unmatched_files: true
 
   release:
     needs: [python, node, node-macos, node-linux, rust]

Cross.toml

@@ -1,39 +1,14 @@
 # These make sure our builds are compatible with old glibc versions.
 [target.x86_64-unknown-linux-gnu]
 pre-build = [
-    # Install newer gfortran
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "yum install -y openssl-devel unzip gcc-gfortran",
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
-    "scl enable devtoolset-11 bash",
-    # protobuf is too old, so we directly download binaries
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
 ]
 image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos"
 
 [target.aarch64-unknown-linux-gnu]
 pre-build = [
-    "yum install -y openssl-devel unzip",
+    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    # protobuf is too old, so we directly download binaries
+    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler",
-    "PB_REL=https://github.com/protocolbuffers/protobuf/releases",
-    "PB_VERSION=23.1",
-    "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip",
-    "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local",
 ]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos
 image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos"
-
-[target.x86_64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]
-
-[target.aarch64-unknown-linux-musl]
-# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl
-pre-build = [
-    "dpkg --add-architecture $CROSS_DEB_ARCH",
-    "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH",
-]

ci/build_linux_artifacts.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 # Builds the Linux artifacts (node binaries).
 # Usage: ./build_linux_artifacts.sh [target]
 # Targets supported: 
@@ -7,78 +6,35 @@
 # - aarch64-unknown-linux-musl
 # - x86_64-unknown-linux-musl
 
-# TODO: refactor this into a Docker container we can pull
+# On MacOS, need to run in a linux container:
+# docker run -v $(pwd):/io -w /io 
+
+# Must run rustup toolchain install stable-x86_64-unknown-linux-gnu --force-non-host
 
 set -e
 
-setup_dependencies() {
+build_node_binaries() {
-    echo "Installing system dependencies..."
-    if [[ $1 == *musl ]]; then
-        # musllinux
-        apk add openssl-dev
-    else
-        # manylinux2014
-        yum install -y openssl-devel unzip   
-    fi
-
-    if [[ $1 == x86_64* ]]; then
-        ARCH=x86_64
-    else
-        # gnu target
-        ARCH=aarch_64
-    fi
-    
-    # Install new enough protobuf (yum-provided is old)
-    PB_REL=https://github.com/protocolbuffers/protobuf/releases
-    PB_VERSION=23.1
-    curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip
-    unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local
-}
-
-install_node() {
-    echo "Installing node..."
-    curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-    source "$HOME"/.bashrc
-
-    if [[ $1 == *musl ]]; then
-        # This node version is 15, we need 16 or higher:
-        # apk add nodejs-current npm 
-        # So instead we install from source (nvm doesn't provide binaries for musl):
-        nvm install -s 17
-    else
-        nvm install 17 # latest that supports glibc 2.17
-    fi
-}
-
-install_rust() {
-    echo "Installing rust..."
-    curl https://sh.rustup.rs -sSf | bash -s -- -y
-    export PATH="$PATH:/root/.cargo/bin"
-}
-
-build_node_binary() {
-    echo "Building node library for $1..."
     pushd node
     
-    if [[ $1 == *musl ]]; then
+    for target in $1
-        # This is needed for cargo to allow build cdylibs with musl
+    do
-        export RUSTFLAGS="-C target-feature=-crt-static"
+        echo "Building node library for $target"
-    fi
+        # cross doesn't yet pass this down to Docker, so we do it ourselves.
-    # We don't pass in target, since the native target here already matches
+        if [[ $target == x86_64* ]]; then
-    # and openblas-src doesn't do well with cross-compilation.
+            export CROSS_CONTAINER_OPTS="--platform linux/amd64"
-    npm run build-release
+        else
-    npm run pack-build
+            export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8"
-
+        fi
+        npm run cross-release -- --target $target
+        npm run pack-build -- --target $target
+    done
     popd
 }
 
-TARGET=${1:-x86_64-unknown-linux-gnu}
+if [ -n "$1" ]; then
-# Others:
+    targets=$1
-# aarch64-unknown-linux-gnu
+else
-# x86_64-unknown-linux-musl
+    # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl"
-# aarch64-unknown-linux-musl
+    targets="aarch64-unknown-linux-gnu"
-
+fi
-setup_dependencies $TARGET
+build_node_binaries $targets
-install_node $TARGET
-install_rust
-build_node_binary $TARGET

ci/ubuntu_build.dockerfile

@@ -0,0 +1,31 @@
+# On MacOS, need to run in a linux container:
+# cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
+# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd):/io -w /io lancedb-node-build bash ci/build_linux_artifacts.sh
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Moscow
+
+RUN apt update && apt install -y protobuf-compiler libssl-dev build-essential curl \
+    software-properties-common npm docker.io
+
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install cross
+# https://github.com/cross-rs/cross/issues/1257#issuecomment-1544553706
+RUN cargo install cross --git https://github.com/cross-rs/cross
+
+# Install additional build targets
+RUN rustup target add x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl
+
+# Install node
+RUN npm install npm@latest -g && \
+    npm install n -g && \
+    n latest
+
+# set CROSS_CONTAINER_IN_CONTAINER to inform `cross` that it is executed from within a container
+ENV CROSS_CONTAINER_IN_CONTAINER=true
+ENV CROSS_CONTAINER_ENGINE_NO_BUILDKIT=1

rust/ffi/node/release_process.md

@@ -68,44 +68,18 @@ bash ci/build_macos_artifacts.sh
 
 ### Build the Linux release libraries
 
-To build a Linux library, we need to use docker with a different build script:
+One-time setup, building the Docker container
 
 ```shell
-ARCH=aarch64
+cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build -
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/manylinux2014_$ARCH \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu
 ```
 
-You can change `ARCH` to `x86_64`.
+To build:
-
-Similar script for musl binaries:
 
 ```shell
-ARCH=aarch64
 docker run \
+    -v /var/run/docker.sock:/var/run/docker.sock \
     -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
+    lancedb-node-build \
-    bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl
+    bash ci/build_linux_artifacts.sh
 ```
-
-<!--
-
-For debugging, use this snippet:
-
-```shell
-ARCH=aarch64
-docker run -it \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_$ARCH \
-    bash
-```
--->
-
-```
-docker run \
-    -v $(pwd):/io -w /io \
-    quay.io/pypa/musllinux_1_1_aarch64 \
-    bash alpine_repro.sh
-```