diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 46748329..272595bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,7 +120,7 @@ jobs: files: node/dist/vectordb-darwin*.tgz node-linux: - name: Linux ${{ matrix.settings.arch}} native node module + name: node-linux (${{ matrix.arch}}-unknown-linux-${{ matrix.libc }}) runs-on: ubuntu-latest needs: draft-release strategy: @@ -129,49 +129,26 @@ jobs: libc: - gnu - musl - settings: - - container: quay.io/pypa/manylinux2014_x86_64 - arch: x86_64 - protoc_arch: x86_64 - # TODO: find a way to build aarch64 on GA - # - container: quay.io/pypa/manylinux2014_aarch64 - # arch: aarch64 - # protoc_arch: aarch_64 - container: - image: ${{ matrix.settings.container }} - options: --platform linux/amd64 - defaults: - run: - shell: bash - working-directory: node + arch: + - x86_64 + - aarch64 steps: - name: Checkout uses: actions/checkout@v2 - - name: Install system dependencies + - name: Build Linux GNU native node modules + if: ${{ matrix.libc == 'gnu' }} run: | - yum install -y openssl-devel unzip - # Install new enough protobuf (yum-provided is old) - PB_REL=https://github.com/protocolbuffers/protobuf/releases - PB_VERSION=23.1 - curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip - unzip protoc-$PB_VERSION-linux-${{ matrix.settings.protoc_arch }}.zip -d /usr/local - - name: Setup Node - uses: actions/setup-node@v3 - with: - node-version: 17 # Last that support glibc 2.17 - - name: Setup Rust + docker run \ + -v $(pwd):/io -w /io \ + quay.io/pypa/manylinux2014_${{ matrix.arch }} \ + bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-gnu + - name: Build musl Linux native node modules + if: ${{ matrix.libc == 'musl' }} run: | - curl https://sh.rustup.rs -sSf | bash -s -- -y - echo "/root/.cargo/bin" >> $GITHUB_PATH - - name: Install npm dependencies - run: npm ci - - name: Build ${{ matrix.settings.libc }} library - run: | - BUILD_TARGET=${{ matrix.settings.arch }}-unknown-linux-${{ matrix.settings.libc }} - rustup target add $BUILD_TARGET - export RUSTFLAGS="-C target-feature=-crt-static" # for musl - npm run build-release -- --target $BUILD_TARGET - npm run pack-build -- --target $BUILD_TARGET + docker run \ + -v $(pwd):/io -w /io \ + quay.io/pypa/musllinux_1_1_${{ matrix.arch }} \ + bash ci/build_linux_artifacts.sh ${{ matrix.arch }}-unknown-linux-musl - uses: softprops/action-gh-release@v1 with: draft: true diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh index ba7826ec..515e9923 100644 --- a/ci/build_linux_artifacts.sh +++ b/ci/build_linux_artifacts.sh @@ -1,3 +1,4 @@ +#!/bin/bash # Builds the Linux artifacts (node binaries). # Usage: ./build_linux_artifacts.sh [target] # Targets supported: @@ -13,28 +14,69 @@ set -e -build_node_binaries() { +setup_dependencies() { + echo "Installing system dependencies..." + if [[ $1 == *musl ]]; then + # musllinux + apk add openssl-dev + else + # manylinux2014 + yum install -y openssl-devel unzip + fi + + if [[ $1 == x86_64* ]]; then + ARCH=x86_64 + else + # gnu target + ARCH=aarch_64 + fi + + # Install new enough protobuf (yum-provided is old) + PB_REL=https://github.com/protocolbuffers/protobuf/releases + PB_VERSION=23.1 + curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-$ARCH.zip + unzip protoc-$PB_VERSION-linux-$ARCH.zip -d /usr/local +} + +install_node() { + echo "Installing node..." + if [[ $1 == *musl ]]; then + apk add nodejs-current npm + else + curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash + source "$HOME"/.nvm/nvm.sh + nvm install 17 # latest that supports glibc 2.17 + fi +} + +install_rust() { + echo "Installing rust..." + curl https://sh.rustup.rs -sSf | bash -s -- -y + export PATH="$PATH:/root/.cargo/bin" + rustup target add $1 +} + +build_node_binary() { + echo "Building node library for $1..." pushd node - for target in $1 - do - echo "Building node library for $target" - # cross doesn't yet pass this down to Docker, so we do it ourselves. - export CROSS_CONTAINER_OPTS="--platform linux/amd64" - if [[ $target == *musl ]]; then - # This is needed for cargo to allow build cdylibs with musl - RUSTFLAGS="-C target-feature=-crt-static" - fi - npm run cross-release -- --target $target - npm run pack-build -- --target $target - done + if [[ $1 == *musl ]]; then + # This is needed for cargo to allow build cdylibs with musl + export RUSTFLAGS="-C target-feature=-crt-static" + fi + npm run build-release -- --target $1 + npm run pack-build -- --target $1 + popd } -if [ -n "$1" ]; then - targets=$1 -else - # targets="x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu aarch64-unknown-linux-musl x86_64-unknown-linux-musl" - targets="aarch64-unknown-linux-gnu" -fi -build_node_binaries $targets \ No newline at end of file +TARGET=${1:-x86_64-unknown-linux-gnu} +# Others: +# aarch64-unknown-linux-gnu +# x86_64-unknown-linux-musl +# aarch64-unknown-linux-musl + +setup_dependencies $TARGET +install_node +install_rust $TARGET +build_node_binary $TARGET diff --git a/rust/ffi/node/release_process.md b/rust/ffi/node/release_process.md index 2dcdfdaa..99b688c0 100644 --- a/rust/ffi/node/release_process.md +++ b/rust/ffi/node/release_process.md @@ -68,18 +68,37 @@ bash ci/build_macos_artifacts.sh ### Build the Linux release libraries -One-time setup, building the Docker container - -```shell -cat ci/ubuntu_build.dockerfile | docker build -t lancedb-node-build - -``` - -To build: +To build a Linux library, we need to use docker with a different build script: ```shell +ARCH=aarch64 docker run \ - -v /var/run/docker.sock:/var/run/docker.sock \ -v $(pwd):/io -w /io \ - lancedb-node-build \ - bash ci/build_linux_artifacts.sh + quay.io/pypa/manylinux2014_$ARCH \ + bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-gnu ``` + +You can change `ARCH` to `x86_64`. + +Similar script for musl binaries: + +```shell +ARCH=aarch64 +docker run \ + -v $(pwd):/io -w /io \ + quay.io/pypa/musllinux_1_1_$ARCH \ + bash ci/build_linux_artifacts.sh $ARCH-unknown-linux-musl +``` + +