From c4237c61d5d7f2fde2a577aa030d676f48dab9e7 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 23 May 2023 18:59:45 -0700 Subject: [PATCH] fixes for action --- .github/workflows/release.yml | 43 +++++++++++++++++++++++++++-------- Cross.toml | 34 +++++++++++++++++++++++---- ci/build_linux_artifacts.sh | 8 +++---- 3 files changed, 67 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d0f85628..8d393ada 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,8 +37,8 @@ jobs: run: cargo package --all-features - uses: softprops/action-gh-release@v1 with: - files: | - rust/target/vectordb-*.crate + draft: true + files: target/vectordb-*.crate python: runs-on: ubuntu-latest @@ -55,11 +55,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.10 + python-version: "3.10" - name: Build wheel - run: python setup.py sdist bdist_wheel + run: | + pip install wheel + python setup.py sdist bdist_wheel - uses: softprops/action-gh-release@v1 with: + draft: true files: | python/dist/lancedb-*.tar.gz python/dist/lancedb-*.whl @@ -79,7 +82,6 @@ jobs: node-version: 20 cache: 'npm' cache-dependency-path: node/package-lock.json - - uses: Swatinem/rust-cache@v2 - name: Install dependencies run: | sudo apt update @@ -91,12 +93,14 @@ jobs: npm pack - uses: softprops/action-gh-release@v1 with: + draft: true files: node/vectordb-*.tgz node-macos: runs-on: macos-12 needs: draft-release strategy: + fail-fast: false matrix: target: [x86_64-apple-darwin, aarch64-apple-darwin] steps: @@ -104,35 +108,54 @@ jobs: uses: actions/checkout@v2 - name: Install system dependencies run: brew install protobuf + - name: Install npm dependencies + run: | + cd node + npm ci - name: Build MacOS native node modules run: bash ci/build_macos_artifacts.sh ${{ matrix.target }} - uses: softprops/action-gh-release@v1 with: + draft: true files: node/dist/vectordb-darwin*.tgz node-linux: runs-on: ubuntu-latest needs: draft-release strategy: + fail-fast: false matrix: target: - - x86_64-unknown-linux-gnu:centos - - aarch64-unknown-linux-gnu:centos - - aarch64-unknown-linux-musl - - x86_64-unknown-linux-musl + - x86_64-unknown-linux-gnu + # TODO: get release working for these targets + # - aarch64-unknown-linux-gnu + # - aarch64-unknown-linux-musl + # - x86_64-unknown-linux-musl steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Rust + uses: ATiltedTree/setup-rust@v1 + with: + rust-version: stable - name: Install system dependencies run: | sudo apt update sudo apt install -y protobuf-compiler libssl-dev + cargo install cross + - name: Install npm dependencies + run: | + cd node + npm ci - name: Build Linux native node modules run: bash ci/build_linux_artifacts.sh ${{ matrix.target }} - uses: softprops/action-gh-release@v1 with: + draft: true files: node/dist/vectordb-linux*.tgz release: - needs: [python, node, node-macos, node-linux] + needs: [python, node, node-macos, node-linux, rust] runs-on: ubuntu-latest steps: - uses: actions/download-artifact@v3 diff --git a/Cross.toml b/Cross.toml index 6ec737f6..0ee2864f 100644 --- a/Cross.toml +++ b/Cross.toml @@ -1,14 +1,40 @@ # These make sure our builds are compatible with old glibc versions. [target.x86_64-unknown-linux-gnu] pre-build = [ - "dpkg --add-architecture $CROSS_DEB_ARCH", - "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler", + # Install newer gfortran + "yum install -y centos-release-scl", + "yum install -y openssl-devel unzip devtoolset-11-gcc-gfortran", + "scl enable devtoolset-11 bash", + # protobuf is too old, so we directly download binaries + "PB_REL=https://github.com/protocolbuffers/protobuf/releases", + "PB_VERSION=23.1", + "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", + "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local", ] image = "ghcr.io/cross-rs/x86_64-unknown-linux-gnu:main-centos" [target.aarch64-unknown-linux-gnu] pre-build = [ - "dpkg --add-architecture $CROSS_DEB_ARCH", - "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH protobuf-compiler", + "yum install -y openssl-devel unzip", + # protobuf is too old, so we directly download binaries + "PB_REL=https://github.com/protocolbuffers/protobuf/releases", + "PB_VERSION=23.1", + "curl -LO $PB_REL/download/v$PB_VERSION/protoc-$PB_VERSION-linux-x86_64.zip", + "unzip protoc-$PB_VERSION-linux-x86_64.zip -d /usr/local", ] +# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-gnu.centos image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main-centos" + +[target.x86_64-unknown-linux-musl] +# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.x86_64-unknown-linux-musl +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH", +] + +[target.aarch64-unknown-linux-musl] +# https://github.com/cross-rs/cross/blob/main/docker/Dockerfile.aarch64-unknown-linux-musl +pre-build = [ + "dpkg --add-architecture $CROSS_DEB_ARCH", + "apt-get update && apt-get install --assume-yes libssl-dev:$CROSS_DEB_ARCH", +] diff --git a/ci/build_linux_artifacts.sh b/ci/build_linux_artifacts.sh index 200f73ea..ba7826ec 100644 --- a/ci/build_linux_artifacts.sh +++ b/ci/build_linux_artifacts.sh @@ -20,10 +20,10 @@ build_node_binaries() { do echo "Building node library for $target" # cross doesn't yet pass this down to Docker, so we do it ourselves. - if [[ $target == x86_64* ]]; then - export CROSS_CONTAINER_OPTS="--platform linux/amd64" - else - export CROSS_CONTAINER_OPTS="--platform linux/arm64/v8" + export CROSS_CONTAINER_OPTS="--platform linux/amd64" + if [[ $target == *musl ]]; then + # This is needed for cargo to allow build cdylibs with musl + RUSTFLAGS="-C target-feature=-crt-static" fi npm run cross-release -- --target $target npm run pack-build -- --target $target