diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml new file mode 100644 index 000000000..a99cf3320 --- /dev/null +++ b/.github/actions/setup-protoc/action.yml @@ -0,0 +1,80 @@ +name: Setup protoc +description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves + +inputs: + install: + description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there." + required: false + default: "true" + +outputs: + version: + description: "The pinned protoc version, e.g. 36.2" + value: ${{ steps.pin.outputs.version }} + linux-x86_64-sha256: + description: "SHA-256 of the protoc linux-x86_64 zip" + value: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + linux-aarch_64-sha256: + description: "SHA-256 of the protoc linux-aarch_64 zip" + value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + +runs: + using: composite + steps: + # protobuf does not publish checksum files for protoc. These are the digests + # GitHub records for the release assets; when bumping, copy them from + # `gh api repos/protocolbuffers/protobuf/releases/tags/v`. + - name: Set protoc pin + id: pin + shell: bash + run: | + { + echo "version=36.2" + echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278" + echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a" + echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b" + echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3" + } >> "$GITHUB_OUTPUT" + - name: Install protoc + if: ${{ inputs.install == 'true' }} + shell: bash + env: + PROTOC_VERSION: ${{ steps.pin.outputs.version }} + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }} + WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }} + run: | + set -euo pipefail + case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in + Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;; + Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;; + macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;; + Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;; + *) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;; + esac + zip="$RUNNER_TEMP/protoc.zip" + dest="$RUNNER_TEMP/protoc" + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip" + # Hash stdin rather than a path: Windows paths contain backslashes, + # which make sha256sum escape its output. + if [ "$RUNNER_OS_NAME" = "macOS" ]; then + actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1) + else + actual=$(sha256sum < "$zip" | cut -d' ' -f1) + fi + if [ "$actual" != "$sha256" ]; then + echo "protoc checksum mismatch: expected $sha256, got $actual" >&2 + exit 1 + fi + if [ "$RUNNER_OS_NAME" = "Windows" ]; then + # Git Bash on Windows runners does not reliably ship unzip. + 7z x -y -o"$dest" "$zip" > /dev/null + else + unzip -q -o "$zip" -d "$dest" + fi + rm "$zip" + echo "$dest/bin" >> "$GITHUB_PATH" + "$dest/bin/protoc" --version diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index 8fa5b1f79..44ce82ee5 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -29,6 +29,11 @@ inputs: runs: using: "composite" steps: + - name: Get protoc version + id: protoc + uses: ./.github/actions/setup-protoc + with: + install: "false" - name: CONFIRM ARM BUILD shell: bash env: @@ -54,13 +59,14 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version @@ -71,14 +77,15 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e yum install -y clang - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml index 75d942c96..727cd190c 100644 --- a/.github/workflows/build_mac_wheel/action.yml +++ b/.github/workflows/build_mac_wheel/action.yml @@ -12,10 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install macos dependency - shell: bash - run: | - brew install protobuf + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 18f39818f..954ee7d35 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -12,15 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 4f883aec1..d16203979 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -11,6 +11,7 @@ on: - .github/workflows/build_linux_wheel/action.yml - .github/workflows/build_mac_wheel/action.yml - .github/workflows/build_windows_wheel/action.yml + - .github/actions/setup-protoc/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3594bd826..3608c24ca 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -15,6 +15,7 @@ on: - .github/workflows/build_linux_wheel/** - .github/workflows/build_mac_wheel/** - .github/workflows/build_windows_wheel/** + - .github/actions/setup-protoc/** - .github/workflows/run_tests/** permissions: