From eabec507a874d8b7ff643f181a815ff6e0ae5046 Mon Sep 17 00:00:00 2001 From: Will Jones Date: Wed, 14 Oct 2026 10:58:12 -0700 Subject: [PATCH] ci: pin protoc v36.2 with checksums across wheel builds (#4535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The Python wheel builds used different protoc versions: v24.4 on Linux, v21.12 on Windows, and whatever Homebrew shipped on macOS. None of them verified the download. A new composite action, `.github/actions/setup-protoc`, now holds one pin, v36.2, with a SHA-256 for each platform's zip. - macOS and Windows: the action downloads protoc, checks its hash, and installs it. - Linux: protoc has to be installed inside the manylinux container, so the build reads the version and hash from the action's outputs and checks the hash there. protobuf does not publish checksum files for protoc. The pinned hashes are the digests GitHub recorded for the v36.2 release assets. Older releases such as v24.4 have none, which is why this moves to v36.2. protoc only runs at build time, through `prost-build` in the Lance crates. The Linux protoc binary is statically linked, so it runs in the manylinux 2_17 container. The action is also added to the path triggers of `pypi-publish.yml` and `python.yml`, so a change to the pin runs the wheel builds. Follow-up to https://github.com/lancedb/lancedb/pull/4522#issuecomment-6160848344. The sdist job in that PR can use the same action once this merges. ## Evidence Running the action's install script locally on linux-aarch_64 installs `libprotoc 36.2`, and a wrong hash fails with `protoc checksum mismatch`. `cargo check -p lancedb` with `PROTOC` set to protoc 36.2 builds the Lance protos. The PyPI Publish dry run on this PR covers the Linux, macOS and Windows builds. ## Merge Risk **Door:** Two-way. CI only. **Blast Radius:** Python wheel builds. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .github/actions/setup-protoc/action.yml | 80 +++++++++++++++++++ .../workflows/build_linux_wheel/action.yml | 15 +++- .github/workflows/build_mac_wheel/action.yml | 5 +- .../workflows/build_windows_wheel/action.yml | 10 +-- .github/workflows/pypi-publish.yml | 1 + .github/workflows/python.yml | 1 + 6 files changed, 95 insertions(+), 17 deletions(-) create mode 100644 .github/actions/setup-protoc/action.yml diff --git a/.github/actions/setup-protoc/action.yml b/.github/actions/setup-protoc/action.yml new file mode 100644 index 000000000..a99cf3320 --- /dev/null +++ b/.github/actions/setup-protoc/action.yml @@ -0,0 +1,80 @@ +name: Setup protoc +description: Install the protoc version pinned for wheel builds, and expose it for builds that install protoc themselves + +inputs: + install: + description: "Install protoc on the runner. Set to false when the build runs in a container (e.g. manylinux) and installs protoc there." + required: false + default: "true" + +outputs: + version: + description: "The pinned protoc version, e.g. 36.2" + value: ${{ steps.pin.outputs.version }} + linux-x86_64-sha256: + description: "SHA-256 of the protoc linux-x86_64 zip" + value: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + linux-aarch_64-sha256: + description: "SHA-256 of the protoc linux-aarch_64 zip" + value: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + +runs: + using: composite + steps: + # protobuf does not publish checksum files for protoc. These are the digests + # GitHub records for the release assets; when bumping, copy them from + # `gh api repos/protocolbuffers/protobuf/releases/tags/v`. + - name: Set protoc pin + id: pin + shell: bash + run: | + { + echo "version=36.2" + echo "linux-x86_64-sha256=121f6c7afe1d4d0e3ea6aab9432038599250134cbf4474cb1167d2c7decd4278" + echo "linux-aarch_64-sha256=8b8f18bd2b30346efbc698dd5a73dd7c805f3ef8380f6dfc95c768f3f1852f6a" + echo "osx-universal_binary-sha256=83d0dc80672f53486f54299bca6177159f9cad1fd54d4d3143eeea53c4423e1b" + echo "win64-sha256=f0c128dc0d8492eceece83bb459a4c0e316764b929ffbf1aa416357fd644edd3" + } >> "$GITHUB_OUTPUT" + - name: Install protoc + if: ${{ inputs.install == 'true' }} + shell: bash + env: + PROTOC_VERSION: ${{ steps.pin.outputs.version }} + RUNNER_OS_NAME: ${{ runner.os }} + RUNNER_ARCH_NAME: ${{ runner.arch }} + LINUX_X86_64_SHA256: ${{ steps.pin.outputs.linux-x86_64-sha256 }} + LINUX_AARCH_64_SHA256: ${{ steps.pin.outputs.linux-aarch_64-sha256 }} + OSX_SHA256: ${{ steps.pin.outputs.osx-universal_binary-sha256 }} + WIN64_SHA256: ${{ steps.pin.outputs.win64-sha256 }} + run: | + set -euo pipefail + case "$RUNNER_OS_NAME/$RUNNER_ARCH_NAME" in + Linux/X64) platform="linux-x86_64"; sha256="$LINUX_X86_64_SHA256" ;; + Linux/ARM64) platform="linux-aarch_64"; sha256="$LINUX_AARCH_64_SHA256" ;; + macOS/*) platform="osx-universal_binary"; sha256="$OSX_SHA256" ;; + Windows/X64) platform="win64"; sha256="$WIN64_SHA256" ;; + *) echo "Unsupported runner: $RUNNER_OS_NAME/$RUNNER_ARCH_NAME" >&2; exit 1 ;; + esac + zip="$RUNNER_TEMP/protoc.zip" + dest="$RUNNER_TEMP/protoc" + curl -fsSL "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${platform}.zip" -o "$zip" + # Hash stdin rather than a path: Windows paths contain backslashes, + # which make sha256sum escape its output. + if [ "$RUNNER_OS_NAME" = "macOS" ]; then + actual=$(shasum -a 256 < "$zip" | cut -d' ' -f1) + else + actual=$(sha256sum < "$zip" | cut -d' ' -f1) + fi + if [ "$actual" != "$sha256" ]; then + echo "protoc checksum mismatch: expected $sha256, got $actual" >&2 + exit 1 + fi + if [ "$RUNNER_OS_NAME" = "Windows" ]; then + # Git Bash on Windows runners does not reliably ship unzip. + 7z x -y -o"$dest" "$zip" > /dev/null + else + unzip -q -o "$zip" -d "$dest" + fi + rm "$zip" + echo "$dest/bin" >> "$GITHUB_PATH" + "$dest/bin/protoc" --version diff --git a/.github/workflows/build_linux_wheel/action.yml b/.github/workflows/build_linux_wheel/action.yml index 8fa5b1f79..44ce82ee5 100644 --- a/.github/workflows/build_linux_wheel/action.yml +++ b/.github/workflows/build_linux_wheel/action.yml @@ -29,6 +29,11 @@ inputs: runs: using: "composite" steps: + - name: Get protoc version + id: protoc + uses: ./.github/actions/setup-protoc + with: + install: "false" - name: CONFIRM ARM BUILD shell: bash env: @@ -54,13 +59,14 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-x86_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: x86_64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-x86_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version @@ -71,14 +77,15 @@ runs: maturin-version: "1.12.4" command: build working-directory: python - docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" + docker-options: "-e PIP_EXTRA_INDEX_URL='https://pypi.fury.io/lance-format/ https://pypi.fury.io/lancedb/' -e PROTOC=/usr/local/bin/protoc -e PROTOC_VERSION=${{ steps.protoc.outputs.version }} -e PROTOC_SHA256=${{ steps.protoc.outputs.linux-aarch_64-sha256 }} ${{ inputs.rustflags != '' && format('-e RUSTFLAGS={0}', inputs.rustflags) || '' }}" target: aarch64-unknown-linux-gnu manylinux: ${{ inputs.manylinux }} args: ${{ inputs.args }} before-script-linux: | set -e yum install -y clang - curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v24.4/protoc-24.4-linux-aarch_64.zip -o /tmp/protoc.zip + curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-aarch_64.zip -o /tmp/protoc.zip + echo "${PROTOC_SHA256} /tmp/protoc.zip" | sha256sum -c - unzip /tmp/protoc.zip -d /usr/local rm /tmp/protoc.zip /usr/local/bin/protoc --version diff --git a/.github/workflows/build_mac_wheel/action.yml b/.github/workflows/build_mac_wheel/action.yml index 75d942c96..727cd190c 100644 --- a/.github/workflows/build_mac_wheel/action.yml +++ b/.github/workflows/build_mac_wheel/action.yml @@ -12,10 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install macos dependency - shell: bash - run: | - brew install protobuf + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/build_windows_wheel/action.yml b/.github/workflows/build_windows_wheel/action.yml index 18f39818f..954ee7d35 100644 --- a/.github/workflows/build_windows_wheel/action.yml +++ b/.github/workflows/build_windows_wheel/action.yml @@ -12,15 +12,7 @@ inputs: runs: using: "composite" steps: - - name: Install Protoc v21.12 - working-directory: C:\ - run: | - New-Item -Path 'C:\protoc' -ItemType Directory - Set-Location C:\protoc - Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip - 7z x protoc.zip - Add-Content $env:GITHUB_PATH "C:\protoc\bin" - shell: powershell + - uses: ./.github/actions/setup-protoc - name: Build wheel uses: PyO3/maturin-action@v1 with: diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 4f883aec1..d16203979 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -11,6 +11,7 @@ on: - .github/workflows/build_linux_wheel/action.yml - .github/workflows/build_mac_wheel/action.yml - .github/workflows/build_windows_wheel/action.yml + - .github/actions/setup-protoc/action.yml - Cargo.toml # Change in dependency frequently breaks builds - Cargo.lock diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 3594bd826..3608c24ca 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -15,6 +15,7 @@ on: - .github/workflows/build_linux_wheel/** - .github/workflows/build_mac_wheel/** - .github/workflows/build_windows_wheel/** + - .github/actions/setup-protoc/** - .github/workflows/run_tests/** permissions: