diff --git a/.github/actions/create-failure-issue/action.yml b/.github/actions/create-failure-issue/action.yml index 832dc8def..33e3b80ae 100644 --- a/.github/actions/create-failure-issue/action.yml +++ b/.github/actions/create-failure-issue/action.yml @@ -27,19 +27,31 @@ runs: # Extract failed job names FAILED_JOBS=$(echo "$JOB_RESULTS" | jq -r 'to_entries | map(select(.value.result == "failure")) | map(.key) | join(", ")') - # Create issue with workflow name, failed jobs, and run URL - gh issue create \ - --title "$WORKFLOW_NAME Failed ($FAILED_JOBS)" \ - --body "The workflow **$WORKFLOW_NAME** failed during execution. + TITLE="$WORKFLOW_NAME Failed ($FAILED_JOBS)" + + # This action now also runs on nightly schedules, so a breakage that + # persists for a few days would otherwise file one issue per night. + # Comment on the open report instead when one already exists. + EXISTING=$(gh issue list --state open --label ci --limit 100 --json number,title \ + | jq -r --arg title "$TITLE" 'map(select(.title == $title)) | .[0].number // empty') + + if [ -n "$EXISTING" ]; then + gh issue comment "$EXISTING" --body "Failed again: $RUN_URL" + echo "Commented on existing issue #$EXISTING" + else + gh issue create \ + --title "$TITLE" \ + --body "The workflow **$WORKFLOW_NAME** failed during execution. **Failed jobs:** $FAILED_JOBS **Run URL:** $RUN_URL Please investigate the failed jobs and address any issues." \ - --label "ci" + --label "ci" - echo "Issue created successfully" + echo "Issue created successfully" + fi else echo "No job failures detected, skipping issue creation" fi diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 3452168a5..e82cf71c9 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -61,6 +61,11 @@ jobs: sudo apt update sudo apt install -y protobuf-compiler libssl-dev - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Format Rust run: cargo fmt --all -- --check - name: Lint Rust @@ -103,6 +108,11 @@ jobs: cache: 'pnpm' cache-dependency-path: nodejs/pnpm-lock.yaml - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: | sudo apt update @@ -182,6 +192,11 @@ jobs: cache-dependency-path: nodejs/pnpm-lock.yaml - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: | brew install protobuf diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 8a049ee0f..72ef5ad13 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -10,10 +10,16 @@ permissions: on: push: - branches: - - main tags: - "v*" + # The cross-compiled targets (musl especially) break from toolchain and + # dependency changes that nothing else in CI catches, and discovering that + # mid-release is expensive. A nightly run keeps that signal while dropping + # the full 8-target release matrix from all ~90 pushes to main each month. + # `report-failure` files an issue when a nightly breaks. + schedule: + - cron: "0 8 * * *" + workflow_dispatch: pull_request: # This should trigger a dry run (we skip the final publish step) paths: @@ -34,9 +40,18 @@ jobs: - target: aarch64-apple-darwin host: macos-latest features: fp16kernels - pre_build: brew install protobuf + pre_build: |- + brew install protobuf + # Fat LTO (the workspace default in .cargo/config.toml) is + # single-threaded and is the peak-memory step of the build. On + # this runner it accounted for ~111 of the job's ~113 minutes, + # making it the critical path of the entire publish pipeline. + # ThinLTO parallelizes it across the runner's cores, for a few + # percent of runtime performance. + export CARGO_PROFILE_RELEASE_LTO=thin + export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 - target: x86_64-pc-windows-msvc - host: windows-2025-8x-x64 + host: windows-2025 features: "," pre_build: |- choco install --no-progress protoc ninja nasm @@ -44,19 +59,19 @@ jobs: # There is an issue where choco doesn't add nasm to the path export PATH="$PATH:/c/Program Files/NASM" nasm -v - # Fat LTO of the cdylib is single-threaded and the peak-memory - # step of the build, and had started hitting rustc-LLVM OOM on the - # Windows runners. ThinLTO parallelizes it across the runner's - # cores and keeps peak memory well under the limit. + # See the ThinLTO note on aarch64-apple-darwin above. Keeping + # peak memory down is also what lets this run on the standard + # 4-core runner: the 8-core larger runner was only needed to + # stop fat LTO from OOMing rustc-LLVM. export CARGO_PROFILE_RELEASE_LTO=thin export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 - target: aarch64-pc-windows-msvc - host: windows-2025-8x-x64 + host: windows-2025 features: "," pre_build: |- choco install --no-progress protoc rustup target add aarch64-pc-windows-msvc - # See ThinLTO note on the x86_64-pc-windows-msvc target above. + # See the ThinLTO note on aarch64-apple-darwin above. export CARGO_PROFILE_RELEASE_LTO=thin export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16 - target: x86_64-unknown-linux-gnu @@ -131,16 +146,49 @@ jobs: with: toolchain: stable targets: ${{ matrix.settings.target }} - - name: Cache cargo - uses: actions/cache@v5 + # These builds were entirely uncached: the old key was static, so + # `actions/cache` (which only writes on a miss) could never refresh it, + # and the multi-GB whole-`target/` copy it tried to store never fit the + # repo's cache budget, so no entry was ever saved. rust-cache prunes + # `target/` to dependency artifacts and keys on Cargo.lock plus the rustc + # version, which both fixes the key and keeps entries a sane size. + # + # This caches dependency *compilation* only. The LTO link of the cdylib + # re-runs regardless, since the local crate changes every time, so the + # win is larger on the non-LTO jobs than here. + - name: Cache cargo (native builds) + uses: Swatinem/rust-cache@v2 + if: ${{ !matrix.settings.docker }} with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - .cargo-cache - target/ - key: nodejs-${{ matrix.settings.target }}-cargo-${{ matrix.settings.host }} + # The release profile and per-target dirs differ from what the test + # workflows cache, so these need to be separate entries. + key: release-${{ matrix.settings.target }} + # Only the nightly run on main writes, so tag and PR runs restore a + # warm entry without every dependabot PR writing its own (which would + # be unreadable elsewhere anyway, since GitHub scopes caches to the + # creating ref). The nightly cadence also keeps entries inside + # GitHub's 7-day eviction window, which a tag-only trigger would not. + save-if: ${{ github.ref == 'refs/heads/main' }} + # Docker builds can use rust-cache too. `target/` already lives on the + # host because the whole workspace is bind-mounted into the container, and + # rust-cache's prune and save run host-side, so they can manage it -- which + # is what keeps the entry to dependency artifacts rather than a multi-GB + # copy of everything. + # + # Two differences from the native builds. The container's CARGO_HOME is + # bind-mounted from `.cargo-cache` rather than the host's ~/.cargo, so that + # has to be cached explicitly. And the key is derived from the *host* rustc + # version, which is not the compiler that produced these artifacts; that is + # safe because cargo fingerprints the real compiler and rebuilds on a + # mismatch, it just means a base-image toolchain bump costs one cold build + # instead of invalidating the key. + - name: Cache cargo (docker builds) + uses: Swatinem/rust-cache@v2 + if: ${{ matrix.settings.docker }} + with: + key: docker-${{ matrix.settings.target }} + cache-directories: .cargo-cache + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: pnpm install --frozen-lockfile - name: Install Zig @@ -158,9 +206,13 @@ jobs: if: ${{ matrix.settings.docker }} with: image: ${{ matrix.settings.docker }} + # All three mounts must live under `.cargo-cache`, which is what the + # cache step above saves. Previously the registry mounts pointed at + # `.cargo/...`, a path nothing cached, so the container re-downloaded + # the whole crate registry on every run. options: "--user 0:0 -v ${{ github.workspace }}/.cargo-cache/git/db:/usr/local/cargo/git/db \ - -v ${{ github.workspace }}/.cargo/registry/cache:/usr/local/cargo/registry/cache \ - -v ${{ github.workspace }}/.cargo/registry/index:/usr/local/cargo/registry/index \ + -v ${{ github.workspace }}/.cargo-cache/registry/cache:/usr/local/cargo/registry/cache \ + -v ${{ github.workspace }}/.cargo-cache/registry/index:/usr/local/cargo/registry/index \ -v ${{ github.workspace }}:/build -w /build/nodejs" run: | set -e @@ -172,6 +224,16 @@ jobs: --js ../lancedb/native.js \ --strip \ --output-dir dist/ + # The container runs as root (`--user 0:0`), so everything it wrote to the + # mounted cache dirs is root-owned. rust-cache's post step runs as the + # runner user and has to both read these and delete from them while + # pruning, so hand them back before it runs. + - name: Take ownership of docker build output + if: ${{ matrix.settings.docker }} + run: | + sudo chown -R "$(id -u):$(id -g)" \ + "${{ github.workspace }}/.cargo-cache" \ + "${{ github.workspace }}/target" - name: Build run: | ${{ matrix.settings.pre_build }} @@ -185,6 +247,15 @@ jobs: --output-dir dist/ if: ${{ !matrix.settings.docker }} shell: bash + # The standard Windows runners have ~14 GB free, and a release `target/` + # for this workspace is a large fraction of that. Report the remaining + # headroom so a build that only just fits is visible before a dependency + # bump turns it into a failed release. `always()` so the numbers are + # still there when the build is what ran out of space. + - name: Report disk headroom + if: always() + run: df -h + shell: bash - name: Upload artifact uses: actions/upload-artifact@v7 with: @@ -335,7 +406,9 @@ jobs: name: Report Workflow Failure runs-on: ubuntu-latest needs: [build-lancedb, test-lancedb, publish] - if: always() && failure() && startsWith(github.ref, 'refs/tags/v') + # Nightly runs are the only thing watching the cross-compiled targets now, + # so they have to report failures too or the signal is silently lost. + if: always() && failure() && (startsWith(github.ref, 'refs/tags/v') || github.event_name == 'schedule') permissions: contents: read issues: write diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 8825e7fb0..74b7d05e6 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -20,6 +20,12 @@ env: permissions: contents: read +# Without this, a force-push to a PR leaves the previous run going -- including +# a ~74 minute Windows job and a billed arm64 wheel build. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: linux: name: Python ${{ matrix.config.package_name }} ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }} @@ -122,6 +128,13 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.13" + # NOTE: caching cargo here would be a no-op. This workflow only runs on + # tags and PRs, and GitHub only lets a run restore caches from its own ref + # or the default branch -- so with no run on main there is nothing that + # can populate an entry the release build would be allowed to read. Fixing + # this needs a main/nightly trigger (which would also catch wheel-build + # breakage before a release); the ~74 minutes here is otherwise dominated + # by the fat-LTO link, which no cache avoids. - uses: ./.github/workflows/build_windows_wheel with: python-minor-version: 10 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ee8b319bb..52582395f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -108,6 +108,15 @@ jobs: run: | sudo apt update sudo apt install -y protobuf-compiler + # `pip install -e .` builds the extension with maturin, which is most of + # this job's ~33 minutes. It had no Rust cache, so every dependency was + # recompiled from scratch on every run. + - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install run: | pip install --extra-index-url https://pypi.fury.io/lance-format/ --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings] @@ -168,6 +177,14 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.13" + # maturin runs cargo natively on macOS (docker is Linux-only), so the host + # target dir is cacheable. This job had no Rust cache. + - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - uses: ./.github/workflows/build_mac_wheel with: args: --profile ci @@ -197,6 +214,14 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.13" + # maturin runs cargo natively on Windows (docker is Linux-only), so the + # host target dir is cacheable. This job had no Rust cache at all and so + # rebuilt every dependency from scratch on every run. + - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. The repo sits at + # GitHub's cache cap, so per-PR saves just evict main's entries. + save-if: ${{ github.ref == 'refs/heads/main' }} - uses: ./.github/workflows/build_windows_wheel with: args: --profile ci @@ -224,6 +249,14 @@ jobs: uses: actions/setup-python@v6 with: python-version: "3.10" + # As with Doctest, `pip install -e .` compiles the extension and this job + # had no Rust cache, which is most of its ~37 minutes. + - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install lancedb run: | pip install "pydantic<2" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 3bc67d2ec..1a0b65c4a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -48,6 +48,11 @@ jobs: with: components: rustfmt, clippy - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: | sudo apt update @@ -89,6 +94,11 @@ jobs: run: rm -f Cargo.lock - uses: rui314/setup-mold@v1 - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: | sudo apt update @@ -118,6 +128,11 @@ jobs: fetch-depth: 0 lfs: true - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: | sudo apt update @@ -175,6 +190,11 @@ jobs: - name: CPU features run: sysctl -a | grep cpu - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install dependencies run: brew install protobuf - name: Run tests @@ -187,12 +207,19 @@ jobs: cargo test --profile ci --features $ALL_FEATURES --locked windows: - runs-on: windows-2022 strategy: + fail-fast: false matrix: - target: - - x86_64-pc-windows-msvc - - aarch64-pc-windows-msvc + include: + - target: x86_64-pc-windows-msvc + runner: windows-2022 + # windows-11-arm is a standard runner, so it is free on public repos. + # Running natively lets the aarch64 tests actually execute -- this + # job used to cross-compile them and then skip the test step, paying + # full codegen and link cost for a compile check. + - target: aarch64-pc-windows-msvc + runner: windows-11-arm + runs-on: ${{ matrix.runner }} defaults: run: working-directory: rust/lancedb @@ -201,6 +228,11 @@ jobs: - name: Set target run: rustup target add ${{ matrix.target }} - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Install Protoc v21.12 run: choco install --no-progress protoc - name: Build @@ -208,11 +240,12 @@ jobs: $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT cargo build --profile ci --features aws,remote --tests --locked --target ${{ matrix.target }} - name: Run tests - # Can only run tests when target matches host - if: ${{ matrix.target == 'x86_64-pc-windows-msvc' }} run: | $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT - cargo test --profile ci --features aws,remote --locked + # `--target` has to match the build step above. Without it cargo uses + # target/ci/ rather than target//ci/ and rebuilds the entire + # dependency graph a second time. + cargo test --profile ci --features aws,remote --locked --target ${{ matrix.target }} msrv: # Check the minimum supported Rust version @@ -238,6 +271,11 @@ jobs: with: toolchain: ${{ matrix.msrv }} - uses: Swatinem/rust-cache@v2 + with: + # Restore everywhere, but only save from main. Per-PR saves are + # unreadable outside their own branch anyway, since GitHub scopes + # caches to the creating ref. + save-if: ${{ github.ref == 'refs/heads/main' }} - name: Downgrade dependencies # These packages have newer requirements for MSRV run: |