From 16217ff567052b020a07083f07036ed8d139325d Mon Sep 17 00:00:00 2001 From: discord9 Date: Wed, 15 Jul 2026 09:46:40 +0800 Subject: [PATCH] ci: add persistent query regression cache (#8474) * ci: add persistent query regression cache Signed-off-by: discord9 * ci: add sccache to query regression runner Signed-off-by: discord9 * ci: pin query regression label revision Signed-off-by: discord9 * ci: isolate query regression toolchain state Signed-off-by: discord9 --------- Signed-off-by: discord9 --- .../query-regression/Dockerfile | 51 +++ .../query-regression/README.md | 344 +++++++++----- .../query-regression/cache-pvc.yaml | 15 + .../query-regression/values-8-cores.yaml | 101 ++++- .../query-regression/values-paused.yaml | 2 + .github/scripts/query-regression-comment.cjs | 4 +- .../scripts/query-regression-pr-metadata.py | 5 +- .github/workflows/query-regression.yml | 424 +++++++++++++++++- 8 files changed, 817 insertions(+), 129 deletions(-) create mode 100644 .github/runner-scale-sets/query-regression/cache-pvc.yaml create mode 100644 .github/runner-scale-sets/query-regression/values-paused.yaml diff --git a/.github/runner-scale-sets/query-regression/Dockerfile b/.github/runner-scale-sets/query-regression/Dockerfile index 1f9f7203b9..95161f6093 100644 --- a/.github/runner-scale-sets/query-regression/Dockerfile +++ b/.github/runner-scale-sets/query-regression/Dockerfile @@ -3,6 +3,19 @@ FROM ghcr.io/actions/actions-runner@sha256:08c30b0a7105f64bddfc485d2487a22aa0393 USER root +ARG SCCACHE_VERSION=0.16.0 +ARG SCCACHE_SHA256=aec995a83ad3dff3d14b6314e08858b7b73d35ca85a5bcf3d3a9ec07dee35588 +ARG RUSTUP_INIT_VERSION=1.29.0 +ARG RUSTUP_INIT_TARGET=x86_64-unknown-linux-gnu +ARG RUSTUP_INIT_SHA256=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10 +ARG RUST_TOOLCHAIN=nightly-2026-03-21 + +ENV RUSTUP_HOME=/opt/rustup \ + CARGO_HOME=/opt/cargo \ + RUSTUP_TOOLCHAIN=${RUST_TOOLCHAIN} \ + RUSTUP_AUTO_INSTALL=0 \ + PATH=/opt/cargo/bin:${PATH} + RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ build-essential \ @@ -30,4 +43,42 @@ RUN apt-get update \ COPY --from=uv /uv /uvx /usr/local/bin/ +RUN curl --fail --location --silent --show-error \ + --output /tmp/sccache.tar.gz \ + "https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \ + && echo "${SCCACHE_SHA256} /tmp/sccache.tar.gz" | sha256sum --check --status - \ + && mkdir --parents /tmp/sccache \ + && tar --extract --gzip --file /tmp/sccache.tar.gz \ + --directory /tmp/sccache --strip-components=1 \ + "sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" \ + && install --mode=0755 /tmp/sccache/sccache /usr/local/bin/sccache \ + && rm -rf /tmp/sccache.tar.gz /tmp/sccache + +RUN curl --fail --location --silent --show-error \ + --output /tmp/rustup-init \ + "https://static.rust-lang.org/rustup/archive/${RUSTUP_INIT_VERSION}/${RUSTUP_INIT_TARGET}/rustup-init" \ + && echo "${RUSTUP_INIT_SHA256} /tmp/rustup-init" | sha256sum --check --status - \ + && chmod 0755 /tmp/rustup-init \ + && /tmp/rustup-init -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" --no-modify-path \ + && rm -f /tmp/rustup-init \ + && chown -R root:root /opt/rustup /opt/cargo \ + && chmod -R go-w /opt/rustup /opt/cargo \ + && find /opt/rustup /opt/cargo -type d -exec chmod 0755 {} + \ + && find /opt/rustup /opt/cargo -type f -exec chmod a+r {} + + USER runner + +RUN set -eu \ + && test "$(id -u)" = "1001" \ + && temporary_cargo_home="$(mktemp --directory)" \ + && trap 'rm -rf "${temporary_cargo_home}"' 0 \ + && export CARGO_HOME="${temporary_cargo_home}" \ + && test "${RUSTUP_AUTO_INSTALL}" = "0" \ + && rustup --version \ + && cargo --version \ + && rustc --version \ + && active_toolchain="$(rustup show active-toolchain)" \ + && printf 'Active toolchain: %s\n' "${active_toolchain}" \ + && case "${active_toolchain}" in "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu"|"${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu "*) ;; *) exit 1;; esac \ + && test ! -w /opt/rustup \ + && test ! -w /opt/cargo/bin diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md index eb1a7e12f7..fd68b4c832 100644 --- a/.github/runner-scale-sets/query-regression/README.md +++ b/.github/runner-scale-sets/query-regression/README.md @@ -1,21 +1,14 @@ # Query regression self-hosted runners -The `Query Regression` workflow targets self-hosted GitHub Actions runners via -runner labels or ARC runner scale set names: +The `Query Regression` workflow uses the dedicated ARC runner scale set +`perf-regression-8-cores`. ARC runner Pods run in the target Kubernetes cluster +and connect outbound to GitHub. The live scale set is currently **paused**: +`minRunners=0`, `maxRunners=0`, and no runner Pods. Do not resume it without +explicit approval. -- `perf-regression-8-cores` +## Prerequisites and trust admission -The name intentionally avoids generic labels such as `ubuntu-22.04-8-cores`, -which may already be used by GitHub-hosted larger runners or other runner pools. - -For Kubernetes-based runners, ARC runner pods run inside the target Kubernetes -cluster and connect outbound to GitHub; GitHub can then dispatch jobs whose -`runs-on` value matches the runner scale set name or runner labels. - -## Prerequisites - -Install the ARC scale set controller in the Kubernetes cluster if it is not -already installed: +Install the ARC scale set controller if it is not already installed: ```bash helm upgrade --install arc \ @@ -25,8 +18,8 @@ helm upgrade --install arc \ --version 0.14.2 ``` -Create the GitHub App or PAT secret in the namespace that will host the runner -scale sets. Prefer a GitHub App with access limited to `GreptimeTeam/greptimedb`. +Create the GitHub App secret in the runner namespace. Prefer an App limited to +`GreptimeTeam/greptimedb`: ```bash kubectl -n arc-runners create secret generic greptimedb-arc-github-app \ @@ -35,14 +28,46 @@ kubectl -n arc-runners create secret generic greptimedb-arc-github-app \ --from-file=github_app_private_key= ``` -The values files in this directory reference that secret by name. +The values files here reference that secret by name. -## Build the runner image +A maintainer applying the `query-regression` label is **trust admission for +that exact PR revision**. The admitted job may use this scale set's dedicated, +writable persistent cache. `pull_request: labeled` is the only PR trigger: the +label event snapshots its merge, head, and base SHAs. A queued job fetches that +immutable event merge SHA directly, verifies it is a two-parent merge whose +parents include the snapshotted head exactly once, and uses its other parent as +the actual base build revision. The snapshotted event base is retained for audit +only, so a difference from the merge's non-head parent is not a failure. The job +never follows a newer mutable PR merge ref. An unavailable event merge, or one +that does not contain exactly one snapshotted head parent, fails closed. A later +PR head change does not retarget an already queued run: it may execute only its +previously trusted event revision if that revision remains fetchable. To run the +new revision, the maintainer must review it, remove the label, and re-add +`query-regression`; cancel the old run if it is no longer wanted. An existing +label does not automatically rerun the benchmark. -Use a derived ARC runner image instead of the minimal upstream runner image. The -image keeps the official `/home/runner/run.sh` entrypoint layout and adds the -tools this workflow expects to be present on a normal CI host, including `wget`, -`uv`, C/C++ build tools, OpenSSL headers, and protobuf tooling. +Admission does not relax runner hardening or GitHub permissions. Keep +service-account token mounting disabled; do not mount host paths, the Docker +socket, kubeconfig, or long-lived credentials. The runner and cache initializer +use UID/GID 1001, disallow privilege escalation, drop all capabilities, and use +the RuntimeDefault seccomp profile. Keep GitHub tokens least-privilege and +review workflow changes before admission. Where the CNI supports it, restrict +egress to required GitHub Actions, artifact/cache, Rust/crate/toolchain, DNS, +and image-registry endpoints; block unrelated cluster services, private ranges, +and metadata endpoints unless a case requires them. + +### Office routing prerequisite + +Direct split routing through `.2` is an **external office-network prerequisite**. +The gateway at `192.168.50.2` must route GitHub Actions, GitHub content, +artifact/cache, crates.io, Rust toolchain, and image-registry traffic directly +rather than through the VPN. Neither this repository nor Kubernetes configures +that route. Verify it with the responsible network operator before any canary. + +## Runner image and workflow tools + +Build and push the derived runner image; it preserves the official +`/home/runner/run.sh` entrypoint and supplies CI tools needed at runtime: ```bash docker build \ @@ -53,100 +78,219 @@ docker build \ docker push greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest ``` -Deploy the runner image by digest rather than by a mutable tag. Update -`values-8-cores.yaml` after pushing a rebuilt image. +Deploy by digest, not mutable tag, by updating `values-8-cores.yaml` after a +rebuild. If the registry is private, use a dedicated read-only pull secret only +as `imagePullSecrets`; never expose registry credentials to runner containers. +Both digest-pinned init and runner containers use `IfNotPresent`: the immutable +digest makes a cached image safe and avoids adding a registry dependency to every +runner startup. -The workflow still runs setup actions for pinned Rust and `uv` behavior. `mold` -is installed in the image and selected through `CARGO_BUILD_RUSTFLAGS`, so jobs -do not need privileged package installation at runtime. +Before builds, the workflow asserts UID/GID 1001 and exact image tool versions: +`libprotoc 3.21.12`, `uv 0.11.26`, `mold 2.30.0`, `Python 3.12.3`, `sccache +0.16.0`, root-owned `rustup 1.29.0`, and the image-baked +`nightly-2026-03-21` Rust toolchain. Rustup, Cargo, and Rustc must resolve from +`/opt/cargo/bin`; the runner cannot write `/opt/rustup` or `/opt/cargo/bin`. +`actions-rust-lang/setup-rust-toolchain@v1` is intentionally removed. The +workflow sets its warning-denying mold `RUSTFLAGS` directly, disables automatic +Rustup installation, and performs no runtime toolchain downloads. -Decide whether the registry repository is public or private. Public pull access -avoids distributing registry credentials to the runner namespace. If the image -must be private, create a dedicated read-only image-pull secret for this image -and attach it only as `imagePullSecrets`; do not expose registry credentials to -runner containers. +The workflow no longer uses GitHub `rust-cache`, `setup-protoc`, `setup-uv`, or +runtime Rust setup: the image establishes immutable executable state and the PVC +supplies only reusable Cargo data. Do not reintroduce those actions unless the +corresponding cache or image contract changes. -## Install the query-regression scale set +## Capacity and persistent cache -Install the runner scale set. The Helm release name and `runnerScaleSetName` -should match the `runs-on` value used by the workflow. +`values-8-cores.yaml` is normal operation: `minRunners=0`, `maxRunners=1`. +`values-paused.yaml` is the mandatory pause overlay: `minRunners=0`, +`maxRunners=0`. The job uses group `query-regression-persistent-cache-v1`, +`queue: max`, and `cancel-in-progress: false`; admitted jobs queue rather than +replacing older pending jobs. During maintenance, cancel admitted queued runs as +well as pausing ARC. Runner Pods have `activeDeadlineSeconds=12600`. + +The cache claim `query-regression-build-cache` is a nominal 600Gi `local-path` +PVC in `arc-runners`. It is `ReadWriteOnce`; `local-path` uses +WaitForFirstConsumer binding and Delete reclaim behavior, produces a +node-affine local PV, is non-expandable, and the 600Gi request is not a hard +storage quota. The runner's `minipc-3` selector is its only consumer candidate. + +The initializer mounts the PVC root at `/cache`, creates and write-tests these +versioned subpaths as non-root UID/GID 1001, and the runner mounts them as: + +| Persistent state | PVC subpath | Runner mount | +| --- | --- | --- | +| Ephemeral Cargo home | `emptyDir` | `/home/runner/.cargo` | +| Cargo registry data | `cargo-registry-v1` | `/home/runner/.cargo/registry` | +| Cargo Git data | `cargo-git-v1` | `/home/runner/.cargo/git` | +| Cargo target | `query-regression-target-v1` | `/home/runner/query-regression-target` | +| Cache metadata | `meta-v1` | `/home/runner/query-regression-cache-meta` | +| sccache local disk cache | `sccache-v1` | `/home/runner/.cache/sccache` | +| Immutable Rust toolchain | image-owned | `/opt/rustup`, `/opt/cargo/bin` | + +The Pod security context uses UID/GID and `fsGroup` 1001 with +`fsGroupChangePolicy: OnRootMismatch`; no privileged `chown` or raw `hostPath` +is used. `CARGO_HOME` is a per-Pod `emptyDir`; only its nested `registry` and +`git` mounts are persistent. `RUSTUP_HOME=/opt/rustup` and `/opt/cargo/bin` are +image-owned immutable paths, while `CARGO_TARGET_DIR`, cache metadata, and +`SCCACHE_DIR` are persistent absolute paths. The runner sets +`RUSTC_WRAPPER=/usr/local/bin/sccache`, +`SCCACHE_DIR=/home/runner/.cache/sccache`, `SCCACHE_CACHE_SIZE=40G`, and +`CARGO_INCREMENTAL=0`. sccache uses its local PVC disk backend and self-evicts +at 40G; do not add runtime downloads, object storage, or a shared backend. + +The repository's `.cargo/config.toml` remains a trusted per-revision build input. +In contrast, `$CARGO_HOME/config*`, credentials, installed bins, and Cargo +metadata outside the persistent `registry` and `git` data mounts are ephemeral +and cannot survive to another Pod. + +The local disk backend has a one-server constraint. `maxRunners=1` and the +unchanged `query-regression-persistent-cache-v1` workflow concurrency group +serialize runs; do not increase runner capacity or relax that serialization +while this backend is in use. Base and candidate builds share the target; Cargo +fingerprints invalidate source and dependency changes. The workflow records the +sccache version and relevant environment in the target ABI marker, starts and +zeros sccache after cache and toolchain checks, shows initial/base/candidate +statistics, and resets statistics between base and candidate builds. + +### Disk preflight and cleanup contract + +Before applying or unpausing, verify the backing filesystem on `minipc-3` has +at least 900GiB free. The current local-path provisioner source is +`/opt/local-path-provisioner`; measure the filesystem containing it: + +```bash +df -PB1G /opt/local-path-provisioner +``` + +The workflow reports `du`, `df -P`, human-readable free space, and inode +availability before builds and in an always-run report. Its cleanup is narrow +and non-destructive: + +- warn at target size 400GiB; at 450GiB clear only the complete target root; +- warn at Cargo registry-plus-Git data size 60GiB; at 80GiB remove only + `registry/src` and `git/checkouts`, then abort if that persistent data remains + at least 80GiB; +- below 300GiB backing free space, clear the complete target root first, + remeasure, then remove only those Cargo extracted trees and checkouts; abort + if free space is still below 300GiB; +- never automatically remove Cargo registry cache/index, Git database, the + image-owned Cargo bin or Rustup toolchain, cache metadata, the self-evicting + sccache directory, or the PVC. + +The target clear uses fixed absolute roots and removes all entries, including +dotfiles. After migration validation, remove obsolete `cargo-home-v1` and +`rustup-home-v1` only in explicit maintenance while ARC is 0/0 and no runner Pod +exists; they are not mounted by the current configuration. + +## Deploy and pause safely + +First verify external `.2` routing and the disk preflight. Apply the PVC; while +the scale set is paused, it is expected to remain `Pending` because +WaitForFirstConsumer has no scheduled runner: + +```bash +kubectl apply --dry-run=server \ + -f .github/runner-scale-sets/query-regression/cache-pvc.yaml +kubectl apply -f .github/runner-scale-sets/query-regression/cache-pvc.yaml +``` + +Render normal and paused configurations. Normal values are always first; the +pause overlay is always last: + +```bash +helm template perf-regression-8-cores \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --namespace arc-runners --version 0.14.2 \ + --set controllerServiceAccount.name=arc-gha-rs-controller \ + --set controllerServiceAccount.namespace=arc-systems \ + -f .github/runner-scale-sets/query-regression/values-8-cores.yaml + +helm template perf-regression-8-cores \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --namespace arc-runners --version 0.14.2 \ + --set controllerServiceAccount.name=arc-gha-rs-controller \ + --set controllerServiceAccount.namespace=arc-systems \ + -f .github/runner-scale-sets/query-regression/values-8-cores.yaml \ + -f .github/runner-scale-sets/query-regression/values-paused.yaml +``` + +The **first post-merge Helm deployment must reconcile the release in paused +mode**. Keep the pause overlay last: + +```bash +# First post-merge deployment and every return to paused mode: 0/0. +helm upgrade --install perf-regression-8-cores \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --namespace arc-runners --create-namespace --version 0.14.2 \ + --reset-values --wait \ + -f .github/runner-scale-sets/query-regression/values-8-cores.yaml \ + -f .github/runner-scale-sets/query-regression/values-paused.yaml + +# Expect 0/0 and no runner resources before considering normal mode. +kubectl -n arc-runners get autoscalingrunnerset perf-regression-8-cores \ + -o jsonpath='{.spec.minRunners}{"/"}{.spec.maxRunners}{"\n"}' +kubectl -n arc-runners get ephemeralrunners,pods \ + -l actions.github.com/scale-set-name=perf-regression-8-cores +``` + +Only after that verification and separate explicit approval, apply normal 0/1 +operation without the pause overlay: ```bash helm upgrade --install perf-regression-8-cores \ oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ - --namespace arc-runners \ - --create-namespace \ - --version 0.14.2 \ + --namespace arc-runners --create-namespace --version 0.14.2 \ + --reset-values --wait \ -f .github/runner-scale-sets/query-regression/values-8-cores.yaml ``` -Check registration and pods: +Do not use bare `helm rollback`, `--atomic`, or `--reuse-values`: a stored +revision can restore nonzero runner capacity. Inspect rendered manifests for +capacity, the `minipc-3` selector, cache claim and mounts, initializer, +security context, and resources. After approved normal mode receives its first +canary, the PVC binds to `minipc-3`. -```bash -kubectl -n arc-runners get pods -kubectl -n arc-runners get autoscalingrunnersets -``` +For local-PV node loss, cache recovery is intentionally disposable: return to +0/0, recreate the PVC on a healthy node, cold-fill it, and run a new canary. -The scale sets should also appear under repository Actions runner settings. +## Canary and rollback -## Security notes for fork PRs +With explicit approval, run two identical `workflow_dispatch` canaries on +`perf-regression-8-cores`, using immutable full base and candidate commit SHAs +and `cargo_profile=nightly`. The first is the cold fill; the second verifies warm +reuse. Record the workflow's base/candidate build elapsed logs and cache +ABI-marker output, initial/base/candidate sccache statistics, and cache report. +Confirm the image tool contract (root-owned Rustup/Cargo paths, baked nightly, +and non-writable `/opt` roots) and that the ephemeral Cargo home contains only +the mounted registry/Git data before Cargo creates per-Pod state. +Obtain dependency and tool network byte counters from the +identified `.2` counter source, filtered to `minipc-3` and the dependency/tool +destinations. -Maintainer-approved fork PRs can run on self-hosted runners. Approval only lets -the workflow execute; it does not make the fork code trusted. +Accept the canary only when all of the following hold: -The `query-regression` label is the explicit trigger for PR runs. Updating a PR -does not rerun the benchmark automatically; remove and re-add the label after -reviewing the updated changes. +- exactly one runner Pod runs on `minipc-3`, and both jobs use the same bound PV; +- UID/GID 1001 cache mounts are writable; the warm run does not invalidate the + target or bulk-redownload crates or toolchains; sccache reports separate base + and candidate build statistics without server or cache-path errors; immutable + Rustup/Cargo roots remain non-writable and only registry/Git data persists; +- warm base build time is at most 50% of cold base build time; +- warm dependency/tool network bytes are at most 10% of cold fill bytes; +- cache sizes remain below soft watermarks, node free space remains at least + 300GiB, and the benchmark is correct without TLS EOFs or timeouts; +- `.2` confirms this traffic is outside VPN accounting. -These runners execute PR code and should be treated as untrusted execution -capacity: +Immediately return to 0/0 after either canary unless ongoing normal operation +has been explicitly approved; return immediately on any traffic, cache, disk, +TLS, or correctness failure. To roll back, use the paused Helm upgrade above, +or another explicit `helm upgrade --install` with known-good values followed by +`values-paused.yaml`, `--reset-values`, and `--wait`. Do not delete the PVC +automatically; preserve it for diagnosis unless intentionally discarding cache. -- keep them isolated from sensitive internal services unless explicitly required; -- do not mount host paths, Docker socket, kubeconfig, or long-lived credentials; -- use ephemeral runner pods and no shared work directory with trusted jobs; -- disable service account token mounting in runner pods unless Kubernetes API - access is intentionally required; -- use a runner image whose default user is non-root, disable privilege - escalation, drop Linux capabilities, and use the runtime-default seccomp - profile; -- keep GitHub tokens least-privilege and rely on normal `pull_request` behavior - for fork PRs, where repository secrets are withheld and `GITHUB_TOKEN` is - read-only; -- review fork workflow changes before approving the run. +## Future optional phases -If stronger isolation is required, install a separate runner group/namespace just -for query-regression PR workloads and restrict repository/workflow access to the -`Query Regression` workflow. - -Use namespace or cluster network policy to restrict runner egress where the CNI -supports the needed controls. Query-regression runners need outbound access to -GitHub Actions services, GitHub artifact/cache endpoints, Rust/crate/toolchain -endpoints, DNS, and the configured image registries. Block access to unrelated -cluster services, private network ranges, and cloud metadata endpoints unless a -case explicitly needs them. - -## Build cache - -The workflow builds base and candidate in the same job, the same source path, -and a shared `CARGO_TARGET_DIR`. It checks out the base ref into `src`, builds -and copies the base binary aside, then resets that same `src` checkout to the -candidate ref before building the candidate binary. Keeping the workspace path -stable improves Cargo incremental reuse for local workspace crates compared with -building separate `base-src` and `candidate-src` checkouts. - -The workflow also uses the GitHub Actions Rust cache for restore-only cache -reuse. PR and dispatch runs do not save cache entries. Refresh shared caches from -trusted maintenance workflows only. - -The ARC values in this directory do not configure a cross-run runner-local -compiler cache. If cross-run compile time still dominates the benchmark, -deployers may add one of the following at the runner infrastructure layer: - -- a custom runner image with Rust tooling, `mold`, and `sccache` preinstalled; -- `RUSTC_WRAPPER=sccache` plus a shared `sccache` backend such as object storage, - Redis, or a Kubernetes storage class that is safe for concurrent runner pods; -- separate cache namespaces/buckets for untrusted PR code and trusted branches to - avoid cache poisoning across trust boundaries. - -Do not mount a shared writable host path, Docker socket, kubeconfig, or other -privileged credentials into runners that execute PR code. +The current phase uses a digest-pinned image with sccache 0.16.0 and no shared +cache service. Optional follow-ups are an image additionally seeded with the +exact Rust toolchain and `cargo fetch --locked`; or an internal read/write +sccache backend or Cargo/Git mirror. Evaluate them only if persistent PVC reuse +is insufficient. diff --git a/.github/runner-scale-sets/query-regression/cache-pvc.yaml b/.github/runner-scale-sets/query-regression/cache-pvc.yaml new file mode 100644 index 0000000000..692f3a2d6e --- /dev/null +++ b/.github/runner-scale-sets/query-regression/cache-pvc.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: query-regression-build-cache + namespace: arc-runners + labels: + app.kubernetes.io/part-of: perf-regression-8-cores +spec: + volumeMode: Filesystem + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 600Gi diff --git a/.github/runner-scale-sets/query-regression/values-8-cores.yaml b/.github/runner-scale-sets/query-regression/values-8-cores.yaml index 54ade99bad..77ec4055e5 100644 --- a/.github/runner-scale-sets/query-regression/values-8-cores.yaml +++ b/.github/runner-scale-sets/query-regression/values-8-cores.yaml @@ -3,21 +3,96 @@ githubConfigSecret: greptimedb-arc-github-app runnerScaleSetName: "perf-regression-8-cores" minRunners: 0 -maxRunners: 4 +maxRunners: 1 template: spec: automountServiceAccountToken: false - activeDeadlineSeconds: 11400 + activeDeadlineSeconds: 12600 + nodeSelector: + kubernetes.io/hostname: minipc-3 securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + fsGroupChangePolicy: OnRootMismatch seccompProfile: type: RuntimeDefault + volumes: + - name: cargo-home + emptyDir: {} + - name: build-cache + persistentVolumeClaim: + claimName: query-regression-build-cache + initContainers: + - name: initialize-build-cache + image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -ec + - | + umask 0002 + for directory in cargo-registry-v1 cargo-git-v1 query-regression-target-v1 meta-v1 sccache-v1; do + cache_directory="/cache/${directory}" + mkdir -p "${cache_directory}" + test -w "${cache_directory}" + probe_file="${cache_directory}/.write-probe" + : > "${probe_file}" + test -f "${probe_file}" + rm "${probe_file}" + done + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: 100m + memory: 128Mi + ephemeral-storage: 1Gi + limits: + cpu: 500m + memory: 512Mi + ephemeral-storage: 1Gi + volumeMounts: + - name: build-cache + mountPath: /cache containers: - name: runner - image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:108b7357532e5cf3329853373f561b0c674e954d6307b53d37075df5a189b0b2 - imagePullPolicy: Always + image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e + imagePullPolicy: IfNotPresent command: ["/home/runner/run.sh"] + env: + - name: CARGO_HOME + value: /home/runner/.cargo + - name: RUSTUP_HOME + value: /opt/rustup + - name: RUSTUP_TOOLCHAIN + value: nightly-2026-03-21 + - name: RUSTUP_AUTO_INSTALL + value: "0" + - name: CARGO_TARGET_DIR + value: /home/runner/query-regression-target + - name: QUERY_REGRESSION_CACHE_META + value: /home/runner/query-regression-cache-meta + - name: RUSTC_WRAPPER + value: /usr/local/bin/sccache + - name: SCCACHE_DIR + value: /home/runner/.cache/sccache + - name: SCCACHE_CACHE_SIZE + value: 40G + - name: CARGO_INCREMENTAL + value: "0" securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 allowPrivilegeEscalation: false capabilities: drop: @@ -31,3 +106,21 @@ template: cpu: "8" memory: 24Gi ephemeral-storage: 80Gi + volumeMounts: + - name: cargo-home + mountPath: /home/runner/.cargo + - name: build-cache + mountPath: /home/runner/.cargo/registry + subPath: cargo-registry-v1 + - name: build-cache + mountPath: /home/runner/.cargo/git + subPath: cargo-git-v1 + - name: build-cache + mountPath: /home/runner/query-regression-target + subPath: query-regression-target-v1 + - name: build-cache + mountPath: /home/runner/query-regression-cache-meta + subPath: meta-v1 + - name: build-cache + mountPath: /home/runner/.cache/sccache + subPath: sccache-v1 diff --git a/.github/runner-scale-sets/query-regression/values-paused.yaml b/.github/runner-scale-sets/query-regression/values-paused.yaml new file mode 100644 index 0000000000..fe734ef761 --- /dev/null +++ b/.github/runner-scale-sets/query-regression/values-paused.yaml @@ -0,0 +1,2 @@ +minRunners: 0 +maxRunners: 0 diff --git a/.github/scripts/query-regression-comment.cjs b/.github/scripts/query-regression-comment.cjs index 5a98e0b816..6f4386fc3b 100644 --- a/.github/scripts/query-regression-comment.cjs +++ b/.github/scripts/query-regression-comment.cjs @@ -241,8 +241,10 @@ module.exports = async function validateQueryRegressionComment({ github, context '> Rendered by a trusted workflow from JSON artifacts produced by the query-regression run. Results from untrusted PR code are advisory until reviewed.', '', `- **Workflow run:** ${serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${expectedRunId}`, - `- **Base SHA:** \`${text(metadata.base_sha)}\``, + `- **Built base SHA:** \`${text(metadata.built_base_sha)}\``, + `- **Event base SHA:** \`${text(metadata.event_base_sha)}\``, `- **Head SHA:** \`${text(metadata.head_sha)}\``, + `- **Candidate merge SHA:** \`${text(metadata.candidate_sha)}\``, '', ].join('\n'); diff --git a/.github/scripts/query-regression-pr-metadata.py b/.github/scripts/query-regression-pr-metadata.py index 2d8ed3f2e7..ba7ab113b6 100644 --- a/.github/scripts/query-regression-pr-metadata.py +++ b/.github/scripts/query-regression-pr-metadata.py @@ -31,7 +31,10 @@ def main() -> int: metadata = { "pr_number": int(os.environ["PR_NUMBER"]), "head_sha": os.environ["HEAD_SHA"], - "base_sha": os.environ["BASE_SHA"], + "base_sha": os.environ["EVENT_BASE_SHA"], + "event_base_sha": os.environ["EVENT_BASE_SHA"], + "built_base_sha": os.environ["VERIFIED_BASE_SHA"], + "candidate_sha": os.environ["VERIFIED_CANDIDATE_SHA"], "head_repo": os.environ["HEAD_REPO"], "base_repo": os.environ["BASE_REPO"], "run_id": int(os.environ["RUN_ID"]), diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index 619442e2f9..aa6a7724b2 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -45,10 +45,6 @@ on: permissions: contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} - cancel-in-progress: true - jobs: query-regression: if: >- @@ -59,12 +55,29 @@ jobs: (github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }} runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || 'perf-regression-8-cores' }} timeout-minutes: 180 + concurrency: + group: query-regression-persistent-cache-v1 + queue: max + cancel-in-progress: false env: CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }} - CARGO_TARGET_DIR: ${{ github.workspace }}/query-regression-target - CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold" + CARGO_HOME: /home/runner/.cargo + RUSTUP_HOME: /opt/rustup + RUSTUP_TOOLCHAIN: nightly-2026-03-21 + RUSTUP_AUTO_INSTALL: "0" + CARGO_TARGET_DIR: /home/runner/query-regression-target + QUERY_REGRESSION_CACHE_META: /home/runner/query-regression-cache-meta + RUSTC_WRAPPER: /usr/local/bin/sccache + SCCACHE_DIR: /home/runner/.cache/sccache + SCCACHE_CACHE_SIZE: 40G + CARGO_INCREMENTAL: "0" + RUSTFLAGS: -D warnings -C link-arg=-fuse-ld=mold + QUERY_REGRESSION_CACHE_EPOCH: "1" BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }} - CANDIDATE_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }} + CANDIDATE_REF: ${{ github.event_name == 'workflow_dispatch' && (inputs.candidate_ref || github.ref) || '' }} + EVENT_MERGE_SHA: ${{ github.event_name == 'pull_request' && github.sha || '' }} + EVENT_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }} + EVENT_BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }} steps: - name: Checkout base source uses: actions/checkout@v4 @@ -82,28 +95,338 @@ jobs: cp src/.github/scripts/query-regression-summary.py \ query-regression-trusted-scripts/query-regression-summary.py - - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Resolve immutable candidate and build base + working-directory: src + shell: bash + run: | + set -euo pipefail - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - cache: false + fail_closed() { + printf '%s\n' \ + "PR changed or merge result unavailable; maintainer must remove and re-add query-regression after reviewing current revision. $1" >&2 + exit 1 + } - - uses: astral-sh/setup-uv@v5 + is_full_sha() { + [[ "$1" =~ ^[[:xdigit:]]{40}$ ]] + } - - name: Rust cache - uses: Swatinem/rust-cache@v2 - with: - workspaces: src - shared-key: query-regression - cache-all-crates: "true" - save-if: false + initially_checked_out_base_sha="$(git rev-parse --verify HEAD)" + is_full_sha "${initially_checked_out_base_sha}" || { + printf 'Could not resolve initially checked-out BASE_REF commit.\n' >&2 + exit 1 + } + + if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + for identity in EVENT_MERGE_SHA EVENT_HEAD_SHA EVENT_BASE_SHA; do + value="${!identity-}" + is_full_sha "${value}" || fail_closed "Missing or invalid ${identity}." + done + + if ! git fetch --no-tags --prune --depth=2 origin "${EVENT_MERGE_SHA}"; then + fail_closed "Could not fetch event merge SHA ${EVENT_MERGE_SHA}." + fi + fetched_sha="$(git rev-parse --verify FETCH_HEAD 2>/dev/null)" \ + || fail_closed "Fetched event merge SHA could not be resolved." + [[ "${fetched_sha,,}" == "${EVENT_MERGE_SHA,,}" ]] \ + || fail_closed "FETCH_HEAD ${fetched_sha} does not match event merge SHA ${EVENT_MERGE_SHA}." + + read -r -a parents <<< "$(git show -s --format=%P FETCH_HEAD)" + (( ${#parents[@]} == 2 )) \ + || fail_closed "Event merge SHA ${EVENT_MERGE_SHA} is not a two-parent merge commit." + head_parent_count=0 + for parent in "${parents[@]}"; do + if [[ "${parent,,}" == "${EVENT_HEAD_SHA,,}" ]]; then + ((head_parent_count += 1)) + else + VERIFIED_BASE_SHA="${parent,,}" + fi + done + (( head_parent_count == 1 )) \ + || fail_closed "Event merge SHA must have exactly one EVENT_HEAD_SHA parent." + is_full_sha "${VERIFIED_BASE_SHA-}" \ + || fail_closed "Event merge SHA has no non-head parent for the base build." + VERIFIED_CANDIDATE_SHA="${fetched_sha,,}" + else + git fetch --no-tags --prune --depth=1 origin "${CANDIDATE_REF}" + VERIFIED_CANDIDATE_SHA="$(git rev-parse --verify FETCH_HEAD)" + VERIFIED_BASE_SHA="${initially_checked_out_base_sha}" + fi + + is_full_sha "${VERIFIED_CANDIDATE_SHA}" || { + printf 'Could not resolve candidate SHA.\n' >&2 + exit 1 + } + is_full_sha "${VERIFIED_BASE_SHA}" || { + printf 'Could not resolve build base SHA.\n' >&2 + exit 1 + } + printf 'VERIFIED_CANDIDATE_SHA=%s\n' "${VERIFIED_CANDIDATE_SHA,,}" >> "${GITHUB_ENV}" + printf 'VERIFIED_BASE_SHA=%s\n' "${VERIFIED_BASE_SHA,,}" >> "${GITHUB_ENV}" + printf 'Verified candidate SHA: %s\n' "${VERIFIED_CANDIDATE_SHA,,}" + printf 'Verified build base SHA: %s\n' "${VERIFIED_BASE_SHA,,}" + printf -- '- Verified candidate SHA: `%s`; built base SHA: `%s`\n' \ + "${VERIFIED_CANDIDATE_SHA,,}" "${VERIFIED_BASE_SHA,,}" >> "${GITHUB_STEP_SUMMARY}" + git reset --hard "${VERIFIED_BASE_SHA}" + git clean -ffdx + + - name: Verify runner image tools + shell: bash + run: | + set -euo pipefail + [[ "$(id -u)" == "1001" ]] + [[ "$(id -g)" == "1001" ]] + [[ "$(protoc --version)" == "libprotoc 3.21.12" ]] + [[ "$(uv --version)" =~ ^uv[[:space:]]0\.11\.26([[:space:]]|$) ]] + mold_version="$(mold --version)" + [[ "${mold_version}" =~ ^mold[[:space:]]2\.30\.0([[:space:]]|$) ]] + [[ "$(python3 --version)" == "Python 3.12.3" ]] + sccache_version="$(sccache --version)" + [[ "${sccache_version}" =~ ^sccache[[:space:]]0\.16\.0([[:space:]]|$) ]] + [[ "$(command -v rustup)" == "/opt/cargo/bin/rustup" ]] + [[ "$(command -v cargo)" == "/opt/cargo/bin/cargo" ]] + [[ "$(command -v rustc)" == "/opt/cargo/bin/rustc" ]] + [[ "$(rustup --version)" =~ ^rustup[[:space:]]1\.29\.0([[:space:]]|$) ]] + cargo_version="$(cargo --version)" + [[ "${cargo_version}" =~ ^cargo[[:space:]]1\.96\.0-nightly[[:space:]]\(cbb9bb8bd[[:space:]][0-9]{4}-[0-9]{2}-[0-9]{2}\)$ ]] + rustc_version="$(rustc --version)" + [[ "${rustc_version}" =~ ^rustc[[:space:]]1\.96\.0-nightly[[:space:]]\(ac7f9ec7d[[:space:]][0-9]{4}-[0-9]{2}-[0-9]{2}\)$ ]] + active_toolchain="$(rustup show active-toolchain)" + [[ "${active_toolchain}" =~ ^nightly-2026-03-21-x86_64-unknown-linux-gnu([[:space:]]|$) ]] + [[ "${RUSTUP_HOME}" == "/opt/rustup" ]] + [[ "${RUSTUP_TOOLCHAIN}" == "nightly-2026-03-21" ]] + [[ "${RUSTUP_AUTO_INSTALL}" == "0" ]] + test -r /opt/rustup && test -x /opt/rustup + test ! -w /opt/rustup + test ! -w /opt/cargo/bin + for entry in config config.toml credentials credentials.toml bin .crates.toml .crates2.json .global-cache .package-cache; do + test ! -e "${CARGO_HOME}/${entry}" + done + mkdir -p "${CARGO_HOME}/registry" "${CARGO_HOME}/git" + + - name: Prepare persistent query regression cache + shell: bash + working-directory: src + run: | + set -euo pipefail + + readonly EXPECTED_CARGO_HOME="/home/runner/.cargo" + readonly EXPECTED_CARGO_REGISTRY="/home/runner/.cargo/registry" + readonly EXPECTED_CARGO_GIT="/home/runner/.cargo/git" + readonly EXPECTED_RUSTUP_HOME="/opt/rustup" + readonly EXPECTED_TARGET_DIR="/home/runner/query-regression-target" + readonly EXPECTED_CACHE_META="/home/runner/query-regression-cache-meta" + readonly EXPECTED_SCCACHE_DIR="/home/runner/.cache/sccache" + readonly EXPECTED_RUSTC_WRAPPER="/usr/local/bin/sccache" + readonly RUNNER_IMAGE_DIGEST="sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e" + readonly RUNNER_IMAGE_EPOCH="3" + + require_expected_root() { + local name="$1" + local actual="$2" + local expected="$3" + [[ "${actual}" == /* && "${actual}" == "${expected}" ]] || { + printf 'Refusing unexpected %s root: %s\n' "${name}" "${actual}" >&2 + exit 1 + } + } + + clear_directory() { + local root="$1" + [[ "${root}" == /* && "${root}" != "/" ]] || { + printf 'Refusing to clear non-absolute or root path: %s\n' "${root}" >&2 + exit 1 + } + case "${root}" in + "${EXPECTED_TARGET_DIR}"|"${EXPECTED_TARGET_DIR}/debug"|"${EXPECTED_TARGET_DIR}/release"|"${EXPECTED_TARGET_DIR}/nightly") ;; + *) + printf 'Refusing to clear unexpected cache path: %s\n' "${root}" >&2 + exit 1 + ;; + esac + find "${root}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + + } + + clear_cargo_extracted_trees() { + require_expected_root CARGO_REGISTRY "${CARGO_HOME}/registry" "${EXPECTED_CARGO_REGISTRY}" + require_expected_root CARGO_GIT "${CARGO_HOME}/git" "${EXPECTED_CARGO_GIT}" + rm -rf -- "${CARGO_HOME}/registry/src" "${CARGO_HOME}/git/checkouts" + } + + cargo_size_kib() { + du -sk -- "${CARGO_HOME}/registry" "${CARGO_HOME}/git" | awk '{ total += $1 } END { print total }' + } + + target_size_kib() { + du -sk -- "${CARGO_TARGET_DIR}" | cut -f1 + } + + free_kib() { + df -Pk "${CARGO_TARGET_DIR}" | awk 'NR == 2 { print $4 }' + } + + report_cache_usage() { + du -sh -- "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \ + "${RUSTUP_HOME}" "${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}" + df -P "${CARGO_TARGET_DIR}" + df -hP "${CARGO_TARGET_DIR}" + df -Pi "${CARGO_TARGET_DIR}" + } + + require_expected_root CARGO_HOME "${CARGO_HOME}" "${EXPECTED_CARGO_HOME}" + require_expected_root CARGO_REGISTRY "${CARGO_HOME}/registry" "${EXPECTED_CARGO_REGISTRY}" + require_expected_root CARGO_GIT "${CARGO_HOME}/git" "${EXPECTED_CARGO_GIT}" + require_expected_root RUSTUP_HOME "${RUSTUP_HOME}" "${EXPECTED_RUSTUP_HOME}" + require_expected_root CARGO_TARGET_DIR "${CARGO_TARGET_DIR}" "${EXPECTED_TARGET_DIR}" + require_expected_root QUERY_REGRESSION_CACHE_META "${QUERY_REGRESSION_CACHE_META}" "${EXPECTED_CACHE_META}" + require_expected_root SCCACHE_DIR "${SCCACHE_DIR}" "${EXPECTED_SCCACHE_DIR}" + [[ "${RUSTC_WRAPPER}" == "${EXPECTED_RUSTC_WRAPPER}" ]] || { + printf 'Refusing unexpected RUSTC_WRAPPER: %s\n' "${RUSTC_WRAPPER}" >&2 + exit 1 + } + [[ "${SCCACHE_CACHE_SIZE}" == "40G" ]] || { + printf 'Refusing unexpected SCCACHE_CACHE_SIZE: %s\n' "${SCCACHE_CACHE_SIZE}" >&2 + exit 1 + } + [[ "${CARGO_INCREMENTAL}" == "0" ]] || { + printf 'Refusing unexpected CARGO_INCREMENTAL: %s\n' "${CARGO_INCREMENTAL}" >&2 + exit 1 + } + test -r "${RUSTUP_HOME}" + test -x "${RUSTUP_HOME}" + test ! -w "${RUSTUP_HOME}" + mkdir -p "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \ + "${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}" + for root in "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \ + "${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}"; do + test -w "${root}" + touch "${root}/.query-regression-write-test" + rm -f "${root}/.query-regression-write-test" + done + + case "${CARGO_PROFILE}" in + dev) target_profile="debug" ;; + release|nightly) target_profile="${CARGO_PROFILE}" ;; + *) printf 'Unsupported CARGO_PROFILE: %s\n' "${CARGO_PROFILE}" >&2; exit 1 ;; + esac + + rustc_vv="$(rustc -vV)" + if cargo -Vv >/dev/null 2>&1; then + cargo_vv="$(cargo -Vv)" + else + cargo_vv="$(cargo -V)" + fi + host_triple="$(printf '%s\n' "${rustc_vv}" | grep '^host: ' | cut -d' ' -f2-)" + cc_version="$(cc --version)" + mold_version="$(mold --version | grep -m1 .)" + protoc_version="$(protoc --version)" + sccache_version="$(sccache --version)" + rustup_version="$(rustup --version)" + active_toolchain="$(rustup show active-toolchain)" + rustup_path="$(command -v rustup)" + cargo_path="$(command -v cargo)" + rustc_path="$(command -v rustc)" + profile_target_dir="${CARGO_TARGET_DIR}/${target_profile}" + marker="${QUERY_REGRESSION_CACHE_META}/target-${target_profile}.abi" + new_marker="$(mktemp "${QUERY_REGRESSION_CACHE_META}/.${target_profile}.abi.XXXXXX")" + { + printf 'cache_epoch=%s\n' "${QUERY_REGRESSION_CACHE_EPOCH}" + printf 'runner_image_digest=%s\n' "${RUNNER_IMAGE_DIGEST}" + printf 'runner_image_epoch=%s\n' "${RUNNER_IMAGE_EPOCH}" + printf 'host_triple=%s\n' "${host_triple}" + printf 'architecture=%s\n' "$(uname -m)" + printf 'cargo_profile=%s\n' "${CARGO_PROFILE}" + printf 'target_profile=%s\n' "${target_profile}" + printf 'RUSTFLAGS=%s\n' "${RUSTFLAGS-}" + printf 'CARGO_ENCODED_RUSTFLAGS=%s\n' "${CARGO_ENCODED_RUSTFLAGS-}" + printf 'CARGO_INCREMENTAL=%s\n' "${CARGO_INCREMENTAL-}" + printf 'RUSTC_WRAPPER=%s\n' "${RUSTC_WRAPPER-}" + printf 'SCCACHE_DIR=%s\n' "${SCCACHE_DIR-}" + printf 'SCCACHE_CACHE_SIZE=%s\n' "${SCCACHE_CACHE_SIZE-}" + printf 'RUSTUP_HOME=%s\n' "${RUSTUP_HOME-}" + printf 'RUSTUP_TOOLCHAIN=%s\n' "${RUSTUP_TOOLCHAIN-}" + printf 'RUSTUP_AUTO_INSTALL=%s\n' "${RUSTUP_AUTO_INSTALL-}" + printf 'PATH=%s\n' "${PATH-}" + printf '%s\n' "rustc_vV=${rustc_vv}" + printf '%s\n' "cargo_vV=${cargo_vv}" + printf '%s\n' "cc_version=${cc_version}" + printf '%s\n' "mold_version=${mold_version}" + printf '%s\n' "protoc_version=${protoc_version}" + printf '%s\n' "sccache_version=${sccache_version}" + printf '%s\n' "rustup_version=${rustup_version}" + printf '%s\n' "active_toolchain=${active_toolchain}" + printf '%s\n' "rustup_path=${rustup_path}" + printf '%s\n' "cargo_path=${cargo_path}" + printf '%s\n' "rustc_path=${rustc_path}" + } > "${new_marker}" + + if ! cmp -s "${marker}" "${new_marker}"; then + printf 'Cache ABI marker mismatch; invalidating target profile: %s\n' "${target_profile}" + mkdir -p "${profile_target_dir}" + clear_directory "${profile_target_dir}" + else + printf 'Cache ABI marker match; reusing target profile: %s\n' "${target_profile}" + fi + mv -f "${new_marker}" "${marker}" + + report_cache_usage + + target_size="$(target_size_kib)" + if (( target_size >= 400 * 1024 * 1024 )); then + printf 'Warning: target cache is at least 400 GiB (%s KiB)\n' "${target_size}" >&2 + fi + if (( target_size >= 450 * 1024 * 1024 )); then + printf 'Target cache is at least 450 GiB; clearing complete target root\n' >&2 + clear_directory "${CARGO_TARGET_DIR}" + fi + + cargo_size="$(cargo_size_kib)" + if (( cargo_size >= 60 * 1024 * 1024 )); then + printf 'Warning: Cargo cache is at least 60 GiB (%s KiB)\n' "${cargo_size}" >&2 + fi + if (( cargo_size >= 80 * 1024 * 1024 )); then + printf 'Cargo cache is at least 80 GiB; removing extracted sources and checkouts\n' >&2 + clear_cargo_extracted_trees + cargo_size="$(cargo_size_kib)" + if (( cargo_size >= 80 * 1024 * 1024 )); then + printf 'Cargo cache remains at least 80 GiB after cleanup (%s KiB)\n' "${cargo_size}" >&2 + exit 1 + fi + fi + + free_space="$(free_kib)" + if (( free_space < 300 * 1024 * 1024 )); then + printf 'Free space is below 300 GiB; clearing complete target root\n' >&2 + clear_directory "${CARGO_TARGET_DIR}" + free_space="$(free_kib)" + if (( free_space < 300 * 1024 * 1024 )); then + printf 'Free space remains below 300 GiB; removing Cargo extracted sources and checkouts\n' >&2 + clear_cargo_extracted_trees + free_space="$(free_kib)" + if (( free_space < 300 * 1024 * 1024 )); then + printf 'Free space remains below 300 GiB after cleanup (%s KiB)\n' "${free_space}" >&2 + exit 1 + fi + fi + fi + + report_cache_usage + + sccache --start-server + sccache --zero-stats + sccache --show-stats - name: Build base greptime working-directory: src run: | + set -euo pipefail + SECONDS=0 cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + base_build_elapsed="${SECONDS}" + printf 'Base greptime cargo build elapsed: %s seconds\n' "${base_build_elapsed}" + printf -- '- Base greptime cargo build: %s seconds\n' "${base_build_elapsed}" >> "${GITHUB_STEP_SUMMARY}" + sccache --show-stats + sccache --zero-stats target_dir="${CARGO_PROFILE}" if [[ "${CARGO_PROFILE}" == "dev" ]]; then target_dir="debug" @@ -114,9 +437,18 @@ jobs: - name: Switch source to candidate working-directory: src + shell: bash run: | - git fetch --no-tags --prune --depth=1 origin "${CANDIDATE_REF}" - git reset --hard FETCH_HEAD + set -euo pipefail + [[ "${VERIFIED_CANDIDATE_SHA}" =~ ^[[:xdigit:]]{40}$ ]] || { + printf 'Verified candidate SHA is missing or invalid.\n' >&2 + exit 1 + } + git cat-file -e "${VERIFIED_CANDIDATE_SHA}^{commit}" || { + printf 'Verified candidate SHA is unavailable in the local repository.\n' >&2 + exit 1 + } + git reset --hard "${VERIFIED_CANDIDATE_SHA}" git clean -ffdx - name: Test query regression runner lifecycle @@ -126,8 +458,14 @@ jobs: - name: Build candidate greptime and fixture generators working-directory: src run: | + set -euo pipefail + SECONDS=0 cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools + candidate_build_elapsed="${SECONDS}" + printf 'Candidate greptime and fixture generator cargo builds elapsed: %s seconds\n' "${candidate_build_elapsed}" + printf -- '- Candidate greptime and fixture generator cargo builds: %s seconds\n' "${candidate_build_elapsed}" >> "${GITHUB_STEP_SUMMARY}" + sccache --show-stats target_dir="${CARGO_PROFILE}" if [[ "${CARGO_PROFILE}" == "dev" ]]; then target_dir="debug" @@ -192,6 +530,46 @@ jobs: if-no-files-found: warn retention-days: 7 + - name: Report persistent cache usage + if: ${{ always() }} + shell: bash + run: | + set -euo pipefail + for root in \ + /home/runner/.cargo \ + /home/runner/.cargo/registry \ + /home/runner/.cargo/git \ + /opt/rustup \ + /home/runner/query-regression-target \ + /home/runner/query-regression-cache-meta \ + /home/runner/.cache/sccache; do + if [[ -e "${root}" ]]; then + du -sh -- "${root}" + else + printf 'Missing cache path (report only): %s\n' "${root}" + fi + done + if [[ -e /opt/rustup ]]; then + if [[ -w /opt/rustup ]]; then + printf 'Immutable Rustup path is unexpectedly writable: /opt/rustup\n' >&2 + else + printf 'Immutable Rustup path is read-only to runner: /opt/rustup\n' + fi + fi + report_root=/home/runner/query-regression-target + if [[ ! -e "${report_root}" ]]; then + report_root=/home/runner + printf 'Target cache path is missing; reporting its filesystem through %s\n' "${report_root}" + fi + df -P "${report_root}" + df -hP "${report_root}" + df -Pi "${report_root}" + if command -v sccache >/dev/null 2>&1; then + sccache --show-stats || printf 'Unable to show sccache statistics (report only)\n' >&2 + else + printf 'sccache is unavailable (report only)\n' >&2 + fi + - name: Fail on regression failure if: ${{ steps.run.outputs.status != '0' }} run: exit 1