mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-08-18 12:08:22 +00:00
ci: add persistent query regression cache (#8474)
* ci: add persistent query regression cache Signed-off-by: discord9 <discord9@163.com> * ci: add sccache to query regression runner Signed-off-by: discord9 <discord9@163.com> * ci: pin query regression label revision Signed-off-by: discord9 <discord9@163.com> * ci: isolate query regression toolchain state Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
GitHub
@@ -3,6 +3,19 @@ FROM ghcr.io/actions/actions-runner@sha256:08c30b0a7105f64bddfc485d2487a22aa0393
|
||||
|
||||
USER root
|
||||
|
||||
ARG SCCACHE_VERSION=0.16.0
|
||||
ARG SCCACHE_SHA256=aec995a83ad3dff3d14b6314e08858b7b73d35ca85a5bcf3d3a9ec07dee35588
|
||||
ARG RUSTUP_INIT_VERSION=1.29.0
|
||||
ARG RUSTUP_INIT_TARGET=x86_64-unknown-linux-gnu
|
||||
ARG RUSTUP_INIT_SHA256=4acc9acc76d5079515b46346a485974457b5a79893cfb01112423c89aeb5aa10
|
||||
ARG RUST_TOOLCHAIN=nightly-2026-03-21
|
||||
|
||||
ENV RUSTUP_HOME=/opt/rustup \
|
||||
CARGO_HOME=/opt/cargo \
|
||||
RUSTUP_TOOLCHAIN=${RUST_TOOLCHAIN} \
|
||||
RUSTUP_AUTO_INSTALL=0 \
|
||||
PATH=/opt/cargo/bin:${PATH}
|
||||
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
@@ -30,4 +43,42 @@ RUN apt-get update \
|
||||
|
||||
COPY --from=uv /uv /uvx /usr/local/bin/
|
||||
|
||||
RUN curl --fail --location --silent --show-error \
|
||||
--output /tmp/sccache.tar.gz \
|
||||
"https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" \
|
||||
&& echo "${SCCACHE_SHA256} /tmp/sccache.tar.gz" | sha256sum --check --status - \
|
||||
&& mkdir --parents /tmp/sccache \
|
||||
&& tar --extract --gzip --file /tmp/sccache.tar.gz \
|
||||
--directory /tmp/sccache --strip-components=1 \
|
||||
"sccache-v${SCCACHE_VERSION}-x86_64-unknown-linux-musl/sccache" \
|
||||
&& install --mode=0755 /tmp/sccache/sccache /usr/local/bin/sccache \
|
||||
&& rm -rf /tmp/sccache.tar.gz /tmp/sccache
|
||||
|
||||
RUN curl --fail --location --silent --show-error \
|
||||
--output /tmp/rustup-init \
|
||||
"https://static.rust-lang.org/rustup/archive/${RUSTUP_INIT_VERSION}/${RUSTUP_INIT_TARGET}/rustup-init" \
|
||||
&& echo "${RUSTUP_INIT_SHA256} /tmp/rustup-init" | sha256sum --check --status - \
|
||||
&& chmod 0755 /tmp/rustup-init \
|
||||
&& /tmp/rustup-init -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" --no-modify-path \
|
||||
&& rm -f /tmp/rustup-init \
|
||||
&& chown -R root:root /opt/rustup /opt/cargo \
|
||||
&& chmod -R go-w /opt/rustup /opt/cargo \
|
||||
&& find /opt/rustup /opt/cargo -type d -exec chmod 0755 {} + \
|
||||
&& find /opt/rustup /opt/cargo -type f -exec chmod a+r {} +
|
||||
|
||||
USER runner
|
||||
|
||||
RUN set -eu \
|
||||
&& test "$(id -u)" = "1001" \
|
||||
&& temporary_cargo_home="$(mktemp --directory)" \
|
||||
&& trap 'rm -rf "${temporary_cargo_home}"' 0 \
|
||||
&& export CARGO_HOME="${temporary_cargo_home}" \
|
||||
&& test "${RUSTUP_AUTO_INSTALL}" = "0" \
|
||||
&& rustup --version \
|
||||
&& cargo --version \
|
||||
&& rustc --version \
|
||||
&& active_toolchain="$(rustup show active-toolchain)" \
|
||||
&& printf 'Active toolchain: %s\n' "${active_toolchain}" \
|
||||
&& case "${active_toolchain}" in "${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu"|"${RUST_TOOLCHAIN}-x86_64-unknown-linux-gnu "*) ;; *) exit 1;; esac \
|
||||
&& test ! -w /opt/rustup \
|
||||
&& test ! -w /opt/cargo/bin
|
||||
|
||||
@@ -1,21 +1,14 @@
|
||||
# Query regression self-hosted runners
|
||||
|
||||
The `Query Regression` workflow targets self-hosted GitHub Actions runners via
|
||||
runner labels or ARC runner scale set names:
|
||||
The `Query Regression` workflow uses the dedicated ARC runner scale set
|
||||
`perf-regression-8-cores`. ARC runner Pods run in the target Kubernetes cluster
|
||||
and connect outbound to GitHub. The live scale set is currently **paused**:
|
||||
`minRunners=0`, `maxRunners=0`, and no runner Pods. Do not resume it without
|
||||
explicit approval.
|
||||
|
||||
- `perf-regression-8-cores`
|
||||
## Prerequisites and trust admission
|
||||
|
||||
The name intentionally avoids generic labels such as `ubuntu-22.04-8-cores`,
|
||||
which may already be used by GitHub-hosted larger runners or other runner pools.
|
||||
|
||||
For Kubernetes-based runners, ARC runner pods run inside the target Kubernetes
|
||||
cluster and connect outbound to GitHub; GitHub can then dispatch jobs whose
|
||||
`runs-on` value matches the runner scale set name or runner labels.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Install the ARC scale set controller in the Kubernetes cluster if it is not
|
||||
already installed:
|
||||
Install the ARC scale set controller if it is not already installed:
|
||||
|
||||
```bash
|
||||
helm upgrade --install arc \
|
||||
@@ -25,8 +18,8 @@ helm upgrade --install arc \
|
||||
--version 0.14.2
|
||||
```
|
||||
|
||||
Create the GitHub App or PAT secret in the namespace that will host the runner
|
||||
scale sets. Prefer a GitHub App with access limited to `GreptimeTeam/greptimedb`.
|
||||
Create the GitHub App secret in the runner namespace. Prefer an App limited to
|
||||
`GreptimeTeam/greptimedb`:
|
||||
|
||||
```bash
|
||||
kubectl -n arc-runners create secret generic greptimedb-arc-github-app \
|
||||
@@ -35,14 +28,46 @@ kubectl -n arc-runners create secret generic greptimedb-arc-github-app \
|
||||
--from-file=github_app_private_key=<private-key.pem>
|
||||
```
|
||||
|
||||
The values files in this directory reference that secret by name.
|
||||
The values files here reference that secret by name.
|
||||
|
||||
## Build the runner image
|
||||
A maintainer applying the `query-regression` label is **trust admission for
|
||||
that exact PR revision**. The admitted job may use this scale set's dedicated,
|
||||
writable persistent cache. `pull_request: labeled` is the only PR trigger: the
|
||||
label event snapshots its merge, head, and base SHAs. A queued job fetches that
|
||||
immutable event merge SHA directly, verifies it is a two-parent merge whose
|
||||
parents include the snapshotted head exactly once, and uses its other parent as
|
||||
the actual base build revision. The snapshotted event base is retained for audit
|
||||
only, so a difference from the merge's non-head parent is not a failure. The job
|
||||
never follows a newer mutable PR merge ref. An unavailable event merge, or one
|
||||
that does not contain exactly one snapshotted head parent, fails closed. A later
|
||||
PR head change does not retarget an already queued run: it may execute only its
|
||||
previously trusted event revision if that revision remains fetchable. To run the
|
||||
new revision, the maintainer must review it, remove the label, and re-add
|
||||
`query-regression`; cancel the old run if it is no longer wanted. An existing
|
||||
label does not automatically rerun the benchmark.
|
||||
|
||||
Use a derived ARC runner image instead of the minimal upstream runner image. The
|
||||
image keeps the official `/home/runner/run.sh` entrypoint layout and adds the
|
||||
tools this workflow expects to be present on a normal CI host, including `wget`,
|
||||
`uv`, C/C++ build tools, OpenSSL headers, and protobuf tooling.
|
||||
Admission does not relax runner hardening or GitHub permissions. Keep
|
||||
service-account token mounting disabled; do not mount host paths, the Docker
|
||||
socket, kubeconfig, or long-lived credentials. The runner and cache initializer
|
||||
use UID/GID 1001, disallow privilege escalation, drop all capabilities, and use
|
||||
the RuntimeDefault seccomp profile. Keep GitHub tokens least-privilege and
|
||||
review workflow changes before admission. Where the CNI supports it, restrict
|
||||
egress to required GitHub Actions, artifact/cache, Rust/crate/toolchain, DNS,
|
||||
and image-registry endpoints; block unrelated cluster services, private ranges,
|
||||
and metadata endpoints unless a case requires them.
|
||||
|
||||
### Office routing prerequisite
|
||||
|
||||
Direct split routing through `.2` is an **external office-network prerequisite**.
|
||||
The gateway at `192.168.50.2` must route GitHub Actions, GitHub content,
|
||||
artifact/cache, crates.io, Rust toolchain, and image-registry traffic directly
|
||||
rather than through the VPN. Neither this repository nor Kubernetes configures
|
||||
that route. Verify it with the responsible network operator before any canary.
|
||||
|
||||
## Runner image and workflow tools
|
||||
|
||||
Build and push the derived runner image; it preserves the official
|
||||
`/home/runner/run.sh` entrypoint and supplies CI tools needed at runtime:
|
||||
|
||||
```bash
|
||||
docker build \
|
||||
@@ -53,100 +78,219 @@ docker build \
|
||||
docker push greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest
|
||||
```
|
||||
|
||||
Deploy the runner image by digest rather than by a mutable tag. Update
|
||||
`values-8-cores.yaml` after pushing a rebuilt image.
|
||||
Deploy by digest, not mutable tag, by updating `values-8-cores.yaml` after a
|
||||
rebuild. If the registry is private, use a dedicated read-only pull secret only
|
||||
as `imagePullSecrets`; never expose registry credentials to runner containers.
|
||||
Both digest-pinned init and runner containers use `IfNotPresent`: the immutable
|
||||
digest makes a cached image safe and avoids adding a registry dependency to every
|
||||
runner startup.
|
||||
|
||||
The workflow still runs setup actions for pinned Rust and `uv` behavior. `mold`
|
||||
is installed in the image and selected through `CARGO_BUILD_RUSTFLAGS`, so jobs
|
||||
do not need privileged package installation at runtime.
|
||||
Before builds, the workflow asserts UID/GID 1001 and exact image tool versions:
|
||||
`libprotoc 3.21.12`, `uv 0.11.26`, `mold 2.30.0`, `Python 3.12.3`, `sccache
|
||||
0.16.0`, root-owned `rustup 1.29.0`, and the image-baked
|
||||
`nightly-2026-03-21` Rust toolchain. Rustup, Cargo, and Rustc must resolve from
|
||||
`/opt/cargo/bin`; the runner cannot write `/opt/rustup` or `/opt/cargo/bin`.
|
||||
`actions-rust-lang/setup-rust-toolchain@v1` is intentionally removed. The
|
||||
workflow sets its warning-denying mold `RUSTFLAGS` directly, disables automatic
|
||||
Rustup installation, and performs no runtime toolchain downloads.
|
||||
|
||||
Decide whether the registry repository is public or private. Public pull access
|
||||
avoids distributing registry credentials to the runner namespace. If the image
|
||||
must be private, create a dedicated read-only image-pull secret for this image
|
||||
and attach it only as `imagePullSecrets`; do not expose registry credentials to
|
||||
runner containers.
|
||||
The workflow no longer uses GitHub `rust-cache`, `setup-protoc`, `setup-uv`, or
|
||||
runtime Rust setup: the image establishes immutable executable state and the PVC
|
||||
supplies only reusable Cargo data. Do not reintroduce those actions unless the
|
||||
corresponding cache or image contract changes.
|
||||
|
||||
## Install the query-regression scale set
|
||||
## Capacity and persistent cache
|
||||
|
||||
Install the runner scale set. The Helm release name and `runnerScaleSetName`
|
||||
should match the `runs-on` value used by the workflow.
|
||||
`values-8-cores.yaml` is normal operation: `minRunners=0`, `maxRunners=1`.
|
||||
`values-paused.yaml` is the mandatory pause overlay: `minRunners=0`,
|
||||
`maxRunners=0`. The job uses group `query-regression-persistent-cache-v1`,
|
||||
`queue: max`, and `cancel-in-progress: false`; admitted jobs queue rather than
|
||||
replacing older pending jobs. During maintenance, cancel admitted queued runs as
|
||||
well as pausing ARC. Runner Pods have `activeDeadlineSeconds=12600`.
|
||||
|
||||
The cache claim `query-regression-build-cache` is a nominal 600Gi `local-path`
|
||||
PVC in `arc-runners`. It is `ReadWriteOnce`; `local-path` uses
|
||||
WaitForFirstConsumer binding and Delete reclaim behavior, produces a
|
||||
node-affine local PV, is non-expandable, and the 600Gi request is not a hard
|
||||
storage quota. The runner's `minipc-3` selector is its only consumer candidate.
|
||||
|
||||
The initializer mounts the PVC root at `/cache`, creates and write-tests these
|
||||
versioned subpaths as non-root UID/GID 1001, and the runner mounts them as:
|
||||
|
||||
| Persistent state | PVC subpath | Runner mount |
|
||||
| --- | --- | --- |
|
||||
| Ephemeral Cargo home | `emptyDir` | `/home/runner/.cargo` |
|
||||
| Cargo registry data | `cargo-registry-v1` | `/home/runner/.cargo/registry` |
|
||||
| Cargo Git data | `cargo-git-v1` | `/home/runner/.cargo/git` |
|
||||
| Cargo target | `query-regression-target-v1` | `/home/runner/query-regression-target` |
|
||||
| Cache metadata | `meta-v1` | `/home/runner/query-regression-cache-meta` |
|
||||
| sccache local disk cache | `sccache-v1` | `/home/runner/.cache/sccache` |
|
||||
| Immutable Rust toolchain | image-owned | `/opt/rustup`, `/opt/cargo/bin` |
|
||||
|
||||
The Pod security context uses UID/GID and `fsGroup` 1001 with
|
||||
`fsGroupChangePolicy: OnRootMismatch`; no privileged `chown` or raw `hostPath`
|
||||
is used. `CARGO_HOME` is a per-Pod `emptyDir`; only its nested `registry` and
|
||||
`git` mounts are persistent. `RUSTUP_HOME=/opt/rustup` and `/opt/cargo/bin` are
|
||||
image-owned immutable paths, while `CARGO_TARGET_DIR`, cache metadata, and
|
||||
`SCCACHE_DIR` are persistent absolute paths. The runner sets
|
||||
`RUSTC_WRAPPER=/usr/local/bin/sccache`,
|
||||
`SCCACHE_DIR=/home/runner/.cache/sccache`, `SCCACHE_CACHE_SIZE=40G`, and
|
||||
`CARGO_INCREMENTAL=0`. sccache uses its local PVC disk backend and self-evicts
|
||||
at 40G; do not add runtime downloads, object storage, or a shared backend.
|
||||
|
||||
The repository's `.cargo/config.toml` remains a trusted per-revision build input.
|
||||
In contrast, `$CARGO_HOME/config*`, credentials, installed bins, and Cargo
|
||||
metadata outside the persistent `registry` and `git` data mounts are ephemeral
|
||||
and cannot survive to another Pod.
|
||||
|
||||
The local disk backend has a one-server constraint. `maxRunners=1` and the
|
||||
unchanged `query-regression-persistent-cache-v1` workflow concurrency group
|
||||
serialize runs; do not increase runner capacity or relax that serialization
|
||||
while this backend is in use. Base and candidate builds share the target; Cargo
|
||||
fingerprints invalidate source and dependency changes. The workflow records the
|
||||
sccache version and relevant environment in the target ABI marker, starts and
|
||||
zeros sccache after cache and toolchain checks, shows initial/base/candidate
|
||||
statistics, and resets statistics between base and candidate builds.
|
||||
|
||||
### Disk preflight and cleanup contract
|
||||
|
||||
Before applying or unpausing, verify the backing filesystem on `minipc-3` has
|
||||
at least 900GiB free. The current local-path provisioner source is
|
||||
`/opt/local-path-provisioner`; measure the filesystem containing it:
|
||||
|
||||
```bash
|
||||
df -PB1G /opt/local-path-provisioner
|
||||
```
|
||||
|
||||
The workflow reports `du`, `df -P`, human-readable free space, and inode
|
||||
availability before builds and in an always-run report. Its cleanup is narrow
|
||||
and non-destructive:
|
||||
|
||||
- warn at target size 400GiB; at 450GiB clear only the complete target root;
|
||||
- warn at Cargo registry-plus-Git data size 60GiB; at 80GiB remove only
|
||||
`registry/src` and `git/checkouts`, then abort if that persistent data remains
|
||||
at least 80GiB;
|
||||
- below 300GiB backing free space, clear the complete target root first,
|
||||
remeasure, then remove only those Cargo extracted trees and checkouts; abort
|
||||
if free space is still below 300GiB;
|
||||
- never automatically remove Cargo registry cache/index, Git database, the
|
||||
image-owned Cargo bin or Rustup toolchain, cache metadata, the self-evicting
|
||||
sccache directory, or the PVC.
|
||||
|
||||
The target clear uses fixed absolute roots and removes all entries, including
|
||||
dotfiles. After migration validation, remove obsolete `cargo-home-v1` and
|
||||
`rustup-home-v1` only in explicit maintenance while ARC is 0/0 and no runner Pod
|
||||
exists; they are not mounted by the current configuration.
|
||||
|
||||
## Deploy and pause safely
|
||||
|
||||
First verify external `.2` routing and the disk preflight. Apply the PVC; while
|
||||
the scale set is paused, it is expected to remain `Pending` because
|
||||
WaitForFirstConsumer has no scheduled runner:
|
||||
|
||||
```bash
|
||||
kubectl apply --dry-run=server \
|
||||
-f .github/runner-scale-sets/query-regression/cache-pvc.yaml
|
||||
kubectl apply -f .github/runner-scale-sets/query-regression/cache-pvc.yaml
|
||||
```
|
||||
|
||||
Render normal and paused configurations. Normal values are always first; the
|
||||
pause overlay is always last:
|
||||
|
||||
```bash
|
||||
helm template perf-regression-8-cores \
|
||||
oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
|
||||
--namespace arc-runners --version 0.14.2 \
|
||||
--set controllerServiceAccount.name=arc-gha-rs-controller \
|
||||
--set controllerServiceAccount.namespace=arc-systems \
|
||||
-f .github/runner-scale-sets/query-regression/values-8-cores.yaml
|
||||
|
||||
helm template perf-regression-8-cores \
|
||||
oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
|
||||
--namespace arc-runners --version 0.14.2 \
|
||||
--set controllerServiceAccount.name=arc-gha-rs-controller \
|
||||
--set controllerServiceAccount.namespace=arc-systems \
|
||||
-f .github/runner-scale-sets/query-regression/values-8-cores.yaml \
|
||||
-f .github/runner-scale-sets/query-regression/values-paused.yaml
|
||||
```
|
||||
|
||||
The **first post-merge Helm deployment must reconcile the release in paused
|
||||
mode**. Keep the pause overlay last:
|
||||
|
||||
```bash
|
||||
# First post-merge deployment and every return to paused mode: 0/0.
|
||||
helm upgrade --install perf-regression-8-cores \
|
||||
oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
|
||||
--namespace arc-runners --create-namespace --version 0.14.2 \
|
||||
--reset-values --wait \
|
||||
-f .github/runner-scale-sets/query-regression/values-8-cores.yaml \
|
||||
-f .github/runner-scale-sets/query-regression/values-paused.yaml
|
||||
|
||||
# Expect 0/0 and no runner resources before considering normal mode.
|
||||
kubectl -n arc-runners get autoscalingrunnerset perf-regression-8-cores \
|
||||
-o jsonpath='{.spec.minRunners}{"/"}{.spec.maxRunners}{"\n"}'
|
||||
kubectl -n arc-runners get ephemeralrunners,pods \
|
||||
-l actions.github.com/scale-set-name=perf-regression-8-cores
|
||||
```
|
||||
|
||||
Only after that verification and separate explicit approval, apply normal 0/1
|
||||
operation without the pause overlay:
|
||||
|
||||
```bash
|
||||
helm upgrade --install perf-regression-8-cores \
|
||||
oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \
|
||||
--namespace arc-runners \
|
||||
--create-namespace \
|
||||
--version 0.14.2 \
|
||||
--namespace arc-runners --create-namespace --version 0.14.2 \
|
||||
--reset-values --wait \
|
||||
-f .github/runner-scale-sets/query-regression/values-8-cores.yaml
|
||||
```
|
||||
|
||||
Check registration and pods:
|
||||
Do not use bare `helm rollback`, `--atomic`, or `--reuse-values`: a stored
|
||||
revision can restore nonzero runner capacity. Inspect rendered manifests for
|
||||
capacity, the `minipc-3` selector, cache claim and mounts, initializer,
|
||||
security context, and resources. After approved normal mode receives its first
|
||||
canary, the PVC binds to `minipc-3`.
|
||||
|
||||
```bash
|
||||
kubectl -n arc-runners get pods
|
||||
kubectl -n arc-runners get autoscalingrunnersets
|
||||
```
|
||||
For local-PV node loss, cache recovery is intentionally disposable: return to
|
||||
0/0, recreate the PVC on a healthy node, cold-fill it, and run a new canary.
|
||||
|
||||
The scale sets should also appear under repository Actions runner settings.
|
||||
## Canary and rollback
|
||||
|
||||
## Security notes for fork PRs
|
||||
With explicit approval, run two identical `workflow_dispatch` canaries on
|
||||
`perf-regression-8-cores`, using immutable full base and candidate commit SHAs
|
||||
and `cargo_profile=nightly`. The first is the cold fill; the second verifies warm
|
||||
reuse. Record the workflow's base/candidate build elapsed logs and cache
|
||||
ABI-marker output, initial/base/candidate sccache statistics, and cache report.
|
||||
Confirm the image tool contract (root-owned Rustup/Cargo paths, baked nightly,
|
||||
and non-writable `/opt` roots) and that the ephemeral Cargo home contains only
|
||||
the mounted registry/Git data before Cargo creates per-Pod state.
|
||||
Obtain dependency and tool network byte counters from the
|
||||
identified `.2` counter source, filtered to `minipc-3` and the dependency/tool
|
||||
destinations.
|
||||
|
||||
Maintainer-approved fork PRs can run on self-hosted runners. Approval only lets
|
||||
the workflow execute; it does not make the fork code trusted.
|
||||
Accept the canary only when all of the following hold:
|
||||
|
||||
The `query-regression` label is the explicit trigger for PR runs. Updating a PR
|
||||
does not rerun the benchmark automatically; remove and re-add the label after
|
||||
reviewing the updated changes.
|
||||
- exactly one runner Pod runs on `minipc-3`, and both jobs use the same bound PV;
|
||||
- UID/GID 1001 cache mounts are writable; the warm run does not invalidate the
|
||||
target or bulk-redownload crates or toolchains; sccache reports separate base
|
||||
and candidate build statistics without server or cache-path errors; immutable
|
||||
Rustup/Cargo roots remain non-writable and only registry/Git data persists;
|
||||
- warm base build time is at most 50% of cold base build time;
|
||||
- warm dependency/tool network bytes are at most 10% of cold fill bytes;
|
||||
- cache sizes remain below soft watermarks, node free space remains at least
|
||||
300GiB, and the benchmark is correct without TLS EOFs or timeouts;
|
||||
- `.2` confirms this traffic is outside VPN accounting.
|
||||
|
||||
These runners execute PR code and should be treated as untrusted execution
|
||||
capacity:
|
||||
Immediately return to 0/0 after either canary unless ongoing normal operation
|
||||
has been explicitly approved; return immediately on any traffic, cache, disk,
|
||||
TLS, or correctness failure. To roll back, use the paused Helm upgrade above,
|
||||
or another explicit `helm upgrade --install` with known-good values followed by
|
||||
`values-paused.yaml`, `--reset-values`, and `--wait`. Do not delete the PVC
|
||||
automatically; preserve it for diagnosis unless intentionally discarding cache.
|
||||
|
||||
- keep them isolated from sensitive internal services unless explicitly required;
|
||||
- do not mount host paths, Docker socket, kubeconfig, or long-lived credentials;
|
||||
- use ephemeral runner pods and no shared work directory with trusted jobs;
|
||||
- disable service account token mounting in runner pods unless Kubernetes API
|
||||
access is intentionally required;
|
||||
- use a runner image whose default user is non-root, disable privilege
|
||||
escalation, drop Linux capabilities, and use the runtime-default seccomp
|
||||
profile;
|
||||
- keep GitHub tokens least-privilege and rely on normal `pull_request` behavior
|
||||
for fork PRs, where repository secrets are withheld and `GITHUB_TOKEN` is
|
||||
read-only;
|
||||
- review fork workflow changes before approving the run.
|
||||
## Future optional phases
|
||||
|
||||
If stronger isolation is required, install a separate runner group/namespace just
|
||||
for query-regression PR workloads and restrict repository/workflow access to the
|
||||
`Query Regression` workflow.
|
||||
|
||||
Use namespace or cluster network policy to restrict runner egress where the CNI
|
||||
supports the needed controls. Query-regression runners need outbound access to
|
||||
GitHub Actions services, GitHub artifact/cache endpoints, Rust/crate/toolchain
|
||||
endpoints, DNS, and the configured image registries. Block access to unrelated
|
||||
cluster services, private network ranges, and cloud metadata endpoints unless a
|
||||
case explicitly needs them.
|
||||
|
||||
## Build cache
|
||||
|
||||
The workflow builds base and candidate in the same job, the same source path,
|
||||
and a shared `CARGO_TARGET_DIR`. It checks out the base ref into `src`, builds
|
||||
and copies the base binary aside, then resets that same `src` checkout to the
|
||||
candidate ref before building the candidate binary. Keeping the workspace path
|
||||
stable improves Cargo incremental reuse for local workspace crates compared with
|
||||
building separate `base-src` and `candidate-src` checkouts.
|
||||
|
||||
The workflow also uses the GitHub Actions Rust cache for restore-only cache
|
||||
reuse. PR and dispatch runs do not save cache entries. Refresh shared caches from
|
||||
trusted maintenance workflows only.
|
||||
|
||||
The ARC values in this directory do not configure a cross-run runner-local
|
||||
compiler cache. If cross-run compile time still dominates the benchmark,
|
||||
deployers may add one of the following at the runner infrastructure layer:
|
||||
|
||||
- a custom runner image with Rust tooling, `mold`, and `sccache` preinstalled;
|
||||
- `RUSTC_WRAPPER=sccache` plus a shared `sccache` backend such as object storage,
|
||||
Redis, or a Kubernetes storage class that is safe for concurrent runner pods;
|
||||
- separate cache namespaces/buckets for untrusted PR code and trusted branches to
|
||||
avoid cache poisoning across trust boundaries.
|
||||
|
||||
Do not mount a shared writable host path, Docker socket, kubeconfig, or other
|
||||
privileged credentials into runners that execute PR code.
|
||||
The current phase uses a digest-pinned image with sccache 0.16.0 and no shared
|
||||
cache service. Optional follow-ups are an image additionally seeded with the
|
||||
exact Rust toolchain and `cargo fetch --locked`; or an internal read/write
|
||||
sccache backend or Cargo/Git mirror. Evaluate them only if persistent PVC reuse
|
||||
is insufficient.
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: query-regression-build-cache
|
||||
namespace: arc-runners
|
||||
labels:
|
||||
app.kubernetes.io/part-of: perf-regression-8-cores
|
||||
spec:
|
||||
volumeMode: Filesystem
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: local-path
|
||||
resources:
|
||||
requests:
|
||||
storage: 600Gi
|
||||
@@ -3,21 +3,96 @@ githubConfigSecret: greptimedb-arc-github-app
|
||||
|
||||
runnerScaleSetName: "perf-regression-8-cores"
|
||||
minRunners: 0
|
||||
maxRunners: 4
|
||||
maxRunners: 1
|
||||
|
||||
template:
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
activeDeadlineSeconds: 11400
|
||||
activeDeadlineSeconds: 12600
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: minipc-3
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
fsGroup: 1001
|
||||
fsGroupChangePolicy: OnRootMismatch
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumes:
|
||||
- name: cargo-home
|
||||
emptyDir: {}
|
||||
- name: build-cache
|
||||
persistentVolumeClaim:
|
||||
claimName: query-regression-build-cache
|
||||
initContainers:
|
||||
- name: initialize-build-cache
|
||||
image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- /bin/sh
|
||||
- -ec
|
||||
- |
|
||||
umask 0002
|
||||
for directory in cargo-registry-v1 cargo-git-v1 query-regression-target-v1 meta-v1 sccache-v1; do
|
||||
cache_directory="/cache/${directory}"
|
||||
mkdir -p "${cache_directory}"
|
||||
test -w "${cache_directory}"
|
||||
probe_file="${cache_directory}/.write-probe"
|
||||
: > "${probe_file}"
|
||||
test -f "${probe_file}"
|
||||
rm "${probe_file}"
|
||||
done
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
- ALL
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
ephemeral-storage: 1Gi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 512Mi
|
||||
ephemeral-storage: 1Gi
|
||||
volumeMounts:
|
||||
- name: build-cache
|
||||
mountPath: /cache
|
||||
containers:
|
||||
- name: runner
|
||||
image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:108b7357532e5cf3329853373f561b0c674e954d6307b53d37075df5a189b0b2
|
||||
imagePullPolicy: Always
|
||||
image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e
|
||||
imagePullPolicy: IfNotPresent
|
||||
command: ["/home/runner/run.sh"]
|
||||
env:
|
||||
- name: CARGO_HOME
|
||||
value: /home/runner/.cargo
|
||||
- name: RUSTUP_HOME
|
||||
value: /opt/rustup
|
||||
- name: RUSTUP_TOOLCHAIN
|
||||
value: nightly-2026-03-21
|
||||
- name: RUSTUP_AUTO_INSTALL
|
||||
value: "0"
|
||||
- name: CARGO_TARGET_DIR
|
||||
value: /home/runner/query-regression-target
|
||||
- name: QUERY_REGRESSION_CACHE_META
|
||||
value: /home/runner/query-regression-cache-meta
|
||||
- name: RUSTC_WRAPPER
|
||||
value: /usr/local/bin/sccache
|
||||
- name: SCCACHE_DIR
|
||||
value: /home/runner/.cache/sccache
|
||||
- name: SCCACHE_CACHE_SIZE
|
||||
value: 40G
|
||||
- name: CARGO_INCREMENTAL
|
||||
value: "0"
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 1001
|
||||
runAsGroup: 1001
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop:
|
||||
@@ -31,3 +106,21 @@ template:
|
||||
cpu: "8"
|
||||
memory: 24Gi
|
||||
ephemeral-storage: 80Gi
|
||||
volumeMounts:
|
||||
- name: cargo-home
|
||||
mountPath: /home/runner/.cargo
|
||||
- name: build-cache
|
||||
mountPath: /home/runner/.cargo/registry
|
||||
subPath: cargo-registry-v1
|
||||
- name: build-cache
|
||||
mountPath: /home/runner/.cargo/git
|
||||
subPath: cargo-git-v1
|
||||
- name: build-cache
|
||||
mountPath: /home/runner/query-regression-target
|
||||
subPath: query-regression-target-v1
|
||||
- name: build-cache
|
||||
mountPath: /home/runner/query-regression-cache-meta
|
||||
subPath: meta-v1
|
||||
- name: build-cache
|
||||
mountPath: /home/runner/.cache/sccache
|
||||
subPath: sccache-v1
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
minRunners: 0
|
||||
maxRunners: 0
|
||||
@@ -241,8 +241,10 @@ module.exports = async function validateQueryRegressionComment({ github, context
|
||||
'> Rendered by a trusted workflow from JSON artifacts produced by the query-regression run. Results from untrusted PR code are advisory until reviewed.',
|
||||
'',
|
||||
`- **Workflow run:** ${serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${expectedRunId}`,
|
||||
`- **Base SHA:** \`${text(metadata.base_sha)}\``,
|
||||
`- **Built base SHA:** \`${text(metadata.built_base_sha)}\``,
|
||||
`- **Event base SHA:** \`${text(metadata.event_base_sha)}\``,
|
||||
`- **Head SHA:** \`${text(metadata.head_sha)}\``,
|
||||
`- **Candidate merge SHA:** \`${text(metadata.candidate_sha)}\``,
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
|
||||
@@ -31,7 +31,10 @@ def main() -> int:
|
||||
metadata = {
|
||||
"pr_number": int(os.environ["PR_NUMBER"]),
|
||||
"head_sha": os.environ["HEAD_SHA"],
|
||||
"base_sha": os.environ["BASE_SHA"],
|
||||
"base_sha": os.environ["EVENT_BASE_SHA"],
|
||||
"event_base_sha": os.environ["EVENT_BASE_SHA"],
|
||||
"built_base_sha": os.environ["VERIFIED_BASE_SHA"],
|
||||
"candidate_sha": os.environ["VERIFIED_CANDIDATE_SHA"],
|
||||
"head_repo": os.environ["HEAD_REPO"],
|
||||
"base_repo": os.environ["BASE_REPO"],
|
||||
"run_id": int(os.environ["RUN_ID"]),
|
||||
|
||||
@@ -45,10 +45,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
query-regression:
|
||||
if: >-
|
||||
@@ -59,12 +55,29 @@ jobs:
|
||||
(github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }}
|
||||
runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || 'perf-regression-8-cores' }}
|
||||
timeout-minutes: 180
|
||||
concurrency:
|
||||
group: query-regression-persistent-cache-v1
|
||||
queue: max
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }}
|
||||
CARGO_TARGET_DIR: ${{ github.workspace }}/query-regression-target
|
||||
CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold"
|
||||
CARGO_HOME: /home/runner/.cargo
|
||||
RUSTUP_HOME: /opt/rustup
|
||||
RUSTUP_TOOLCHAIN: nightly-2026-03-21
|
||||
RUSTUP_AUTO_INSTALL: "0"
|
||||
CARGO_TARGET_DIR: /home/runner/query-regression-target
|
||||
QUERY_REGRESSION_CACHE_META: /home/runner/query-regression-cache-meta
|
||||
RUSTC_WRAPPER: /usr/local/bin/sccache
|
||||
SCCACHE_DIR: /home/runner/.cache/sccache
|
||||
SCCACHE_CACHE_SIZE: 40G
|
||||
CARGO_INCREMENTAL: "0"
|
||||
RUSTFLAGS: -D warnings -C link-arg=-fuse-ld=mold
|
||||
QUERY_REGRESSION_CACHE_EPOCH: "1"
|
||||
BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }}
|
||||
CANDIDATE_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }}
|
||||
CANDIDATE_REF: ${{ github.event_name == 'workflow_dispatch' && (inputs.candidate_ref || github.ref) || '' }}
|
||||
EVENT_MERGE_SHA: ${{ github.event_name == 'pull_request' && github.sha || '' }}
|
||||
EVENT_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
|
||||
EVENT_BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }}
|
||||
steps:
|
||||
- name: Checkout base source
|
||||
uses: actions/checkout@v4
|
||||
@@ -82,28 +95,338 @@ jobs:
|
||||
cp src/.github/scripts/query-regression-summary.py \
|
||||
query-regression-trusted-scripts/query-regression-summary.py
|
||||
|
||||
- uses: arduino/setup-protoc@v3
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Resolve immutable candidate and build base
|
||||
working-directory: src
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache: false
|
||||
fail_closed() {
|
||||
printf '%s\n' \
|
||||
"PR changed or merge result unavailable; maintainer must remove and re-add query-regression after reviewing current revision. $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
is_full_sha() {
|
||||
[[ "$1" =~ ^[[:xdigit:]]{40}$ ]]
|
||||
}
|
||||
|
||||
- name: Rust cache
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: src
|
||||
shared-key: query-regression
|
||||
cache-all-crates: "true"
|
||||
save-if: false
|
||||
initially_checked_out_base_sha="$(git rev-parse --verify HEAD)"
|
||||
is_full_sha "${initially_checked_out_base_sha}" || {
|
||||
printf 'Could not resolve initially checked-out BASE_REF commit.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
|
||||
for identity in EVENT_MERGE_SHA EVENT_HEAD_SHA EVENT_BASE_SHA; do
|
||||
value="${!identity-}"
|
||||
is_full_sha "${value}" || fail_closed "Missing or invalid ${identity}."
|
||||
done
|
||||
|
||||
if ! git fetch --no-tags --prune --depth=2 origin "${EVENT_MERGE_SHA}"; then
|
||||
fail_closed "Could not fetch event merge SHA ${EVENT_MERGE_SHA}."
|
||||
fi
|
||||
fetched_sha="$(git rev-parse --verify FETCH_HEAD 2>/dev/null)" \
|
||||
|| fail_closed "Fetched event merge SHA could not be resolved."
|
||||
[[ "${fetched_sha,,}" == "${EVENT_MERGE_SHA,,}" ]] \
|
||||
|| fail_closed "FETCH_HEAD ${fetched_sha} does not match event merge SHA ${EVENT_MERGE_SHA}."
|
||||
|
||||
read -r -a parents <<< "$(git show -s --format=%P FETCH_HEAD)"
|
||||
(( ${#parents[@]} == 2 )) \
|
||||
|| fail_closed "Event merge SHA ${EVENT_MERGE_SHA} is not a two-parent merge commit."
|
||||
head_parent_count=0
|
||||
for parent in "${parents[@]}"; do
|
||||
if [[ "${parent,,}" == "${EVENT_HEAD_SHA,,}" ]]; then
|
||||
((head_parent_count += 1))
|
||||
else
|
||||
VERIFIED_BASE_SHA="${parent,,}"
|
||||
fi
|
||||
done
|
||||
(( head_parent_count == 1 )) \
|
||||
|| fail_closed "Event merge SHA must have exactly one EVENT_HEAD_SHA parent."
|
||||
is_full_sha "${VERIFIED_BASE_SHA-}" \
|
||||
|| fail_closed "Event merge SHA has no non-head parent for the base build."
|
||||
VERIFIED_CANDIDATE_SHA="${fetched_sha,,}"
|
||||
else
|
||||
git fetch --no-tags --prune --depth=1 origin "${CANDIDATE_REF}"
|
||||
VERIFIED_CANDIDATE_SHA="$(git rev-parse --verify FETCH_HEAD)"
|
||||
VERIFIED_BASE_SHA="${initially_checked_out_base_sha}"
|
||||
fi
|
||||
|
||||
is_full_sha "${VERIFIED_CANDIDATE_SHA}" || {
|
||||
printf 'Could not resolve candidate SHA.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
is_full_sha "${VERIFIED_BASE_SHA}" || {
|
||||
printf 'Could not resolve build base SHA.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
printf 'VERIFIED_CANDIDATE_SHA=%s\n' "${VERIFIED_CANDIDATE_SHA,,}" >> "${GITHUB_ENV}"
|
||||
printf 'VERIFIED_BASE_SHA=%s\n' "${VERIFIED_BASE_SHA,,}" >> "${GITHUB_ENV}"
|
||||
printf 'Verified candidate SHA: %s\n' "${VERIFIED_CANDIDATE_SHA,,}"
|
||||
printf 'Verified build base SHA: %s\n' "${VERIFIED_BASE_SHA,,}"
|
||||
printf -- '- Verified candidate SHA: `%s`; built base SHA: `%s`\n' \
|
||||
"${VERIFIED_CANDIDATE_SHA,,}" "${VERIFIED_BASE_SHA,,}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
git reset --hard "${VERIFIED_BASE_SHA}"
|
||||
git clean -ffdx
|
||||
|
||||
- name: Verify runner image tools
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$(id -u)" == "1001" ]]
|
||||
[[ "$(id -g)" == "1001" ]]
|
||||
[[ "$(protoc --version)" == "libprotoc 3.21.12" ]]
|
||||
[[ "$(uv --version)" =~ ^uv[[:space:]]0\.11\.26([[:space:]]|$) ]]
|
||||
mold_version="$(mold --version)"
|
||||
[[ "${mold_version}" =~ ^mold[[:space:]]2\.30\.0([[:space:]]|$) ]]
|
||||
[[ "$(python3 --version)" == "Python 3.12.3" ]]
|
||||
sccache_version="$(sccache --version)"
|
||||
[[ "${sccache_version}" =~ ^sccache[[:space:]]0\.16\.0([[:space:]]|$) ]]
|
||||
[[ "$(command -v rustup)" == "/opt/cargo/bin/rustup" ]]
|
||||
[[ "$(command -v cargo)" == "/opt/cargo/bin/cargo" ]]
|
||||
[[ "$(command -v rustc)" == "/opt/cargo/bin/rustc" ]]
|
||||
[[ "$(rustup --version)" =~ ^rustup[[:space:]]1\.29\.0([[:space:]]|$) ]]
|
||||
cargo_version="$(cargo --version)"
|
||||
[[ "${cargo_version}" =~ ^cargo[[:space:]]1\.96\.0-nightly[[:space:]]\(cbb9bb8bd[[:space:]][0-9]{4}-[0-9]{2}-[0-9]{2}\)$ ]]
|
||||
rustc_version="$(rustc --version)"
|
||||
[[ "${rustc_version}" =~ ^rustc[[:space:]]1\.96\.0-nightly[[:space:]]\(ac7f9ec7d[[:space:]][0-9]{4}-[0-9]{2}-[0-9]{2}\)$ ]]
|
||||
active_toolchain="$(rustup show active-toolchain)"
|
||||
[[ "${active_toolchain}" =~ ^nightly-2026-03-21-x86_64-unknown-linux-gnu([[:space:]]|$) ]]
|
||||
[[ "${RUSTUP_HOME}" == "/opt/rustup" ]]
|
||||
[[ "${RUSTUP_TOOLCHAIN}" == "nightly-2026-03-21" ]]
|
||||
[[ "${RUSTUP_AUTO_INSTALL}" == "0" ]]
|
||||
test -r /opt/rustup && test -x /opt/rustup
|
||||
test ! -w /opt/rustup
|
||||
test ! -w /opt/cargo/bin
|
||||
for entry in config config.toml credentials credentials.toml bin .crates.toml .crates2.json .global-cache .package-cache; do
|
||||
test ! -e "${CARGO_HOME}/${entry}"
|
||||
done
|
||||
mkdir -p "${CARGO_HOME}/registry" "${CARGO_HOME}/git"
|
||||
|
||||
- name: Prepare persistent query regression cache
|
||||
shell: bash
|
||||
working-directory: src
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
readonly EXPECTED_CARGO_HOME="/home/runner/.cargo"
|
||||
readonly EXPECTED_CARGO_REGISTRY="/home/runner/.cargo/registry"
|
||||
readonly EXPECTED_CARGO_GIT="/home/runner/.cargo/git"
|
||||
readonly EXPECTED_RUSTUP_HOME="/opt/rustup"
|
||||
readonly EXPECTED_TARGET_DIR="/home/runner/query-regression-target"
|
||||
readonly EXPECTED_CACHE_META="/home/runner/query-regression-cache-meta"
|
||||
readonly EXPECTED_SCCACHE_DIR="/home/runner/.cache/sccache"
|
||||
readonly EXPECTED_RUSTC_WRAPPER="/usr/local/bin/sccache"
|
||||
readonly RUNNER_IMAGE_DIGEST="sha256:7022b551771dc82ec4c493f0c3db6ee3050ececd336598690a2165240ebe953e"
|
||||
readonly RUNNER_IMAGE_EPOCH="3"
|
||||
|
||||
require_expected_root() {
|
||||
local name="$1"
|
||||
local actual="$2"
|
||||
local expected="$3"
|
||||
[[ "${actual}" == /* && "${actual}" == "${expected}" ]] || {
|
||||
printf 'Refusing unexpected %s root: %s\n' "${name}" "${actual}" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
clear_directory() {
|
||||
local root="$1"
|
||||
[[ "${root}" == /* && "${root}" != "/" ]] || {
|
||||
printf 'Refusing to clear non-absolute or root path: %s\n' "${root}" >&2
|
||||
exit 1
|
||||
}
|
||||
case "${root}" in
|
||||
"${EXPECTED_TARGET_DIR}"|"${EXPECTED_TARGET_DIR}/debug"|"${EXPECTED_TARGET_DIR}/release"|"${EXPECTED_TARGET_DIR}/nightly") ;;
|
||||
*)
|
||||
printf 'Refusing to clear unexpected cache path: %s\n' "${root}" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
find "${root}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
}
|
||||
|
||||
clear_cargo_extracted_trees() {
|
||||
require_expected_root CARGO_REGISTRY "${CARGO_HOME}/registry" "${EXPECTED_CARGO_REGISTRY}"
|
||||
require_expected_root CARGO_GIT "${CARGO_HOME}/git" "${EXPECTED_CARGO_GIT}"
|
||||
rm -rf -- "${CARGO_HOME}/registry/src" "${CARGO_HOME}/git/checkouts"
|
||||
}
|
||||
|
||||
cargo_size_kib() {
|
||||
du -sk -- "${CARGO_HOME}/registry" "${CARGO_HOME}/git" | awk '{ total += $1 } END { print total }'
|
||||
}
|
||||
|
||||
target_size_kib() {
|
||||
du -sk -- "${CARGO_TARGET_DIR}" | cut -f1
|
||||
}
|
||||
|
||||
free_kib() {
|
||||
df -Pk "${CARGO_TARGET_DIR}" | awk 'NR == 2 { print $4 }'
|
||||
}
|
||||
|
||||
report_cache_usage() {
|
||||
du -sh -- "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \
|
||||
"${RUSTUP_HOME}" "${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}"
|
||||
df -P "${CARGO_TARGET_DIR}"
|
||||
df -hP "${CARGO_TARGET_DIR}"
|
||||
df -Pi "${CARGO_TARGET_DIR}"
|
||||
}
|
||||
|
||||
require_expected_root CARGO_HOME "${CARGO_HOME}" "${EXPECTED_CARGO_HOME}"
|
||||
require_expected_root CARGO_REGISTRY "${CARGO_HOME}/registry" "${EXPECTED_CARGO_REGISTRY}"
|
||||
require_expected_root CARGO_GIT "${CARGO_HOME}/git" "${EXPECTED_CARGO_GIT}"
|
||||
require_expected_root RUSTUP_HOME "${RUSTUP_HOME}" "${EXPECTED_RUSTUP_HOME}"
|
||||
require_expected_root CARGO_TARGET_DIR "${CARGO_TARGET_DIR}" "${EXPECTED_TARGET_DIR}"
|
||||
require_expected_root QUERY_REGRESSION_CACHE_META "${QUERY_REGRESSION_CACHE_META}" "${EXPECTED_CACHE_META}"
|
||||
require_expected_root SCCACHE_DIR "${SCCACHE_DIR}" "${EXPECTED_SCCACHE_DIR}"
|
||||
[[ "${RUSTC_WRAPPER}" == "${EXPECTED_RUSTC_WRAPPER}" ]] || {
|
||||
printf 'Refusing unexpected RUSTC_WRAPPER: %s\n' "${RUSTC_WRAPPER}" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "${SCCACHE_CACHE_SIZE}" == "40G" ]] || {
|
||||
printf 'Refusing unexpected SCCACHE_CACHE_SIZE: %s\n' "${SCCACHE_CACHE_SIZE}" >&2
|
||||
exit 1
|
||||
}
|
||||
[[ "${CARGO_INCREMENTAL}" == "0" ]] || {
|
||||
printf 'Refusing unexpected CARGO_INCREMENTAL: %s\n' "${CARGO_INCREMENTAL}" >&2
|
||||
exit 1
|
||||
}
|
||||
test -r "${RUSTUP_HOME}"
|
||||
test -x "${RUSTUP_HOME}"
|
||||
test ! -w "${RUSTUP_HOME}"
|
||||
mkdir -p "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \
|
||||
"${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}"
|
||||
for root in "${CARGO_HOME}" "${CARGO_HOME}/registry" "${CARGO_HOME}/git" \
|
||||
"${CARGO_TARGET_DIR}" "${QUERY_REGRESSION_CACHE_META}" "${SCCACHE_DIR}"; do
|
||||
test -w "${root}"
|
||||
touch "${root}/.query-regression-write-test"
|
||||
rm -f "${root}/.query-regression-write-test"
|
||||
done
|
||||
|
||||
case "${CARGO_PROFILE}" in
|
||||
dev) target_profile="debug" ;;
|
||||
release|nightly) target_profile="${CARGO_PROFILE}" ;;
|
||||
*) printf 'Unsupported CARGO_PROFILE: %s\n' "${CARGO_PROFILE}" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
rustc_vv="$(rustc -vV)"
|
||||
if cargo -Vv >/dev/null 2>&1; then
|
||||
cargo_vv="$(cargo -Vv)"
|
||||
else
|
||||
cargo_vv="$(cargo -V)"
|
||||
fi
|
||||
host_triple="$(printf '%s\n' "${rustc_vv}" | grep '^host: ' | cut -d' ' -f2-)"
|
||||
cc_version="$(cc --version)"
|
||||
mold_version="$(mold --version | grep -m1 .)"
|
||||
protoc_version="$(protoc --version)"
|
||||
sccache_version="$(sccache --version)"
|
||||
rustup_version="$(rustup --version)"
|
||||
active_toolchain="$(rustup show active-toolchain)"
|
||||
rustup_path="$(command -v rustup)"
|
||||
cargo_path="$(command -v cargo)"
|
||||
rustc_path="$(command -v rustc)"
|
||||
profile_target_dir="${CARGO_TARGET_DIR}/${target_profile}"
|
||||
marker="${QUERY_REGRESSION_CACHE_META}/target-${target_profile}.abi"
|
||||
new_marker="$(mktemp "${QUERY_REGRESSION_CACHE_META}/.${target_profile}.abi.XXXXXX")"
|
||||
{
|
||||
printf 'cache_epoch=%s\n' "${QUERY_REGRESSION_CACHE_EPOCH}"
|
||||
printf 'runner_image_digest=%s\n' "${RUNNER_IMAGE_DIGEST}"
|
||||
printf 'runner_image_epoch=%s\n' "${RUNNER_IMAGE_EPOCH}"
|
||||
printf 'host_triple=%s\n' "${host_triple}"
|
||||
printf 'architecture=%s\n' "$(uname -m)"
|
||||
printf 'cargo_profile=%s\n' "${CARGO_PROFILE}"
|
||||
printf 'target_profile=%s\n' "${target_profile}"
|
||||
printf 'RUSTFLAGS=%s\n' "${RUSTFLAGS-}"
|
||||
printf 'CARGO_ENCODED_RUSTFLAGS=%s\n' "${CARGO_ENCODED_RUSTFLAGS-}"
|
||||
printf 'CARGO_INCREMENTAL=%s\n' "${CARGO_INCREMENTAL-}"
|
||||
printf 'RUSTC_WRAPPER=%s\n' "${RUSTC_WRAPPER-}"
|
||||
printf 'SCCACHE_DIR=%s\n' "${SCCACHE_DIR-}"
|
||||
printf 'SCCACHE_CACHE_SIZE=%s\n' "${SCCACHE_CACHE_SIZE-}"
|
||||
printf 'RUSTUP_HOME=%s\n' "${RUSTUP_HOME-}"
|
||||
printf 'RUSTUP_TOOLCHAIN=%s\n' "${RUSTUP_TOOLCHAIN-}"
|
||||
printf 'RUSTUP_AUTO_INSTALL=%s\n' "${RUSTUP_AUTO_INSTALL-}"
|
||||
printf 'PATH=%s\n' "${PATH-}"
|
||||
printf '%s\n' "rustc_vV=${rustc_vv}"
|
||||
printf '%s\n' "cargo_vV=${cargo_vv}"
|
||||
printf '%s\n' "cc_version=${cc_version}"
|
||||
printf '%s\n' "mold_version=${mold_version}"
|
||||
printf '%s\n' "protoc_version=${protoc_version}"
|
||||
printf '%s\n' "sccache_version=${sccache_version}"
|
||||
printf '%s\n' "rustup_version=${rustup_version}"
|
||||
printf '%s\n' "active_toolchain=${active_toolchain}"
|
||||
printf '%s\n' "rustup_path=${rustup_path}"
|
||||
printf '%s\n' "cargo_path=${cargo_path}"
|
||||
printf '%s\n' "rustc_path=${rustc_path}"
|
||||
} > "${new_marker}"
|
||||
|
||||
if ! cmp -s "${marker}" "${new_marker}"; then
|
||||
printf 'Cache ABI marker mismatch; invalidating target profile: %s\n' "${target_profile}"
|
||||
mkdir -p "${profile_target_dir}"
|
||||
clear_directory "${profile_target_dir}"
|
||||
else
|
||||
printf 'Cache ABI marker match; reusing target profile: %s\n' "${target_profile}"
|
||||
fi
|
||||
mv -f "${new_marker}" "${marker}"
|
||||
|
||||
report_cache_usage
|
||||
|
||||
target_size="$(target_size_kib)"
|
||||
if (( target_size >= 400 * 1024 * 1024 )); then
|
||||
printf 'Warning: target cache is at least 400 GiB (%s KiB)\n' "${target_size}" >&2
|
||||
fi
|
||||
if (( target_size >= 450 * 1024 * 1024 )); then
|
||||
printf 'Target cache is at least 450 GiB; clearing complete target root\n' >&2
|
||||
clear_directory "${CARGO_TARGET_DIR}"
|
||||
fi
|
||||
|
||||
cargo_size="$(cargo_size_kib)"
|
||||
if (( cargo_size >= 60 * 1024 * 1024 )); then
|
||||
printf 'Warning: Cargo cache is at least 60 GiB (%s KiB)\n' "${cargo_size}" >&2
|
||||
fi
|
||||
if (( cargo_size >= 80 * 1024 * 1024 )); then
|
||||
printf 'Cargo cache is at least 80 GiB; removing extracted sources and checkouts\n' >&2
|
||||
clear_cargo_extracted_trees
|
||||
cargo_size="$(cargo_size_kib)"
|
||||
if (( cargo_size >= 80 * 1024 * 1024 )); then
|
||||
printf 'Cargo cache remains at least 80 GiB after cleanup (%s KiB)\n' "${cargo_size}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
free_space="$(free_kib)"
|
||||
if (( free_space < 300 * 1024 * 1024 )); then
|
||||
printf 'Free space is below 300 GiB; clearing complete target root\n' >&2
|
||||
clear_directory "${CARGO_TARGET_DIR}"
|
||||
free_space="$(free_kib)"
|
||||
if (( free_space < 300 * 1024 * 1024 )); then
|
||||
printf 'Free space remains below 300 GiB; removing Cargo extracted sources and checkouts\n' >&2
|
||||
clear_cargo_extracted_trees
|
||||
free_space="$(free_kib)"
|
||||
if (( free_space < 300 * 1024 * 1024 )); then
|
||||
printf 'Free space remains below 300 GiB after cleanup (%s KiB)\n' "${free_space}" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
report_cache_usage
|
||||
|
||||
sccache --start-server
|
||||
sccache --zero-stats
|
||||
sccache --show-stats
|
||||
|
||||
- name: Build base greptime
|
||||
working-directory: src
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SECONDS=0
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime
|
||||
base_build_elapsed="${SECONDS}"
|
||||
printf 'Base greptime cargo build elapsed: %s seconds\n' "${base_build_elapsed}"
|
||||
printf -- '- Base greptime cargo build: %s seconds\n' "${base_build_elapsed}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
sccache --show-stats
|
||||
sccache --zero-stats
|
||||
target_dir="${CARGO_PROFILE}"
|
||||
if [[ "${CARGO_PROFILE}" == "dev" ]]; then
|
||||
target_dir="debug"
|
||||
@@ -114,9 +437,18 @@ jobs:
|
||||
|
||||
- name: Switch source to candidate
|
||||
working-directory: src
|
||||
shell: bash
|
||||
run: |
|
||||
git fetch --no-tags --prune --depth=1 origin "${CANDIDATE_REF}"
|
||||
git reset --hard FETCH_HEAD
|
||||
set -euo pipefail
|
||||
[[ "${VERIFIED_CANDIDATE_SHA}" =~ ^[[:xdigit:]]{40}$ ]] || {
|
||||
printf 'Verified candidate SHA is missing or invalid.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
git cat-file -e "${VERIFIED_CANDIDATE_SHA}^{commit}" || {
|
||||
printf 'Verified candidate SHA is unavailable in the local repository.\n' >&2
|
||||
exit 1
|
||||
}
|
||||
git reset --hard "${VERIFIED_CANDIDATE_SHA}"
|
||||
git clean -ffdx
|
||||
|
||||
- name: Test query regression runner lifecycle
|
||||
@@ -126,8 +458,14 @@ jobs:
|
||||
- name: Build candidate greptime and fixture generators
|
||||
working-directory: src
|
||||
run: |
|
||||
set -euo pipefail
|
||||
SECONDS=0
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools
|
||||
candidate_build_elapsed="${SECONDS}"
|
||||
printf 'Candidate greptime and fixture generator cargo builds elapsed: %s seconds\n' "${candidate_build_elapsed}"
|
||||
printf -- '- Candidate greptime and fixture generator cargo builds: %s seconds\n' "${candidate_build_elapsed}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
sccache --show-stats
|
||||
target_dir="${CARGO_PROFILE}"
|
||||
if [[ "${CARGO_PROFILE}" == "dev" ]]; then
|
||||
target_dir="debug"
|
||||
@@ -192,6 +530,46 @@ jobs:
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Report persistent cache usage
|
||||
if: ${{ always() }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for root in \
|
||||
/home/runner/.cargo \
|
||||
/home/runner/.cargo/registry \
|
||||
/home/runner/.cargo/git \
|
||||
/opt/rustup \
|
||||
/home/runner/query-regression-target \
|
||||
/home/runner/query-regression-cache-meta \
|
||||
/home/runner/.cache/sccache; do
|
||||
if [[ -e "${root}" ]]; then
|
||||
du -sh -- "${root}"
|
||||
else
|
||||
printf 'Missing cache path (report only): %s\n' "${root}"
|
||||
fi
|
||||
done
|
||||
if [[ -e /opt/rustup ]]; then
|
||||
if [[ -w /opt/rustup ]]; then
|
||||
printf 'Immutable Rustup path is unexpectedly writable: /opt/rustup\n' >&2
|
||||
else
|
||||
printf 'Immutable Rustup path is read-only to runner: /opt/rustup\n'
|
||||
fi
|
||||
fi
|
||||
report_root=/home/runner/query-regression-target
|
||||
if [[ ! -e "${report_root}" ]]; then
|
||||
report_root=/home/runner
|
||||
printf 'Target cache path is missing; reporting its filesystem through %s\n' "${report_root}"
|
||||
fi
|
||||
df -P "${report_root}"
|
||||
df -hP "${report_root}"
|
||||
df -Pi "${report_root}"
|
||||
if command -v sccache >/dev/null 2>&1; then
|
||||
sccache --show-stats || printf 'Unable to show sccache statistics (report only)\n' >&2
|
||||
else
|
||||
printf 'sccache is unavailable (report only)\n' >&2
|
||||
fi
|
||||
|
||||
- name: Fail on regression failure
|
||||
if: ${{ steps.run.outputs.status != '0' }}
|
||||
run: exit 1
|
||||
|
||||
Reference in New Issue
Block a user