diff --git a/.github/runner-scale-sets/query-regression/Dockerfile b/.github/runner-scale-sets/query-regression/Dockerfile new file mode 100644 index 0000000000..1f9f7203b9 --- /dev/null +++ b/.github/runner-scale-sets/query-regression/Dockerfile @@ -0,0 +1,33 @@ +FROM ghcr.io/astral-sh/uv@sha256:3d868e555f8f1dbc324afa005066cd11e1053fc4743b9808ca8025283e65efa5 AS uv +FROM ghcr.io/actions/actions-runner@sha256:08c30b0a7105f64bddfc485d2487a22aa03932a791402393352fdf674bda2c29 + +USER root + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + git \ + gzip \ + jq \ + libssl-dev \ + mold \ + openssh-client \ + pkg-config \ + protobuf-compiler \ + python3 \ + sudo \ + tar \ + unzip \ + wget \ + xz-utils \ + zip \ + zstd \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=uv /uv /uvx /usr/local/bin/ + +USER runner diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md new file mode 100644 index 0000000000..eb1a7e12f7 --- /dev/null +++ b/.github/runner-scale-sets/query-regression/README.md @@ -0,0 +1,152 @@ +# Query regression self-hosted runners + +The `Query Regression` workflow targets self-hosted GitHub Actions runners via +runner labels or ARC runner scale set names: + +- `perf-regression-8-cores` + +The name intentionally avoids generic labels such as `ubuntu-22.04-8-cores`, +which may already be used by GitHub-hosted larger runners or other runner pools. + +For Kubernetes-based runners, ARC runner pods run inside the target Kubernetes +cluster and connect outbound to GitHub; GitHub can then dispatch jobs whose +`runs-on` value matches the runner scale set name or runner labels. + +## Prerequisites + +Install the ARC scale set controller in the Kubernetes cluster if it is not +already installed: + +```bash +helm upgrade --install arc \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller \ + --namespace arc-systems \ + --create-namespace \ + --version 0.14.2 +``` + +Create the GitHub App or PAT secret in the namespace that will host the runner +scale sets. Prefer a GitHub App with access limited to `GreptimeTeam/greptimedb`. + +```bash +kubectl -n arc-runners create secret generic greptimedb-arc-github-app \ + --from-literal=github_app_id= \ + --from-literal=github_app_installation_id= \ + --from-file=github_app_private_key= +``` + +The values files in this directory reference that secret by name. + +## Build the runner image + +Use a derived ARC runner image instead of the minimal upstream runner image. The +image keeps the official `/home/runner/run.sh` entrypoint layout and adds the +tools this workflow expects to be present on a normal CI host, including `wget`, +`uv`, C/C++ build tools, OpenSSL headers, and protobuf tooling. + +```bash +docker build \ + -f .github/runner-scale-sets/query-regression/Dockerfile \ + -t greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest \ + .github/runner-scale-sets/query-regression + +docker push greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest +``` + +Deploy the runner image by digest rather than by a mutable tag. Update +`values-8-cores.yaml` after pushing a rebuilt image. + +The workflow still runs setup actions for pinned Rust and `uv` behavior. `mold` +is installed in the image and selected through `CARGO_BUILD_RUSTFLAGS`, so jobs +do not need privileged package installation at runtime. + +Decide whether the registry repository is public or private. Public pull access +avoids distributing registry credentials to the runner namespace. If the image +must be private, create a dedicated read-only image-pull secret for this image +and attach it only as `imagePullSecrets`; do not expose registry credentials to +runner containers. + +## Install the query-regression scale set + +Install the runner scale set. The Helm release name and `runnerScaleSetName` +should match the `runs-on` value used by the workflow. + +```bash +helm upgrade --install perf-regression-8-cores \ + oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set \ + --namespace arc-runners \ + --create-namespace \ + --version 0.14.2 \ + -f .github/runner-scale-sets/query-regression/values-8-cores.yaml +``` + +Check registration and pods: + +```bash +kubectl -n arc-runners get pods +kubectl -n arc-runners get autoscalingrunnersets +``` + +The scale sets should also appear under repository Actions runner settings. + +## Security notes for fork PRs + +Maintainer-approved fork PRs can run on self-hosted runners. Approval only lets +the workflow execute; it does not make the fork code trusted. + +The `query-regression` label is the explicit trigger for PR runs. Updating a PR +does not rerun the benchmark automatically; remove and re-add the label after +reviewing the updated changes. + +These runners execute PR code and should be treated as untrusted execution +capacity: + +- keep them isolated from sensitive internal services unless explicitly required; +- do not mount host paths, Docker socket, kubeconfig, or long-lived credentials; +- use ephemeral runner pods and no shared work directory with trusted jobs; +- disable service account token mounting in runner pods unless Kubernetes API + access is intentionally required; +- use a runner image whose default user is non-root, disable privilege + escalation, drop Linux capabilities, and use the runtime-default seccomp + profile; +- keep GitHub tokens least-privilege and rely on normal `pull_request` behavior + for fork PRs, where repository secrets are withheld and `GITHUB_TOKEN` is + read-only; +- review fork workflow changes before approving the run. + +If stronger isolation is required, install a separate runner group/namespace just +for query-regression PR workloads and restrict repository/workflow access to the +`Query Regression` workflow. + +Use namespace or cluster network policy to restrict runner egress where the CNI +supports the needed controls. Query-regression runners need outbound access to +GitHub Actions services, GitHub artifact/cache endpoints, Rust/crate/toolchain +endpoints, DNS, and the configured image registries. Block access to unrelated +cluster services, private network ranges, and cloud metadata endpoints unless a +case explicitly needs them. + +## Build cache + +The workflow builds base and candidate in the same job, the same source path, +and a shared `CARGO_TARGET_DIR`. It checks out the base ref into `src`, builds +and copies the base binary aside, then resets that same `src` checkout to the +candidate ref before building the candidate binary. Keeping the workspace path +stable improves Cargo incremental reuse for local workspace crates compared with +building separate `base-src` and `candidate-src` checkouts. + +The workflow also uses the GitHub Actions Rust cache for restore-only cache +reuse. PR and dispatch runs do not save cache entries. Refresh shared caches from +trusted maintenance workflows only. + +The ARC values in this directory do not configure a cross-run runner-local +compiler cache. If cross-run compile time still dominates the benchmark, +deployers may add one of the following at the runner infrastructure layer: + +- a custom runner image with Rust tooling, `mold`, and `sccache` preinstalled; +- `RUSTC_WRAPPER=sccache` plus a shared `sccache` backend such as object storage, + Redis, or a Kubernetes storage class that is safe for concurrent runner pods; +- separate cache namespaces/buckets for untrusted PR code and trusted branches to + avoid cache poisoning across trust boundaries. + +Do not mount a shared writable host path, Docker socket, kubeconfig, or other +privileged credentials into runners that execute PR code. diff --git a/.github/runner-scale-sets/query-regression/values-8-cores.yaml b/.github/runner-scale-sets/query-regression/values-8-cores.yaml new file mode 100644 index 0000000000..54ade99bad --- /dev/null +++ b/.github/runner-scale-sets/query-regression/values-8-cores.yaml @@ -0,0 +1,33 @@ +githubConfigUrl: "https://github.com/GreptimeTeam/greptimedb" +githubConfigSecret: greptimedb-arc-github-app + +runnerScaleSetName: "perf-regression-8-cores" +minRunners: 0 +maxRunners: 4 + +template: + spec: + automountServiceAccountToken: false + activeDeadlineSeconds: 11400 + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - name: runner + image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:108b7357532e5cf3329853373f561b0c674e954d6307b53d37075df5a189b0b2 + imagePullPolicy: Always + command: ["/home/runner/run.sh"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + requests: + cpu: "8" + memory: 24Gi + ephemeral-storage: 80Gi + limits: + cpu: "8" + memory: 24Gi + ephemeral-storage: 80Gi diff --git a/.github/scripts/query-regression-comment.cjs b/.github/scripts/query-regression-comment.cjs new file mode 100644 index 0000000000..100b1d12d2 --- /dev/null +++ b/.github/scripts/query-regression-comment.cjs @@ -0,0 +1,277 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const fs = require('fs'); +const path = require('path'); + +function skip(core, message) { + core.info(message); + core.setOutput('should_post', 'false'); +} + +function findReports(dir) { + const reports = []; + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + reports.push(...findReports(full)); + } else if (entry.isFile() && entry.name === 'query-regression-report.json') { + reports.push(full); + } + } + return reports.sort(); +} + +function text(value) { + if (value === null || value === undefined || value === '') return 'N/A'; + const result = String(value) + .replace(//g, '') + .replace(/\\/g, '\\\\') + .replace(/`/g, '`') + .replace(//g, '>') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/\(/g, '\\(') + .replace(/\)/g, '\\)') + .replace(/!/g, '\\!') + .replace(/@/g, '@\u200b') + .replace(/\|/g, '\\|') + .replace(/\r?\n/g, ' '); + return result.length > 512 ? `${result.slice(0, 509)}...` : result; +} + +function statusEmoji(status) { + return { ok: 'āœ…', measured: 'āœ…', failed: 'āŒ', planned: 'šŸ“', 'fixture-ready': 'šŸ§Ŗ' }[status] || 'āš ļø'; +} + +function fmtMs(value) { + const number = Number(value); + return Number.isFinite(number) ? number.toFixed(2) : 'N/A'; +} + +function measurementsByName(target) { + const result = new Map(); + const measurements = Array.isArray(target?.measurements) ? target.measurements : []; + measurements.forEach((measurement, index) => { + result.set(String(measurement?.name || `query-${index}`), measurement || {}); + }); + return result; +} + +function regression(base, candidate) { + const b = Number(base); + const c = Number(candidate); + if (!Number.isFinite(b) || !Number.isFinite(c) || b === 0) return 'N/A'; + return `${(((c - b) / b) * 100).toFixed(1)}%`; +} + +function renderReport(report, reportPath) { + const caseInfo = report.case || {}; + const name = caseInfo.name || path.basename(path.dirname(reportPath)); + const status = report.status || 'missing'; + const lines = [ + `### ${statusEmoji(status)} ${text(name)}`, + '', + `- **Status:** \`${text(status)}\``, + `- **Case path:** \`${text(report.case_path)}\``, + `- **Query mode:** \`${text(report.query_mode)}\``, + ]; + if (report.error) { + lines.push(`- **Error:** \`${text(report.error)}\``); + } + + const targets = Array.isArray(report.targets) ? report.targets : []; + lines.push('', '| Target | Status | Validation errors | Region |', '| --- | --- | ---: | --- |'); + for (const target of targets) { + const discovered = target?.discovered || {}; + const region = Array.isArray(discovered) + ? discovered.map(item => item?.region_id).filter(Boolean).join(', ') + : discovered.region_id; + lines.push( + `| ${text(target?.name)} | ${statusEmoji(target?.status)} \`${text(target?.status)}\` | ${(target?.validation_errors || []).length} | ${text(region)} |` + ); + } + + if (targets.length >= 2) { + const base = measurementsByName(targets[0]); + const candidate = measurementsByName(targets[1]); + const names = Array.from(new Set([...base.keys(), ...candidate.keys()])).sort(); + lines.push('', '| Query | Base median ms | Candidate median ms | Regression |', '| --- | ---: | ---: | ---: |'); + for (const query of names) { + const b = base.get(query) || {}; + const c = candidate.get(query) || {}; + lines.push( + `| ${text(query)} | ${fmtMs(b.latency_ms_median)} | ${fmtMs(c.latency_ms_median)} | ${regression(b.latency_ms_median, c.latency_ms_median)} |` + ); + } + } + return lines.join('\n'); +} + +module.exports = async function validateQueryRegressionComment({ github, context, core }) { + const artifactDir = 'query-regression-comment'; + const metadataPath = path.join(artifactDir, 'query-regression-pr.json'); + const summaryPath = path.join(artifactDir, 'query-regression-summary.md'); + + if (!fs.existsSync(metadataPath)) { + return skip(core, 'Missing query-regression-pr.json; skipping sticky comment.'); + } + + let metadata; + try { + metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); + } catch (error) { + core.warning(`Invalid PR metadata JSON: ${error.message}`); + return skip(core, 'Invalid PR metadata JSON; skipping sticky comment.'); + } + + const expectedRunId = Number(process.env.WORKFLOW_RUN_ID); + const expectedRunAttempt = Number(process.env.WORKFLOW_RUN_ATTEMPT); + if (metadata.run_id !== expectedRunId || metadata.run_attempt !== expectedRunAttempt) { + return skip(core, 'Artifact metadata does not match this workflow_run; skipping.'); + } + + if (metadata.base_repo !== `${context.repo.owner}/${context.repo.repo}`) { + return skip(core, `PR targets ${metadata.base_repo}, not this repository; skipping.`); + } + + const prNumber = Number(metadata.pr_number); + if (!Number.isInteger(prNumber) || prNumber <= 0) { + return skip(core, 'Invalid PR number in metadata; skipping.'); + } + + const run = context.payload.workflow_run; + if (run.event !== 'pull_request') { + return skip(core, `Workflow run event is ${run.event}, not pull_request; skipping.`); + } + if (run.head_sha !== metadata.head_sha) { + return skip(core, 'Workflow run head SHA differs from artifact metadata; skipping.'); + } + const runHeadRepo = run.head_repository?.full_name; + if (!runHeadRepo) { + return skip(core, 'Workflow run head repository is missing; skipping.'); + } + if (runHeadRepo !== metadata.head_repo) { + return skip(core, 'Workflow run head repository differs from artifact metadata; skipping.'); + } + + // GitHub leaves workflow_run.pull_requests empty for fork PRs. When present, + // use it as an extra guard; otherwise resolve the unique open PR from trusted + // workflow_run head repo/branch/SHA metadata before accepting the artifact PR. + const workflowPrNumbers = new Set( + (run.pull_requests || []).map(pr => Number(pr.number)).filter(Number.isInteger) + ); + if (workflowPrNumbers.size > 0) { + if (!workflowPrNumbers.has(prNumber)) { + return skip(core, `PR #${prNumber} is not listed in workflow_run ${run.id}; skipping.`); + } + } else { + const runHeadOwner = run.head_repository?.owner?.login; + const runHeadBranch = run.head_branch; + if (!runHeadOwner || !runHeadBranch) { + return skip(core, 'Workflow run head owner or branch is missing; skipping.'); + } + + let matchingPrs; + try { + const { data: pullRequests } = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + head: `${runHeadOwner}:${runHeadBranch}`, + per_page: 100, + }); + matchingPrs = pullRequests.filter(pr => ( + pr.head.repo?.full_name === runHeadRepo && + pr.head.sha === run.head_sha && + pr.base.repo?.full_name === metadata.base_repo + )); + } catch (error) { + core.warning(`Could not resolve PR from workflow_run metadata: ${error.message}`); + return skip(core, 'Could not resolve PR from workflow_run metadata; skipping.'); + } + + if (matchingPrs.length !== 1) { + return skip(core, `Workflow run matched ${matchingPrs.length} open PRs; skipping.`); + } + if (Number(matchingPrs[0].number) !== prNumber) { + return skip(core, `Artifact PR #${prNumber} does not match workflow_run PR #${matchingPrs[0].number}; skipping.`); + } + } + + let pull; + try { + ({ data: pull } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + })); + } catch (error) { + core.warning(`Could not read PR #${prNumber}: ${error.message}`); + return skip(core, `Could not read PR #${prNumber}; skipping.`); + } + + if (pull.state !== 'open') { + return skip(core, `PR #${prNumber} is ${pull.state}; skipping.`); + } + if (pull.base.repo.full_name !== metadata.base_repo || pull.head.repo.full_name !== metadata.head_repo) { + return skip(core, 'Current PR repository metadata does not match artifact; skipping.'); + } + if (pull.head.sha !== metadata.head_sha) { + return skip(core, 'Current PR head SHA differs from artifact; skipping stale run.'); + } + + const reportPaths = findReports(artifactDir); + const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com'; + let body = [ + '## Query regression report', + '', + '> Rendered by a trusted workflow from JSON artifacts produced by the query-regression run. Results from untrusted PR code are advisory until reviewed.', + '', + `- **Workflow run:** ${serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${expectedRunId}`, + `- **Base SHA:** \`${text(metadata.base_sha)}\``, + `- **Head SHA:** \`${text(metadata.head_sha)}\``, + '', + ].join('\n'); + + if (reportPaths.length === 0) { + body += 'No query-regression JSON reports were found in the artifact.\n'; + } else { + const rendered = []; + for (const reportPath of reportPaths) { + const size = fs.statSync(reportPath).size; + if (size > 1024 * 1024) { + return skip(core, `Report file too large: ${reportPath} (${size} bytes).`); + } + let report; + try { + report = JSON.parse(fs.readFileSync(reportPath, 'utf8')); + } catch (error) { + return skip(core, `Invalid report JSON in ${reportPath}: ${error.message}`); + } + rendered.push(renderReport(report, reportPath)); + } + body += rendered.join('\n\n---\n\n') + '\n'; + } + + if (body.length > 59999) { + body = `${body.slice(0, 59500)}\n\nā€¦\n\n_Comment truncated because it exceeded 60000 characters._\n`; + } + fs.writeFileSync(summaryPath, body); + + core.setOutput('should_post', 'true'); + core.setOutput('pr_number', String(prNumber)); + core.setOutput('summary_path', summaryPath); +}; diff --git a/.github/scripts/query-regression-pr-metadata.py b/.github/scripts/query-regression-pr-metadata.py index 0ce87e4582..2d8ed3f2e7 100644 --- a/.github/scripts/query-regression-pr-metadata.py +++ b/.github/scripts/query-regression-pr-metadata.py @@ -31,6 +31,7 @@ def main() -> int: metadata = { "pr_number": int(os.environ["PR_NUMBER"]), "head_sha": os.environ["HEAD_SHA"], + "base_sha": os.environ["BASE_SHA"], "head_repo": os.environ["HEAD_REPO"], "base_repo": os.environ["BASE_REPO"], "run_id": int(os.environ["RUN_ID"]), diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index 7af0ccd1fe..4714b5dd0f 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -54,6 +54,12 @@ def profile_dir(cargo_profile: str) -> str: return cargo_profile +def configured_path(value: str | None) -> Path | None: + if not value or not value.strip(): + return None + return Path(value.strip()) + + def resolve_case_path(candidate_src: Path, case: str) -> Path: path = Path(case) if path.is_absolute() or path.parts[:1] == (candidate_src.name,): @@ -83,6 +89,9 @@ def append_step_summary(summary: Path) -> None: def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int: target_dir = profile_dir(args.cargo_profile) + base_bin = args.base_bin or args.base_src / "target" / target_dir / "greptime" + candidate_bin = args.candidate_bin or args.candidate_src / "target" / target_dir / "greptime" + fixture_generator = args.fixture_generator or args.candidate_src / "target" / target_dir / "query_perf_fixture" cmd = [ "uv", "run", @@ -92,11 +101,11 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int: "--case", str(case_path), "--base-bin", - str(args.base_src / "target" / target_dir / "greptime"), + str(base_bin), "--candidate-bin", - str(args.candidate_src / "target" / target_dir / "greptime"), + str(candidate_bin), "--fixture-generator", - str(args.candidate_src / "target" / target_dir / "query_perf_fixture"), + str(fixture_generator), "--work-dir", str(work_dir), "--http-timeout", @@ -138,6 +147,13 @@ def main() -> int: parser.add_argument("--cases", action="append", help="'all' or comma/space separated case paths") parser.add_argument("--base-src", type=Path, default=Path("base-src")) parser.add_argument("--candidate-src", type=Path, default=Path("candidate-src")) + parser.add_argument("--base-bin", type=Path, default=configured_path(os.environ.get("BASE_BIN"))) + parser.add_argument("--candidate-bin", type=Path, default=configured_path(os.environ.get("CANDIDATE_BIN"))) + parser.add_argument( + "--fixture-generator", + type=Path, + default=configured_path(os.environ.get("FIXTURE_GENERATOR")), + ) parser.add_argument("--cargo-profile", default=os.environ.get("CARGO_PROFILE", "nightly")) parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path) parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300")) diff --git a/.github/workflows/query-regression-comment.yml b/.github/workflows/query-regression-comment.yml index 573eda85ca..b320f78e17 100644 --- a/.github/workflows/query-regression-comment.yml +++ b/.github/workflows/query-regression-comment.yml @@ -19,6 +19,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: + - name: Checkout trusted scripts + uses: actions/checkout@v4 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - name: Locate query regression comment artifact id: artifact uses: actions/github-script@v7 @@ -55,11 +61,12 @@ jobs: uses: actions/download-artifact@v4 continue-on-error: true with: - name: query-regression-comment + artifact-ids: ${{ steps.artifact.outputs.id }} path: query-regression-comment repository: ${{ github.repository }} github-token: ${{ secrets.GITHUB_TOKEN }} run-id: ${{ github.event.workflow_run.id }} + merge-multiple: true - name: Validate PR metadata and prepare comment id: validate @@ -70,103 +77,8 @@ jobs: WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} with: script: | - const fs = require('fs'); - const path = require('path'); - - const artifactDir = 'query-regression-comment'; - const metadataPath = path.join(artifactDir, 'query-regression-pr.json'); - const summaryPath = path.join(artifactDir, 'query-regression-summary.md'); - - function skip(message) { - core.info(message); - core.setOutput('should_post', 'false'); - } - - if (!fs.existsSync(metadataPath)) { - return skip('Missing query-regression-pr.json; skipping sticky comment.'); - } - - let metadata; - try { - metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8')); - } catch (error) { - core.warning(`Invalid PR metadata JSON: ${error.message}`); - return skip('Invalid PR metadata JSON; skipping sticky comment.'); - } - - const expectedRunId = Number(process.env.WORKFLOW_RUN_ID); - const expectedRunAttempt = Number(process.env.WORKFLOW_RUN_ATTEMPT); - if (metadata.run_id !== expectedRunId || metadata.run_attempt !== expectedRunAttempt) { - return skip('Artifact metadata does not match this workflow_run; skipping.'); - } - - if (metadata.base_repo !== context.repo.owner + '/' + context.repo.repo) { - return skip(`PR targets ${metadata.base_repo}, not this repository; skipping.`); - } - - const prNumber = Number(metadata.pr_number); - if (!Number.isInteger(prNumber) || prNumber <= 0) { - return skip('Invalid PR number in metadata; skipping.'); - } - - const run = context.payload.workflow_run; - const trustedPrNumbers = new Set( - (run.pull_requests || []) - .map(pr => Number(pr.number)) - .filter(Number.isInteger) - ); - try { - const { data: associatedPrs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({ - owner: context.repo.owner, - repo: context.repo.repo, - commit_sha: run.head_sha, - }); - for (const pr of associatedPrs) { - trustedPrNumbers.add(Number(pr.number)); - } - } catch (error) { - core.warning(`Could not list PRs associated with workflow_run head SHA: ${error.message}`); - } - if (!trustedPrNumbers.has(prNumber)) { - return skip(`PR #${prNumber} is not associated with workflow_run ${run.id}; skipping.`); - } - - const { data: pull } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - - if (pull.state !== 'open') { - return skip(`PR #${prNumber} is ${pull.state}; skipping.`); - } - if (pull.base.repo.full_name !== metadata.base_repo || pull.head.repo.full_name !== metadata.head_repo) { - return skip('Current PR repository metadata does not match artifact; skipping.'); - } - if (pull.head.sha !== metadata.head_sha) { - return skip('Current PR head SHA differs from artifact; skipping stale run.'); - } - - let body = 'Query regression completed, but no summary artifact was found.'; - if (fs.existsSync(summaryPath)) { - const maxSummaryBytes = 1024 * 1024; - const summarySize = fs.statSync(summaryPath).size; - if (summarySize > maxSummaryBytes) { - return skip(`Summary file too large: ${summarySize} bytes.`); - } - body = fs.readFileSync(summaryPath, 'utf8'); - } - body = body - .replace(//g, '') - .replace(/@/g, '@\u200b'); - if (body.length > 59999) { - body = body.slice(0, 59500) + '\n\nā€¦\n\n_Comment truncated because it exceeded 60000 characters._\n'; - } - fs.writeFileSync(summaryPath, body); - - core.setOutput('should_post', 'true'); - core.setOutput('pr_number', String(prNumber)); - core.setOutput('summary_path', summaryPath); + const validate = require('./.github/scripts/query-regression-comment.cjs'); + await validate({ github, context, core }); - name: Post sticky PR comment if: ${{ steps.validate.outputs.should_post == 'true' }} diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index 6e8a4618b3..2f11462d09 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -32,8 +32,15 @@ on: - nightly - release - dev + runner: + description: Self-hosted runner label or ARC runner scale set for this query regression run + required: true + type: choice + default: perf-regression-8-cores + options: + - perf-regression-8-cores pull_request: - types: [labeled, ready_for_review, reopened] + types: [labeled] permissions: contents: read @@ -46,62 +53,85 @@ jobs: query-regression: if: >- ${{ github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && - !github.event.pull_request.draft && - contains(github.event.pull_request.labels.*.name, 'query-regression') && - (github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }} - runs-on: ubuntu-latest + (github.event_name == 'pull_request' && + !github.event.pull_request.draft && + contains(github.event.pull_request.labels.*.name, 'query-regression') && + (github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || 'perf-regression-8-cores' }} timeout-minutes: 180 env: CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }} + CARGO_TARGET_DIR: ${{ github.workspace }}/query-regression-target + CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=mold" + BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }} + CANDIDATE_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }} steps: - name: Checkout base source uses: actions/checkout@v4 with: repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }} - path: base-src + ref: ${{ env.BASE_REF }} + path: src persist-credentials: false - - name: Checkout candidate source - uses: actions/checkout@v4 - with: - repository: ${{ github.repository }} - ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }} - path: candidate-src - persist-credentials: false + - name: Preserve trusted query regression scripts + run: | + mkdir -p query-regression-trusted-scripts + cp src/.github/scripts/query-regression-pr-metadata.py \ + query-regression-trusted-scripts/query-regression-pr-metadata.py + cp src/.github/scripts/query-regression-summary.py \ + query-regression-trusted-scripts/query-regression-summary.py - uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - uses: astral-sh/setup-uv@v5 - - name: Rust cache (base) + - name: Rust cache uses: Swatinem/rust-cache@v2 with: - workspaces: base-src - shared-key: query-regression-base - cache-all-crates: "true" - save-if: false - - - name: Rust cache (candidate) - uses: Swatinem/rust-cache@v2 - with: - workspaces: candidate-src - shared-key: query-regression-candidate + workspaces: src + shared-key: query-regression cache-all-crates: "true" save-if: false - name: Build base greptime - working-directory: base-src - run: cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + working-directory: src + run: | + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + target_dir="${CARGO_PROFILE}" + if [[ "${CARGO_PROFILE}" == "dev" ]]; then + target_dir="debug" + fi + mkdir -p "${GITHUB_WORKSPACE}/query-regression-bins/base" + cp "${CARGO_TARGET_DIR}/${target_dir}/greptime" \ + "${GITHUB_WORKSPACE}/query-regression-bins/base/greptime" + + - name: Switch source to candidate + working-directory: src + run: | + git fetch --no-tags --prune --depth=1 origin "${CANDIDATE_REF}" + git reset --hard FETCH_HEAD + git clean -ffdx - name: Build candidate greptime and fixture generator - working-directory: candidate-src - run: cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture + working-directory: src + run: | + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture + target_dir="${CARGO_PROFILE}" + if [[ "${CARGO_PROFILE}" == "dev" ]]; then + target_dir="debug" + fi + mkdir -p "${GITHUB_WORKSPACE}/query-regression-bins/candidate" + cp "${CARGO_TARGET_DIR}/${target_dir}/greptime" \ + "${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime" + cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \ + "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture" - name: Run query regression id: run @@ -111,20 +141,26 @@ jobs: ALLOW_LARGE_FIXTURE: ${{ github.event_name == 'pull_request' && 'true' || inputs.allow_large_fixture }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} CASE_NAME: ${{ github.event_name == 'pull_request' && 'default case set' || inputs.case }} - BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }} - CANDIDATE_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }} - run: uv run --no-project python candidate-src/.github/scripts/query-regression-run.py + BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime + CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime + FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture + run: >- + uv run --no-project python src/.github/scripts/query-regression-run.py + --base-src src + --candidate-src src + --summary-script query-regression-trusted-scripts/query-regression-summary.py - name: Write PR metadata for trusted comment workflow if: ${{ always() && github.event_name == 'pull_request' }} env: PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }} RUN_ID: ${{ github.run_id }} RUN_ATTEMPT: ${{ github.run_attempt }} - run: uv run --no-project python candidate-src/.github/scripts/query-regression-pr-metadata.py + run: uv run --no-project python query-regression-trusted-scripts/query-regression-pr-metadata.py - name: Upload query regression artifacts if: always() @@ -143,7 +179,7 @@ jobs: name: query-regression-comment path: | query-regression-pr.json - query-regression-summary.md + query-regression-work/**/query-regression-report.json if-no-files-found: warn retention-days: 7