mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-07-07 06:20:39 +00:00
feat: add query regression perf harness (#8406)
* feat: add query regression perf harness Signed-off-by: discord9 <discord9@163.com> * feat: extend query regression cases Signed-off-by: discord9 <discord9@163.com> * ci: harden query regression workflows Signed-off-by: discord9 <discord9@163.com> * fix: address query regression review comments Signed-off-by: discord9 <discord9@163.com> * ci: limit query regression PR triggers Signed-off-by: discord9 <discord9@163.com> * ci: run full query regression case set Signed-off-by: discord9 <discord9@163.com> * refactor: model query regression scenarios Signed-off-by: discord9 <discord9@163.com> * fix: avoid unenforced query regression thresholds Signed-off-by: discord9 <discord9@163.com> --------- Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
GitHub
177
.github/workflows/query-regression-comment.yml
vendored
Normal file
177
.github/workflows/query-regression-comment.yml
vendored
Normal file
@@ -0,0 +1,177 @@
|
||||
name: Query Regression Comment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["Query Regression"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
comment:
|
||||
if: >-
|
||||
${{ github.event.workflow_run.event == 'pull_request' &&
|
||||
github.event.workflow_run.conclusion != 'cancelled' &&
|
||||
github.event.workflow_run.conclusion != 'skipped' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Locate query regression comment artifact
|
||||
id: artifact
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const run_id = context.payload.workflow_run.id;
|
||||
const { data } = await github.rest.actions.listWorkflowRunArtifacts({
|
||||
owner,
|
||||
repo,
|
||||
run_id,
|
||||
per_page: 100,
|
||||
});
|
||||
const artifact = data.artifacts.find(
|
||||
item => item.name === 'query-regression-comment' && !item.expired
|
||||
);
|
||||
if (!artifact) {
|
||||
core.info('No query-regression-comment artifact found; skipping.');
|
||||
core.setOutput('found', 'false');
|
||||
return;
|
||||
}
|
||||
const maxBytes = 1024 * 1024;
|
||||
if (artifact.size_in_bytes > maxBytes) {
|
||||
core.warning(`Comment artifact too large: ${artifact.size_in_bytes} bytes`);
|
||||
core.setOutput('found', 'false');
|
||||
return;
|
||||
}
|
||||
core.setOutput('found', 'true');
|
||||
core.setOutput('id', String(artifact.id));
|
||||
|
||||
- name: Download query regression comment artifact
|
||||
id: download
|
||||
if: ${{ steps.artifact.outputs.found == 'true' }}
|
||||
uses: actions/download-artifact@v4
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: query-regression-comment
|
||||
path: query-regression-comment
|
||||
repository: ${{ github.repository }}
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
|
||||
- name: Validate PR metadata and prepare comment
|
||||
id: validate
|
||||
if: ${{ steps.download.outcome == 'success' }}
|
||||
uses: actions/github-script@v7
|
||||
env:
|
||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const artifactDir = 'query-regression-comment';
|
||||
const metadataPath = path.join(artifactDir, 'query-regression-pr.json');
|
||||
const summaryPath = path.join(artifactDir, 'query-regression-summary.md');
|
||||
|
||||
function skip(message) {
|
||||
core.info(message);
|
||||
core.setOutput('should_post', 'false');
|
||||
}
|
||||
|
||||
if (!fs.existsSync(metadataPath)) {
|
||||
return skip('Missing query-regression-pr.json; skipping sticky comment.');
|
||||
}
|
||||
|
||||
let metadata;
|
||||
try {
|
||||
metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
|
||||
} catch (error) {
|
||||
core.warning(`Invalid PR metadata JSON: ${error.message}`);
|
||||
return skip('Invalid PR metadata JSON; skipping sticky comment.');
|
||||
}
|
||||
|
||||
const expectedRunId = Number(process.env.WORKFLOW_RUN_ID);
|
||||
const expectedRunAttempt = Number(process.env.WORKFLOW_RUN_ATTEMPT);
|
||||
if (metadata.run_id !== expectedRunId || metadata.run_attempt !== expectedRunAttempt) {
|
||||
return skip('Artifact metadata does not match this workflow_run; skipping.');
|
||||
}
|
||||
|
||||
if (metadata.base_repo !== context.repo.owner + '/' + context.repo.repo) {
|
||||
return skip(`PR targets ${metadata.base_repo}, not this repository; skipping.`);
|
||||
}
|
||||
|
||||
const prNumber = Number(metadata.pr_number);
|
||||
if (!Number.isInteger(prNumber) || prNumber <= 0) {
|
||||
return skip('Invalid PR number in metadata; skipping.');
|
||||
}
|
||||
|
||||
const run = context.payload.workflow_run;
|
||||
const trustedPrNumbers = new Set(
|
||||
(run.pull_requests || [])
|
||||
.map(pr => Number(pr.number))
|
||||
.filter(Number.isInteger)
|
||||
);
|
||||
try {
|
||||
const { data: associatedPrs } = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
commit_sha: run.head_sha,
|
||||
});
|
||||
for (const pr of associatedPrs) {
|
||||
trustedPrNumbers.add(Number(pr.number));
|
||||
}
|
||||
} catch (error) {
|
||||
core.warning(`Could not list PRs associated with workflow_run head SHA: ${error.message}`);
|
||||
}
|
||||
if (!trustedPrNumbers.has(prNumber)) {
|
||||
return skip(`PR #${prNumber} is not associated with workflow_run ${run.id}; skipping.`);
|
||||
}
|
||||
|
||||
const { data: pull } = await github.rest.pulls.get({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: prNumber,
|
||||
});
|
||||
|
||||
if (pull.state !== 'open') {
|
||||
return skip(`PR #${prNumber} is ${pull.state}; skipping.`);
|
||||
}
|
||||
if (pull.base.repo.full_name !== metadata.base_repo || pull.head.repo.full_name !== metadata.head_repo) {
|
||||
return skip('Current PR repository metadata does not match artifact; skipping.');
|
||||
}
|
||||
if (pull.head.sha !== metadata.head_sha) {
|
||||
return skip('Current PR head SHA differs from artifact; skipping stale run.');
|
||||
}
|
||||
|
||||
let body = 'Query regression completed, but no summary artifact was found.';
|
||||
if (fs.existsSync(summaryPath)) {
|
||||
const maxSummaryBytes = 1024 * 1024;
|
||||
const summarySize = fs.statSync(summaryPath).size;
|
||||
if (summarySize > maxSummaryBytes) {
|
||||
return skip(`Summary file too large: ${summarySize} bytes.`);
|
||||
}
|
||||
body = fs.readFileSync(summaryPath, 'utf8');
|
||||
}
|
||||
body = body
|
||||
.replace(/<!--[\s\S]*?-->/g, '')
|
||||
.replace(/@/g, '@\u200b');
|
||||
if (body.length > 59999) {
|
||||
body = body.slice(0, 59500) + '\n\n…\n\n_Comment truncated because it exceeded 60000 characters._\n';
|
||||
}
|
||||
fs.writeFileSync(summaryPath, body);
|
||||
|
||||
core.setOutput('should_post', 'true');
|
||||
core.setOutput('pr_number', String(prNumber));
|
||||
core.setOutput('summary_path', summaryPath);
|
||||
|
||||
- name: Post sticky PR comment
|
||||
if: ${{ steps.validate.outputs.should_post == 'true' }}
|
||||
uses: marocchino/sticky-pull-request-comment@v2
|
||||
with:
|
||||
header: query-regression-report
|
||||
number: ${{ steps.validate.outputs.pr_number }}
|
||||
path: ${{ steps.validate.outputs.summary_path }}
|
||||
152
.github/workflows/query-regression.yml
vendored
Normal file
152
.github/workflows/query-regression.yml
vendored
Normal file
@@ -0,0 +1,152 @@
|
||||
name: Query Regression
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
case:
|
||||
description: Query perf case path(s) in candidate checkout, or all
|
||||
required: true
|
||||
default: all
|
||||
base_ref:
|
||||
description: Base ref/sha to build
|
||||
required: true
|
||||
default: main
|
||||
candidate_ref:
|
||||
description: Candidate ref/sha to build (empty = current ref)
|
||||
required: false
|
||||
default: ""
|
||||
allow_large_fixture:
|
||||
description: Pass --allow-large-fixture to runner
|
||||
type: boolean
|
||||
default: true
|
||||
http_timeout:
|
||||
description: Runner HTTP timeout seconds
|
||||
required: true
|
||||
default: "300"
|
||||
cargo_profile:
|
||||
description: Cargo profile to build base and candidate binaries with
|
||||
required: true
|
||||
type: choice
|
||||
default: nightly
|
||||
options:
|
||||
- nightly
|
||||
- release
|
||||
- dev
|
||||
pull_request:
|
||||
types: [labeled, ready_for_review, reopened]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
query-regression:
|
||||
if: >-
|
||||
${{ github.event_name == 'workflow_dispatch' ||
|
||||
(github.event_name == 'pull_request' &&
|
||||
!github.event.pull_request.draft &&
|
||||
contains(github.event.pull_request.labels.*.name, 'query-regression') &&
|
||||
(github.event.action != 'labeled' || github.event.label.name == 'query-regression')) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }}
|
||||
steps:
|
||||
- name: Checkout base source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ github.repository }}
|
||||
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }}
|
||||
path: base-src
|
||||
persist-credentials: false
|
||||
|
||||
- name: Checkout candidate source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: ${{ github.repository }}
|
||||
ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }}
|
||||
path: candidate-src
|
||||
persist-credentials: false
|
||||
|
||||
- uses: arduino/setup-protoc@v3
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
|
||||
- uses: astral-sh/setup-uv@v5
|
||||
|
||||
- name: Rust cache (base)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: base-src
|
||||
shared-key: query-regression-base
|
||||
cache-all-crates: "true"
|
||||
save-if: false
|
||||
|
||||
- name: Rust cache (candidate)
|
||||
uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
workspaces: candidate-src
|
||||
shared-key: query-regression-candidate
|
||||
cache-all-crates: "true"
|
||||
save-if: false
|
||||
|
||||
- name: Build base greptime
|
||||
working-directory: base-src
|
||||
run: cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime
|
||||
|
||||
- name: Build candidate greptime and fixture generator
|
||||
working-directory: candidate-src
|
||||
run: cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture
|
||||
|
||||
- name: Run query regression
|
||||
id: run
|
||||
env:
|
||||
CASE_PATHS: ${{ github.event_name == 'pull_request' && 'all' || inputs.case }}
|
||||
HTTP_TIMEOUT: ${{ github.event_name == 'pull_request' && '300' || inputs.http_timeout }}
|
||||
ALLOW_LARGE_FIXTURE: ${{ github.event_name == 'pull_request' && 'true' || inputs.allow_large_fixture }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
CASE_NAME: ${{ github.event_name == 'pull_request' && 'default case set' || inputs.case }}
|
||||
BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }}
|
||||
CANDIDATE_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || inputs.candidate_ref || github.ref }}
|
||||
run: uv run --no-project python candidate-src/.github/scripts/query-regression-run.py
|
||||
|
||||
- name: Write PR metadata for trusted comment workflow
|
||||
if: ${{ always() && github.event_name == 'pull_request' }}
|
||||
env:
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
RUN_ATTEMPT: ${{ github.run_attempt }}
|
||||
run: uv run --no-project python candidate-src/.github/scripts/query-regression-pr-metadata.py
|
||||
|
||||
- name: Upload query regression artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: query-regression-report
|
||||
path: |
|
||||
query-regression-work/**
|
||||
query-regression-summary.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload trusted comment artifact
|
||||
if: ${{ always() && github.event_name == 'pull_request' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: query-regression-comment
|
||||
path: |
|
||||
query-regression-pr.json
|
||||
query-regression-summary.md
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Fail on regression failure
|
||||
if: ${{ steps.run.outputs.status != '0' }}
|
||||
run: exit 1
|
||||
Reference in New Issue
Block a user