chore(ci): Implement /query-regression command handling and admission workflow (#8975)

* Implement `/query-regression` command handling and admission workflow

- Add `query-regression-slash.py` script for processing `/query-regression` commands in PR comments, validating case arguments, and checking permissions.
- Update `checks.yml` to include tests for the new slash command functionality.
- Modify `query-regression-comment.yml` to trigger on the new `Query Regression Command` workflow.
- Create `query-regression-slash.yml` to handle the dispatched command, validate allowlist and permissions, and initiate the regression workflow.
- Enhance `query-regression.yml` to support additional inputs for PR admission and SHA verification.
- Introduce `slash-command-dispatch.yml` to parse and dispatch commands from PR comments.
- Document the new command admission process in `AGENTS.md` and `README.md`.
- Add unit tests in `test_query_regression_slash.py` to cover command parsing and admission logic.

* refactor: enhance query-regression command handling with comment validation and identity checks

* feat: implement admission identity handling for query regression workflows

* refactor: update PR admission logic in query regression workflow

* refactor: update token usage in slash command dispatch and README for clarity

* test: add cases for handling re-run failed jobs and stale runner artifacts

* refactor: improve repository metadata handling in query regression scripts

* chore: enable overwrite for artifact uploads to handle re-run failed jobs

* chore: enable overwrite for query regression admission uploads

* feat: enhance query-regression admission with HMAC signing and verification

- Introduced HMAC signing for admission markers in query-regression workflows to ensure integrity and authenticity.
- Updated `query-regression-comment.test.cjs` to include tests for signing and verifying admission markers.
- Modified `query-regression-slash.py` to handle admission marker signing and verification, including checks for dispatch sender and head SHA consistency.
- Enhanced workflows to securely manage admission markers and HMAC secrets, ensuring they are not exposed to untrusted contexts.
- Improved documentation to clarify the admission process and the role of HMAC in securing the workflow.

* test: add case to find newly posted marker among newer comments

* test: add case to verify multiline output handling in write_outputs function
This commit is contained in:
localhost
2026-09-07 07:23:05 +00:00
committed by GitHub
parent f9df4def74
commit 4d65e8984a
12 changed files with 2515 additions and 193 deletions
+1
View File
@@ -87,6 +87,7 @@ jobs:
python3 tests/perf/test_query_regression_summary_otlp.py
python3 tests/perf/test_query_regression_case_selection.py
python3 tests/perf/test_query_regression_nightly_refs.py
python3 tests/perf/test_query_regression_slash.py
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
check:
+32 -12
View File
@@ -2,20 +2,22 @@ name: Query Regression Comment
on:
workflow_run:
workflows: ["Query Regression"]
workflows:
- Query Regression Command
types: [completed]
permissions:
contents: read
actions: read
issues: read
pull-requests: write
jobs:
comment:
if: >-
${{ github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion != 'cancelled' &&
github.event.workflow_run.conclusion != 'skipped' }}
${{ github.event.workflow_run.conclusion != 'cancelled' &&
github.event.workflow_run.conclusion != 'skipped' &&
github.event.workflow_run.event == 'repository_dispatch' }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -25,7 +27,7 @@ jobs:
ref: ${{ github.event.repository.default_branch }}
persist-credentials: false
- name: Locate query regression comment artifact
- name: Locate query regression artifacts
id: artifact
uses: actions/github-script@v7
with:
@@ -38,24 +40,41 @@ jobs:
run_id,
per_page: 100,
});
const artifact = data.artifacts.find(
const admission = data.artifacts.find(
item => item.name === 'query-regression-admission' && !item.expired
);
const comment = data.artifacts.find(
item => item.name === 'query-regression-comment' && !item.expired
);
if (!artifact) {
core.info('No query-regression-comment artifact found; skipping.');
if (!admission || !comment) {
core.info('Trusted admission identity or comment artifact missing; skipping.');
core.setOutput('found', 'false');
return;
}
core.setOutput('found', 'true');
core.setOutput('id', String(artifact.id));
core.setOutput('admission_id', String(admission.id));
core.setOutput('comment_id', String(comment.id));
- name: Download query regression comment artifact
id: download
- name: Download admission identity
id: download-admission
if: ${{ steps.artifact.outputs.found == 'true' }}
uses: actions/download-artifact@v4
continue-on-error: true
with:
artifact-ids: ${{ steps.artifact.outputs.id }}
artifact-ids: ${{ steps.artifact.outputs.admission_id }}
path: query-regression-admission
repository: ${{ github.repository }}
github-token: ${{ secrets.GITHUB_TOKEN }}
run-id: ${{ github.event.workflow_run.id }}
merge-multiple: true
- name: Download query regression comment artifact
id: download
if: ${{ steps.download-admission.outcome == 'success' }}
uses: actions/download-artifact@v4
continue-on-error: true
with:
artifact-ids: ${{ steps.artifact.outputs.comment_id }}
path: query-regression-comment
repository: ${{ github.repository }}
github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -69,6 +88,7 @@ jobs:
env:
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
WORKFLOW_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
QUERY_REGRESSION_ADMISSION_HMAC: ${{ secrets.QUERY_REGRESSION_ADMISSION_HMAC }}
with:
script: |
const validate = require('./.github/scripts/query-regression-comment.cjs');
@@ -0,0 +1,121 @@
name: Query Regression Command
# Handler for `/query-regression`, dispatched by slash-command-dispatch.yml.
# Requires github-actions[bot] as the dispatch sender, re-fetches the
# comment by id, requires the current PR head to match the dispatcher
# snapshot, admits an allowlisted admin, snapshots merge/head/base SHAs,
# and calls the reusable Query Regression workflow.
on:
repository_dispatch:
types: [query-regression-command]
permissions:
contents: read
issues: write
pull-requests: write
jobs:
parse:
name: Admit dispatched query-regression command
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
skip: ${{ steps.admit.outputs.skip }}
reason: ${{ steps.admit.outputs.reason }}
case: ${{ steps.admit.outputs.case }}
pr_number: ${{ steps.admit.outputs.pr_number }}
base_sha: ${{ steps.admit.outputs.base_sha }}
candidate_sha: ${{ steps.admit.outputs.candidate_sha }}
head_sha: ${{ steps.admit.outputs.head_sha }}
head_repo: ${{ steps.admit.outputs.head_repo }}
base_repo: ${{ steps.admit.outputs.base_repo }}
steps:
- name: Checkout trusted admission script
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Admit dispatched command
id: admit
env:
GITHUB_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_API_URL: ${{ github.api_url }}
COMMENT_ID: ${{ github.event.client_payload.github.payload.comment.id }}
COMMENT_ACTOR: ${{ github.event.client_payload.github.payload.comment.user.login }}
COMMAND_ARGS: ${{ github.event.client_payload.slash_command.args.unnamed.all || '' }}
PR_NUMBER: ${{ github.event.client_payload.github.payload.issue.number }}
DISPATCH_SENDER: ${{ github.event.sender.login }}
DISPATCH_HEAD_SHA: ${{ github.event.client_payload.pull_request.head.sha }}
QUERY_REGRESSION_COMMENT_ALLOWLIST: ${{ vars.QUERY_REGRESSION_COMMENT_ALLOWLIST }}
# Never reference this secret in query-regression.yml: the ECS job
# would inherit it and could forge the admission marker.
QUERY_REGRESSION_ADMISSION_HMAC: ${{ secrets.QUERY_REGRESSION_ADMISSION_HMAC }}
run: python3 .github/scripts/query-regression-slash.py
- name: Summarize admission
if: always()
env:
SKIP: ${{ steps.admit.outputs.skip }}
REASON: ${{ steps.admit.outputs.reason }}
CASE: ${{ steps.admit.outputs.case }}
PR_NUMBER: ${{ steps.admit.outputs.pr_number }}
BASE_SHA: ${{ steps.admit.outputs.base_sha }}
CANDIDATE_SHA: ${{ steps.admit.outputs.candidate_sha }}
HEAD_SHA: ${{ steps.admit.outputs.head_sha }}
run: |
set -euo pipefail
if [[ "${SKIP}" == "true" ]]; then
printf 'Skipping query-regression command: %s\n' "${REASON}" | tee -a "${GITHUB_STEP_SUMMARY}"
elif [[ "${SKIP}" == "false" ]]; then
{
printf 'Admitted `/query-regression` (`%s`) on PR #%s\n' "${CASE}" "${PR_NUMBER}"
printf -- '- Head SHA: `%s`\n' "${HEAD_SHA}"
printf -- '- Merge SHA: `%s`\n' "${CANDIDATE_SHA}"
printf -- '- Base SHA: `%s`\n' "${BASE_SHA}"
} | tee -a "${GITHUB_STEP_SUMMARY}"
else
printf 'Admission failed without a decision (the admit step crashed); see its logs.\n' \
| tee -a "${GITHUB_STEP_SUMMARY}" >&2
exit 1
fi
- name: Upload admission identity
if: ${{ steps.admit.outputs.skip == 'false' && steps.admit.outputs.candidate_sha != '' }}
uses: actions/upload-artifact@v4
with:
name: query-regression-admission
path: query-regression-admission.json
if-no-files-found: error
retention-days: 7
overwrite: true
- name: Reply with admission result
if: ${{ !cancelled() && steps.admit.outputs.reply != '' }}
env:
GH_TOKEN: ${{ github.token }}
REPLY: ${{ steps.admit.outputs.reply }}
PR_NUMBER: ${{ steps.admit.outputs.pr_number || github.event.client_payload.github.payload.issue.number }}
run: |
set -euo pipefail
printf '%s\n' "${REPLY}" | gh pr comment "${PR_NUMBER}" --body-file -
query-regression:
name: Query regression from comment
needs: [parse]
if: ${{ needs.parse.outputs.skip == 'false' && needs.parse.outputs.candidate_sha != '' }}
uses: ./.github/workflows/query-regression.yml
# inherit is for Aliyun/GH PAT used by provision/teardown. Do not
# reference QUERY_REGRESSION_ADMISSION_HMAC in the reusable workflow.
secrets: inherit
with:
case: ${{ needs.parse.outputs.case }}
base_ref: ${{ needs.parse.outputs.base_sha }}
candidate_ref: ${{ needs.parse.outputs.candidate_sha }}
cargo_profile: nightly
runner: ${{ vars.QUERY_REGRESSION_PR_RUNNER || 'aliyun-ecs' }}
pr_number: ${{ needs.parse.outputs.pr_number }}
head_sha: ${{ needs.parse.outputs.head_sha }}
head_repo: ${{ needs.parse.outputs.head_repo }}
base_repo: ${{ needs.parse.outputs.base_repo }}
+66 -56
View File
@@ -36,6 +36,28 @@ on:
required: false
type: string
default: aliyun-ecs
pr_number:
description: >-
When set, treat this reusable call as PR admission (merge-SHA
verification, trusted helper scripts, sticky comment artifact)
required: false
type: string
default: ""
head_sha:
description: PR head SHA snapshotted at admission (required with pr_number)
required: false
type: string
default: ""
head_repo:
description: PR head repository full_name (required with pr_number)
required: false
type: string
default: ""
base_repo:
description: PR base repository full_name (required with pr_number)
required: false
type: string
default: ""
workflow_dispatch:
inputs:
case:
@@ -82,21 +104,20 @@ on:
required: false
type: boolean
default: false
pull_request:
types: [labeled]
permissions:
# Load-bearing: every job, including ECS, gets this GITHUB_TOKEN.
# Do not grant issues or pull-requests write; candidate code could
# otherwise forge the HMAC admission marker on a victim PR.
contents: read
jobs:
test-tooling:
# Stdlib unittests for the workflow Python (case selection, report
# helpers, nightly SHA picking, and rendered ECS user-data). They do
# not talk to Aliyun or the Actions runner process; running them on
# ubuntu-latest fails fast before any ECS spend. Not gated on the
# regression labels: those labels boot a VM, these tests should not.
# Ordinary PRs also run the same tests from checks.yml, because this
# workflow only starts on `labeled` (or dispatch / workflow_call).
# helpers, nightly SHA picking, comment-command admission, and rendered
# ECS user-data). They do not talk to Aliyun or the Actions runner
# process; running them on ubuntu-latest fails fast before any ECS spend.
# Ordinary PRs also run the same tests from checks.yml.
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
@@ -112,24 +133,16 @@ jobs:
python3 tests/perf/test_query_regression_summary_otlp.py
python3 tests/perf/test_query_regression_case_selection.py
python3 tests/perf/test_query_regression_nightly_refs.py
python3 tests/perf/test_query_regression_slash.py
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
provision:
# Runs when the aliyun-ecs path is selected: explicitly via the runner
# input, or for PR labels by default (a QUERY_REGRESSION_PR_RUNNER
# repository variable set to another value redirects PRs to that literal
# runner label instead). Uses trusted scripts from the PR base (or the
# dispatched ref), never from candidate code. Waits for test-tooling so
# a broken user-data template does not still create a VM.
# Runs when the aliyun-ecs path is selected via the runner input (PR
# comment admission can redirect with QUERY_REGRESSION_PR_RUNNER). Uses
# trusted scripts from the caller ref, never from candidate code. Waits
# for test-tooling so a broken user-data template does not still create a VM.
needs: [test-tooling]
if: >-
${{ !failure() && !cancelled() &&
((github.event_name != 'pull_request' && inputs.runner == 'aliyun-ecs') ||
(github.event_name == 'pull_request' &&
!github.event.pull_request.draft &&
(github.event.label.name == 'query-regression' ||
github.event.label.name == 'heavy-regression') &&
(vars.QUERY_REGRESSION_PR_RUNNER || 'aliyun-ecs') == 'aliyun-ecs')) }}
if: ${{ !failure() && !cancelled() && inputs.runner == 'aliyun-ecs' }}
runs-on: ubuntu-latest
timeout-minutes: 45
outputs:
@@ -141,7 +154,7 @@ jobs:
uses: actions/checkout@v4
with:
repository: ${{ github.repository }}
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}
ref: ${{ github.sha }}
persist-credentials: false
- name: Install uv
@@ -170,20 +183,11 @@ jobs:
# lets the job run when provision was intentionally skipped because a
# literal runner label was selected; a failed or cancelled provision still
# blocks the run because no ECS runner would be waiting.
if: >-
${{ !failure() && !cancelled() &&
(github.event_name != 'pull_request' ||
(github.event_name == 'pull_request' &&
!github.event.pull_request.draft &&
(github.event.label.name == 'query-regression' ||
github.event.label.name == 'heavy-regression'))) }}
runs-on: >-
${{ needs.provision.outputs.label ||
(github.event_name != 'pull_request' && inputs.runner ||
(vars.QUERY_REGRESSION_PR_RUNNER || 'aliyun-ecs')) }}
if: ${{ !failure() && !cancelled() }}
runs-on: ${{ needs.provision.outputs.label || inputs.runner }}
timeout-minutes: 180
env:
CARGO_PROFILE: ${{ github.event_name == 'pull_request' && 'nightly' || inputs.cargo_profile }}
CARGO_PROFILE: ${{ inputs.cargo_profile }}
CARGO_HOME: /home/runner/.cargo
UV_CACHE_DIR: /home/runner/.cargo/uv-cache
RUSTUP_HOME: /opt/rustup
@@ -198,11 +202,18 @@ jobs:
CARGO_INCREMENTAL: "0"
RUSTFLAGS: -D warnings -C link-arg=-fuse-ld=mold
QUERY_REGRESSION_CACHE_EPOCH: "1"
BASE_REF: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || inputs.base_ref }}
CANDIDATE_REF: ${{ github.event_name != 'pull_request' && (inputs.candidate_ref || github.ref) || '' }}
EVENT_MERGE_SHA: ${{ github.event_name == 'pull_request' && github.sha || '' }}
EVENT_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }}
EVENT_BASE_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || '' }}
# Reusable workflows keep the caller's github.event_name (slash is
# repository_dispatch; nightly is workflow_run), never "workflow_call".
# PR admission is the explicit pr_number input from the slash wrapper.
PR_ADMISSION: ${{ inputs.pr_number != '' }}
PR_NUMBER: ${{ inputs.pr_number }}
HEAD_REPO: ${{ inputs.head_repo }}
BASE_REPO: ${{ inputs.base_repo }}
BASE_REF: ${{ inputs.base_ref }}
CANDIDATE_REF: ${{ inputs.candidate_ref || github.ref }}
EVENT_MERGE_SHA: ${{ inputs.pr_number != '' && inputs.candidate_ref || '' }}
EVENT_HEAD_SHA: ${{ inputs.head_sha }}
EVENT_BASE_SHA: ${{ inputs.pr_number != '' && inputs.base_ref || '' }}
# Runner identity contract. The ECS image uses 1001; a manually prepared
# host may override via repo variables when 1001 is already taken.
EXPECTED_RUNNER_UID: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
@@ -276,7 +287,7 @@ jobs:
fail_closed() {
printf '%s\n' \
"PR changed or merge result unavailable; maintainer must remove and re-add the regression label after reviewing current revision. $1" >&2
"PR changed or merge result unavailable; maintainer must re-admit the current revision by commenting /query-regression. $1" >&2
exit 1
}
@@ -318,7 +329,7 @@ jobs:
exit 1
}
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
if [[ "${PR_ADMISSION}" == "true" ]]; then
for identity in EVENT_MERGE_SHA EVENT_HEAD_SHA EVENT_BASE_SHA; do
value="${!identity-}"
is_full_sha "${value}" || fail_closed "Missing or invalid ${identity}."
@@ -392,7 +403,7 @@ jobs:
run: |
set -euo pipefail
helper_ref="${VERIFIED_CANDIDATE_SHA}"
if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
if [[ "${PR_ADMISSION}" == "true" ]]; then
helper_ref="${VERIFIED_BASE_SHA}"
fi
mkdir -p "${GITHUB_WORKSPACE}/query-regression-trusted-scripts"
@@ -736,16 +747,16 @@ jobs:
- name: Run query regression
id: run
env:
CASE_PATHS: ${{ github.event_name == 'pull_request' && (github.event.label.name == 'heavy-regression' && 'heavy' || 'all') || inputs.case }}
HTTP_TIMEOUT: ${{ github.event_name == 'pull_request' && '300' || inputs.http_timeout }}
ALLOW_LARGE_FIXTURE: ${{ github.event_name == 'pull_request' && 'true' || inputs.allow_large_fixture }}
CASE_PATHS: ${{ inputs.case }}
HTTP_TIMEOUT: ${{ inputs.http_timeout }}
ALLOW_LARGE_FIXTURE: ${{ inputs.allow_large_fixture }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
CASE_NAME: ${{ github.event_name == 'pull_request' && (github.event.label.name == 'heavy-regression' && 'heavy case set' || 'default case set') || inputs.case }}
CASE_NAME: ${{ inputs.case }}
BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime
CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime
FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture
OTELGEN_BIN: /usr/local/bin/otelgen
SUMMARY_SCRIPT: ${{ github.event_name == 'pull_request' && 'query-regression-trusted-scripts/query-regression-summary.py' || 'src/.github/scripts/query-regression-summary.py' }}
SUMMARY_SCRIPT: ${{ env.PR_ADMISSION == 'true' && 'query-regression-trusted-scripts/query-regression-summary.py' || 'src/.github/scripts/query-regression-summary.py' }}
QUERY_REGRESSION_RUNNER: ${{ github.workspace }}/query-regression-bins/candidate/query_regression_runner
run: >-
uv run --no-project python src/.github/scripts/query-regression-run.py
@@ -754,13 +765,9 @@ jobs:
--summary-script "${SUMMARY_SCRIPT}"
- name: Write PR metadata for trusted comment workflow
if: ${{ always() && github.event_name == 'pull_request' }}
if: ${{ always() && env.PR_ADMISSION == 'true' }}
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
BASE_REPO: ${{ github.event.pull_request.base.repo.full_name }}
HEAD_SHA: ${{ inputs.head_sha }}
RUN_ID: ${{ github.run_id }}
RUN_ATTEMPT: ${{ github.run_attempt }}
run: uv run --no-project python query-regression-trusted-scripts/query-regression-pr-metadata.py
@@ -780,9 +787,11 @@ jobs:
machine-telemetry.log
if-no-files-found: warn
retention-days: 7
# Re-run failed jobs uploads again in the same run; v4 rejects duplicate names.
overwrite: true
- name: Upload trusted comment artifact
if: ${{ always() && github.event_name == 'pull_request' }}
if: ${{ always() && env.PR_ADMISSION == 'true' }}
uses: actions/upload-artifact@v4
with:
name: query-regression-comment
@@ -791,6 +800,7 @@ jobs:
query-regression-work/**/query-regression-report.json
if-no-files-found: warn
retention-days: 7
overwrite: true
- name: Report cache usage
if: ${{ always() }}
@@ -883,7 +893,7 @@ jobs:
uses: actions/checkout@v4
with:
repository: ${{ github.repository }}
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.sha }}
ref: ${{ github.sha }}
persist-credentials: false
- name: Install uv
@@ -0,0 +1,36 @@
name: Slash Command Dispatch
# ChatOps front door: parse `/command` on PR comments, check admin
# permission, and repository_dispatch to a per-command handler. Handlers
# own allowlists, SHA admission, and the actual work. Same-repo dispatch
# uses github.token with contents: write; GitHub starts the handler run
# for GITHUB_TOKEN-created repository_dispatch events. Do not pass the
# long-lived GH_PERSONAL_ACCESS_TOKEN into this third-party action.
#
# To add a command: list it under `commands` and add a workflow with
# `on.repository_dispatch.types: ["<command>-command"]`.
on:
issue_comment:
types: [created]
permissions:
contents: write
issues: write
pull-requests: write
jobs:
slash-command-dispatch:
name: Dispatch slash command
if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/') }}
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Slash Command Dispatch
uses: peter-evans/slash-command-dispatch@9bdcd7914ec1b75590b790b844aa3b8eee7c683a # v5.0.2
with:
token: ${{ github.token }}
permission: admin
issue-type: pull-request
commands: |
query-regression