mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-03 16:10:21 +00:00
ci: one shared path classification decides which suites run (#14150)
This commit is contained in:
Hengfei Yang
@@ -0,0 +1,29 @@
|
||||
name: Classify changed paths
|
||||
description: One shared answer to "which suites does this change need", replacing per-workflow path regexes
|
||||
inputs:
|
||||
base:
|
||||
description: Ref to diff against; the caller must have fetched it (fetch-depth 0)
|
||||
default: origin/${{ github.base_ref || 'main' }}
|
||||
head:
|
||||
description: Commit to diff to
|
||||
default: ${{ github.sha }}
|
||||
outputs:
|
||||
run_unit_backend: { value: "${{ steps.c.outputs.run_unit_backend }}" }
|
||||
run_unit_ui: { value: "${{ steps.c.outputs.run_unit_ui }}" }
|
||||
run_api: { value: "${{ steps.c.outputs.run_api }}" }
|
||||
run_db: { value: "${{ steps.c.outputs.run_db }}" }
|
||||
run_playwright: { value: "${{ steps.c.outputs.run_playwright }}" }
|
||||
run_enterprise: { value: "${{ steps.c.outputs.run_enterprise }}" }
|
||||
changed_workflows: { value: "${{ steps.c.outputs.changed_workflows }}" }
|
||||
unknown: { value: "${{ steps.c.outputs.unknown }}" }
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- id: c
|
||||
shell: bash
|
||||
env:
|
||||
BASE: ${{ inputs.base }}
|
||||
HEAD: ${{ inputs.head }}
|
||||
run: |
|
||||
CHANGED=$(git -c core.quotepath=false diff --name-only "$BASE" "$HEAD" 2>/dev/null || true)
|
||||
printf '%s\n' "$CHANGED" | "$GITHUB_ACTION_PATH/classify.sh" | tee -a "$GITHUB_OUTPUT"
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
# Classify a list of changed paths (one per line on stdin) into the suites that must run.
|
||||
# Prints key=value lines. Unknown paths run everything: the merge queue uses the same rules,
|
||||
# so an allow-list miss would mean a path that is never tested.
|
||||
set -euo pipefail
|
||||
|
||||
docs=false ci=false build_support=false ops=false backend=false frontend=false
|
||||
api_tests=false ui_tests=false db_tests=false test_data=false mobile=false unknown=false
|
||||
workflows=""
|
||||
n=0
|
||||
while IFS= read -r f; do
|
||||
[ -z "$f" ] && continue
|
||||
n=$((n+1))
|
||||
case "$f" in
|
||||
*.md|docs/*|screenshots/*|qa-reports/*|*.png|*.jpg|*.jpeg|*.gif|*.svg|*.webp|LICENSE|.github/ISSUE_TEMPLATE/*) docs=true ;;
|
||||
.github/workflows/*) ci=true; workflows="$workflows ${f#.github/workflows/}" ;;
|
||||
.github/actions/*|.github/scripts/*|.github/protoc.sh|.github/dependabot.yml|.github/pr-title-checker-config.json) build_support=true ;;
|
||||
scripts/*|deploy/*|cross/*|config/*|download.sh|downloadO2.sh|Cross.*.toml|deny.toml|.devcontainer/*|.agents/*|.claude/*|.opencode/*|opencode.jsonc|.claudeignore|.cursorignore|.typos.toml|.gitattributes|.gitignore|.env.example|openobserve.cdx.xml|package-lock.json) ops=true ;;
|
||||
src/*|proto/*|Cargo.toml|Cargo.lock|build.rs|rust-toolchain.toml|.cargo/*|tests/integration_test.rs|coverage.sh|clippy.toml|rustfmt.toml) backend=true ;;
|
||||
web/*) frontend=true ;;
|
||||
tests/api-testing/*) api_tests=true ;;
|
||||
tests/ui-testing/*) ui_tests=true ;;
|
||||
tests/db-testing/*) db_tests=true ;;
|
||||
tests/mobile-testing/*) mobile=true ;;
|
||||
tests/test-data/*|tests/*.json|tests/sourcemaps.zip|tests/.gitignore) test_data=true ;;
|
||||
*) unknown=true ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# No diff at all (no base to compare against) is treated as unknown: run everything.
|
||||
[ "$n" -eq 0 ] && unknown=true
|
||||
|
||||
or() { for v in "$@"; do [ "$v" = true ] && { echo true; return; }; done; echo false; }
|
||||
run_unit_backend=$(or "$backend" "$build_support" "$unknown")
|
||||
run_unit_ui=$(or "$frontend" "$unknown")
|
||||
run_api=$(or "$backend" "$api_tests" "$test_data" "$unknown")
|
||||
run_db=$(or "$backend" "$db_tests" "$unknown")
|
||||
run_playwright=$(or "$backend" "$frontend" "$ui_tests" "$test_data" "$unknown")
|
||||
run_enterprise=$(or "$backend" "$frontend" "$unknown")
|
||||
|
||||
for k in docs ci build_support ops backend frontend api_tests ui_tests db_tests test_data mobile unknown \
|
||||
run_unit_backend run_unit_ui run_api run_db run_playwright run_enterprise; do
|
||||
echo "$k=${!k}"
|
||||
done
|
||||
echo "changed_workflows=${workflows# }"
|
||||
@@ -49,27 +49,20 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# One shared classification for every suite (see .github/actions/classify-changes); a change to
|
||||
# this workflow file re-tests itself on top of it.
|
||||
- uses: ./.github/actions/classify-changes
|
||||
id: classify
|
||||
- id: check
|
||||
env:
|
||||
HAS_CHANGES: ${{ steps.classify.outputs.run_api }}
|
||||
CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
|
||||
run: |
|
||||
# Skip when a PR touches only docs, images, frontend/UI-test files, or unrelated
|
||||
# workflow files (e.g. a dependabot bump to another workflow) — none affect the API.
|
||||
# Source and this workflow (api-testing.yml) still run the suite.
|
||||
IGNORE_RE='(\.md$|^docs/|^web/|^tests/ui-testing/|^tests/playwright-tests/|\.(png|jpe?g|gif|svg|webp)$|^\.github/workflows/)'
|
||||
CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
|
||||
# Files not matching IGNORE_RE. Captured (not `grep -qv`) so the check doesn't depend
|
||||
# on grep -qv exit semantics, which differ across grep builds; the `|| true` keeps
|
||||
# `set -e` from aborting when grep matches nothing (exit 1). The `grep -qE` below is a
|
||||
# loop-safe `if`-condition term (`set -e` doesn't fire there), so it needs no `|| true`.
|
||||
RELEVANT=$(printf '%s' "$CHANGED" | grep -vE "$IGNORE_RE" || true)
|
||||
if [ -z "$CHANGED" ] || [ -n "$RELEVANT" ] || printf '%s' "$CHANGED" | grep -qE '^\.github/workflows/api-testing\.yml$'; then
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
if [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' api-testing.yml '; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Build the exact PR/merge-group binary once, then fan it out to every API
|
||||
# test shard. The artifact is scoped to this workflow run and SHA; Rust cache
|
||||
# remains an input to this single build, not a substitute for the binary.
|
||||
build_api_binary:
|
||||
name: build_api_binary
|
||||
needs: [check_changes]
|
||||
|
||||
@@ -44,16 +44,19 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# One shared classification for every suite (see .github/actions/classify-changes); a change to
|
||||
# this workflow file re-tests itself on top of it.
|
||||
- uses: ./.github/actions/classify-changes
|
||||
id: classify
|
||||
- id: check
|
||||
env:
|
||||
HAS_CHANGES: ${{ steps.classify.outputs.run_db }}
|
||||
CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
|
||||
run: |
|
||||
CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
|
||||
# Same contract as playwright/api: other workflows' files are irrelevant, this file re-tests itself.
|
||||
if [ -z "$CHANGED" ] || echo "$CHANGED" | grep -qE '^\.github/workflows/db-testing\.yml$'; then
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
elif echo "$CHANGED" | grep -qvE '(\.md$|^docs/|^web/|^tests/ui-testing/|^tests/playwright-tests/|\.(png|jpe?g|gif|svg|webp)$|^\.github/workflows/)'; then
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
if [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' db-testing.yml '; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
db_validation_tests:
|
||||
|
||||
@@ -21,11 +21,28 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
pull-requests: write
|
||||
contents: read
|
||||
statuses: write
|
||||
steps:
|
||||
# Debounce: a push inside this window cancels the run (concurrency) before anything paid starts.
|
||||
- if: github.event_name == 'pull_request'
|
||||
run: sleep 60
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: ./.github/actions/classify-changes
|
||||
id: classify
|
||||
# ent-build-status-check is required; when no backend or frontend code changed there is nothing to build,
|
||||
# so report it as passed here instead of dispatching.
|
||||
- name: Report not applicable
|
||||
if: ${{ steps.classify.outputs.run_enterprise != 'true' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh api -X POST "repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }}" \
|
||||
-f state=success -f context=ent-build-status-check -f description="no backend or frontend change; enterprise build not needed"
|
||||
- name: Generate token
|
||||
if: ${{ steps.classify.outputs.run_enterprise == 'true' }}
|
||||
id: generate_token
|
||||
uses: actions/create-github-app-token@v3.2.0
|
||||
with:
|
||||
@@ -34,6 +51,7 @@ jobs:
|
||||
# Set the owner, so the token can be used in all repositories
|
||||
owner: ${{ github.repository_owner }}
|
||||
- name: Set build short sha
|
||||
if: ${{ steps.classify.outputs.run_enterprise == 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
unset SHORT_COMMIT_ID
|
||||
@@ -41,9 +59,11 @@ jobs:
|
||||
echo "COMMIT_ID=${GITHUB_HEAD_SHA}" >> $GITHUB_ENV
|
||||
|
||||
- name: Confirm git commit SHA output
|
||||
if: ${{ steps.classify.outputs.run_enterprise == 'true' }}
|
||||
run: echo ${{ env.SHORT_COMMIT_ID }}
|
||||
|
||||
- name: Repository Dispatch
|
||||
if: ${{ steps.classify.outputs.run_enterprise == 'true' }}
|
||||
uses: peter-evans/repository-dispatch@v4
|
||||
env:
|
||||
TOKEN: ${{ steps.generate_token.outputs.token }}
|
||||
|
||||
@@ -87,26 +87,19 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# One shared classification for every suite (see .github/actions/classify-changes); a change to
|
||||
# this workflow file re-tests itself on top of it.
|
||||
- uses: ./.github/actions/classify-changes
|
||||
id: classify
|
||||
- id: check
|
||||
env:
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
HAS_CHANGES: ${{ steps.classify.outputs.run_playwright }}
|
||||
CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
|
||||
run: |
|
||||
# Skip when a PR touches only docs, images, or unrelated workflow files (e.g. a
|
||||
# dependabot bump to another workflow) — none can affect the app. Source, web/, and
|
||||
# this workflow (playwright.yml) still run the suite.
|
||||
# Two-dot on purpose: base-branch drift over-reports has_changes, which keeps the
|
||||
# merge_group full run firing whenever the merged tree differs from what PRs tested.
|
||||
IGNORE_RE='(\.md$|^docs/|\.(png|jpe?g|gif|svg|webp)$|^\.github/workflows/)'
|
||||
CHANGED=$(git diff --name-only "origin/${BASE_REF:-main}" ${{ github.sha }})
|
||||
# Files not matching IGNORE_RE. Captured (not `grep -qv`) so the check doesn't depend
|
||||
# on grep -qv exit semantics, which differ across grep builds; the `|| true` keeps
|
||||
# `set -e` from aborting when grep matches nothing (exit 1). The `grep -qE` below is a
|
||||
# loop-safe `if`-condition term (`set -e` doesn't fire there), so it needs no `|| true`.
|
||||
RELEVANT=$(printf '%s' "$CHANGED" | grep -vE "$IGNORE_RE" || true)
|
||||
if [ -z "$CHANGED" ] || [ -n "$RELEVANT" ] || printf '%s' "$CHANGED" | grep -qE '^\.github/workflows/playwright\.yml$'; then
|
||||
echo "has_changes=true" >> $GITHUB_OUTPUT
|
||||
if [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' playwright.yml '; then
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
build_binary:
|
||||
|
||||
@@ -41,18 +41,25 @@ jobs:
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# One shared classification for every suite (see .github/actions/classify-changes); a change to
|
||||
# this workflow file re-tests itself on top of it.
|
||||
- uses: ./.github/actions/classify-changes
|
||||
id: classify
|
||||
- id: check
|
||||
env:
|
||||
HAS_BACKEND_CHANGES: ${{ steps.classify.outputs.run_unit_backend }}
|
||||
HAS_UI_CHANGES: ${{ steps.classify.outputs.run_unit_ui }}
|
||||
CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
|
||||
run: |
|
||||
CHANGED=$(git diff --name-only origin/${{ github.base_ref || 'main' }} ${{ github.sha }})
|
||||
if [ -z "$CHANGED" ] || echo "$CHANGED" | grep -qE '^src/|^proto/|^\.cargo/|^Cargo\.(toml|lock)$|^build\.rs$|^rust-toolchain\.toml$|^coverage\.sh$|^\.github/workflows/unit-tests\.yml$'; then
|
||||
echo "has_backend_changes=true" >> $GITHUB_OUTPUT
|
||||
if [ "$HAS_BACKEND_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' unit-tests.yml '; then
|
||||
echo "has_backend_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_backend_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "has_backend_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
if [ -z "$CHANGED" ] || echo "$CHANGED" | grep -q '^web/'; then
|
||||
echo "has_ui_changes=true" >> $GITHUB_OUTPUT
|
||||
if [ "$HAS_UI_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' unit-tests.yml '; then
|
||||
echo "has_ui_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_ui_changes=false" >> $GITHUB_OUTPUT
|
||||
echo "has_ui_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
backend_static_checks:
|
||||
|
||||
Reference in New Issue
Block a user