mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 00:09:35 +00:00
ci: split merge / TestDino / OpenObserve into separate jobs + harden merge (#12942)
## Why The scheduled OSS regression kept failing in `Merge Reports and Upload to TestDino` even after the reporter fix (#12939): in [run 28417106847](https://github.com/openobserve/openobserve/actions/runs/28417106847) all 11 shards passed but `merge-reports` **bailed mid-merge** on the `ubicloud-standard-4` runner (extracted ~4 of 10 blobs, exited 0 without building the report). The blobs are valid 1.50.0 and merge cleanly locally — it's a runner-resource issue. ENT's EKS runner completes the same merge, which is why it passed there. The job also folded merge + TestDino upload + OpenObserve metrics together, so any hiccup cascaded. ## What Split the one job into **three independent jobs** (in both `playwright.yml` and `playwright_regression.yml`): | Job | Role | |-----|------| | **merge_reports** | merge blobs → extract stats/flaky → upload merged report as artifact. Bumped to `ubicloud-standard-8` + `merge-reports` wrapped in a 3× retry. | | **upload_testdino** | own job, only on test failure, consumes the merged report via artifact. | | **report_openobserve** | own job, `continue-on-error` metrics, consumes stats/flaky via job outputs. | `playwright_summary` now `needs:` only `[build_binary, ui_integration_tests, merge_reports]` — **never** TestDino or OpenObserve, so reporting plumbing can't block the run's pass/fail signal. Pairs with ENT `test/ci-split-report-jobs`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Shrinath Rao
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
43db0f4de0
commit
6081bcd3d8
@@ -876,16 +876,18 @@ jobs:
|
||||
if: always()
|
||||
run: cat o2.log
|
||||
|
||||
merge_and_upload_reports:
|
||||
merge_reports:
|
||||
timeout-minutes: 15
|
||||
name: Merge Reports and Upload to TestDino
|
||||
name: Merge Reports
|
||||
needs: [ui_integration_tests]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
||||
# eks runner: `playwright merge-reports` silently stops mid-merge (exits 0, ~3 of N blobs, no
|
||||
# output) on the ubicloud and github-hosted runners — but works on the eks runners (same as ENT).
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
labels: eks-openobserve-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
actions: read # the no-blobs investigation reads run/job state from the Actions API
|
||||
outputs:
|
||||
# Compact Playwright stats (expected/unexpected/flaky/skipped) from the merged report.json,
|
||||
# consumed by report_to_openobserve to record per-test counts in ci_test_runs.
|
||||
@@ -1029,10 +1031,32 @@ jobs:
|
||||
# Log a count only — avoid dumping test file paths/titles into public CI logs.
|
||||
echo "flaky/failed test count: $(printf '%s' "$FLAKY" | jq 'length' 2>/dev/null || echo '?')"
|
||||
|
||||
# Non-fatal metrics sidecar — runs HERE inside the report flow (like the TestDino
|
||||
# step below) so it never trails / gates playwright_summary (which doesn't need this
|
||||
# job). One retry-aware run-summary doc to OpenObserve (ci_test_runs, suite=ui) plus
|
||||
# per-test flaky/failed rows to ci_test_results. continue-on-error: never fails the run.
|
||||
- name: Upload merged report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: merged-report-${{ github.run_attempt }}
|
||||
path: tests/ui-testing/playwright-results
|
||||
retention-days: 1
|
||||
if-no-files-found: warn
|
||||
|
||||
# OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one
|
||||
# retry-aware run-summary doc (ci_test_runs, suite=ui) + per-test flaky/failed rows
|
||||
# (ci_test_results). The step is continue-on-error and the poster scripts never fail.
|
||||
report_openobserve:
|
||||
timeout-minutes: 10
|
||||
name: Report to OpenObserve
|
||||
needs: [ui_integration_tests, merge_reports]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read # reads run/job timings from the Actions API
|
||||
steps:
|
||||
- name: Clone the current repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Report run metrics to OpenObserve
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -1041,8 +1065,8 @@ jobs:
|
||||
O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
|
||||
O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
|
||||
TESTS: ${{ needs.ui_integration_tests.result }}
|
||||
MERGE_STATS: ${{ steps.report_stats.outputs.stats }}
|
||||
MERGE_FLAKY: ${{ steps.report_flaky.outputs.flaky }}
|
||||
MERGE_STATS: ${{ needs.merge_reports.outputs.stats }}
|
||||
MERGE_FLAKY: ${{ needs.merge_reports.outputs.flaky }}
|
||||
run: |
|
||||
# This job runs only when tests actually ran (merge `if:` gates on !cancelled &&
|
||||
# ui != skipped), so TESTS is success|failure here.
|
||||
@@ -1068,8 +1092,37 @@ jobs:
|
||||
fi
|
||||
fi
|
||||
|
||||
# TestDino upload — its own job so it never gates playwright_summary. Only runs when tests
|
||||
# failed (and uploads enabled); consumes the merged report via artifact.
|
||||
upload_testdino:
|
||||
timeout-minutes: 15
|
||||
name: Upload to TestDino
|
||||
needs: [ui_integration_tests, merge_reports]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }}
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read # download-artifact fetches the merged-report artifact
|
||||
steps:
|
||||
- name: Clone the current repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 22
|
||||
|
||||
- name: Install project dependencies
|
||||
run: cd tests/ui-testing && npm ci
|
||||
|
||||
- name: Download merged report
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: merged-report-${{ github.run_attempt }}
|
||||
path: tests/ui-testing/playwright-results
|
||||
|
||||
- name: Cache test failures to TestDino
|
||||
if: ${{ needs.ui_integration_tests.result == 'failure' }}
|
||||
env:
|
||||
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
|
||||
run: |
|
||||
@@ -1113,7 +1166,7 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Upload to TestDino
|
||||
if: ${{ needs.ui_integration_tests.result == 'failure' && env.UPLOAD_TO_TESTDINO == 'true' }}
|
||||
if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }}
|
||||
run: |
|
||||
cd tests/ui-testing
|
||||
|
||||
|
||||
@@ -538,19 +538,21 @@ jobs:
|
||||
if: always()
|
||||
run: cat o2.log
|
||||
|
||||
merge_and_upload_reports:
|
||||
merge_reports:
|
||||
timeout-minutes: 15
|
||||
name: Merge Reports and Upload to TestDino
|
||||
name: Merge Reports
|
||||
needs: [ui_integration_tests]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
||||
# eks runner: `playwright merge-reports` silently stops mid-merge (exits 0, ~3 of N blobs, no
|
||||
# output) on the ubicloud and github-hosted runners — but works on the eks runners (same as ENT).
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
labels: eks-openobserve-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read # report step reads run/job timings from the Actions API
|
||||
actions: read # the no-blobs investigation reads run/job state from the Actions API
|
||||
outputs:
|
||||
# Compact Playwright stats (expected/unexpected/flaky/skipped/duration) from the merged
|
||||
# report.json — consumed by report_to_openobserve to record reliability metrics.
|
||||
# report.json — consumed by the report_openobserve job to record reliability metrics.
|
||||
stats: ${{ steps.report_stats.outputs.stats }}
|
||||
|
||||
steps:
|
||||
@@ -656,8 +658,46 @@ jobs:
|
||||
echo "stats=$STATS" >> "$GITHUB_OUTPUT"
|
||||
echo "report stats: $STATS"
|
||||
|
||||
- name: Upload merged report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: merged-report-${{ github.run_attempt }}
|
||||
path: tests/ui-testing/playwright-results
|
||||
retention-days: 1
|
||||
if-no-files-found: warn
|
||||
|
||||
# TestDino upload — its own job so it never gates playwright_summary. Only runs when tests
|
||||
# failed (and uploads are enabled); consumes the merged report via artifact.
|
||||
upload_testdino:
|
||||
timeout-minutes: 15
|
||||
name: Upload to TestDino
|
||||
needs: [ui_integration_tests, merge_reports]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }}
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read # download-artifact fetches the merged-report artifact
|
||||
steps:
|
||||
- name: Clone the current repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Install project dependencies
|
||||
run: cd tests/ui-testing && npm ci
|
||||
|
||||
- name: Download merged report
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: merged-report-${{ github.run_attempt }}
|
||||
path: tests/ui-testing/playwright-results
|
||||
|
||||
- name: Cache test failures to TestDino
|
||||
if: ${{ needs.ui_integration_tests.result == 'failure' }}
|
||||
env:
|
||||
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
|
||||
run: |
|
||||
@@ -693,7 +733,7 @@ jobs:
|
||||
echo "::endgroup::"
|
||||
|
||||
- name: Upload to TestDino
|
||||
if: ${{ needs.ui_integration_tests.result == 'failure' && env.UPLOAD_TO_TESTDINO == 'true' }}
|
||||
if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }}
|
||||
run: |
|
||||
cd tests/ui-testing
|
||||
|
||||
@@ -736,13 +776,23 @@ jobs:
|
||||
--verbose \
|
||||
--token="${{ secrets.TESTDINO_API_TOKEN }}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Non-fatal metrics sidecar — runs HERE inside the report flow (like the TestDino
|
||||
# step above) so it never trails / gates playwright_summary. One run-summary
|
||||
# document to OpenObserve (stream: ci_regression) + per-shard rows
|
||||
# (ci_regression_shards). continue-on-error + a never-fail poster script mean
|
||||
# this can NEVER break the regression run.
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one
|
||||
# run-summary doc (stream: ci_regression) + per-shard rows (ci_regression_shards). The step
|
||||
# is continue-on-error and the poster script never fails, so this can never break the run.
|
||||
report_openobserve:
|
||||
timeout-minutes: 10
|
||||
name: Report to OpenObserve
|
||||
needs: [build_binary, ui_integration_tests, merge_reports]
|
||||
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read # reads run/job timings from the Actions API
|
||||
steps:
|
||||
- name: Clone the current repo
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Report run metrics to OpenObserve
|
||||
continue-on-error: true
|
||||
env:
|
||||
@@ -751,13 +801,10 @@ jobs:
|
||||
O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
|
||||
O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
|
||||
# We are now inside merge_and_upload_reports (needs: [ui_integration_tests]), which
|
||||
# runs only when tests actually ran. build_binary precedes ui_integration_tests, so
|
||||
# build is effectively success here; merge is in-progress (treated success).
|
||||
BUILD_RESULT: success
|
||||
BUILD_RESULT: ${{ needs.build_binary.result }}
|
||||
UI_RESULT: ${{ needs.ui_integration_tests.result }}
|
||||
MERGE_RESULT: success
|
||||
MERGE_STATS: ${{ steps.report_stats.outputs.stats }}
|
||||
MERGE_RESULT: ${{ needs.merge_reports.result }}
|
||||
MERGE_STATS: ${{ needs.merge_reports.outputs.stats }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# Forks / unconfigured repos: skip entirely (no API call, no rate-limit spend).
|
||||
@@ -896,7 +943,7 @@ jobs:
|
||||
runs-on:
|
||||
labels: ubicloud-standard-4
|
||||
permissions: {}
|
||||
needs: [build_binary, ui_integration_tests, merge_and_upload_reports]
|
||||
needs: [build_binary, ui_integration_tests, merge_reports]
|
||||
if: always()
|
||||
steps:
|
||||
- name: Check test results
|
||||
@@ -913,7 +960,9 @@ jobs:
|
||||
UI_OK=true
|
||||
fi
|
||||
|
||||
if [ "${{ needs.merge_and_upload_reports.result }}" == "success" ] || [ "${{ needs.merge_and_upload_reports.result }}" == "skipped" ] || [ "${{ needs.merge_and_upload_reports.result }}" == "failure" ]; then
|
||||
# A merge/report failure is a non-blocker: the tests already ran and passed, so only
|
||||
# a cancelled merge should fail the summary.
|
||||
if [ "${{ needs.merge_reports.result }}" == "success" ] || [ "${{ needs.merge_reports.result }}" == "skipped" ] || [ "${{ needs.merge_reports.result }}" == "failure" ]; then
|
||||
MERGE_OK=true
|
||||
fi
|
||||
|
||||
@@ -924,6 +973,6 @@ jobs:
|
||||
echo "Regression tests failed:"
|
||||
echo " build_binary: ${{ needs.build_binary.result }}"
|
||||
echo " ui_integration_tests: ${{ needs.ui_integration_tests.result }}"
|
||||
echo " merge_and_upload_reports: ${{ needs.merge_and_upload_reports.result }}"
|
||||
echo " merge_reports: ${{ needs.merge_reports.result }}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user