mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 16:09:05 +00:00
Shrinath RaoandClaude Opus 4.8
df68a26fcf
## Why Our OSS Playwright + API CI compute (ubicloud build + eks matrix) is a major cost driver. Two sources of avoidable spend: 1. **Every merged PR ran the full heavy suite up to 3×** — on the PR (label-gated), in the merge queue (`merge_group`), and *again* on `push: main`. The `push: main` run is a duplicate of the `merge_group` run (both build/test the exact commit landing on main) **and** it bypasses the `e2e` label gate, clawing back the gate's savings. 2. **Regression runs every 4h regardless of whether `main` moved** — rebuilding and re-running the full matrix on identical code overnight/weekends. ## Changes **`playwright.yml` + `api-testing.yml`** - Remove the `push: main` trigger. `merge_group` remains the trusted main-ward validation. - **Cache fix (important):** the Rust cache was only *saved* on `push: main` runs (`save-if: github.ref == 'refs/heads/main'`). Deleting `push` alone would stop the cache ever refreshing → cold builds → *more* compute. So `save-if` now also fires on the `merge_group` run, which builds the exact soon-to-be-main commit. **`playwright_regression.yml`** - New `check_main_changed` gate: compares `main`'s HEAD to the last run's `head_sha` and skips build+matrix when nothing merged since. Manual `workflow_dispatch` always runs. The skip cascades cleanly (`playwright_summary` already treats skipped build/ui as success → stays green). ## Impact - ~1 full build + 146-shard matrix eliminated per merged PR (the `push:main` duplicate). - Idle-period regression runs (no merges) skipped instead of running the full matrix. - No coverage loss: merge queue still gates every merge; cache stays warm. ## Not in this PR (deferred per discussion) - `db-testing.yml` / `unit-tests.yml` `push:main` removal (cheaper, later). - ENT merge-queue adoption. - OSS→ENT dispatch (semgrep / endpoint-auth-probe) gating. Paired ENT change (regression gate): openobserve/o2-enterprise `ci/trim-redundant-runs`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1070 lines
51 KiB
YAML
1070 lines
51 KiB
YAML
name: Playwright Regression Tests
|
|
|
|
on:
|
|
schedule:
|
|
# Daytime IST only (9 AM, 1 PM, 5 PM, 9 PM IST) — the 1 AM and 5 AM IST slots were
|
|
# dropped: main rarely moves overnight, so those runs mostly re-tested unchanged code.
|
|
- cron: '30 3,7,11,15 * * *' # 9 AM, 1 PM, 5 PM, 9 PM IST (3:30, 7:30, 11:30, 15:30 UTC)
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
COLUMNS: 150
|
|
ZO_ROOT_USER_EMAIL: root@example.com
|
|
ZO_ROOT_USER_PASSWORD: Complexpass#123
|
|
ZO_BASE_URL: http://localhost:5080
|
|
WS_ZO_BASE_URL: ws://localhost:5080
|
|
ZO_BASE_URL_SC: http://localhost:5080
|
|
ZO_BASE_URL_SC_UI: http://localhost:5080
|
|
INGESTION_URL: http://localhost:5080
|
|
ORGNAME: default
|
|
ZO_WEB_URL: "http://localhost:5080"
|
|
ZO_QUICK_MODE_NUM_FIELDS: 100
|
|
ZO_QUICK_MODE_STRATEGY: first
|
|
ZO_ALLOW_USER_DEFINED_SCHEMAS: true
|
|
ZO_INGEST_ALLOWED_UPTO: 5
|
|
ZO_FEATURE_QUERY_EXCLUDE_ALL: false
|
|
ZO_USAGE_BATCH_SIZE: 200
|
|
ZO_USAGE_PUBLISH_INTERVAL: 2
|
|
ZO_USAGE_REPORTING_ENABLED: true
|
|
ZO_MIN_AUTO_REFRESH_INTERVAL: 5
|
|
ZO_STREAMING_ENABLED: true
|
|
ZO_COLS_PER_RECORD_LIMIT: "80000"
|
|
ZO_SMTP_ENABLED: true
|
|
ZO_FORMAT_STREAM_NAME_TO_LOWERCASE: false
|
|
ZO_CREATE_ORG_THROUGH_INGESTION: true
|
|
ZO_UTF8_VIEW_ENABLED: false
|
|
ZO_ENABLE_CROSS_LINKING: true
|
|
# Timechart (visualize) tab is hidden by default; enable it so visualize E2E tests can reach it.
|
|
ZO_TIMECHART_ENABLED: true
|
|
ZO_SSRF_ALLOW_LOOPBACK: true
|
|
UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }}
|
|
|
|
jobs:
|
|
# Skip the whole suite when main has not moved since the last regression run.
|
|
# Scheduled runs fire every 4h; if nothing merged in between, rebuilding and
|
|
# re-running the full matrix on identical code is pure waste. Manual dispatch
|
|
# always runs.
|
|
check_main_changed:
|
|
timeout-minutes: 5
|
|
name: check_main_changed
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
actions: read # for `gh run list` (last run's head_sha)
|
|
outputs:
|
|
should_run: ${{ steps.decide.outputs.should_run }}
|
|
steps:
|
|
- id: decide
|
|
env:
|
|
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
run: |
|
|
# Manual dispatch always runs.
|
|
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::Manual dispatch — running regardless of main changes."
|
|
exit 0
|
|
fi
|
|
# On a scheduled run github.sha is main's tip, so no checkout is needed.
|
|
CURRENT_SHA="${{ github.sha }}"
|
|
# head_sha of the most recent completed previous run of THIS workflow on
|
|
# main, excluding the current run. Limit 50 so the current run is always in
|
|
# the window even under bursty history. On any failure LAST_SHA stays empty
|
|
# and we fall through to should_run=true (fail-safe: run rather than skip).
|
|
LAST_SHA=$(gh run list \
|
|
--workflow "${{ github.workflow }}" \
|
|
--branch main \
|
|
--limit 50 \
|
|
--json databaseId,headSha,status \
|
|
--jq "[.[] | select(.databaseId != ${{ github.run_id }} and .status == \"completed\")] | .[0].headSha" \
|
|
2>/dev/null || echo "")
|
|
if [ -z "$LAST_SHA" ]; then
|
|
echo "::warning::Could not determine last run SHA (no prior completed run, or gh query failed) — defaulting to run."
|
|
fi
|
|
echo "Current main SHA: ${CURRENT_SHA:-<none>}"
|
|
echo "Last run SHA: ${LAST_SHA:-<none>}"
|
|
if [ -n "$LAST_SHA" ] && [ "$CURRENT_SHA" = "$LAST_SHA" ]; then
|
|
echo "should_run=false" >> "$GITHUB_OUTPUT"
|
|
echo "::notice::main unchanged since last regression run ($CURRENT_SHA) — skipping."
|
|
else
|
|
echo "should_run=true" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
|
|
build_binary:
|
|
timeout-minutes: 45
|
|
name: build_binary
|
|
needs: [check_main_changed]
|
|
if: ${{ needs.check_main_changed.outputs.should_run == 'true' }}
|
|
runs-on:
|
|
labels: ubicloud-standard-16
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- name: Remove unused tools
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf "/usr/local/share/boost"
|
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@master
|
|
with:
|
|
toolchain: nightly-2026-05-20
|
|
targets: x86_64-unknown-linux-gnu
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
cache-on-failure: true
|
|
shared-key: playwright-release-ci
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Install Protoc
|
|
run: |
|
|
bash ./.github/protoc.sh
|
|
|
|
- uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Lint frontend code
|
|
run: cd web && npm ci && npm run lint:ci
|
|
|
|
- name: Build binary and frontend
|
|
env:
|
|
NODE_OPTIONS: "--max-old-space-size=8192"
|
|
GIT_HASH_OVERRIDE: ${{ github.sha }}
|
|
run: |
|
|
bash -c "cd web && npm run build && cp -r ./dist /tmp/dist" &
|
|
sed -i 's|folder = "web/dist/"|folder = "/tmp/dist/"|' src/handler/http/router/ui.rs
|
|
cargo build --profile release-ci --target x86_64-unknown-linux-gnu
|
|
wait
|
|
|
|
- name: Upload binary artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: release-ci-binary
|
|
path: target/x86_64-unknown-linux-gnu/release-ci/openobserve
|
|
retention-days: 1
|
|
|
|
- name: Upload frontend artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: frontend-dist
|
|
path: /tmp/dist
|
|
retention-days: 1
|
|
if-no-files-found: error
|
|
|
|
ui_integration_tests:
|
|
timeout-minutes: 45
|
|
name: e2e / ${{ matrix.testfolder }}
|
|
needs: [build_binary]
|
|
runs-on:
|
|
labels: eks-openobserve-standard-8
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- testfolder: "Logs-Regression"
|
|
run_files:
|
|
[
|
|
"logs-regression.spec.js",
|
|
"logs-bugs.spec.js",
|
|
"logs-9754.spec.js",
|
|
"logs-9044-7354.spec.js",
|
|
]
|
|
- testfolder: "Alerts-Regression"
|
|
run_files:
|
|
[
|
|
"alerts-regression.spec.js",
|
|
"alerts-stream-switching.spec.js",
|
|
"alerts-bugs.spec.js",
|
|
]
|
|
- testfolder: "Dashboard-Regression"
|
|
run_files:
|
|
[
|
|
"dashboard-regression.spec.js",
|
|
]
|
|
- testfolder: "Streams-Regression"
|
|
run_files:
|
|
[
|
|
"streams-regression.spec.js",
|
|
]
|
|
- testfolder: "Pipelines-Regression"
|
|
run_files:
|
|
[
|
|
"enrichment-regression.spec.js",
|
|
"pipeline-regression.spec.js",
|
|
]
|
|
- testfolder: "Traces-Regression"
|
|
run_files:
|
|
[
|
|
"traces-regression.spec.js",
|
|
"traces-bugs.spec.js",
|
|
]
|
|
- testfolder: "Metrics-Regression"
|
|
run_files:
|
|
[
|
|
"metrics-regression.spec.js",
|
|
]
|
|
- testfolder: "GeneralTests-Regression"
|
|
run_files:
|
|
[
|
|
"ui-regression.spec.js",
|
|
"landing-regression.spec.js",
|
|
]
|
|
- testfolder: "DataSources-Regression"
|
|
run_files:
|
|
[
|
|
"datasources-regression.spec.js",
|
|
]
|
|
- testfolder: "Reports-Regression"
|
|
run_files:
|
|
[
|
|
"reports-regression-bugs.spec.js",
|
|
]
|
|
|
|
steps:
|
|
- name: Kill background apt processes
|
|
run: |
|
|
sudo systemctl stop apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
|
|
sudo systemctl stop apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
|
|
sudo systemctl disable apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
|
|
sudo flock --wait 30 /var/lib/dpkg/lock-frontend true 2>/dev/null || true
|
|
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Download binary artifact
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: release-ci-binary
|
|
path: release-ci-binary
|
|
|
|
- name: Download frontend artifact
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: frontend-dist
|
|
path: /tmp/dist
|
|
|
|
- name: Start OpenObserve
|
|
run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 &
|
|
|
|
- name: Wait for start
|
|
run: |
|
|
for i in $(seq 1 120); do
|
|
if curl -sf http://localhost:5080/web/login > /dev/null 2>&1; then
|
|
echo "Server ready after ${i}s"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "Server failed to start in 120s"
|
|
exit 1
|
|
|
|
- name: Ensure we are getting a reply from the server
|
|
run: curl http://localhost:5080/web/login
|
|
|
|
- uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Cache Playwright browser
|
|
id: playwright-cache
|
|
uses: actions/cache@v6
|
|
with:
|
|
path: ~/.cache/ms-playwright
|
|
key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }}
|
|
|
|
- name: Install npm dependencies
|
|
run: cd tests/ui-testing && npm ci
|
|
|
|
- name: Install Playwright browser
|
|
run: |
|
|
cd tests/ui-testing
|
|
if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then
|
|
echo "Browser cache hit — installing system deps only"
|
|
npx playwright install-deps chromium
|
|
else
|
|
echo "Browser cache miss — installing browser + system deps"
|
|
npx playwright install --with-deps chromium
|
|
fi
|
|
|
|
- name: Check if this is a rerun
|
|
id: check_rerun
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
echo "::notice::GitHub run attempt: ${{ github.run_attempt }}"
|
|
if [ "${{ github.run_attempt }}" -gt "1" ]; then
|
|
echo "is_rerun=true" >> $GITHUB_OUTPUT
|
|
echo "::notice::This is a rerun (attempt ${{ github.run_attempt }}). Will attempt to run only failed tests."
|
|
echo "::group::Detecting rerun type (all jobs vs failed only)"
|
|
CURRENT_ATTEMPT=${{ github.run_attempt }}
|
|
ALL_JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=all&per_page=100")
|
|
CURRENT_ATTEMPT_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == $CURRENT_ATTEMPT) | select(.status != \"completed\")] | length")
|
|
TOTAL_MATRIX_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == 1) | .name] | unique | length")
|
|
echo "::notice::Total unique matrix job names: $TOTAL_MATRIX_JOBS"
|
|
echo "::notice::Jobs running in current attempt $CURRENT_ATTEMPT: $CURRENT_ATTEMPT_JOBS"
|
|
if [ "$CURRENT_ATTEMPT_JOBS" -eq "$TOTAL_MATRIX_JOBS" ]; then
|
|
echo "::notice::✅ DETECTED: Re-run ALL jobs (all $TOTAL_MATRIX_JOBS matrix jobs running)"
|
|
echo "rerun_type=all" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "::notice::✅ DETECTED: Re-run FAILED jobs only ($CURRENT_ATTEMPT_JOBS out of $TOTAL_MATRIX_JOBS jobs)"
|
|
echo "rerun_type=failed" >> $GITHUB_OUTPUT
|
|
fi
|
|
echo "::endgroup::"
|
|
else
|
|
echo "is_rerun=false" >> $GITHUB_OUTPUT
|
|
echo "rerun_type=none" >> $GITHUB_OUTPUT
|
|
echo "::notice::This is the first run. Will run all tests in matrix."
|
|
fi
|
|
|
|
- name: Write .env and run ui-tests
|
|
env:
|
|
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
touch .env
|
|
echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env
|
|
echo "ZO_ROOT_USER_PASSWORD=${ZO_ROOT_USER_PASSWORD}" >> .env
|
|
echo "ZO_BASE_URL=${ZO_BASE_URL}" >> .env
|
|
echo "WS_ZO_BASE_URL=${WS_ZO_BASE_URL}" >> .env
|
|
echo "ZO_BASE_URL_SC=${ZO_BASE_URL_SC}" >> .env
|
|
echo "ZO_BASE_URL_SC_UI=${ZO_BASE_URL_SC_UI}" >> .env
|
|
echo "INGESTION_URL=${INGESTION_URL}" >> .env
|
|
echo "ORGNAME=${ORGNAME}" >> .env
|
|
echo "ZO_SMTP_ENABLED=${ZO_SMTP_ENABLED}" >> .env
|
|
mv .env tests/ui-testing
|
|
cd tests/ui-testing
|
|
|
|
# Check if this is a rerun
|
|
IS_RERUN="${{ steps.check_rerun.outputs.is_rerun }}"
|
|
RERUN_TYPE="${{ steps.check_rerun.outputs.rerun_type }}"
|
|
|
|
if [ "$IS_RERUN" = "true" ]; then
|
|
if [ "$RERUN_TYPE" = "all" ]; then
|
|
echo "::notice::🔄 Re-run ALL jobs detected - skipping TestDino optimization"
|
|
echo "::notice::Will run ALL tests in this shard regardless of previous failures"
|
|
RERUN_MODE="normal"
|
|
EXTRA_PW_FLAGS=""
|
|
else
|
|
echo "::group::Fetching last failed tests from TestDino"
|
|
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
|
|
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
|
|
echo "::notice::Using cache ID: $CACHE_ID"
|
|
TEMP_OUTPUT="/tmp/testdino_output_$$.txt"
|
|
if npx tdpw last-failed --cache-id "$CACHE_ID" > "$TEMP_OUTPUT" 2>&1; then
|
|
LAST_FAILED_FLAGS="$(grep -E '^[A-Za-z_-]+/[a-zA-Z0-9_-]+\.spec\.js' "$TEMP_OUTPUT" | tail -1)"
|
|
rm -f "$TEMP_OUTPUT"
|
|
if [ -n "$LAST_FAILED_FLAGS" ] && [ "$LAST_FAILED_FLAGS" != "No failed tests found" ]; then
|
|
echo "::notice::✅ Found failed tests from previous run."
|
|
echo "::notice::TestDino flags: $LAST_FAILED_FLAGS"
|
|
RERUN_MODE="optimized"
|
|
EXTRA_PW_FLAGS="$LAST_FAILED_FLAGS"
|
|
else
|
|
echo "::warning::TestDino returned no failed tests. Falling back to running all tests in this matrix job."
|
|
echo "::warning::This means you'll be billed for all tests. Please investigate why last-failed returned empty."
|
|
RERUN_MODE="fallback"
|
|
EXTRA_PW_FLAGS=""
|
|
fi
|
|
else
|
|
echo "::error::Failed to fetch last failed tests from TestDino (exit code: $?)."
|
|
echo "::warning::Falling back to running all tests in this matrix job."
|
|
echo "::warning::This means you'll be billed for all tests. Please check TestDino integration."
|
|
RERUN_MODE="fallback"
|
|
EXTRA_PW_FLAGS=""
|
|
fi
|
|
echo "::endgroup::"
|
|
fi
|
|
else
|
|
RERUN_MODE="normal"
|
|
EXTRA_PW_FLAGS=""
|
|
fi
|
|
|
|
FILE_LIST="${{ join(matrix.run_files, ' ') }}"
|
|
echo "DEBUG: FILE_LIST = $FILE_LIST"
|
|
echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}"
|
|
|
|
if [ -n "$FILE_LIST" ]; then
|
|
# Derive feature subfolder from testfolder (maps testfolder name → directory under RegressionSet/)
|
|
case "${{ matrix.testfolder }}" in
|
|
"Dashboard-Regression") FEATURE="Dashboards" ;;
|
|
"Pipelines-Regression") FEATURE="Pipelines" ;;
|
|
"Alerts-Regression") FEATURE="Alerts" ;;
|
|
"Logs-Regression") FEATURE="Logs" ;;
|
|
"Traces-Regression") FEATURE="Traces" ;;
|
|
"Metrics-Regression") FEATURE="Metrics" ;;
|
|
"Streams-Regression") FEATURE="Streams" ;;
|
|
"Reports-Regression") FEATURE="Reports" ;;
|
|
"DataSources-Regression") FEATURE="DataSources" ;;
|
|
"GeneralTests-Regression") FEATURE="General" ;;
|
|
*) FEATURE="RegressionSet" ;;
|
|
esac
|
|
ACTUAL_FOLDER="RegressionSet/$FEATURE"
|
|
echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER"
|
|
echo "DEBUG: FEATURE = $FEATURE"
|
|
|
|
FILE_PATHS=""
|
|
for file in $FILE_LIST; do
|
|
FILE_PATH="./playwright-tests/$ACTUAL_FOLDER/$file"
|
|
echo "DEBUG: Will run file: $FILE_PATH"
|
|
FILE_PATHS="$FILE_PATHS $FILE_PATH"
|
|
done
|
|
|
|
if [ "$RERUN_MODE" = "optimized" ]; then
|
|
echo "DEBUG: Full TestDino output (all failed tests): $EXTRA_PW_FLAGS"
|
|
echo "DEBUG: Current shard folder: $ACTUAL_FOLDER"
|
|
|
|
SHARD_TESTS=""
|
|
|
|
if echo "$EXTRA_PW_FLAGS" | grep -iq "$ACTUAL_FOLDER/.*\.spec\.js"; then
|
|
echo "DEBUG: Found test(s) for this shard in TestDino output"
|
|
|
|
TEST_FILE=$(echo "$EXTRA_PW_FLAGS" | grep -ioE "$ACTUAL_FOLDER/[^ ]+\.spec\.js" | head -1)
|
|
|
|
if [ -n "$TEST_FILE" ]; then
|
|
if [[ "${TEST_FILE,,}" == "${ACTUAL_FOLDER,,}/"* ]]; then
|
|
TEST_FILENAME=$(basename "$TEST_FILE")
|
|
|
|
if echo "$FILE_LIST" | grep -qw "$TEST_FILENAME"; then
|
|
TEST_FILE_PATH=$(echo "$EXTRA_PW_FLAGS" | sed "s/ -g.*//" | sed "s|^|./playwright-tests/|")
|
|
|
|
if echo "$EXTRA_PW_FLAGS" | grep -q " -g "; then
|
|
TEST_NAME=$(echo "$EXTRA_PW_FLAGS" | sed 's/.*-g //' | sed 's/^"\(.*\)"$/\1/')
|
|
echo "DEBUG: File path: $TEST_FILE_PATH"
|
|
echo "DEBUG: Test name (unquoted): $TEST_NAME"
|
|
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test from this shard"
|
|
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\" -g \"$TEST_NAME\""
|
|
npx playwright test "$TEST_FILE_PATH" -g "$TEST_NAME"
|
|
else
|
|
echo "DEBUG: File path: $TEST_FILE_PATH (no -g flag)"
|
|
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test file from this shard"
|
|
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\""
|
|
npx playwright test "$TEST_FILE_PATH"
|
|
fi
|
|
else
|
|
echo "::notice::⏭️ Test file '$TEST_FILENAME' not in this shard's run_files list."
|
|
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
|
|
|
|
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
|
|
|
|
if [ "$PREV_ATTEMPT" -ge 1 ]; then
|
|
JOB_NAME="e2e / ${{ matrix.testfolder }}"
|
|
|
|
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
|
|
|
|
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
|
|
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
|
|
|
|
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
|
|
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended - running FULL test suite"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
elif [ "$PREV_JOB_STATUS" = "success" ]; then
|
|
echo "::notice::✅ Previous shard completed successfully - no tests to rerun"
|
|
exit 0
|
|
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
|
|
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded - running FULL test suite"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
else
|
|
echo "::warning::⚠️ Unknown job status - running FULL test suite for safety"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
fi
|
|
else
|
|
echo "::error::Failed to fetch job status from GitHub API - running FULL test suite"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
fi
|
|
else
|
|
echo "::notice::✅ This is the first run, no previous attempt to check"
|
|
echo "::notice::No failed tests to rerun. Exiting successfully."
|
|
exit 0
|
|
fi
|
|
fi
|
|
else
|
|
echo "::notice::⏭️ Test file doesn't match shard folder exactly. Skipping."
|
|
exit 0
|
|
fi
|
|
else
|
|
echo "::warning::Failed to extract test file path. Falling back to skip."
|
|
exit 0
|
|
fi
|
|
else
|
|
echo "::notice::⏭️ OPTIMIZED RERUN: No failed tests found for '$ACTUAL_FOLDER' shard in TestDino output"
|
|
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
|
|
|
|
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
|
|
|
|
if [ "$PREV_ATTEMPT" -ge 1 ]; then
|
|
JOB_NAME="e2e / ${{ matrix.testfolder }}"
|
|
|
|
echo "DEBUG: Checking previous attempt #$PREV_ATTEMPT for job: $JOB_NAME"
|
|
|
|
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
|
|
|
|
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
|
|
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
|
|
|
|
echo "DEBUG: Previous job status: $PREV_JOB_STATUS"
|
|
|
|
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
|
|
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended (status: ${PREV_JOB_STATUS:-not found})"
|
|
echo "::warning::Treating as FAILED - running FULL test suite for this shard"
|
|
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
elif [ "$PREV_JOB_STATUS" = "success" ]; then
|
|
echo "::notice::✅ Previous shard completed successfully - all tests passed"
|
|
echo "::notice::No failed tests to rerun. Exiting successfully."
|
|
exit 0
|
|
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
|
|
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded"
|
|
echo "::warning::This shouldn't happen - running FULL test suite for safety"
|
|
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
else
|
|
echo "::warning::⚠️ Unknown job status: $PREV_JOB_STATUS"
|
|
echo "::warning::Running FULL test suite for safety"
|
|
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
fi
|
|
else
|
|
echo "::error::Failed to fetch job status from GitHub API"
|
|
echo "::warning::Falling back to running FULL test suite for safety"
|
|
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
exit $?
|
|
fi
|
|
else
|
|
echo "::notice::✅ This is the first run, no previous attempt to check"
|
|
echo "::notice::No failed tests to rerun. Exiting successfully."
|
|
exit 0
|
|
fi
|
|
fi
|
|
else
|
|
# Normal run or fallback
|
|
if [ "$RERUN_MODE" = "fallback" ]; then
|
|
echo "::warning::⚠️ FALLBACK RERUN: Running all tests in matrix (TestDino optimization failed)"
|
|
fi
|
|
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
fi
|
|
else
|
|
echo "No files specified to run for ${{ matrix.testfolder }} folder"
|
|
fi
|
|
|
|
- name: Upload blob report to GitHub Actions Artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: blob-report-${{ matrix.testfolder }}-${{ github.run_attempt }}
|
|
path: tests/ui-testing/blob-report
|
|
retention-days: 1
|
|
|
|
- name: Check OpenObserve logs
|
|
if: always()
|
|
run: cat o2.log
|
|
|
|
merge_reports:
|
|
timeout-minutes: 15
|
|
name: Merge Reports
|
|
needs: [ui_integration_tests]
|
|
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
|
# eks runner: `playwright merge-reports` silently stops mid-merge (exits 0, ~3 of N blobs, no
|
|
# output) on the ubicloud and github-hosted runners — but works on the eks runners (same as ENT).
|
|
runs-on:
|
|
labels: eks-openobserve-standard-4
|
|
permissions:
|
|
contents: read
|
|
actions: read # the no-blobs investigation reads run/job state from the Actions API
|
|
outputs:
|
|
# Compact Playwright stats (expected/unexpected/flaky/skipped/duration) from the merged
|
|
# report.json — consumed by the report_openobserve job to record reliability metrics.
|
|
stats: ${{ steps.report_stats.outputs.stats }}
|
|
# One JSON array of flaky+failed tests (per-test rows) for the ci_test_results stream,
|
|
# so the unified UI+Regression dashboard's Latest-failures / Flakiest panels cover regression.
|
|
flaky: ${{ steps.report_flaky.outputs.flaky }}
|
|
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Install project dependencies
|
|
run: cd tests/ui-testing && npm ci
|
|
|
|
- name: Download blob reports from GitHub Actions Artifacts
|
|
uses: actions/download-artifact@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: tests/ui-testing/all-blob-reports
|
|
pattern: blob-report-*-${{ github.run_attempt }}
|
|
merge-multiple: true
|
|
|
|
- name: Merge Reports
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
|
|
if [ ! -d "all-blob-reports" ] || [ -z "$(ls -A all-blob-reports 2>/dev/null)" ]; then
|
|
echo "::warning::No blob reports found to merge."
|
|
|
|
echo "::group::Investigating why no blob reports exist"
|
|
|
|
JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs")
|
|
|
|
SUCCEEDED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "success")] | length')
|
|
FAILED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "failure")] | length')
|
|
TOTAL=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /"))] | length')
|
|
|
|
echo "::notice::Test job results: $SUCCEEDED succeeded, $FAILED failed, $TOTAL total"
|
|
|
|
if [ "$SUCCEEDED" -gt 0 ] && [ "$FAILED" -eq 0 ]; then
|
|
echo "::notice::✅ All test jobs succeeded - they likely skipped because tests already passed"
|
|
echo "::notice::Creating empty report structure for downstream jobs"
|
|
mkdir -p playwright-results/html-report
|
|
START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
|
|
echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json
|
|
echo "<html><body><h1>No tests executed - all shards skipped</h1><p>All test shards exited early because tests had already passed in a previous attempt.</p></body></html>" > playwright-results/html-report/index.html
|
|
echo "::endgroup::"
|
|
exit 0
|
|
else
|
|
echo "::error::❌ Some/all test jobs failed but no blob reports were found"
|
|
echo "::error::This indicates a problem with test execution or blob reporter"
|
|
echo "::error::Failed jobs: $FAILED, Succeeded jobs: $SUCCEEDED"
|
|
echo "::endgroup::"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
# Force the html + json reporters explicitly with their output paths instead of
|
|
# relying on `unset CI` to flip playwright.config.js's `process.env.CI ? blob : html/json`
|
|
# ternary. The runner exports CI=true and `unset CI` does not reliably reach the
|
|
# merge subprocess, so the config would intermittently re-select the `blob` reporter —
|
|
# which writes a merged blob-report/ and NOTHING to playwright-results/, making the
|
|
# downstream `ls playwright-results/html-report/` fail with a cryptic exit 2.
|
|
mkdir -p playwright-results
|
|
# These env vars override playwright.config.js's reporter settings:
|
|
# PLAYWRIGHT_HTML_OUTPUT_DIR - html reporter output directory
|
|
# PLAYWRIGHT_HTML_OPEN=never - never auto-open the report (required for CI)
|
|
# PLAYWRIGHT_JSON_OUTPUT_NAME - json reporter output file path
|
|
PLAYWRIGHT_HTML_OUTPUT_DIR=playwright-results/html-report \
|
|
PLAYWRIGHT_HTML_OPEN=never \
|
|
PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-results/report.json \
|
|
npx playwright merge-reports --reporter html,json ./all-blob-reports
|
|
|
|
echo "Contents of playwright-results:"
|
|
ls -la playwright-results/ || true
|
|
if [ ! -f playwright-results/html-report/index.html ] || [ ! -f playwright-results/report.json ]; then
|
|
echo "::error::merge-reports did not produce the expected html-report/index.html and report.json"
|
|
echo "::error::all-blob-reports contents:"
|
|
ls -la all-blob-reports/ || true
|
|
exit 1
|
|
fi
|
|
# Diagnostics only — never fail the step on these (the guard above already
|
|
# confirmed the outputs exist).
|
|
echo "Contents of html-report:"
|
|
ls -la playwright-results/html-report/ || true
|
|
echo "Checking report.json:"
|
|
ls -lh playwright-results/report.json || true
|
|
|
|
- name: Extract report stats for OpenObserve
|
|
id: report_stats
|
|
if: always()
|
|
run: |
|
|
cd tests/ui-testing
|
|
if [ -f playwright-results/report.json ]; then
|
|
STATS=$(jq -c '.stats // {}' playwright-results/report.json 2>/dev/null || echo '{}')
|
|
else
|
|
STATS='{}'
|
|
fi
|
|
echo "stats=$STATS" >> "$GITHUB_OUTPUT"
|
|
echo "report stats: $STATS"
|
|
|
|
- name: Extract flaky/failed tests for OpenObserve
|
|
id: report_flaky
|
|
if: always()
|
|
run: |
|
|
# One JSON array of flaky+failed tests for the ci_test_results stream (per-test rows).
|
|
# Logic + schema live in extract-flaky.js; prints "[]" if the report is missing/clean.
|
|
FLAKY=$(node .github/scripts/extract-flaky.js tests/ui-testing/playwright-results/report.json)
|
|
printf 'flaky=%s\n' "$FLAKY" >> "$GITHUB_OUTPUT"
|
|
echo "report flaky count: $(printf '%s' "$FLAKY" | jq 'length' 2>/dev/null || echo '?')"
|
|
|
|
- name: Upload merged report
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: merged-report-${{ github.run_attempt }}
|
|
path: tests/ui-testing/playwright-results
|
|
retention-days: 1
|
|
if-no-files-found: warn
|
|
|
|
# TestDino upload — its own job so it never gates playwright_summary. Only runs when tests
|
|
# failed (and uploads are enabled); consumes the merged report via artifact.
|
|
upload_testdino:
|
|
timeout-minutes: 15
|
|
name: Upload to TestDino
|
|
needs: [ui_integration_tests, merge_reports]
|
|
if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }}
|
|
runs-on:
|
|
labels: eks-openobserve-standard-4
|
|
permissions:
|
|
contents: read
|
|
actions: read # download-artifact fetches the merged-report artifact
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 24
|
|
|
|
- name: Install project dependencies
|
|
run: cd tests/ui-testing && npm ci
|
|
|
|
- name: Download merged report
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
name: merged-report-${{ github.run_attempt }}
|
|
path: tests/ui-testing/playwright-results
|
|
|
|
- name: Cache test failures to TestDino
|
|
env:
|
|
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
echo "::group::Caching test failure metadata to TestDino"
|
|
if [ -f "playwright-results/report.json" ]; then
|
|
echo "::notice::JSON report found at playwright-results/report.json"
|
|
ls -lh playwright-results/report.json
|
|
|
|
mkdir -p ../../playwright-results
|
|
cp playwright-results/report.json ../../playwright-results/
|
|
|
|
cd ../../playwright-results
|
|
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
|
|
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
|
|
echo "::notice::Using cache ID: $CACHE_ID"
|
|
|
|
if CACHE_OUTPUT=$(npx tdpw cache --cache-id "$CACHE_ID" 2>&1); then
|
|
SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "")
|
|
if [ -n "$SUCCESS_MSG" ]; then
|
|
echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud"
|
|
else
|
|
echo "::notice::✅ Cache command completed"
|
|
fi
|
|
else
|
|
echo "::warning::⚠️ Failed to cache test metadata to TestDino"
|
|
echo "::warning::Rerun optimization may not work properly"
|
|
echo "$CACHE_OUTPUT"
|
|
fi
|
|
else
|
|
echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)"
|
|
fi
|
|
echo "::endgroup::"
|
|
|
|
- name: Upload to TestDino
|
|
if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
|
|
if [ ! -f "playwright-results/report.json" ]; then
|
|
echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)"
|
|
exit 0
|
|
fi
|
|
|
|
START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json)
|
|
if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then
|
|
echo "Converting startTime from Unix timestamp to ISO 8601 string"
|
|
jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp
|
|
mv playwright-results/report.json.tmp playwright-results/report.json
|
|
fi
|
|
|
|
# Filter report to only include failed tests for TestDino upload
|
|
node -e "
|
|
const fs = require('fs');
|
|
const report = JSON.parse(fs.readFileSync('playwright-results/report.json', 'utf8'));
|
|
function filterSuite(s) {
|
|
const specs = (s.specs||[]).filter(sp=>sp.tests&&sp.tests.some(t=>t.status==='unexpected'));
|
|
const suites = (s.suites||[]).map(filterSuite).filter(sub=>(sub.specs&&sub.specs.length)||(sub.suites&&sub.suites.length));
|
|
return {...s,specs,suites};
|
|
}
|
|
const filtered = {
|
|
...report,
|
|
suites: report.suites.map(filterSuite).filter(s=>(s.specs&&s.specs.length)||(s.suites&&s.suites.length)),
|
|
stats: {...report.stats,expected:0,flaky:0,skipped:0}
|
|
};
|
|
fs.writeFileSync('playwright-results/report-failures-only.json',JSON.stringify(filtered));
|
|
const total=(report.stats.expected||0)+(report.stats.unexpected||0)+(report.stats.skipped||0)+(report.stats.flaky||0);
|
|
console.log('Uploading '+(report.stats.unexpected||0)+' failed tests out of '+total+' total to TestDino');
|
|
"
|
|
echo "::notice::Report filtered to failures only"
|
|
|
|
npx --yes tdpw@latest upload playwright-results \
|
|
--json-report playwright-results/report-failures-only.json \
|
|
--html-report playwright-results/html-report \
|
|
--upload-html \
|
|
--verbose \
|
|
--token="${{ secrets.TESTDINO_API_TOKEN }}"
|
|
|
|
# OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one
|
|
# run-summary doc (stream: ci_regression) + per-shard rows (ci_regression_shards). The step
|
|
# is continue-on-error and the poster script never fails, so this can never break the run.
|
|
report_openobserve:
|
|
timeout-minutes: 10
|
|
name: Report to OpenObserve
|
|
needs: [build_binary, ui_integration_tests, merge_reports]
|
|
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
|
runs-on:
|
|
labels: eks-openobserve-standard-4
|
|
permissions:
|
|
contents: read
|
|
actions: read # reads run/job timings from the Actions API
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Report run metrics to OpenObserve
|
|
continue-on-error: true
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }}
|
|
O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
|
|
O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}
|
|
BUILD_RESULT: ${{ needs.build_binary.result }}
|
|
UI_RESULT: ${{ needs.ui_integration_tests.result }}
|
|
MERGE_RESULT: ${{ needs.merge_reports.result }}
|
|
MERGE_STATS: ${{ needs.merge_reports.outputs.stats }}
|
|
MERGE_FLAKY: ${{ needs.merge_reports.outputs.flaky }}
|
|
# Branch + PR for per-test rows (ci_test_results), matching the UI workflow.
|
|
REPORT_BRANCH: ${{ github.head_ref || github.ref_name }}
|
|
REPORT_PR: ${{ github.event.pull_request.number }}
|
|
run: |
|
|
set -uo pipefail
|
|
# Forks / unconfigured repos: skip entirely (no API call, no rate-limit spend).
|
|
if [ -z "${O2_REPORTING_INGEST_BASE:-}" ] || [ -z "${O2_REPORTING_AUTH:-}" ]; then
|
|
echo "::notice::O2_REPORTING_* secrets not set — skipping metrics ingest"; exit 0
|
|
fi
|
|
# Per-job metadata (durations + conclusions) for THIS run attempt.
|
|
curl -s -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs?per_page=100" \
|
|
> jobs.json || echo '{"jobs":[]}' > jobs.json
|
|
|
|
# This attempt's start time — the floor for awall (below). On a RE-RUN, GitHub keeps
|
|
# the jobs that did NOT re-run at their ORIGINAL (earlier-attempt) timestamps, so a
|
|
# naive min(started) spans the idle gap to a much-later manual re-run and inflates the
|
|
# duration by hours. Flooring the start at run_started_at counts only this attempt's work.
|
|
RUN_STARTED=$(curl -s -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" \
|
|
| jq 'if .run_started_at then (.run_started_at|fromdateiso8601) else 0 end' 2>/dev/null || echo 0)
|
|
[ -n "$RUN_STARTED" ] || RUN_STARTED=0
|
|
|
|
STATS="${MERGE_STATS:-}"; [ -z "$STATS" ] && STATS=null
|
|
|
|
# Retry-aware: this attempt's wall-clock + total summed across all attempts (so the
|
|
# latest-attempt doc carries the cumulative compute; dashboards keep latest per run_id).
|
|
ATT="${GITHUB_RUN_ATTEMPT:-1}"
|
|
# $1 = floor epoch (run_started_at); start = max(min job started, floor) so re-run
|
|
# carry-over jobs with old timestamps don't inflate the wall-clock.
|
|
awall(){ jq --argjson floor "${1:-0}" '[.jobs[]|select(.completed_at!=null and .started_at!=null)] as $j | (if ($j|length)>0 then (($j|map(.completed_at|fromdateiso8601)|max)-([($j|map(.started_at|fromdateiso8601)|min), $floor]|max)) else 0 end) | if . < 0 then 0 else . end'; }
|
|
FINAL_DUR=$(awall "$RUN_STARTED" < jobs.json 2>/dev/null || echo 0)
|
|
TOTAL_DUR="$FINAL_DUR"
|
|
if [ "$ATT" -gt 1 ] 2>/dev/null; then
|
|
SUM="$FINAL_DUR"
|
|
for n in $(seq 1 $((ATT-1))); do
|
|
NFLOOR=$(curl -s -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${n}" \
|
|
| jq 'if .run_started_at then (.run_started_at|fromdateiso8601) else 0 end' 2>/dev/null || echo 0)
|
|
W=$(curl -s -H "Authorization: token $GH_TOKEN" -H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${n}/jobs?per_page=100" | awall "${NFLOOR:-0}" 2>/dev/null || echo 0)
|
|
SUM=$(awk -v a="$SUM" -v b="$W" 'BEGIN{print a+b}')
|
|
done
|
|
TOTAL_DUR="$SUM"
|
|
fi
|
|
COMMIT="${GITHUB_SHA:0:12}"
|
|
|
|
dur='if (.completed_at and .started_at) then ((.completed_at|fromdateiso8601)-(.started_at|fromdateiso8601)) else null end'
|
|
jq -n \
|
|
--slurpfile jb jobs.json \
|
|
--arg workflow "regression" \
|
|
--arg repo "$GITHUB_REPOSITORY" \
|
|
--arg run_id "$GITHUB_RUN_ID" \
|
|
--arg run_attempt "$GITHUB_RUN_ATTEMPT" \
|
|
--arg run_url "$RUN_URL" \
|
|
--arg trigger "$GITHUB_EVENT_NAME" \
|
|
--arg actor "$GITHUB_ACTOR" \
|
|
--arg branch "$GITHUB_REF_NAME" \
|
|
--arg build_result "$BUILD_RESULT" \
|
|
--arg ui_result "$UI_RESULT" \
|
|
--arg merge_result "$MERGE_RESULT" \
|
|
--argjson stats "$STATS" \
|
|
--argjson final "${FINAL_DUR:-0}" \
|
|
--argjson total "${TOTAL_DUR:-0}" \
|
|
--arg commit "$COMMIT" \
|
|
"
|
|
(\$jb[0].jobs // []) as \$j
|
|
| [ \$j[] | select(.conclusion != null) ] as \$done
|
|
| {
|
|
workflow: \$workflow,
|
|
repo: \$repo,
|
|
run_id: \$run_id,
|
|
run_attempt: (\$run_attempt|tonumber),
|
|
run_url: \$run_url,
|
|
trigger: \$trigger,
|
|
actor: \$actor,
|
|
author: \$actor,
|
|
branch: \$branch,
|
|
suite: \"regression\",
|
|
ingest_source: \"live\",
|
|
commit_sha: \$commit,
|
|
retries: ((\$run_attempt|tonumber)-1),
|
|
was_retried: ((\$run_attempt|tonumber)>1),
|
|
final_duration_sec: \$final,
|
|
total_duration_sec: \$total,
|
|
retry_wasted_sec: (\$total-\$final),
|
|
conclusion: (
|
|
if (\$build_result==\"cancelled\" or \$ui_result==\"cancelled\" or \$merge_result==\"cancelled\") then \"cancelled\"
|
|
elif (\$build_result==\"success\" and \$ui_result==\"success\") then \"success\"
|
|
elif (\$ui_result==\"skipped\" and \$build_result!=\"failure\") then \"skipped\"
|
|
else \"failure\" end
|
|
),
|
|
build_result: \$build_result,
|
|
ui_result: \$ui_result,
|
|
merge_result: \$merge_result,
|
|
started_at: (\$done | map(.started_at) | min),
|
|
finished_at: (\$done | map(.completed_at) | max),
|
|
duration_sec: ((\$done | map(.completed_at|fromdateiso8601) | max) - (\$done | map(.started_at|fromdateiso8601) | min)),
|
|
runner_seconds: ([ \$done[] | ($dur) ] | add),
|
|
build_duration_sec: ( [ \$j[] | select(.name==\"build_binary\") ] | .[0] | if . then ($dur) else null end ),
|
|
shards_total: ([ \$j[] | select(.name|startswith(\"e2e /\")) ] | length),
|
|
shards_passed: ([ \$j[] | select((.name|startswith(\"e2e /\")) and .conclusion==\"success\") ] | length),
|
|
shards_failed: ([ \$j[] | select((.name|startswith(\"e2e /\")) and .conclusion==\"failure\") ] | length),
|
|
shards_skipped: ([ \$j[] | select((.name|startswith(\"e2e /\")) and (.conclusion==\"skipped\" or .conclusion==\"cancelled\")) ] | length),
|
|
shards: [ \$j[] | select(.name|startswith(\"e2e /\")) | {name: .name, conclusion: .conclusion, duration_sec: ($dur)} ],
|
|
tests_total: (if \$stats then ((\$stats.expected//0)+(\$stats.unexpected//0)+(\$stats.flaky//0)+(\$stats.skipped//0)) else null end),
|
|
tests_passed: (if \$stats then (\$stats.expected//0) else null end),
|
|
tests_failed: (if \$stats then (\$stats.unexpected//0) else null end),
|
|
tests_flaky: (if \$stats then (\$stats.flaky//0) else null end),
|
|
tests_skipped: (if \$stats then (\$stats.skipped//0) else null end)
|
|
}
|
|
" > payload.json || { echo "::warning::payload build failed"; echo '{}' > payload.json; }
|
|
|
|
bash .github/scripts/o2-report.sh ci_regression payload.json
|
|
# Also write the same run-level doc to ci_test_runs (already tagged suite=regression) so
|
|
# the unified UI+Regression dashboard drives one panel set off a `suite` variable. Additive:
|
|
# ci_regression keeps receiving the doc, so existing dashboards/crons are untouched.
|
|
bash .github/scripts/o2-report.sh ci_test_runs payload.json
|
|
|
|
# Also emit one row per shard to the ci_regression_shards stream, flattened from
|
|
# payload.shards (each = {name, conclusion, duration_sec}). Powers per-shard duration /
|
|
# slowest-shard / per-module panels. Row schema:
|
|
# run_id, repo, workflow="regression", ingest_source="live",
|
|
# shard_name (raw job name), module (shard_name minus "e2e / " prefix and "-Regression"
|
|
# suffix; "unknown" if empty), conclusion, duration_sec.
|
|
# _timestamp is intentionally NOT set, so OpenObserve stamps ingest time — same as the run
|
|
# doc above (which also leaves it unset), keeping the two streams time-aligned.
|
|
shards_json=$(jq -c --arg repo "$GITHUB_REPOSITORY" --arg run_id "$GITHUB_RUN_ID" \
|
|
'[.shards[]? | {
|
|
run_id: $run_id, repo: $repo, suite: "regression", workflow: "regression", ingest_source: "live",
|
|
shard_name: .name,
|
|
module: (.name | sub("^e2e / ";"") | sub("-Regression$";"") | if . == "" then "unknown" else . end),
|
|
conclusion: .conclusion, duration_sec: .duration_sec
|
|
}]' payload.json 2>/dev/null) && printf '%s' "$shards_json" > shards.json || echo '[]' > shards.json
|
|
if [ "$(jq 'length' shards.json 2>/dev/null || echo 0)" -gt 0 ]; then
|
|
bash .github/scripts/o2-report.sh ci_regression_shards shards.json
|
|
# Dual-write per-shard rows to the unified stream (now tagged suite=regression).
|
|
bash .github/scripts/o2-report.sh ci_test_runs_shards shards.json
|
|
else
|
|
echo "::notice::no shard rows to emit"
|
|
fi
|
|
|
|
# Per-test flaky/failed rows -> ci_test_results (suite=regression), so the unified
|
|
# dashboard's Latest-failures / Flakiest panels cover regression too. Non-fatal; mirrors
|
|
# the UI workflow's block. branch + pr_number stamped so no cross-stream JOIN is needed.
|
|
FLAKY="${MERGE_FLAKY:-[]}"
|
|
if printf '%s' "$FLAKY" | jq -e 'type=="array" and length>0' >/dev/null 2>&1; then
|
|
FLAKY_FILE=$(mktemp "${TMPDIR:-/tmp}/flaky.XXXXXX")
|
|
trap 'rm -f "$FLAKY_FILE"' EXIT
|
|
printf '%s' "$FLAKY" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg repo "$GITHUB_REPOSITORY" \
|
|
--arg branch "${REPORT_BRANCH:-}" --arg pr "${REPORT_PR:-}" \
|
|
'[.[] | {run_id:$run_id, repo:$repo, suite:"regression", workflow:"regression", ingest_source:"live", status:.status, module:.module, test_title:.title, test_file:.file, branch:(if $branch=="" then null else $branch end), pr_number:(if $pr=="" then null else ($pr|tonumber) end)}]' \
|
|
> "$FLAKY_FILE" 2>/dev/null || echo '[]' > "$FLAKY_FILE"
|
|
if [ "$(jq 'length' "$FLAKY_FILE" 2>/dev/null || echo 0)" -gt 0 ]; then
|
|
bash .github/scripts/o2-report.sh ci_test_results "$FLAKY_FILE"
|
|
fi
|
|
fi
|
|
|
|
playwright_summary:
|
|
timeout-minutes: 5
|
|
runs-on:
|
|
labels: eks-openobserve-standard-4
|
|
permissions: {}
|
|
needs: [build_binary, ui_integration_tests, merge_reports]
|
|
if: always()
|
|
steps:
|
|
- name: Check test results
|
|
run: |
|
|
BUILD_OK=false
|
|
UI_OK=false
|
|
MERGE_OK=false
|
|
|
|
if [ "${{ needs.build_binary.result }}" == "success" ] || [ "${{ needs.build_binary.result }}" == "skipped" ]; then
|
|
BUILD_OK=true
|
|
fi
|
|
|
|
if [ "${{ needs.ui_integration_tests.result }}" == "success" ] || [ "${{ needs.ui_integration_tests.result }}" == "skipped" ]; then
|
|
UI_OK=true
|
|
fi
|
|
|
|
# A merge/report failure is a non-blocker: the tests already ran and passed, so only
|
|
# a cancelled merge should fail the summary.
|
|
if [ "${{ needs.merge_reports.result }}" == "success" ] || [ "${{ needs.merge_reports.result }}" == "skipped" ] || [ "${{ needs.merge_reports.result }}" == "failure" ]; then
|
|
MERGE_OK=true
|
|
fi
|
|
|
|
if [ "$BUILD_OK" == "true" ] && [ "$UI_OK" == "true" ] && [ "$MERGE_OK" == "true" ]; then
|
|
echo "All regression tests completed successfully"
|
|
exit 0
|
|
else
|
|
echo "Regression tests failed:"
|
|
echo " build_binary: ${{ needs.build_binary.result }}"
|
|
echo " ui_integration_tests: ${{ needs.ui_integration_tests.result }}"
|
|
echo " merge_reports: ${{ needs.merge_reports.result }}"
|
|
exit 1
|
|
fi
|