starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-04 00:08:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: remove TestDino from e2e workflows (#13326)

Browse Source 
## What

Removes **all TestDino integration** from the Playwright e2e CI
workflows.

### Removed
- **Reporting/upload to TestDino** — the `upload_testdino` job, the
`UPLOAD_TO_TESTDINO` env/var, and all `TESTDINO_API_TOKEN` usages.
- **Rerun optimization** — the `check_rerun` step and the `tdpw
last-failed` `RERUN_MODE` dispatch.

### Files
- `.github/workflows/playwright.yml` (−444)
- `.github/workflows/playwright_regression.yml` (−350)
- `.github/workflows/playwright-auto-rerun.yml` (stale notice line)
- `tests/ui-testing/playwright.config.js` (comment reword only)

### Behavior change
"Re-run failed jobs" no longer targets only the previously-failed tests
— a re-run now runs the **full matrix job**. Native GitHub
re-run-failed-*jobs* still works.

### Kept
`report.json` / merge-reports generation is untouched — still consumed
by the Kinora and OpenObserve reporters.

### Follow-up (after merge)
Revoke `TESTDINO_API_TOKEN` (repo secret) + delete the
`UPLOAD_TO_TESTDINO` repo variable, and revoke the token in the TestDino
dashboard. Paired ENT PR uses the same branch `test/remove-testdino`.

### Validation
`actionlint` clean (no new findings) and zero TestDino references
remain.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Shrinath Rao
2026-07-21 10:59:02 +00:00
committed by GitHub
co-authored by Claude Opus 4.8
parent 784459fcf4
commit 6dfcb8a44a
4 changed files with 9 additions and 795 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/playwright-auto-rerun.yml
-1
View File
@@ -155,7 +155,6 @@ jobs:
echo "::notice::✅ Successfully triggered rerun of failed jobs"
echo "::notice::Run ID: $RUN_ID"
echo "::notice::Only failed and cancelled jobs will be rerun"
echo "::notice::TestDino will optimize the rerun to only execute failed tests"
echo "$RERUN_OUTPUT"
else
echo "::error::❌ Failed to trigger rerun (exit code: $RERUN_EXIT_CODE)"
.github/workflows/playwright.yml
+6 -443
View File
@@ -50,7 +50,6 @@ env:
# (http://localhost:5080/api/.../..._json) is not blocked by the SSRF guard.
ZO_SSRF_ALLOW_LOOPBACK: true
ZO_MODEL_PRICING_ENABLED: true
UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }}
jobs:
check_changes:
@@ -545,62 +544,7 @@ jobs:
path: ~/.cache/ms-playwright
key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }}
- name: Check if this is a rerun
id: check_rerun
env:
GH_TOKEN: ${{ github.token }}
run: |
echo "::notice::GitHub run attempt: ${{ github.run_attempt }}"
if [ "${{ github.run_attempt }}" -gt "1" ]; then
# This is a rerun - always try TestDino optimization
echo "is_rerun=true" >> $GITHUB_OUTPUT
echo "::notice::This is a rerun (attempt ${{ github.run_attempt }}). Will attempt to run only failed tests."
# Additionally detect if this is "re-run all" vs "re-run failed" for better handling
echo "::group::Detecting rerun type (all jobs vs failed only)"
CURRENT_ATTEMPT=${{ github.run_attempt }}
# Get all jobs from the run
# Fallback to empty jobs array on network/rate-limit errors so the step
# never fails due to a transient GitHub API issue.
ALL_JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=all&per_page=100" \
|| echo '{"jobs":[]}')
# Count jobs that are NOT completed (actually running in this attempt)
# Fall back to "0" if jq fails (e.g. API returned an error body instead of JSON).
CURRENT_ATTEMPT_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == $CURRENT_ATTEMPT) | select(.status != \"completed\")] | length" 2>/dev/null || echo "0")
# Count total unique matrix job names (from attempt 1)
TOTAL_MATRIX_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == 1) | .name] | unique | length" 2>/dev/null || echo "0")
echo "::notice::Total unique matrix job names: $TOTAL_MATRIX_JOBS"
echo "::notice::Jobs running in current attempt $CURRENT_ATTEMPT: $CURRENT_ATTEMPT_JOBS"
# Store rerun type for potential use.
# Guard against non-numeric jq output (e.g. "null") to prevent bash
# "integer expression expected" errors under set -eo pipefail.
if [ "${CURRENT_ATTEMPT_JOBS:-0}" -eq "${TOTAL_MATRIX_JOBS:-0}" ] 2>/dev/null; then
echo "::notice::✅ DETECTED: Re-run ALL jobs (all $TOTAL_MATRIX_JOBS matrix jobs running)"
echo "rerun_type=all" >> $GITHUB_OUTPUT
else
echo "::notice::✅ DETECTED: Re-run FAILED jobs only ($CURRENT_ATTEMPT_JOBS out of $TOTAL_MATRIX_JOBS jobs)"
echo "rerun_type=failed" >> $GITHUB_OUTPUT
fi
echo "::endgroup::"
else
echo "is_rerun=false" >> $GITHUB_OUTPUT
echo "rerun_type=none" >> $GITHUB_OUTPUT
echo "::notice::This is the first run. Will run all tests in matrix."
fi
- name: Install dependencies and run ui-tests
env:
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
GH_TOKEN: ${{ github.token }}
run: |
touch .env
echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env
@@ -656,61 +600,6 @@ jobs:
# so use the local playwright binary (pinned via package-lock.json) to install ffmpeg only.
./node_modules/.bin/playwright install ffmpeg
# Check if this is a rerun
IS_RERUN="${{ steps.check_rerun.outputs.is_rerun }}"
RERUN_TYPE="${{ steps.check_rerun.outputs.rerun_type }}"
if [ "$IS_RERUN" = "true" ]; then
# Check if this is "re-run all" - if so, skip TestDino and run all tests
if [ "$RERUN_TYPE" = "all" ]; then
echo "::notice::🔄 Re-run ALL jobs detected - skipping TestDino optimization"
echo "::notice::Will run ALL tests in this shard regardless of previous failures"
RERUN_MODE="normal"
EXTRA_PW_FLAGS=""
else
# This is "re-run failed" - use TestDino optimization
echo "::group::Fetching last failed tests from TestDino"
# Use custom cache ID that includes run_id to distinguish separate workflow runs
# Format: gh_openobserve_main_12345678 (for main) or gh_openobserve_feature-branch_12345678 (for PRs)
# Sanitize branch name by replacing slashes with underscores
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
echo "::notice::Using cache ID: $CACHE_ID"
# Try to get last failed tests from TestDino
# Use a temp file to avoid shell variable truncation for large output
TEMP_OUTPUT="/tmp/testdino_output_$$.txt"
if npx tdpw last-failed --cache-id "$CACHE_ID" > "$TEMP_OUTPUT" 2>&1; then
# Filter out ALL lines that don't match test file pattern
# Valid format: "FolderName/testfile.spec.js" or "FolderName/testfile.spec.js -g test name"
# This filters out debug lines, empty lines, and anything that's not a test path
LAST_FAILED_FLAGS="$(grep -E '^[A-Za-z_-]+/[a-zA-Z0-9_-]+\.spec\.js' "$TEMP_OUTPUT" | tail -1)"
rm -f "$TEMP_OUTPUT"
if [ -n "$LAST_FAILED_FLAGS" ] && [ "$LAST_FAILED_FLAGS" != "No failed tests found" ]; then
echo "::notice::✅ Found failed tests from previous run."
echo "::notice::TestDino flags: $LAST_FAILED_FLAGS"
RERUN_MODE="optimized"
EXTRA_PW_FLAGS="$LAST_FAILED_FLAGS"
else
echo "::warning::TestDino returned no failed tests. Falling back to running all tests in this matrix job."
echo "::warning::This means you'll be billed for all tests. Please investigate why last-failed returned empty."
RERUN_MODE="fallback"
EXTRA_PW_FLAGS=""
fi
else
echo "::error::Failed to fetch last failed tests from TestDino (exit code: $?)."
echo "::warning::Falling back to running all tests in this matrix job."
echo "::warning::This means you'll be billed for all tests. Please check TestDino integration."
RERUN_MODE="fallback"
EXTRA_PW_FLAGS=""
fi
echo "::endgroup::"
fi
else
RERUN_MODE="normal"
EXTRA_PW_FLAGS=""
fi
# Get list of files to run using join function
FILE_LIST="${{ join(matrix.run_files, ' ') }}"
echo "DEBUG: FILE_LIST = $FILE_LIST"
@@ -743,186 +632,8 @@ jobs:
FILE_PATHS="$FILE_PATHS $FILE_PATH"
done
# Determine what to run based on rerun mode
if [ "$RERUN_MODE" = "optimized" ]; then
# TestDino returns ALL failed tests from the previous run
# Format examples:
# Single: "SDR/test.spec.js -g test name"
# Multiple: "Logs/test1.spec.js -g name1 Pipelines/test2.spec.js -g name2"
echo "DEBUG: Full TestDino output (all failed tests): $EXTRA_PW_FLAGS"
echo "DEBUG: Current shard folder: $ACTUAL_FOLDER"
# Strategy: Extract only tests that start with "$ACTUAL_FOLDER/"
# Use grep to find lines matching our shard folder
SHARD_TESTS=""
# Check if the TestDino output contains tests for this shard (case-insensitive)
# Look for pattern: $ACTUAL_FOLDER/something.spec.js
if echo "$EXTRA_PW_FLAGS" | grep -iq "$ACTUAL_FOLDER/.*\.spec\.js"; then
echo "DEBUG: Found test(s) for this shard in TestDino output"
# IMPORTANT: Based on observation, TestDino returns ONE test at a time
# Format: "Folder/file.spec.js -g test name"
# Simply check if it starts with our folder and pass it through
# Extract file path (everything before -g or end of string) - case-insensitive
TEST_FILE=$(echo "$EXTRA_PW_FLAGS" | grep -ioE "$ACTUAL_FOLDER/[^ ]+\.spec\.js" | head -1)
if [ -n "$TEST_FILE" ]; then
# Check if file actually starts with our folder (case-insensitive)
if [[ "${TEST_FILE,,}" == "${ACTUAL_FOLDER,,}/"* ]]; then
# Extract just the filename from the test file path
TEST_FILENAME=$(basename "$TEST_FILE")
# Check if this file is actually in the current shard's run_files list
if echo "$FILE_LIST" | grep -qw "$TEST_FILENAME"; then
# Parse out the file path and test name separately for proper quoting
# Extract file path (before -g)
TEST_FILE_PATH=$(echo "$EXTRA_PW_FLAGS" | sed "s/ -g.*//" | sed "s|^|./playwright-tests/|")
# Check if there's a -g flag
if echo "$EXTRA_PW_FLAGS" | grep -q " -g "; then
# Extract test name (after -g) and strip any surrounding quotes
TEST_NAME=$(echo "$EXTRA_PW_FLAGS" | sed 's/.*-g //' | sed 's/^"\(.*\)"$/\1/')
echo "DEBUG: File path: $TEST_FILE_PATH"
echo "DEBUG: Test name (unquoted): $TEST_NAME"
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test from this shard"
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\" -g \"$TEST_NAME\""
# Properly quote the test name to handle spaces and special chars
npx playwright test "$TEST_FILE_PATH" -g "$TEST_NAME"
else
# No -g flag, just run the file
echo "DEBUG: File path: $TEST_FILE_PATH (no -g flag)"
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test file from this shard"
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\""
npx playwright test "$TEST_FILE_PATH"
fi
else
echo "::notice::⏭️ Test file '$TEST_FILENAME' not in this shard's run_files list."
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
# Check the status of this matrix job in the previous attempt
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
if [ "$PREV_ATTEMPT" -ge 1 ]; then
JOB_NAME="e2e / ${{ matrix.testfolder }}"
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
elif [ "$PREV_JOB_STATUS" = "success" ]; then
echo "::notice::✅ Previous shard completed successfully - no tests to rerun"
exit 0
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
else
echo "::warning::⚠️ Unknown job status - running FULL test suite for safety"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::error::Failed to fetch job status from GitHub API - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::notice::✅ This is the first run, no previous attempt to check"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
fi
fi
else
echo "::notice::⏭️ Test file doesn't match shard folder exactly. Skipping."
exit 0
fi
else
echo "::warning::Failed to extract test file path. Falling back to skip."
exit 0
fi
else
echo "::notice::⏭️ OPTIMIZED RERUN: No failed tests found for '$ACTUAL_FOLDER' shard in TestDino output"
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
# Check the status of this matrix job in the previous attempt
# Rule: If shard was cancelled/abruptly ended → treat as failed → run full test suite
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
if [ "$PREV_ATTEMPT" -ge 1 ]; then
# Job name format: "e2e / {matrix.testfolder}"
JOB_NAME="e2e / ${{ matrix.testfolder }}"
echo "DEBUG: Checking previous attempt #$PREV_ATTEMPT for job: $JOB_NAME"
# Use GitHub REST API to get job status from specific attempt
# This is guaranteed to work and provides attempt-specific data
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
# Successfully got API response, parse it
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
echo "DEBUG: Previous job status: $PREV_JOB_STATUS"
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended (status: ${PREV_JOB_STATUS:-not found})"
echo "::warning::Treating as FAILED - running FULL test suite for this shard"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
elif [ "$PREV_JOB_STATUS" = "success" ]; then
echo "::notice::✅ Previous shard completed successfully - all tests passed"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded"
echo "::warning::This shouldn't happen - running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
else
echo "::warning::⚠️ Unknown job status: $PREV_JOB_STATUS"
echo "::warning::Running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::error::Failed to fetch job status from GitHub API"
echo "::warning::Falling back to running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::notice::✅ This is the first run, no previous attempt to check"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
fi
fi
else
# Normal run or fallback
if [ "$RERUN_MODE" = "fallback" ]; then
echo "::warning::⚠️ FALLBACK RERUN: Running all tests in matrix (TestDino optimization failed)"
fi
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
fi
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
else
echo "No files specified to run for ${{ matrix.testfolder }} folder"
fi
@@ -1021,8 +732,8 @@ jobs:
echo "::notice::This is VALID behavior for optimized reruns"
echo "::notice::Creating empty report structure for downstream jobs"
mkdir -p playwright-results/html-report
# Add stats object to make report compatible with TestDino
# startTime must be ISO 8601 string (not Unix timestamp number) for TestDino compatibility
# Add stats object so downstream report consumers get a well-formed report.
# startTime is an ISO 8601 string (not a Unix timestamp number).
START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json
echo "<html><body><h1>No tests executed - all shards skipped</h1><p>All test shards exited early because tests had already passed in a previous attempt.</p></body></html>" > playwright-results/html-report/index.html
@@ -1161,8 +872,8 @@ jobs:
fi
fi
# Kinora upload — its own job for clean isolation (alongside report_openobserve and
# upload_testdino). Downloads the raw blob reports and merges them locally so trace.zip
# Kinora upload — its own job for clean isolation (alongside report_openobserve).
# Downloads the raw blob reports and merges them locally so trace.zip
# attachments resolve on THIS runner, then uploads the run (+ traces) to the self-hosted
# Kinora dashboard and updates a single sticky PR comment. Fully non-fatal.
report_kinora:
@@ -1256,154 +967,6 @@ jobs:
gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" -f body="$BODY" >/dev/null && echo "Posted new Kinora comment."
fi
# TestDino upload — its own job so it never gates playwright_summary. Only runs when tests
# failed (and uploads enabled); consumes the merged report via artifact.
upload_testdino:
timeout-minutes: 15
name: Upload to TestDino
needs: [ui_integration_tests, merge_reports]
if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }}
runs-on:
labels: eks-openobserve-standard-4
permissions:
contents: read
actions: read # download-artifact fetches the merged-report artifact
steps:
- name: Clone the current repo
uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 22
- name: Install project dependencies
run: cd tests/ui-testing && npm ci
- name: Download merged report
uses: actions/download-artifact@v4
with:
name: merged-report-${{ github.run_attempt }}
path: tests/ui-testing/playwright-results
- name: Cache test failures to TestDino
env:
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
run: |
cd tests/ui-testing
echo "::group::Caching test failure metadata to TestDino"
if [ -f "playwright-results/report.json" ]; then
echo "::notice::JSON report found at playwright-results/report.json"
ls -lh playwright-results/report.json
# Copy report.json to repo root to ensure correct git metadata
mkdir -p ../../playwright-results
cp playwright-results/report.json ../../playwright-results/
# Run tdpw cache from repo root directory (for correct git metadata)
cd ../../playwright-results
# Use custom cache ID that includes run_id to distinguish separate workflow runs
# Format: gh_openobserve_main_12345678 (for main) or gh_openobserve_feature-branch_12345678 (for PRs)
# Sanitize branch name by replacing slashes with underscores
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
echo "::notice::Using cache ID: $CACHE_ID"
# Suppress verbose debug output from TestDino, only show result
if CACHE_OUTPUT=$(npx --yes tdpw cache --cache-id "$CACHE_ID" 2>&1); then
# Extract only the success message, filter out debug spam
SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "")
if [ -n "$SUCCESS_MSG" ]; then
echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud"
else
echo "::notice::✅ Cache command completed"
fi
else
echo "::warning::⚠️ Failed to cache test metadata to TestDino"
echo "::warning::Rerun optimization may not work properly"
# Show output only on failure for debugging
echo "$CACHE_OUTPUT"
fi
else
echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)"
fi
echo "::endgroup::"
- name: Upload to TestDino
if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }}
run: |
cd tests/ui-testing
# Check if report.json exists (it won't exist if all tests were skipped due to rerun optimization)
if [ ! -f "playwright-results/report.json" ]; then
echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)"
exit 0
fi
# Fix startTime format: convert Unix timestamp (number) to ISO 8601 string for TestDino compatibility
# Check if startTime is a number and convert it
START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json)
if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then
echo "Converting startTime from Unix timestamp to ISO 8601 string"
jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp
mv playwright-results/report.json.tmp playwright-results/report.json
fi
# Filter report to only include failed tests for TestDino upload.
# Retries with identical status+error are stripped to the final attempt only,
# reducing quota usage. Retries with differing status or error are kept in full.
node -e "
const fs = require('fs');
const report = JSON.parse(fs.readFileSync('playwright-results/report.json', 'utf8'));
let strippedRetries = 0; // accumulated once per test via single-visit tree traversal
function getErrorFingerprint(r) {
// First line of error.message is stable across retries (stack/timing in later lines varies).
// Distinguish no-error-object (safe to strip on status alone) from error-without-message
// (unknown content — treat as unique so we never falsely deduplicate).
if (r.error?.message) return r.error.message.split('\n')[0].trim();
return r.status + ':' + (r.error ? 'error-no-msg' : 'no-error');
}
function filterSuite(s) {
const specs = (s.specs||[])
.filter(sp=>sp.tests&&sp.tests.some(t=>t.status==='unexpected'))
.map(sp=>({
...sp,
tests: sp.tests.map(t=>{
const results = t.results||[];
if (!results.length) return {...t};
const first = results[0];
const allIdentical = results.length === 1 || results.every(r=>
r.status === first.status &&
getErrorFingerprint(r) === getErrorFingerprint(first)
);
if (allIdentical && results.length > 1) strippedRetries += results.length - 1;
return {
...t,
results: allIdentical ? [results[results.length-1]] : results
};
})
}));
const suites = (s.suites||[]).map(filterSuite).filter(sub=>(sub.specs&&sub.specs.length)||(sub.suites&&sub.suites.length));
return {...s,specs,suites};
}
const filtered = {
...report,
suites: report.suites.map(filterSuite).filter(s=>(s.specs&&s.specs.length)||(s.suites&&s.suites.length)),
stats: {...report.stats,expected:0,flaky:0,skipped:0}
};
fs.writeFileSync('playwright-results/report-failures-only.json',JSON.stringify(filtered));
const total=(report.stats.expected||0)+(report.stats.unexpected||0)+(report.stats.skipped||0)+(report.stats.flaky||0);
console.log('Uploading '+(report.stats.unexpected||0)+' failed tests out of '+total+' total to TestDino (stripped '+strippedRetries+' identical retry results)');
"
echo "::notice::Report filtered to failures only"
npx --yes tdpw@latest upload playwright-results \
--json-report playwright-results/report-failures-only.json \
--html-report playwright-results/html-report \
--upload-html \
--verbose \
--token="${{ secrets.TESTDINO_API_TOKEN }}"
playwright_summary:
timeout-minutes: 5
runs-on:
.github/workflows/playwright_regression.yml
+2 -350
View File
@@ -44,7 +44,6 @@ env:
# Timechart (visualize) tab is hidden by default; enable it so visualize E2E tests can reach it.
ZO_TIMECHART_ENABLED: true
ZO_SSRF_ALLOW_LOOPBACK: true
UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }}
jobs:
# Skip the whole suite when main has not moved since the last regression run.
@@ -307,57 +306,7 @@ jobs:
npx playwright install --with-deps chromium
fi
- name: Check if this is a rerun
id: check_rerun
env:
GH_TOKEN: ${{ github.token }}
run: |
# Rerun-detection strategy:
# 1. On a rerun, query the GitHub Jobs API for this run.
# 2. Compare non-completed jobs in the current attempt against total unique
# matrix jobs: all present -> rerun ALL tests; a subset -> rerun FAILED only.
# 3. If the API returns an unexpected payload (rate-limit / error body with no
# .jobs key), default to "all tests". Safe (everything runs), just not
# cost-optimized — and it never crashes the step (see guard below).
echo "::notice::GitHub run attempt: ${{ github.run_attempt }}"
if [ "${{ github.run_attempt }}" -gt "1" ]; then
echo "is_rerun=true" >> $GITHUB_OUTPUT
echo "::notice::This is a rerun (attempt ${{ github.run_attempt }}). Will attempt to run only failed tests."
echo "::group::Detecting rerun type (all jobs vs failed only)"
CURRENT_ATTEMPT=${{ github.run_attempt }}
ALL_JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=all&per_page=100")
CURRENT_ATTEMPT_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[(.jobs // [])[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == $CURRENT_ATTEMPT) | select(.status != \"completed\")] | length" 2>/dev/null || echo "0")
TOTAL_MATRIX_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[(.jobs // [])[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == 1) | .name] | unique | length" 2>/dev/null || echo "0")
echo "::notice::Total unique matrix job names: $TOTAL_MATRIX_JOBS"
echo "::notice::Jobs running in current attempt $CURRENT_ATTEMPT: $CURRENT_ATTEMPT_JOBS"
# If the jobs API returned an unexpected payload (a rate-limit/error body has no
# .jobs key, or curl failed) the counts above fall back to 0 via `.jobs // []`.
# Never let that crash the step: previously raw `.jobs[]` hit jq "Cannot iterate
# over null", exited 5 under `bash -e`, and skipped the entire test run. On any
# ambiguity, default to re-running ALL tests (safe, just not cost-optimized).
if ! [[ "$CURRENT_ATTEMPT_JOBS" =~ ^[0-9]+$ ]] || ! [[ "$TOTAL_MATRIX_JOBS" =~ ^[0-9]+$ ]] || [ "$TOTAL_MATRIX_JOBS" -eq 0 ]; then
echo "::warning::Could not determine rerun type from jobs API; defaulting to rerun_type=all (all tests)."
echo "rerun_type=all" >> $GITHUB_OUTPUT
elif [ "$CURRENT_ATTEMPT_JOBS" -eq "$TOTAL_MATRIX_JOBS" ]; then
echo "::notice::✅ DETECTED: Re-run ALL jobs (all $TOTAL_MATRIX_JOBS matrix jobs running)"
echo "rerun_type=all" >> $GITHUB_OUTPUT
else
echo "::notice::✅ DETECTED: Re-run FAILED jobs only ($CURRENT_ATTEMPT_JOBS out of $TOTAL_MATRIX_JOBS jobs)"
echo "rerun_type=failed" >> $GITHUB_OUTPUT
fi
echo "::endgroup::"
else
echo "is_rerun=false" >> $GITHUB_OUTPUT
echo "rerun_type=none" >> $GITHUB_OUTPUT
echo "::notice::This is the first run. Will run all tests in matrix."
fi
- name: Write .env and run ui-tests
env:
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
GH_TOKEN: ${{ github.token }}
run: |
touch .env
echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env
@@ -372,50 +321,6 @@ jobs:
mv .env tests/ui-testing
cd tests/ui-testing
# Check if this is a rerun
IS_RERUN="${{ steps.check_rerun.outputs.is_rerun }}"
RERUN_TYPE="${{ steps.check_rerun.outputs.rerun_type }}"
if [ "$IS_RERUN" = "true" ]; then
if [ "$RERUN_TYPE" = "all" ]; then
echo "::notice::🔄 Re-run ALL jobs detected - skipping TestDino optimization"
echo "::notice::Will run ALL tests in this shard regardless of previous failures"
RERUN_MODE="normal"
EXTRA_PW_FLAGS=""
else
echo "::group::Fetching last failed tests from TestDino"
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
echo "::notice::Using cache ID: $CACHE_ID"
TEMP_OUTPUT="/tmp/testdino_output_$$.txt"
if npx tdpw last-failed --cache-id "$CACHE_ID" > "$TEMP_OUTPUT" 2>&1; then
LAST_FAILED_FLAGS="$(grep -E '^[A-Za-z_-]+/[a-zA-Z0-9_-]+\.spec\.js' "$TEMP_OUTPUT" | tail -1)"
rm -f "$TEMP_OUTPUT"
if [ -n "$LAST_FAILED_FLAGS" ] && [ "$LAST_FAILED_FLAGS" != "No failed tests found" ]; then
echo "::notice::✅ Found failed tests from previous run."
echo "::notice::TestDino flags: $LAST_FAILED_FLAGS"
RERUN_MODE="optimized"
EXTRA_PW_FLAGS="$LAST_FAILED_FLAGS"
else
echo "::warning::TestDino returned no failed tests. Falling back to running all tests in this matrix job."
echo "::warning::This means you'll be billed for all tests. Please investigate why last-failed returned empty."
RERUN_MODE="fallback"
EXTRA_PW_FLAGS=""
fi
else
echo "::error::Failed to fetch last failed tests from TestDino (exit code: $?)."
echo "::warning::Falling back to running all tests in this matrix job."
echo "::warning::This means you'll be billed for all tests. Please check TestDino integration."
RERUN_MODE="fallback"
EXTRA_PW_FLAGS=""
fi
echo "::endgroup::"
fi
else
RERUN_MODE="normal"
EXTRA_PW_FLAGS=""
fi
FILE_LIST="${{ join(matrix.run_files, ' ') }}"
echo "DEBUG: FILE_LIST = $FILE_LIST"
echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}"
@@ -446,152 +351,8 @@ jobs:
FILE_PATHS="$FILE_PATHS $FILE_PATH"
done
if [ "$RERUN_MODE" = "optimized" ]; then
echo "DEBUG: Full TestDino output (all failed tests): $EXTRA_PW_FLAGS"
echo "DEBUG: Current shard folder: $ACTUAL_FOLDER"
SHARD_TESTS=""
if echo "$EXTRA_PW_FLAGS" | grep -iq "$ACTUAL_FOLDER/.*\.spec\.js"; then
echo "DEBUG: Found test(s) for this shard in TestDino output"
TEST_FILE=$(echo "$EXTRA_PW_FLAGS" | grep -ioE "$ACTUAL_FOLDER/[^ ]+\.spec\.js" | head -1)
if [ -n "$TEST_FILE" ]; then
if [[ "${TEST_FILE,,}" == "${ACTUAL_FOLDER,,}/"* ]]; then
TEST_FILENAME=$(basename "$TEST_FILE")
if echo "$FILE_LIST" | grep -qw "$TEST_FILENAME"; then
TEST_FILE_PATH=$(echo "$EXTRA_PW_FLAGS" | sed "s/ -g.*//" | sed "s|^|./playwright-tests/|")
if echo "$EXTRA_PW_FLAGS" | grep -q " -g "; then
TEST_NAME=$(echo "$EXTRA_PW_FLAGS" | sed 's/.*-g //' | sed 's/^"\(.*\)"$/\1/')
echo "DEBUG: File path: $TEST_FILE_PATH"
echo "DEBUG: Test name (unquoted): $TEST_NAME"
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test from this shard"
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\" -g \"$TEST_NAME\""
npx playwright test "$TEST_FILE_PATH" -g "$TEST_NAME"
else
echo "DEBUG: File path: $TEST_FILE_PATH (no -g flag)"
echo "::notice::🎯 OPTIMIZED RERUN: Running failed test file from this shard"
echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\""
npx playwright test "$TEST_FILE_PATH"
fi
else
echo "::notice::⏭️ Test file '$TEST_FILENAME' not in this shard's run_files list."
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
if [ "$PREV_ATTEMPT" -ge 1 ]; then
JOB_NAME="e2e / ${{ matrix.testfolder }}"
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
elif [ "$PREV_JOB_STATUS" = "success" ]; then
echo "::notice::✅ Previous shard completed successfully - no tests to rerun"
exit 0
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
else
echo "::warning::⚠️ Unknown job status - running FULL test suite for safety"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::error::Failed to fetch job status from GitHub API - running FULL test suite"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::notice::✅ This is the first run, no previous attempt to check"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
fi
fi
else
echo "::notice::⏭️ Test file doesn't match shard folder exactly. Skipping."
exit 0
fi
else
echo "::warning::Failed to extract test file path. Falling back to skip."
exit 0
fi
else
echo "::notice::⏭️ OPTIMIZED RERUN: No failed tests found for '$ACTUAL_FOLDER' shard in TestDino output"
echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..."
PREV_ATTEMPT=$((${{ github.run_attempt }} - 1))
if [ "$PREV_ATTEMPT" -ge 1 ]; then
JOB_NAME="e2e / ${{ matrix.testfolder }}"
echo "DEBUG: Checking previous attempt #$PREV_ATTEMPT for job: $JOB_NAME"
API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null)
if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then
PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null)
echo "DEBUG: Previous job status: $PREV_JOB_STATUS"
if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then
echo "::warning::⚠️ Previous shard was cancelled/abruptly ended (status: ${PREV_JOB_STATUS:-not found})"
echo "::warning::Treating as FAILED - running FULL test suite for this shard"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
elif [ "$PREV_JOB_STATUS" = "success" ]; then
echo "::notice::✅ Previous shard completed successfully - all tests passed"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
elif [ "$PREV_JOB_STATUS" = "failure" ]; then
echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded"
echo "::warning::This shouldn't happen - running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
else
echo "::warning::⚠️ Unknown job status: $PREV_JOB_STATUS"
echo "::warning::Running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::error::Failed to fetch job status from GitHub API"
echo "::warning::Falling back to running FULL test suite for safety"
echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
exit $?
fi
else
echo "::notice::✅ This is the first run, no previous attempt to check"
echo "::notice::No failed tests to rerun. Exiting successfully."
exit 0
fi
fi
else
# Normal run or fallback
if [ "$RERUN_MODE" = "fallback" ]; then
echo "::warning::⚠️ FALLBACK RERUN: Running all tests in matrix (TestDino optimization failed)"
fi
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
fi
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
npx playwright test $FILE_PATHS
else
echo "No files specified to run for ${{ matrix.testfolder }} folder"
fi
@@ -750,115 +511,6 @@ jobs:
retention-days: 1
if-no-files-found: warn
# TestDino upload — its own job so it never gates playwright_summary. Only runs when tests
# failed (and uploads are enabled); consumes the merged report via artifact.
upload_testdino:
timeout-minutes: 15
name: Upload to TestDino
needs: [ui_integration_tests, merge_reports]
if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }}
runs-on:
labels: eks-openobserve-standard-4
permissions:
contents: read
actions: read # download-artifact fetches the merged-report artifact
steps:
- name: Clone the current repo
uses: actions/checkout@v5
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
- name: Install project dependencies
run: cd tests/ui-testing && npm ci
- name: Download merged report
uses: actions/download-artifact@v4
with:
name: merged-report-${{ github.run_attempt }}
path: tests/ui-testing/playwright-results
- name: Cache test failures to TestDino
env:
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
run: |
cd tests/ui-testing
echo "::group::Caching test failure metadata to TestDino"
if [ -f "playwright-results/report.json" ]; then
echo "::notice::JSON report found at playwright-results/report.json"
ls -lh playwright-results/report.json
mkdir -p ../../playwright-results
cp playwright-results/report.json ../../playwright-results/
cd ../../playwright-results
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
echo "::notice::Using cache ID: $CACHE_ID"
if CACHE_OUTPUT=$(npx tdpw cache --cache-id "$CACHE_ID" 2>&1); then
SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "")
if [ -n "$SUCCESS_MSG" ]; then
echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud"
else
echo "::notice::✅ Cache command completed"
fi
else
echo "::warning::⚠️ Failed to cache test metadata to TestDino"
echo "::warning::Rerun optimization may not work properly"
echo "$CACHE_OUTPUT"
fi
else
echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)"
fi
echo "::endgroup::"
- name: Upload to TestDino
if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }}
run: |
cd tests/ui-testing
if [ ! -f "playwright-results/report.json" ]; then
echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)"
exit 0
fi
START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json)
if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then
echo "Converting startTime from Unix timestamp to ISO 8601 string"
jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp
mv playwright-results/report.json.tmp playwright-results/report.json
fi
# Filter report to only include failed tests for TestDino upload
node -e "
const fs = require('fs');
const report = JSON.parse(fs.readFileSync('playwright-results/report.json', 'utf8'));
function filterSuite(s) {
const specs = (s.specs||[]).filter(sp=>sp.tests&&sp.tests.some(t=>t.status==='unexpected'));
const suites = (s.suites||[]).map(filterSuite).filter(sub=>(sub.specs&&sub.specs.length)||(sub.suites&&sub.suites.length));
return {...s,specs,suites};
}
const filtered = {
...report,
suites: report.suites.map(filterSuite).filter(s=>(s.specs&&s.specs.length)||(s.suites&&s.suites.length)),
stats: {...report.stats,expected:0,flaky:0,skipped:0}
};
fs.writeFileSync('playwright-results/report-failures-only.json',JSON.stringify(filtered));
const total=(report.stats.expected||0)+(report.stats.unexpected||0)+(report.stats.skipped||0)+(report.stats.flaky||0);
console.log('Uploading '+(report.stats.unexpected||0)+' failed tests out of '+total+' total to TestDino');
"
echo "::notice::Report filtered to failures only"
npx --yes tdpw@latest upload playwright-results \
--json-report playwright-results/report-failures-only.json \
--html-report playwright-results/html-report \
--upload-html \
--verbose \
--token="${{ secrets.TESTDINO_API_TOKEN }}"
# OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one
# run-summary doc (stream: ci_regression) + per-shard rows (ci_regression_shards). The step
# is continue-on-error and the poster script never fails, so this can never break the run.
tests/ui-testing/playwright.config.js
+1 -1
View File
@@ -42,7 +42,7 @@ module.exports = defineConfig({
]
: [
['html', { outputFolder: 'playwright-results/html-report', open: 'never' }], // HTML reporter
['json', { outputFile: 'playwright-results/report.json' }] // JSON reporter for TestDino
['json', { outputFile: 'playwright-results/report.json' }] // JSON reporter for downstream report consumers
],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {