From 6dfcb8a44a2cc4e976b9c772ac3b9d54b846933f Mon Sep 17 00:00:00 2001 From: Shrinath Rao Date: Tue, 21 Jul 2026 16:29:02 +0530 Subject: [PATCH] ci: remove TestDino from e2e workflows (#13326) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Removes **all TestDino integration** from the Playwright e2e CI workflows. ### Removed - **Reporting/upload to TestDino** — the `upload_testdino` job, the `UPLOAD_TO_TESTDINO` env/var, and all `TESTDINO_API_TOKEN` usages. - **Rerun optimization** — the `check_rerun` step and the `tdpw last-failed` `RERUN_MODE` dispatch. ### Files - `.github/workflows/playwright.yml` (−444) - `.github/workflows/playwright_regression.yml` (−350) - `.github/workflows/playwright-auto-rerun.yml` (stale notice line) - `tests/ui-testing/playwright.config.js` (comment reword only) ### Behavior change "Re-run failed jobs" no longer targets only the previously-failed tests — a re-run now runs the **full matrix job**. Native GitHub re-run-failed-*jobs* still works. ### Kept `report.json` / merge-reports generation is untouched — still consumed by the Kinora and OpenObserve reporters. ### Follow-up (after merge) Revoke `TESTDINO_API_TOKEN` (repo secret) + delete the `UPLOAD_TO_TESTDINO` repo variable, and revoke the token in the TestDino dashboard. Paired ENT PR uses the same branch `test/remove-testdino`. ### Validation `actionlint` clean (no new findings) and zero TestDino references remain. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.8 --- .github/workflows/playwright-auto-rerun.yml | 1 - .github/workflows/playwright.yml | 449 +------------------- .github/workflows/playwright_regression.yml | 352 +-------------- tests/ui-testing/playwright.config.js | 2 +- 4 files changed, 9 insertions(+), 795 deletions(-) diff --git a/.github/workflows/playwright-auto-rerun.yml b/.github/workflows/playwright-auto-rerun.yml index 43b28246f5..6144b9f2a1 100644 --- a/.github/workflows/playwright-auto-rerun.yml +++ b/.github/workflows/playwright-auto-rerun.yml @@ -155,7 +155,6 @@ jobs: echo "::notice::✅ Successfully triggered rerun of failed jobs" echo "::notice::Run ID: $RUN_ID" echo "::notice::Only failed and cancelled jobs will be rerun" - echo "::notice::TestDino will optimize the rerun to only execute failed tests" echo "$RERUN_OUTPUT" else echo "::error::❌ Failed to trigger rerun (exit code: $RERUN_EXIT_CODE)" diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 8f0c094210..e76056a25a 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -50,7 +50,6 @@ env: # (http://localhost:5080/api/.../..._json) is not blocked by the SSRF guard. ZO_SSRF_ALLOW_LOOPBACK: true ZO_MODEL_PRICING_ENABLED: true - UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }} jobs: check_changes: @@ -545,62 +544,7 @@ jobs: path: ~/.cache/ms-playwright key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }} - - name: Check if this is a rerun - id: check_rerun - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "::notice::GitHub run attempt: ${{ github.run_attempt }}" - if [ "${{ github.run_attempt }}" -gt "1" ]; then - # This is a rerun - always try TestDino optimization - echo "is_rerun=true" >> $GITHUB_OUTPUT - echo "::notice::This is a rerun (attempt ${{ github.run_attempt }}). Will attempt to run only failed tests." - - # Additionally detect if this is "re-run all" vs "re-run failed" for better handling - echo "::group::Detecting rerun type (all jobs vs failed only)" - - CURRENT_ATTEMPT=${{ github.run_attempt }} - - # Get all jobs from the run - # Fallback to empty jobs array on network/rate-limit errors so the step - # never fails due to a transient GitHub API issue. - ALL_JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=all&per_page=100" \ - || echo '{"jobs":[]}') - - # Count jobs that are NOT completed (actually running in this attempt) - # Fall back to "0" if jq fails (e.g. API returned an error body instead of JSON). - CURRENT_ATTEMPT_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == $CURRENT_ATTEMPT) | select(.status != \"completed\")] | length" 2>/dev/null || echo "0") - - # Count total unique matrix job names (from attempt 1) - TOTAL_MATRIX_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[.jobs[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == 1) | .name] | unique | length" 2>/dev/null || echo "0") - - echo "::notice::Total unique matrix job names: $TOTAL_MATRIX_JOBS" - echo "::notice::Jobs running in current attempt $CURRENT_ATTEMPT: $CURRENT_ATTEMPT_JOBS" - - # Store rerun type for potential use. - # Guard against non-numeric jq output (e.g. "null") to prevent bash - # "integer expression expected" errors under set -eo pipefail. - if [ "${CURRENT_ATTEMPT_JOBS:-0}" -eq "${TOTAL_MATRIX_JOBS:-0}" ] 2>/dev/null; then - echo "::notice::✅ DETECTED: Re-run ALL jobs (all $TOTAL_MATRIX_JOBS matrix jobs running)" - echo "rerun_type=all" >> $GITHUB_OUTPUT - else - echo "::notice::✅ DETECTED: Re-run FAILED jobs only ($CURRENT_ATTEMPT_JOBS out of $TOTAL_MATRIX_JOBS jobs)" - echo "rerun_type=failed" >> $GITHUB_OUTPUT - fi - - echo "::endgroup::" - else - echo "is_rerun=false" >> $GITHUB_OUTPUT - echo "rerun_type=none" >> $GITHUB_OUTPUT - echo "::notice::This is the first run. Will run all tests in matrix." - fi - - name: Install dependencies and run ui-tests - env: - TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }} - GH_TOKEN: ${{ github.token }} run: | touch .env echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env @@ -656,61 +600,6 @@ jobs: # so use the local playwright binary (pinned via package-lock.json) to install ffmpeg only. ./node_modules/.bin/playwright install ffmpeg - # Check if this is a rerun - IS_RERUN="${{ steps.check_rerun.outputs.is_rerun }}" - RERUN_TYPE="${{ steps.check_rerun.outputs.rerun_type }}" - - if [ "$IS_RERUN" = "true" ]; then - # Check if this is "re-run all" - if so, skip TestDino and run all tests - if [ "$RERUN_TYPE" = "all" ]; then - echo "::notice::🔄 Re-run ALL jobs detected - skipping TestDino optimization" - echo "::notice::Will run ALL tests in this shard regardless of previous failures" - RERUN_MODE="normal" - EXTRA_PW_FLAGS="" - else - # This is "re-run failed" - use TestDino optimization - echo "::group::Fetching last failed tests from TestDino" - # Use custom cache ID that includes run_id to distinguish separate workflow runs - # Format: gh_openobserve_main_12345678 (for main) or gh_openobserve_feature-branch_12345678 (for PRs) - # Sanitize branch name by replacing slashes with underscores - BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g') - CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}" - echo "::notice::Using cache ID: $CACHE_ID" - # Try to get last failed tests from TestDino - # Use a temp file to avoid shell variable truncation for large output - TEMP_OUTPUT="/tmp/testdino_output_$$.txt" - if npx tdpw last-failed --cache-id "$CACHE_ID" > "$TEMP_OUTPUT" 2>&1; then - # Filter out ALL lines that don't match test file pattern - # Valid format: "FolderName/testfile.spec.js" or "FolderName/testfile.spec.js -g test name" - # This filters out debug lines, empty lines, and anything that's not a test path - LAST_FAILED_FLAGS="$(grep -E '^[A-Za-z_-]+/[a-zA-Z0-9_-]+\.spec\.js' "$TEMP_OUTPUT" | tail -1)" - rm -f "$TEMP_OUTPUT" - - if [ -n "$LAST_FAILED_FLAGS" ] && [ "$LAST_FAILED_FLAGS" != "No failed tests found" ]; then - echo "::notice::✅ Found failed tests from previous run." - echo "::notice::TestDino flags: $LAST_FAILED_FLAGS" - RERUN_MODE="optimized" - EXTRA_PW_FLAGS="$LAST_FAILED_FLAGS" - else - echo "::warning::TestDino returned no failed tests. Falling back to running all tests in this matrix job." - echo "::warning::This means you'll be billed for all tests. Please investigate why last-failed returned empty." - RERUN_MODE="fallback" - EXTRA_PW_FLAGS="" - fi - else - echo "::error::Failed to fetch last failed tests from TestDino (exit code: $?)." - echo "::warning::Falling back to running all tests in this matrix job." - echo "::warning::This means you'll be billed for all tests. Please check TestDino integration." - RERUN_MODE="fallback" - EXTRA_PW_FLAGS="" - fi - echo "::endgroup::" - fi - else - RERUN_MODE="normal" - EXTRA_PW_FLAGS="" - fi - # Get list of files to run using join function FILE_LIST="${{ join(matrix.run_files, ' ') }}" echo "DEBUG: FILE_LIST = $FILE_LIST" @@ -743,186 +632,8 @@ jobs: FILE_PATHS="$FILE_PATHS $FILE_PATH" done - # Determine what to run based on rerun mode - if [ "$RERUN_MODE" = "optimized" ]; then - # TestDino returns ALL failed tests from the previous run - # Format examples: - # Single: "SDR/test.spec.js -g test name" - # Multiple: "Logs/test1.spec.js -g name1 Pipelines/test2.spec.js -g name2" - - echo "DEBUG: Full TestDino output (all failed tests): $EXTRA_PW_FLAGS" - echo "DEBUG: Current shard folder: $ACTUAL_FOLDER" - - # Strategy: Extract only tests that start with "$ACTUAL_FOLDER/" - # Use grep to find lines matching our shard folder - SHARD_TESTS="" - - # Check if the TestDino output contains tests for this shard (case-insensitive) - # Look for pattern: $ACTUAL_FOLDER/something.spec.js - if echo "$EXTRA_PW_FLAGS" | grep -iq "$ACTUAL_FOLDER/.*\.spec\.js"; then - echo "DEBUG: Found test(s) for this shard in TestDino output" - - # IMPORTANT: Based on observation, TestDino returns ONE test at a time - # Format: "Folder/file.spec.js -g test name" - # Simply check if it starts with our folder and pass it through - - # Extract file path (everything before -g or end of string) - case-insensitive - TEST_FILE=$(echo "$EXTRA_PW_FLAGS" | grep -ioE "$ACTUAL_FOLDER/[^ ]+\.spec\.js" | head -1) - - if [ -n "$TEST_FILE" ]; then - # Check if file actually starts with our folder (case-insensitive) - if [[ "${TEST_FILE,,}" == "${ACTUAL_FOLDER,,}/"* ]]; then - # Extract just the filename from the test file path - TEST_FILENAME=$(basename "$TEST_FILE") - - # Check if this file is actually in the current shard's run_files list - if echo "$FILE_LIST" | grep -qw "$TEST_FILENAME"; then - # Parse out the file path and test name separately for proper quoting - # Extract file path (before -g) - TEST_FILE_PATH=$(echo "$EXTRA_PW_FLAGS" | sed "s/ -g.*//" | sed "s|^|./playwright-tests/|") - - # Check if there's a -g flag - if echo "$EXTRA_PW_FLAGS" | grep -q " -g "; then - # Extract test name (after -g) and strip any surrounding quotes - TEST_NAME=$(echo "$EXTRA_PW_FLAGS" | sed 's/.*-g //' | sed 's/^"\(.*\)"$/\1/') - - echo "DEBUG: File path: $TEST_FILE_PATH" - echo "DEBUG: Test name (unquoted): $TEST_NAME" - echo "::notice::🎯 OPTIMIZED RERUN: Running failed test from this shard" - echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\" -g \"$TEST_NAME\"" - - # Properly quote the test name to handle spaces and special chars - npx playwright test "$TEST_FILE_PATH" -g "$TEST_NAME" - else - # No -g flag, just run the file - echo "DEBUG: File path: $TEST_FILE_PATH (no -g flag)" - echo "::notice::🎯 OPTIMIZED RERUN: Running failed test file from this shard" - echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\"" - - npx playwright test "$TEST_FILE_PATH" - fi - else - echo "::notice::⏭️ Test file '$TEST_FILENAME' not in this shard's run_files list." - echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..." - - # Check the status of this matrix job in the previous attempt - PREV_ATTEMPT=$((${{ github.run_attempt }} - 1)) - - if [ "$PREV_ATTEMPT" -ge 1 ]; then - JOB_NAME="e2e / ${{ matrix.testfolder }}" - - API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null) - - if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then - PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null) - - if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then - echo "::warning::⚠️ Previous shard was cancelled/abruptly ended - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - elif [ "$PREV_JOB_STATUS" = "success" ]; then - echo "::notice::✅ Previous shard completed successfully - no tests to rerun" - exit 0 - elif [ "$PREV_JOB_STATUS" = "failure" ]; then - echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - else - echo "::warning::⚠️ Unknown job status - running FULL test suite for safety" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::error::Failed to fetch job status from GitHub API - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::notice::✅ This is the first run, no previous attempt to check" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - fi - fi - else - echo "::notice::⏭️ Test file doesn't match shard folder exactly. Skipping." - exit 0 - fi - else - echo "::warning::Failed to extract test file path. Falling back to skip." - exit 0 - fi - else - echo "::notice::⏭️ OPTIMIZED RERUN: No failed tests found for '$ACTUAL_FOLDER' shard in TestDino output" - echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..." - - # Check the status of this matrix job in the previous attempt - # Rule: If shard was cancelled/abruptly ended → treat as failed → run full test suite - PREV_ATTEMPT=$((${{ github.run_attempt }} - 1)) - - if [ "$PREV_ATTEMPT" -ge 1 ]; then - # Job name format: "e2e / {matrix.testfolder}" - JOB_NAME="e2e / ${{ matrix.testfolder }}" - - echo "DEBUG: Checking previous attempt #$PREV_ATTEMPT for job: $JOB_NAME" - - # Use GitHub REST API to get job status from specific attempt - # This is guaranteed to work and provides attempt-specific data - API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null) - - if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then - # Successfully got API response, parse it - PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null) - - echo "DEBUG: Previous job status: $PREV_JOB_STATUS" - - if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then - echo "::warning::⚠️ Previous shard was cancelled/abruptly ended (status: ${PREV_JOB_STATUS:-not found})" - echo "::warning::Treating as FAILED - running FULL test suite for this shard" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - elif [ "$PREV_JOB_STATUS" = "success" ]; then - echo "::notice::✅ Previous shard completed successfully - all tests passed" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - elif [ "$PREV_JOB_STATUS" = "failure" ]; then - echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded" - echo "::warning::This shouldn't happen - running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - else - echo "::warning::⚠️ Unknown job status: $PREV_JOB_STATUS" - echo "::warning::Running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::error::Failed to fetch job status from GitHub API" - echo "::warning::Falling back to running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::notice::✅ This is the first run, no previous attempt to check" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - fi - fi - else - # Normal run or fallback - if [ "$RERUN_MODE" = "fallback" ]; then - echo "::warning::⚠️ FALLBACK RERUN: Running all tests in matrix (TestDino optimization failed)" - fi - echo "DEBUG: Final command: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - fi + echo "DEBUG: Final command: npx playwright test $FILE_PATHS" + npx playwright test $FILE_PATHS else echo "No files specified to run for ${{ matrix.testfolder }} folder" fi @@ -1021,8 +732,8 @@ jobs: echo "::notice::This is VALID behavior for optimized reruns" echo "::notice::Creating empty report structure for downstream jobs" mkdir -p playwright-results/html-report - # Add stats object to make report compatible with TestDino - # startTime must be ISO 8601 string (not Unix timestamp number) for TestDino compatibility + # Add stats object so downstream report consumers get a well-formed report. + # startTime is an ISO 8601 string (not a Unix timestamp number). START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z) echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json echo "

No tests executed - all shards skipped

All test shards exited early because tests had already passed in a previous attempt.

" > playwright-results/html-report/index.html @@ -1161,8 +872,8 @@ jobs: fi fi - # Kinora upload — its own job for clean isolation (alongside report_openobserve and - # upload_testdino). Downloads the raw blob reports and merges them locally so trace.zip + # Kinora upload — its own job for clean isolation (alongside report_openobserve). + # Downloads the raw blob reports and merges them locally so trace.zip # attachments resolve on THIS runner, then uploads the run (+ traces) to the self-hosted # Kinora dashboard and updates a single sticky PR comment. Fully non-fatal. report_kinora: @@ -1256,154 +967,6 @@ jobs: gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" -f body="$BODY" >/dev/null && echo "Posted new Kinora comment." fi - # TestDino upload — its own job so it never gates playwright_summary. Only runs when tests - # failed (and uploads enabled); consumes the merged report via artifact. - upload_testdino: - timeout-minutes: 15 - name: Upload to TestDino - needs: [ui_integration_tests, merge_reports] - if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }} - runs-on: - labels: eks-openobserve-standard-4 - permissions: - contents: read - actions: read # download-artifact fetches the merged-report artifact - steps: - - name: Clone the current repo - uses: actions/checkout@v5 - - - name: Setup Node.js - uses: actions/setup-node@v7 - with: - node-version: 22 - - - name: Install project dependencies - run: cd tests/ui-testing && npm ci - - - name: Download merged report - uses: actions/download-artifact@v4 - with: - name: merged-report-${{ github.run_attempt }} - path: tests/ui-testing/playwright-results - - - name: Cache test failures to TestDino - env: - TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }} - run: | - cd tests/ui-testing - echo "::group::Caching test failure metadata to TestDino" - if [ -f "playwright-results/report.json" ]; then - echo "::notice::JSON report found at playwright-results/report.json" - ls -lh playwright-results/report.json - - # Copy report.json to repo root to ensure correct git metadata - mkdir -p ../../playwright-results - cp playwright-results/report.json ../../playwright-results/ - - # Run tdpw cache from repo root directory (for correct git metadata) - cd ../../playwright-results - # Use custom cache ID that includes run_id to distinguish separate workflow runs - # Format: gh_openobserve_main_12345678 (for main) or gh_openobserve_feature-branch_12345678 (for PRs) - # Sanitize branch name by replacing slashes with underscores - BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g') - CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}" - echo "::notice::Using cache ID: $CACHE_ID" - - # Suppress verbose debug output from TestDino, only show result - if CACHE_OUTPUT=$(npx --yes tdpw cache --cache-id "$CACHE_ID" 2>&1); then - # Extract only the success message, filter out debug spam - SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "") - if [ -n "$SUCCESS_MSG" ]; then - echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud" - else - echo "::notice::✅ Cache command completed" - fi - else - echo "::warning::⚠️ Failed to cache test metadata to TestDino" - echo "::warning::Rerun optimization may not work properly" - # Show output only on failure for debugging - echo "$CACHE_OUTPUT" - fi - else - echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)" - fi - echo "::endgroup::" - - - name: Upload to TestDino - if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }} - run: | - cd tests/ui-testing - - # Check if report.json exists (it won't exist if all tests were skipped due to rerun optimization) - if [ ! -f "playwright-results/report.json" ]; then - echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)" - exit 0 - fi - - # Fix startTime format: convert Unix timestamp (number) to ISO 8601 string for TestDino compatibility - # Check if startTime is a number and convert it - START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json) - if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then - echo "Converting startTime from Unix timestamp to ISO 8601 string" - jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp - mv playwright-results/report.json.tmp playwright-results/report.json - fi - - # Filter report to only include failed tests for TestDino upload. - # Retries with identical status+error are stripped to the final attempt only, - # reducing quota usage. Retries with differing status or error are kept in full. - node -e " - const fs = require('fs'); - const report = JSON.parse(fs.readFileSync('playwright-results/report.json', 'utf8')); - let strippedRetries = 0; // accumulated once per test via single-visit tree traversal - function getErrorFingerprint(r) { - // First line of error.message is stable across retries (stack/timing in later lines varies). - // Distinguish no-error-object (safe to strip on status alone) from error-without-message - // (unknown content — treat as unique so we never falsely deduplicate). - if (r.error?.message) return r.error.message.split('\n')[0].trim(); - return r.status + ':' + (r.error ? 'error-no-msg' : 'no-error'); - } - function filterSuite(s) { - const specs = (s.specs||[]) - .filter(sp=>sp.tests&&sp.tests.some(t=>t.status==='unexpected')) - .map(sp=>({ - ...sp, - tests: sp.tests.map(t=>{ - const results = t.results||[]; - if (!results.length) return {...t}; - const first = results[0]; - const allIdentical = results.length === 1 || results.every(r=> - r.status === first.status && - getErrorFingerprint(r) === getErrorFingerprint(first) - ); - if (allIdentical && results.length > 1) strippedRetries += results.length - 1; - return { - ...t, - results: allIdentical ? [results[results.length-1]] : results - }; - }) - })); - const suites = (s.suites||[]).map(filterSuite).filter(sub=>(sub.specs&&sub.specs.length)||(sub.suites&&sub.suites.length)); - return {...s,specs,suites}; - } - const filtered = { - ...report, - suites: report.suites.map(filterSuite).filter(s=>(s.specs&&s.specs.length)||(s.suites&&s.suites.length)), - stats: {...report.stats,expected:0,flaky:0,skipped:0} - }; - fs.writeFileSync('playwright-results/report-failures-only.json',JSON.stringify(filtered)); - const total=(report.stats.expected||0)+(report.stats.unexpected||0)+(report.stats.skipped||0)+(report.stats.flaky||0); - console.log('Uploading '+(report.stats.unexpected||0)+' failed tests out of '+total+' total to TestDino (stripped '+strippedRetries+' identical retry results)'); - " - echo "::notice::Report filtered to failures only" - - npx --yes tdpw@latest upload playwright-results \ - --json-report playwright-results/report-failures-only.json \ - --html-report playwright-results/html-report \ - --upload-html \ - --verbose \ - --token="${{ secrets.TESTDINO_API_TOKEN }}" - playwright_summary: timeout-minutes: 5 runs-on: diff --git a/.github/workflows/playwright_regression.yml b/.github/workflows/playwright_regression.yml index a48e64124b..78f7daeb13 100644 --- a/.github/workflows/playwright_regression.yml +++ b/.github/workflows/playwright_regression.yml @@ -44,7 +44,6 @@ env: # Timechart (visualize) tab is hidden by default; enable it so visualize E2E tests can reach it. ZO_TIMECHART_ENABLED: true ZO_SSRF_ALLOW_LOOPBACK: true - UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }} jobs: # Skip the whole suite when main has not moved since the last regression run. @@ -307,57 +306,7 @@ jobs: npx playwright install --with-deps chromium fi - - name: Check if this is a rerun - id: check_rerun - env: - GH_TOKEN: ${{ github.token }} - run: | - # Rerun-detection strategy: - # 1. On a rerun, query the GitHub Jobs API for this run. - # 2. Compare non-completed jobs in the current attempt against total unique - # matrix jobs: all present -> rerun ALL tests; a subset -> rerun FAILED only. - # 3. If the API returns an unexpected payload (rate-limit / error body with no - # .jobs key), default to "all tests". Safe (everything runs), just not - # cost-optimized — and it never crashes the step (see guard below). - echo "::notice::GitHub run attempt: ${{ github.run_attempt }}" - if [ "${{ github.run_attempt }}" -gt "1" ]; then - echo "is_rerun=true" >> $GITHUB_OUTPUT - echo "::notice::This is a rerun (attempt ${{ github.run_attempt }}). Will attempt to run only failed tests." - echo "::group::Detecting rerun type (all jobs vs failed only)" - CURRENT_ATTEMPT=${{ github.run_attempt }} - ALL_JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/jobs?filter=all&per_page=100") - CURRENT_ATTEMPT_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[(.jobs // [])[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == $CURRENT_ATTEMPT) | select(.status != \"completed\")] | length" 2>/dev/null || echo "0") - TOTAL_MATRIX_JOBS=$(echo "$ALL_JOBS_RESPONSE" | jq -r "[(.jobs // [])[] | select(.name | startswith(\"e2e /\")) | select(.run_attempt == 1) | .name] | unique | length" 2>/dev/null || echo "0") - echo "::notice::Total unique matrix job names: $TOTAL_MATRIX_JOBS" - echo "::notice::Jobs running in current attempt $CURRENT_ATTEMPT: $CURRENT_ATTEMPT_JOBS" - # If the jobs API returned an unexpected payload (a rate-limit/error body has no - # .jobs key, or curl failed) the counts above fall back to 0 via `.jobs // []`. - # Never let that crash the step: previously raw `.jobs[]` hit jq "Cannot iterate - # over null", exited 5 under `bash -e`, and skipped the entire test run. On any - # ambiguity, default to re-running ALL tests (safe, just not cost-optimized). - if ! [[ "$CURRENT_ATTEMPT_JOBS" =~ ^[0-9]+$ ]] || ! [[ "$TOTAL_MATRIX_JOBS" =~ ^[0-9]+$ ]] || [ "$TOTAL_MATRIX_JOBS" -eq 0 ]; then - echo "::warning::Could not determine rerun type from jobs API; defaulting to rerun_type=all (all tests)." - echo "rerun_type=all" >> $GITHUB_OUTPUT - elif [ "$CURRENT_ATTEMPT_JOBS" -eq "$TOTAL_MATRIX_JOBS" ]; then - echo "::notice::✅ DETECTED: Re-run ALL jobs (all $TOTAL_MATRIX_JOBS matrix jobs running)" - echo "rerun_type=all" >> $GITHUB_OUTPUT - else - echo "::notice::✅ DETECTED: Re-run FAILED jobs only ($CURRENT_ATTEMPT_JOBS out of $TOTAL_MATRIX_JOBS jobs)" - echo "rerun_type=failed" >> $GITHUB_OUTPUT - fi - echo "::endgroup::" - else - echo "is_rerun=false" >> $GITHUB_OUTPUT - echo "rerun_type=none" >> $GITHUB_OUTPUT - echo "::notice::This is the first run. Will run all tests in matrix." - fi - - name: Write .env and run ui-tests - env: - TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }} - GH_TOKEN: ${{ github.token }} run: | touch .env echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env @@ -372,50 +321,6 @@ jobs: mv .env tests/ui-testing cd tests/ui-testing - # Check if this is a rerun - IS_RERUN="${{ steps.check_rerun.outputs.is_rerun }}" - RERUN_TYPE="${{ steps.check_rerun.outputs.rerun_type }}" - - if [ "$IS_RERUN" = "true" ]; then - if [ "$RERUN_TYPE" = "all" ]; then - echo "::notice::🔄 Re-run ALL jobs detected - skipping TestDino optimization" - echo "::notice::Will run ALL tests in this shard regardless of previous failures" - RERUN_MODE="normal" - EXTRA_PW_FLAGS="" - else - echo "::group::Fetching last failed tests from TestDino" - BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g') - CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}" - echo "::notice::Using cache ID: $CACHE_ID" - TEMP_OUTPUT="/tmp/testdino_output_$$.txt" - if npx tdpw last-failed --cache-id "$CACHE_ID" > "$TEMP_OUTPUT" 2>&1; then - LAST_FAILED_FLAGS="$(grep -E '^[A-Za-z_-]+/[a-zA-Z0-9_-]+\.spec\.js' "$TEMP_OUTPUT" | tail -1)" - rm -f "$TEMP_OUTPUT" - if [ -n "$LAST_FAILED_FLAGS" ] && [ "$LAST_FAILED_FLAGS" != "No failed tests found" ]; then - echo "::notice::✅ Found failed tests from previous run." - echo "::notice::TestDino flags: $LAST_FAILED_FLAGS" - RERUN_MODE="optimized" - EXTRA_PW_FLAGS="$LAST_FAILED_FLAGS" - else - echo "::warning::TestDino returned no failed tests. Falling back to running all tests in this matrix job." - echo "::warning::This means you'll be billed for all tests. Please investigate why last-failed returned empty." - RERUN_MODE="fallback" - EXTRA_PW_FLAGS="" - fi - else - echo "::error::Failed to fetch last failed tests from TestDino (exit code: $?)." - echo "::warning::Falling back to running all tests in this matrix job." - echo "::warning::This means you'll be billed for all tests. Please check TestDino integration." - RERUN_MODE="fallback" - EXTRA_PW_FLAGS="" - fi - echo "::endgroup::" - fi - else - RERUN_MODE="normal" - EXTRA_PW_FLAGS="" - fi - FILE_LIST="${{ join(matrix.run_files, ' ') }}" echo "DEBUG: FILE_LIST = $FILE_LIST" echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}" @@ -446,152 +351,8 @@ jobs: FILE_PATHS="$FILE_PATHS $FILE_PATH" done - if [ "$RERUN_MODE" = "optimized" ]; then - echo "DEBUG: Full TestDino output (all failed tests): $EXTRA_PW_FLAGS" - echo "DEBUG: Current shard folder: $ACTUAL_FOLDER" - - SHARD_TESTS="" - - if echo "$EXTRA_PW_FLAGS" | grep -iq "$ACTUAL_FOLDER/.*\.spec\.js"; then - echo "DEBUG: Found test(s) for this shard in TestDino output" - - TEST_FILE=$(echo "$EXTRA_PW_FLAGS" | grep -ioE "$ACTUAL_FOLDER/[^ ]+\.spec\.js" | head -1) - - if [ -n "$TEST_FILE" ]; then - if [[ "${TEST_FILE,,}" == "${ACTUAL_FOLDER,,}/"* ]]; then - TEST_FILENAME=$(basename "$TEST_FILE") - - if echo "$FILE_LIST" | grep -qw "$TEST_FILENAME"; then - TEST_FILE_PATH=$(echo "$EXTRA_PW_FLAGS" | sed "s/ -g.*//" | sed "s|^|./playwright-tests/|") - - if echo "$EXTRA_PW_FLAGS" | grep -q " -g "; then - TEST_NAME=$(echo "$EXTRA_PW_FLAGS" | sed 's/.*-g //' | sed 's/^"\(.*\)"$/\1/') - echo "DEBUG: File path: $TEST_FILE_PATH" - echo "DEBUG: Test name (unquoted): $TEST_NAME" - echo "::notice::🎯 OPTIMIZED RERUN: Running failed test from this shard" - echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\" -g \"$TEST_NAME\"" - npx playwright test "$TEST_FILE_PATH" -g "$TEST_NAME" - else - echo "DEBUG: File path: $TEST_FILE_PATH (no -g flag)" - echo "::notice::🎯 OPTIMIZED RERUN: Running failed test file from this shard" - echo "DEBUG: Final command: npx playwright test \"$TEST_FILE_PATH\"" - npx playwright test "$TEST_FILE_PATH" - fi - else - echo "::notice::⏭️ Test file '$TEST_FILENAME' not in this shard's run_files list." - echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..." - - PREV_ATTEMPT=$((${{ github.run_attempt }} - 1)) - - if [ "$PREV_ATTEMPT" -ge 1 ]; then - JOB_NAME="e2e / ${{ matrix.testfolder }}" - - API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null) - - if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then - PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null) - - if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then - echo "::warning::⚠️ Previous shard was cancelled/abruptly ended - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - elif [ "$PREV_JOB_STATUS" = "success" ]; then - echo "::notice::✅ Previous shard completed successfully - no tests to rerun" - exit 0 - elif [ "$PREV_JOB_STATUS" = "failure" ]; then - echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - else - echo "::warning::⚠️ Unknown job status - running FULL test suite for safety" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::error::Failed to fetch job status from GitHub API - running FULL test suite" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::notice::✅ This is the first run, no previous attempt to check" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - fi - fi - else - echo "::notice::⏭️ Test file doesn't match shard folder exactly. Skipping." - exit 0 - fi - else - echo "::warning::Failed to extract test file path. Falling back to skip." - exit 0 - fi - else - echo "::notice::⏭️ OPTIMIZED RERUN: No failed tests found for '$ACTUAL_FOLDER' shard in TestDino output" - echo "::notice::Checking if this shard was cancelled/abruptly ended in previous attempt..." - - PREV_ATTEMPT=$((${{ github.run_attempt }} - 1)) - - if [ "$PREV_ATTEMPT" -ge 1 ]; then - JOB_NAME="e2e / ${{ matrix.testfolder }}" - - echo "DEBUG: Checking previous attempt #$PREV_ATTEMPT for job: $JOB_NAME" - - API_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ - -H "Accept: application/vnd.github.v3+json" \ - "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/$PREV_ATTEMPT/jobs" 2>/dev/null) - - if [ $? -eq 0 ] && [ -n "$API_RESPONSE" ]; then - PREV_JOB_STATUS=$(echo "$API_RESPONSE" | jq -r ".jobs[] | select(.name == \"$JOB_NAME\") | .conclusion" 2>/dev/null) - - echo "DEBUG: Previous job status: $PREV_JOB_STATUS" - - if [ "$PREV_JOB_STATUS" = "cancelled" ] || [ "$PREV_JOB_STATUS" = "null" ] || [ -z "$PREV_JOB_STATUS" ]; then - echo "::warning::⚠️ Previous shard was cancelled/abruptly ended (status: ${PREV_JOB_STATUS:-not found})" - echo "::warning::Treating as FAILED - running FULL test suite for this shard" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - elif [ "$PREV_JOB_STATUS" = "success" ]; then - echo "::notice::✅ Previous shard completed successfully - all tests passed" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - elif [ "$PREV_JOB_STATUS" = "failure" ]; then - echo "::warning::⚠️ Previous shard failed but TestDino has no failures recorded" - echo "::warning::This shouldn't happen - running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - else - echo "::warning::⚠️ Unknown job status: $PREV_JOB_STATUS" - echo "::warning::Running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::error::Failed to fetch job status from GitHub API" - echo "::warning::Falling back to running FULL test suite for safety" - echo "DEBUG: Running all matrix tests: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - exit $? - fi - else - echo "::notice::✅ This is the first run, no previous attempt to check" - echo "::notice::No failed tests to rerun. Exiting successfully." - exit 0 - fi - fi - else - # Normal run or fallback - if [ "$RERUN_MODE" = "fallback" ]; then - echo "::warning::⚠️ FALLBACK RERUN: Running all tests in matrix (TestDino optimization failed)" - fi - echo "DEBUG: Final command: npx playwright test $FILE_PATHS" - npx playwright test $FILE_PATHS - fi + echo "DEBUG: Final command: npx playwright test $FILE_PATHS" + npx playwright test $FILE_PATHS else echo "No files specified to run for ${{ matrix.testfolder }} folder" fi @@ -750,115 +511,6 @@ jobs: retention-days: 1 if-no-files-found: warn - # TestDino upload — its own job so it never gates playwright_summary. Only runs when tests - # failed (and uploads are enabled); consumes the merged report via artifact. - upload_testdino: - timeout-minutes: 15 - name: Upload to TestDino - needs: [ui_integration_tests, merge_reports] - if: ${{ !cancelled() && needs.ui_integration_tests.result == 'failure' && needs.merge_reports.result == 'success' }} - runs-on: - labels: eks-openobserve-standard-4 - permissions: - contents: read - actions: read # download-artifact fetches the merged-report artifact - steps: - - name: Clone the current repo - uses: actions/checkout@v5 - - - name: Setup Node.js - uses: actions/setup-node@v7 - with: - node-version: 24 - - - name: Install project dependencies - run: cd tests/ui-testing && npm ci - - - name: Download merged report - uses: actions/download-artifact@v4 - with: - name: merged-report-${{ github.run_attempt }} - path: tests/ui-testing/playwright-results - - - name: Cache test failures to TestDino - env: - TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }} - run: | - cd tests/ui-testing - echo "::group::Caching test failure metadata to TestDino" - if [ -f "playwright-results/report.json" ]; then - echo "::notice::JSON report found at playwright-results/report.json" - ls -lh playwright-results/report.json - - mkdir -p ../../playwright-results - cp playwright-results/report.json ../../playwright-results/ - - cd ../../playwright-results - BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g') - CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}" - echo "::notice::Using cache ID: $CACHE_ID" - - if CACHE_OUTPUT=$(npx tdpw cache --cache-id "$CACHE_ID" 2>&1); then - SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "") - if [ -n "$SUCCESS_MSG" ]; then - echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud" - else - echo "::notice::✅ Cache command completed" - fi - else - echo "::warning::⚠️ Failed to cache test metadata to TestDino" - echo "::warning::Rerun optimization may not work properly" - echo "$CACHE_OUTPUT" - fi - else - echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)" - fi - echo "::endgroup::" - - - name: Upload to TestDino - if: ${{ env.UPLOAD_TO_TESTDINO == 'true' }} - run: | - cd tests/ui-testing - - if [ ! -f "playwright-results/report.json" ]; then - echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)" - exit 0 - fi - - START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json) - if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then - echo "Converting startTime from Unix timestamp to ISO 8601 string" - jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp - mv playwright-results/report.json.tmp playwright-results/report.json - fi - - # Filter report to only include failed tests for TestDino upload - node -e " - const fs = require('fs'); - const report = JSON.parse(fs.readFileSync('playwright-results/report.json', 'utf8')); - function filterSuite(s) { - const specs = (s.specs||[]).filter(sp=>sp.tests&&sp.tests.some(t=>t.status==='unexpected')); - const suites = (s.suites||[]).map(filterSuite).filter(sub=>(sub.specs&&sub.specs.length)||(sub.suites&&sub.suites.length)); - return {...s,specs,suites}; - } - const filtered = { - ...report, - suites: report.suites.map(filterSuite).filter(s=>(s.specs&&s.specs.length)||(s.suites&&s.suites.length)), - stats: {...report.stats,expected:0,flaky:0,skipped:0} - }; - fs.writeFileSync('playwright-results/report-failures-only.json',JSON.stringify(filtered)); - const total=(report.stats.expected||0)+(report.stats.unexpected||0)+(report.stats.skipped||0)+(report.stats.flaky||0); - console.log('Uploading '+(report.stats.unexpected||0)+' failed tests out of '+total+' total to TestDino'); - " - echo "::notice::Report filtered to failures only" - - npx --yes tdpw@latest upload playwright-results \ - --json-report playwright-results/report-failures-only.json \ - --html-report playwright-results/html-report \ - --upload-html \ - --verbose \ - --token="${{ secrets.TESTDINO_API_TOKEN }}" - # OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one # run-summary doc (stream: ci_regression) + per-shard rows (ci_regression_shards). The step # is continue-on-error and the poster script never fails, so this can never break the run. diff --git a/tests/ui-testing/playwright.config.js b/tests/ui-testing/playwright.config.js index d776fb749d..66d1681522 100644 --- a/tests/ui-testing/playwright.config.js +++ b/tests/ui-testing/playwright.config.js @@ -42,7 +42,7 @@ module.exports = defineConfig({ ] : [ ['html', { outputFolder: 'playwright-results/html-report', open: 'never' }], // HTML reporter - ['json', { outputFile: 'playwright-results/report.json' }] // JSON reporter for TestDino + ['json', { outputFile: 'playwright-results/report.json' }] // JSON reporter for downstream report consumers ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: {