name: Playwright UI Tests on: # No `push: main` trigger: the merge_group (merge-queue) run already builds and # tests the exact commit that lands on main, so a post-merge push run would be a # full-suite duplicate. The merge_group run is now the trusted main-ward build and # also warms the shared Rust cache (see save-if on the rust-cache step below). pull_request: branches: - "**" # labeled/unlabeled so 'ready-for-ci' gates CI; only a new commit cancels an in-flight run (see concurrency); a draft starts no paid CI (an in-flight run still finishes). # auto_merge_enabled: the one full-matrix run per PR, at merge time (see generate_matrix). types: [opened, reopened, synchronize, labeled, unlabeled, auto_merge_enabled, ready_for_review, converted_to_draft] merge_group: schedule: - cron: '30 20 * * *' # 2:00 AM IST # Manual run on any branch (the "Use workflow from" ref); no PR, label or path filter applies. workflow_dispatch: inputs: shards: description: "Comma-separated testfolder names from tests/ui-testing/ci-matrix/ci_matrix.json (empty = all shards)" required: false default: "" type: string concurrency: # A manual run gets its own group so it never cancels, or is cancelled by, a PR, queue or nightly run. group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || (github.event_name == 'workflow_dispatch' && format('dispatch-{0}', github.run_id)) || github.ref }} # Only a new commit cancels: a cancelled same-SHA run leaves a failed required summary that a newer run cannot clear. cancel-in-progress: ${{ github.event_name != 'pull_request' || github.event.action == 'synchronize' }} env: COLUMNS: 150 ZO_ROOT_USER_EMAIL: root@example.com ZO_ROOT_USER_PASSWORD: Complexpass#123 ZO_BASE_URL: http://localhost:5080 WS_ZO_BASE_URL: ws://localhost:5080 ZO_BASE_URL_SC: http://localhost:5080 ZO_BASE_URL_SC_UI: http://localhost:5080 INGESTION_URL: http://localhost:5080 ORGNAME: default ZO_WEB_URL: "http://localhost:5080" ZO_QUICK_MODE_NUM_FIELDS: 100 ZO_QUICK_MODE_STRATEGY: first ZO_ALLOW_USER_DEFINED_SCHEMAS: true ZO_INGEST_ALLOWED_UPTO: 5 ZO_SLO_BACKFILL_CHUNK_SECS: 86400 # matches the server default; per-shard override below ZO_FEATURE_QUERY_EXCLUDE_ALL: false ZO_USAGE_BATCH_SIZE: 200 ZO_USAGE_PUBLISH_INTERVAL: 2 ZO_MIN_AUTO_REFRESH_INTERVAL: 5 ZO_STREAMING_ENABLED: true ZO_COLS_PER_RECORD_LIMIT: "80000" # SMTP was already enabled here, but with no host/port it fell back to the # server defaults (localhost:25) where nothing listens — so destinations could # be SAVED (the enabled-gate passes) while every send failed silently. The # Mailpit step below gives that flag something to talk to, which also makes the # delivery assertions in Alerts/dl-email-destinations.spec.js runnable. ZO_SMTP_ENABLED: true ZO_SMTP_HOST: localhost # 1026 is the DL relay, not Mailpit — it expands DL_ADDRESS and forwards # everything else through untouched, so non-email shards see no difference. ZO_SMTP_PORT: 1026 ZO_SMTP_FROM_EMAIL: alerts@ci.openobserve.test # Read by tests/ui-testing/playwright-tests/utils/mail-sink.js MAILPIT_URL: http://localhost:8025 # A distribution list that exists only inside the runner: one alias, three # member mailboxes. Lets the fan-out case run with no external mail provider, # no secrets and no egress. DL_ADDRESS: ci-dl@ci.openobserve.test DL_MEMBERS: ci-dl-m1@ci.openobserve.test,ci-dl-m2@ci.openobserve.test,ci-dl-m3@ci.openobserve.test ZO_FORMAT_STREAM_NAME_TO_LOWERCASE: false ZO_CREATE_ORG_THROUGH_INGESTION: true ZO_UTF8_VIEW_ENABLED: false ZO_ENABLE_CROSS_LINKING: true # Timechart (visualize) tab is hidden by default; enable it so visualize E2E tests can reach it. ZO_TIMECHART_ENABLED: true # Allow alert destinations to target localhost so the test validation stream # (http://localhost:5080/api/.../..._json) is not blocked by the SSRF guard. ZO_SSRF_ALLOW_LOOPBACK: true jobs: check_changes: # Runs on PRs carrying the 'ready-for-ci' label and on this repo's nightly schedule; the merge queue runs unit tests only (see docs/ci). if: ${{ github.event_name != 'merge_group' && (github.event_name != 'schedule' || github.repository == 'openobserve/openobserve') && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false)) }} timeout-minutes: 5 name: check_changes runs-on: ubuntu-latest permissions: contents: read actions: read # for `gh run list` (last nightly's commit and result) outputs: has_changes: ${{ steps.check.outputs.has_changes }} steps: # Debounce: a push inside this window cancels the run (concurrency) before anything paid starts. - if: github.event_name == 'pull_request' run: sleep 60 - uses: actions/checkout@v5 with: fetch-depth: 0 # Blobless: the diff needs commits and trees only, never file contents. filter: blob:none # One shared classification for every suite (see .github/actions/classify-changes); a change to # this workflow file re-tests itself on top of it. - uses: ./.github/actions/classify-changes id: classify # A nightly on the main commit the last nightly already passed on would only repeat its result. - id: main_changed if: github.event_name == 'schedule' env: GH_TOKEN: ${{ github.token }} GH_REPO: ${{ github.repository }} run: | # On any gh failure LAST stays empty and the nightly runs (fail-safe: run rather than skip). LAST=$(gh run list --workflow "$GITHUB_WORKFLOW" --event schedule --limit 50 \ --json databaseId,headSha,status,conclusion \ --jq "[.[] | select(.databaseId < $GITHUB_RUN_ID and .status == \"completed\")] | .[0] // empty | .headSha + \" \" + .conclusion" \ 2>/dev/null || echo "") read -r LAST_SHA LAST_CONCLUSION <<< "$LAST" || true echo "Current main SHA: $GITHUB_SHA" echo "Last nightly: ${LAST_SHA:-} (${LAST_CONCLUSION:-})" # A red main must stay red, so only a passed nightly on the same commit lets this one skip. if [ -n "$LAST_SHA" ] && [ "$LAST_SHA" = "$GITHUB_SHA" ] && [ "$LAST_CONCLUSION" = "success" ]; then echo "changed=false" >> "$GITHUB_OUTPUT" echo "::notice::main unchanged since the last nightly passed ($GITHUB_SHA); skipping." else echo "changed=true" >> "$GITHUB_OUTPUT" fi # Rejects an unknown shard name here, before build_binary spends a runner on it. - name: Validate requested shards if: github.event_name == 'workflow_dispatch' env: SHARDS: ${{ inputs.shards }} run: node .github/scripts/build-ci-matrix.js tests/ui-testing/ci-matrix/ci_matrix.json --only-shards "$SHARDS" > /dev/null - id: check env: EVENT_NAME: ${{ github.event_name }} MAIN_CHANGED: ${{ steps.main_changed.outputs.changed }} HAS_CHANGES: ${{ steps.classify.outputs.run_playwright }} CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }} run: | if [ "$EVENT_NAME" = "schedule" ]; then echo "has_changes=$([ "$MAIN_CHANGED" = "false" ] && echo false || echo true)" >> "$GITHUB_OUTPUT" elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then echo "has_changes=true" >> "$GITHUB_OUTPUT" elif [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' playwright.yml '; then echo "has_changes=true" >> "$GITHUB_OUTPUT" else echo "has_changes=false" >> "$GITHUB_OUTPUT" fi resolve_binary: timeout-minutes: 5 name: resolve_binary needs: [check_changes] if: needs.check_changes.outputs.has_changes == 'true' runs-on: ubuntu-latest permissions: actions: read contents: read outputs: reuse: ${{ steps.resolve.outputs.reuse }} run_id: ${{ steps.resolve.outputs.run_id }} binary_artifact_name: ${{ steps.resolve.outputs.binary_artifact_name }} frontend_artifact_name: ${{ steps.resolve.outputs.frontend_artifact_name }} steps: - name: Find exact-SHA Playwright runtime id: resolve env: GH_TOKEN: ${{ github.token }} SOURCE_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} ARTIFACT_SHA: ${{ github.sha }} CURRENT_RUN_ID: ${{ github.run_id }} run: | set -euo pipefail binary_artifact="release-ci-binary-${ARTIFACT_SHA}" frontend_artifact="frontend-dist-${ARTIFACT_SHA}" selected_run_id="$CURRENT_RUN_ID" reuse=false # Scope candidates to this workflow and source head. ARTIFACT_SHA is # the exact tested merge/commit SHA, so a base-branch update cannot # reuse a binary built from an older synthetic merge commit. run_ids="$( gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/workflows/playwright.yml/runs?head_sha=${SOURCE_HEAD_SHA}&per_page=100" \ --jq '.workflow_runs | sort_by(.created_at) | reverse | .[].id' \ 2>/dev/null || true )" while IFS= read -r run_id; do [ -n "$run_id" ] || continue artifact_names="$( gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100" \ --jq '.artifacts[] | select(.expired == false) | .name' \ 2>/dev/null || true )" if grep -Fxq "$binary_artifact" <<< "$artifact_names" && grep -Fxq "$frontend_artifact" <<< "$artifact_names"; then selected_run_id="$run_id" reuse=true break fi done <<< "$run_ids" { echo "reuse=$reuse" echo "run_id=$selected_run_id" echo "binary_artifact_name=$binary_artifact" echo "frontend_artifact_name=$frontend_artifact" } >> "$GITHUB_OUTPUT" if [ "$reuse" = "true" ]; then echo "Reusing exact-SHA Playwright runtime from run $selected_run_id" else echo "No reusable exact-SHA Playwright runtime found; build_binary will run" fi build_binary: timeout-minutes: 45 name: build_binary needs: [check_changes, resolve_binary] if: >- always() && needs.check_changes.result == 'success' && needs.check_changes.outputs.has_changes == 'true' && needs.resolve_binary.result == 'success' && needs.resolve_binary.outputs.reuse != 'true' runs-on: labels: ubicloud-standard-16 permissions: contents: read steps: - name: Remove unused tools run: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: Clone the current repo uses: actions/checkout@v5 - uses: ./.github/actions/github-auth - name: Setup Rust Toolchain uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: nightly-2026-10-24 targets: x86_64-unknown-linux-gnu - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: cache-on-failure: true shared-key: playwright-release-ci # Save the shared cache on merge_group runs (the main-ward build now that # push:main is gone); keep the main-ref check for any non-PR fallback. save-if: ${{ github.event_name == 'merge_group' || github.ref == 'refs/heads/main' }} - name: Install Protoc run: | bash ./.github/protoc.sh - uses: actions/setup-node@v7 with: node-version: 24 cache: npm cache-dependency-path: web/package-lock.json - name: Install frontend dependencies run: cd web && npm ci - name: Build binary and frontend env: NODE_OPTIONS: "--max-old-space-size=8192" GIT_HASH_OVERRIDE: ${{ github.sha }} run: | mkdir -p /tmp/dist bash -c "cd web && npm run build && cp -r ./dist/. /tmp/dist/" & NPM_PID=$! sed -i 's|folder = "../../web/dist/"|folder = "/tmp/dist/"|' src/web/src/lib.rs && grep -q 'folder = "/tmp/dist/"' src/web/src/lib.rs cargo build --profile release-ci --target x86_64-unknown-linux-gnu wait $NPM_PID test -f /tmp/dist/index.html - name: Upload binary artifact uses: actions/upload-artifact@v4 with: name: release-ci-binary-${{ github.sha }} path: target/x86_64-unknown-linux-gnu/release-ci/openobserve retention-days: 1 - name: Upload frontend artifact uses: actions/upload-artifact@v4 with: name: frontend-dist-${{ github.sha }} path: /tmp/dist retention-days: 1 if-no-files-found: error # Build the shard matrix from tests/ui-testing/ci-matrix/ci_matrix.json (single source of truth, # shared with ENT via build-ci-matrix.js). Runs under the same gate as build_binary so # ui_integration_tests always has a matrix whenever it is allowed to run. generate_matrix: timeout-minutes: 5 name: generate_matrix needs: [check_changes] if: needs.check_changes.outputs.has_changes == 'true' runs-on: ubuntu-latest permissions: contents: read outputs: matrix: ${{ steps.gen.outputs.matrix }} # Kept from the earlier attempt on "Re-run failed jobs"; equal to run_attempt on "Re-run all jobs". attempt: ${{ github.run_attempt }} smoke_active: ${{ steps.gen.outputs.smoke_active }} shards_selected: ${{ steps.gen.outputs.shards_selected }} steps: - uses: actions/checkout@v5 with: # Full history only where the selection diff needs it; merge_group never diffs. fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }} filter: blob:none - name: Selection contract tests run: node --test .github/scripts/build-ci-matrix.test.js - id: gen env: # Smoke selection runs on PR pushes; the merge queue no longer runs Playwright, so the # full matrix runs once per PR when auto-merge is enabled ("Merge when ready"). # Escape hatches: the 'e2e-full' PR label forces the full matrix on every push; the # PW_SMOKE_DISABLED repo variable ('true') turns smoke off everywhere without a code change. IS_PR: ${{ github.event_name == 'pull_request' }} HAS_FULL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'e2e-full') || github.event.pull_request.auto_merge != null }} SMOKE_DISABLED: ${{ vars.PW_SMOKE_DISABLED }} BASE_REF: ${{ github.base_ref }} SHARDS: ${{ inputs.shards }} run: | SMOKE_DISABLED=$(echo "$SMOKE_DISABLED" | tr '[:upper:]' '[:lower:]') SELECT_ARGS="" if [ "$IS_PR" == "true" ] && [ "$HAS_FULL_LABEL" != "true" ] && [ "$SMOKE_DISABLED" != "true" ]; then # Three-dot: diff from the merge base, so commits that landed on the base branch # after this PR's merge commit was cut never leak into the selection. # --no-renames lists a moved file under BOTH paths, so the shard owning the old # location still runs; quotepath off keeps non-ASCII names matchable. if git -c core.quotepath=false diff --no-renames --name-only "origin/${BASE_REF:-main}...${{ github.sha }}" > /tmp/pw-changed-files.txt; then SELECT_ARGS="--select-for-changes /tmp/pw-changed-files.txt" else echo "::warning::changed-file diff failed — running the full matrix" fi fi # || true: under bash -e a die() would abort before the log below ever prints. MATRIX=$(node .github/scripts/build-ci-matrix.js tests/ui-testing/ci-matrix/ci_matrix.json $SELECT_ARGS --only-shards "$SHARDS" 2> /tmp/pw-select-log.txt) || true cat /tmp/pw-select-log.txt >&2 if [ -z "$MATRIX" ]; then echo "::error::matrix generation failed (see build-ci-matrix log above)"; exit 1; fi echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" COUNT=$(echo "$MATRIX" | node -e "const m=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log('generated',m.include.length,'shards:', m.include.map(s=>s.testfolder).join(', ')); require('fs').appendFileSync(process.env.GITHUB_OUTPUT, 'shards_selected='+m.include.length+'\n')") echo "smoke_active=$([ -n "$SELECT_ARGS" ] && echo true || echo false)" >> "$GITHUB_OUTPUT" # Durable record of what ran and why, visible from the PR checks page. { echo "### Playwright shard selection" echo "Mode: $([ -n "$SELECT_ARGS" ] && echo "smoke (changed-path selection)" || { [ -n "$SHARDS" ] && echo "chosen shards (manual run)"; } || echo "full matrix")" echo '```' echo "$COUNT" cat /tmp/pw-select-log.txt echo '```' } >> "$GITHUB_STEP_SUMMARY" # One sticky comment per PR (found by the hidden marker) so the triage result updates # in place on every push instead of spamming a fresh comment. - name: Upsert shard-selection comment on PR if: ${{ github.event_name == 'pull_request' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.ORG_ADMIN_TOKEN }} # PAT: comment as a human, not github-actions[bot] REPO: ${{ github.repository }} PR_NUMBER: ${{ github.event.pull_request.number }} HEAD_SHA: ${{ github.event.pull_request.head.sha }} MATRIX: ${{ steps.gen.outputs.matrix }} SMOKE_ACTIVE: ${{ steps.gen.outputs.smoke_active }} ATTEMPT: ${{ github.run_attempt }} run: | MARKER="" BODY=$(MARKER="$MARKER" node .github/scripts/pw-triage-comment.js /tmp/pw-select-log.txt) CID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \ -q ".[] | select(.body | contains(\"$MARKER\")) | .id" | head -1) if [ -n "$CID" ]; then gh api -X PATCH "repos/$REPO/issues/comments/$CID" -f body="$BODY" >/dev/null && echo "Updated triage comment ($CID)." else gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" -f body="$BODY" >/dev/null && echo "Posted triage comment." fi # A same-commit run queued behind a passed one would only repeat it; own job so build_binary never waits on generate_matrix. prior_pass: timeout-minutes: 5 name: prior_pass needs: [generate_matrix] # A manual re-run asks for the tests to run again, so only a first attempt may reuse a pass. if: github.event_name == 'pull_request' && github.run_attempt == 1 && needs.generate_matrix.result == 'success' runs-on: ubuntu-latest permissions: actions: read outputs: already_passed: ${{ steps.find.outputs.already_passed }} run_url: ${{ steps.find.outputs.run_url }} steps: - name: Find a passed run of this exact commit id: find env: GH_TOKEN: ${{ github.token }} SOURCE_HEAD_SHA: ${{ github.event.pull_request.head.sha }} ARTIFACT_SHA: ${{ github.sha }} CURRENT_RUN_ID: ${{ github.run_id }} MATRIX: ${{ needs.generate_matrix.outputs.matrix }} run: | set -uo pipefail already_passed=false run_url="" tmp="$(mktemp -d)" shards="$(jq -r '.include[].testfolder' <<< "$MATRIX" 2>/dev/null | sort -u || true)" # Only completed, successful runs count: a failure is never copied and an in-flight run proves nothing. candidates="$( gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/workflows/playwright.yml/runs?head_sha=${SOURCE_HEAD_SHA}&event=pull_request&status=success&per_page=100" \ --jq "[.workflow_runs[] | select(.id != ${CURRENT_RUN_ID} and .status == \"completed\" and .conclusion == \"success\")] | sort_by(.created_at) | reverse | .[] | \"\(.id) \(.run_attempt) \(.html_url)\"" \ 2>/dev/null || true )" while read -r run_id attempt url; do [ -n "$shards" ] && [ -n "${run_id:-}" ] || continue passed="$( gh api --method GET --paginate \ "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?filter=latest&per_page=100" \ --jq '.jobs[] | select(.conclusion == "success" and (.name | startswith("e2e / "))) | .name | ltrimstr("e2e / ")' \ 2>/dev/null | sort -u || true )" if [ -n "$(comm -23 <(printf '%s\n' "$shards") <(printf '%s\n' "$passed"))" ]; then echo "Run $run_id did not pass every selected shard"; continue fi # The head SHA alone is not the tested commit: the PR merge commit moves with the base branch. advice_id="$( gh api --method GET \ "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?name=rerun-advice-attempt-${attempt}" \ --jq '[.artifacts[] | select(.expired == false)][0].id // empty' \ 2>/dev/null || true )" [ -n "$advice_id" ] || { echo "Run $run_id has no rerun advice to prove its tested commit"; continue; } tested_sha="$( gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${advice_id}/zip" > "$tmp/advice.zip" 2>/dev/null && unzip -p "$tmp/advice.zip" rerun-advice.json 2>/dev/null | jq -r '.tested_oss_sha // empty' 2>/dev/null || true )" if [ "$tested_sha" != "$ARTIFACT_SHA" ]; then echo "Run $run_id tested ${tested_sha:-an unknown commit}, not $ARTIFACT_SHA"; continue fi already_passed=true run_url="$url" break done <<< "$candidates" { echo "already_passed=$already_passed" echo "run_url=$run_url" } >> "$GITHUB_OUTPUT" if [ "$already_passed" = "true" ]; then echo "::notice::Every selected shard already passed on $ARTIFACT_SHA in $run_url; not re-running them." else echo "No earlier passed run covers this commit and shard selection; shards will run" fi ui_integration_tests: timeout-minutes: 45 name: e2e / ${{ matrix.testfolder }} needs: [resolve_binary, build_binary, generate_matrix, prior_pass] if: >- always() && !cancelled() && needs.resolve_binary.result == 'success' && needs.generate_matrix.result == 'success' && needs.prior_pass.outputs.already_passed != 'true' && (needs.resolve_binary.outputs.reuse == 'true' || needs.build_binary.result == 'success') runs-on: labels: eks-openobserve-standard-8 permissions: contents: read actions: read # container: # image: mcr.microsoft.com/playwright:v1.50.0-jammy # options: --user root strategy: fail-fast: false # Matrix is generated from tests/ui-testing/ci-matrix/ci_matrix.json by the # generate_matrix job (the single source of truth shared with ENT). To add # or move a spec, edit that JSON file — do NOT hand-edit a matrix here. matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }} steps: - name: Kill background apt processes run: | sudo systemctl stop apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true sudo systemctl stop apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true sudo systemctl disable apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true sudo flock --wait 30 /var/lib/dpkg/lock-frontend true 2>/dev/null || true - name: Clone the current repo uses: actions/checkout@v5 - name: Download binary artifact uses: actions/download-artifact@v5 with: name: ${{ needs.resolve_binary.outputs.binary_artifact_name }} path: release-ci-binary github-token: ${{ github.token }} repository: ${{ github.repository }} run-id: ${{ needs.resolve_binary.outputs.run_id }} - name: Download frontend artifact uses: actions/download-artifact@v5 with: name: ${{ needs.resolve_binary.outputs.frontend_artifact_name }} path: /tmp/dist github-token: ${{ github.token }} repository: ${{ github.repository }} run-id: ${{ needs.resolve_binary.outputs.run_id }} - name: Start mail sink (Mailpit) # A local SMTP server the tests can also READ. Runs as a static binary # rather than a service container: these shards run on a self-hosted EKS # runner where Docker-in-Docker is not guaranteed, and the app itself is # already launched the same way. ~2s, and it makes ZO_SMTP_* real. # # Gated to the one shard that needs it — every other shard would # otherwise pay a GitHub-releases download for nothing. if: matrix.testfolder == 'Alerts' env: # Pinned, not "latest": an upstream release must never change what this # shard does without a commit here. MAILPIT_VERSION: v1.21.8 run: | set +e curl -fsSL -o mailpit.tar.gz \ "https://github.com/axllent/mailpit/releases/download/${MAILPIT_VERSION}/mailpit-linux-amd64.tar.gz" \ && tar -xzf mailpit.tar.gz mailpit && chmod +x ./mailpit if [ ! -x ./mailpit ]; then echo "::warning::Mailpit ${MAILPIT_VERSION} could not be fetched; email delivery assertions will skip" exit 0 fi # Loopback only — this runner is shared, and delivered alert bodies # should not be readable from the rest of its network. ./mailpit --smtp 127.0.0.1:1025 --listen 127.0.0.1:8025 > mailpit.log 2>&1 & for i in $(seq 1 30); do curl -sf http://localhost:8025/api/v1/messages > /dev/null 2>&1 && { echo "Mailpit ready after ${i}s"; exit 0; } sleep 1 done echo "::warning::Mailpit did not start; email delivery assertions will skip" - name: Start OpenObserve # Per-shard override of the workflow-level default. ZO_QUICK_MODE_ENABLED is # read once at server start and is global to the process, so a shard is the # smallest unit that can change it. Set via "quick_mode_enabled" in # tests/ui-testing/ci-matrix/ci_matrix.json; shards that omit it get "false". env: ZO_QUICK_MODE_ENABLED: ${{ matrix.quick_mode_enabled || 'false' }} # Same per-shard rule: /api/{org}/synthetics* routes register only when true at start ("synthetics_enabled"). ZO_SYNTHETICS_ENABLED: ${{ matrix.synthetics_enabled || 'false' }} # Same per-shard rule: how far back (hours) ingestion accepts a record. # The workflow default of 5 DROPS older rows while still answering 200, # so a suite that seeds history (SLO measures a rolling 7-day window) # would fail as "No data" with nothing in the artifacts to explain it. # Set via "ingest_allowed_upto" in ci-matrix/ci_matrix.json; shards that # omit it keep the workflow default. ZO_INGEST_ALLOWED_UPTO: ${{ matrix.ingest_allowed_upto || env.ZO_INGEST_ALLOWED_UPTO }} # Same per-shard rule: how much history one SLO backfill chunk covers. # The production default (86400 = 1 day) makes a 7-day-window test SLO # backfill in 7 sequential chunks under ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1 # (SLO-Measurement's shard note above) — no test here asserts on # per-chunk timing or partial coverage, only on the final measured # state, so a shard-only larger chunk is free wall-clock, not a # behavior change. Set via "slo_backfill_chunk_secs" in # ci-matrix/ci_matrix.json; shards that omit it keep the server default. ZO_SLO_BACKFILL_CHUNK_SECS: ${{ matrix.slo_backfill_chunk_secs || env.ZO_SLO_BACKFILL_CHUNK_SECS }} run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 & - name: Wait for start run: | for i in $(seq 1 120); do if curl -sf http://localhost:5080/web/login > /dev/null 2>&1; then echo "Server ready after ${i}s" exit 0 fi sleep 1 done echo "Server failed to start in 120s" exit 1 - name: Ensure we are getting a reply from the server run: curl http://localhost:5080/web/login - uses: actions/setup-node@v7 with: node-version: 24 cache: npm cache-dependency-path: tests/ui-testing/package-lock.json - name: Start distribution-list relay # Mailpit is a sink: it accepts any address but stores ONE message, so it # can never show what a real distribution list does. This tiny relay sits # in front of it and expands the alias into per-member deliveries, the # same way alerts-content-templates.spec.js stands up an HTTP receiver to # capture what a webhook destination really sent. # # Gated to the one shard that needs it, matching the Mailpit step above. if: matrix.testfolder == 'Alerts' run: | if ! command -v node > /dev/null 2>&1; then echo "::warning::node not on PATH; DL fan-out case will skip" exit 0 fi node tests/ui-testing/scripts/dl-relay.js > dl-relay.log 2>&1 & for i in $(seq 1 20); do (echo > /dev/tcp/127.0.0.1/1026) >/dev/null 2>&1 && { echo "DL relay ready after ${i}s"; exit 0; } sleep 1 done echo "::warning::DL relay did not start; the fan-out case will skip" - name: Cache Playwright browser id: playwright-cache uses: actions/cache@v6 with: path: ~/.cache/ms-playwright key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }} # Expressions have no arithmetic, so the previous attempt number is computed here. - name: Resolve previous attempt id: prev run: echo "attempt=$(( ${{ github.run_attempt }} - 1 ))" >> "$GITHUB_OUTPUT" - name: Download previous attempt's failed-test list if: ${{ github.run_attempt > 1 }} continue-on-error: true uses: actions/download-artifact@v5 with: name: last-run-${{ matrix.testfolder }}-attempt-${{ steps.prev.outputs.attempt }} path: tests/ui-testing/last-run-prev - name: Install dependencies and run ui-tests run: | touch .env echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env echo "ZO_ROOT_USER_PASSWORD=${ZO_ROOT_USER_PASSWORD}" >> .env echo "ZO_BASE_URL=${ZO_BASE_URL}" >> .env echo "WS_ZO_BASE_URL=${WS_ZO_BASE_URL}" >> .env echo "ZO_BASE_URL_SC=${ZO_BASE_URL_SC}" >> .env echo "ZO_BASE_URL_SC_UI=${ZO_BASE_URL_SC_UI}" >> .env echo "INGESTION_URL=${INGESTION_URL}" >> .env echo "ORGNAME=${ORGNAME}" >> .env echo "ZO_SMTP_ENABLED=${ZO_SMTP_ENABLED}" >> .env echo "MAILPIT_URL=${MAILPIT_URL}" >> .env echo "DL_ADDRESS=${DL_ADDRESS}" >> .env echo "DL_MEMBERS=${DL_MEMBERS}" >> .env mv .env tests/ui-testing cd tests/ui-testing && npm ci # Install apt system dependencies (always needed). # azure.archive.ubuntu.com intermittently times out fetching font/lib # packages, which makes `install-deps` exit 100 and fails the whole job. # Give apt its own per-fetch retries and wrap the call in a backoff loop. echo 'Acquire::Retries "5";' | sudo tee /etc/apt/apt.conf.d/99retries >/dev/null for attempt in 1 2 3; do if npx playwright install-deps chromium; then echo "install-deps succeeded on attempt ${attempt}" break fi if [ "${attempt}" = 3 ]; then echo "::error::playwright install-deps chromium failed after 3 attempts" exit 1 fi echo "install-deps attempt ${attempt} failed; retrying in $((attempt * 20))s" sleep $((attempt * 20)) done # Get revision from playwright's own metadata CHROMIUM_REV=$(node -pe " require('./node_modules/playwright-core/browsers.json') .browsers.find(b => b.name === 'chromium').revision ") install_browser() { local NAME="$1" local BINARY="$2" local ZIP_PATH="$3" local DIR="$HOME/.cache/ms-playwright/${NAME}-${CHROMIUM_REV}" if [ -f "${DIR}/INSTALLATION_COMPLETE" ]; then echo "${NAME} already installed (cache hit)"; return 0 fi rm -rf "${DIR}" echo "Cache miss — downloading ${NAME} r${CHROMIUM_REV} via curl + system unzip" curl -fL --progress-bar \ "https://cdn.playwright.dev/dbazure/download/playwright/builds/${ZIP_PATH}" \ -o "/tmp/${NAME}.zip" mkdir -p "${DIR}" unzip -q "/tmp/${NAME}.zip" -d "${DIR}" rm -f "/tmp/${NAME}.zip" find "${DIR}" -name "${BINARY}" -type f -exec chmod 755 {} \; find "${DIR}" -name 'chrome_sandbox' -type f -exec chmod 4755 {} \; 2>/dev/null || true touch "${DIR}/INSTALLATION_COMPLETE" echo "${NAME} installed: $(ls ${DIR}/)" } install_browser "chromium" "chrome" \ "chromium/${CHROMIUM_REV}/chromium-linux.zip" install_browser "chromium_headless_shell" "headless_shell" \ "chromium/${CHROMIUM_REV}/chromium-headless-shell-linux.zip" # Get list of files to run using join function FILE_LIST="${{ join(matrix.run_files, ' ') }}" echo "DEBUG: FILE_LIST = $FILE_LIST" echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}" if [ -n "$FILE_LIST" ]; then # actual_folder (the real directory under playwright-tests/) comes straight # from the matrix now — the logical->directory mapping lives in ci-matrix/ci_matrix.json. ACTUAL_FOLDER="${{ matrix.actual_folder }}" echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER" # Build file paths FILE_PATHS="" for file in $FILE_LIST; do FILE_PATH="./playwright-tests/$ACTUAL_FOLDER/$file" echo "DEBUG: Will run file: $FILE_PATH" FILE_PATHS="$FILE_PATHS $FILE_PATH" done # Optional per-shard worker pin. Empty keeps playwright.config.js's # default; a shard whose specs contend for a shared server-side # resource sets "workers": 1 in ci_matrix.json, because fullyParallel # races separate spec FILES and mode:'serial' only orders tests # within one file. WORKERS_ARG="" if [ -n "${{ matrix.workers }}" ]; then WORKERS_ARG="--workers=${{ matrix.workers }}" echo "DEBUG: pinning workers to ${{ matrix.workers }}" fi # Re-run whole files that had a failure (a serial group needs its earlier tests). PREV=last-run-prev/.last-run.json if [ "$GITHUB_RUN_ATTEMPT" -gt 1 ] && [ "${{ needs.generate_matrix.outputs.attempt }}" = "$GITHUB_RUN_ATTEMPT" ]; then echo "Re-run all jobs: running the full shard" elif [ "$GITHUB_RUN_ATTEMPT" -gt 1 ] && [ -f "$PREV" ] \ && node -e "const r=require('./$PREV'); process.exit(r.status === 'failed' && (r.failedTests||[]).length ? 0 : 1)"; then mkdir -p test-results && cp "$PREV" test-results/.last-run.json LIST_JSON=$(mktemp) read -ra SHARD_FILES <<< "$FILE_PATHS" FAILED_FILES="" if PLAYWRIGHT_JSON_OUTPUT_NAME="$LIST_JSON" npx playwright test "${SHARD_FILES[@]}" --last-failed --list --reporter=json > /dev/null; then FAILED_FILES=$(node -e "const r=JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); console.log([...new Set((r.suites||[]).map(s => s.file).filter(Boolean))].sort().map(f => './playwright-tests/' + f).join(' '))" "$LIST_JSON" || true) fi # An attempt that dies before writing its own list must not upload the previous attempt's as its own. rm -f test-results/.last-run.json if [ -n "$FAILED_FILES" ]; then echo "Re-run: files with failed tests from attempt ${{ steps.prev.outputs.attempt }}: $FAILED_FILES" FILE_PATHS="$FAILED_FILES" else echo "::warning::Could not list the failed tests from attempt ${{ steps.prev.outputs.attempt }} — running the full shard" fi elif [ "$GITHUB_RUN_ATTEMPT" -gt 1 ]; then echo "::warning::No complete failed-test list from attempt ${{ steps.prev.outputs.attempt }} — running the full shard" fi echo "DEBUG: Final command: npx playwright test $FILE_PATHS $WORKERS_ARG" npx playwright test $FILE_PATHS $WORKERS_ARG else echo "No files specified to run for ${{ matrix.testfolder }} folder" fi - name: Upload blob report to GitHub Actions Artifacts if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: name: blob-report-${{ matrix.testfolder }}-attempt-${{ github.run_attempt }} path: tests/ui-testing/blob-report retention-days: 1 # job-status lets merge_reports tell a shard that died before writing results from one that passed. - name: Record shard result if: ${{ !cancelled() }} continue-on-error: true env: JOB_STATUS: ${{ job.status }} run: | cd tests/ui-testing rm -rf shard-record && mkdir shard-record printf '%s\n' "$JOB_STATUS" > shard-record/job-status if [ -f test-results/.last-run.json ]; then cp test-results/.last-run.json shard-record/; fi - name: Upload failed-test list for re-runs if: ${{ !cancelled() }} continue-on-error: true uses: actions/upload-artifact@v4 with: name: last-run-${{ matrix.testfolder }}-attempt-${{ github.run_attempt }} path: tests/ui-testing/shard-record include-hidden-files: true if-no-files-found: ignore retention-days: 1 # Playwright retries only a failed test, so a retry in a passing shard is a flaky test whose first failure the server log explains. - name: Detect flaky tests id: flaky if: success() continue-on-error: true run: | flaky=false for z in tests/ui-testing/blob-report/*.zip; do [ -f "$z" ] || continue retries=$(unzip -p "$z" report.jsonl 2>/dev/null | grep -cE '"method":"onTestBegin".*"retry":[1-9]' || true) if [ "${retries:-0}" -gt 0 ]; then flaky=true; echo "$z: $retries retried test attempt(s)"; fi done echo "flaky=$flaky" >> "$GITHUB_OUTPUT" - name: Show OpenObserve log tail if: failure() || steps.flaky.outputs.flaky == 'true' run: tail -n 200 o2.log || true - name: Upload OpenObserve log if: failure() || steps.flaky.outputs.flaky == 'true' continue-on-error: true uses: actions/upload-artifact@v4 with: name: o2-log-${{ matrix.testfolder }}-attempt-${{ github.run_attempt }} path: o2.log retention-days: 7 if-no-files-found: ignore merge_reports: timeout-minutes: 20 name: Merge Reports needs: [generate_matrix, ui_integration_tests] if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }} # eks runner: `playwright merge-reports` silently stops mid-merge (exits 0, ~3 of N blobs, no # output) on the ubicloud and github-hosted runners — but works on the eks runners (same as ENT). runs-on: labels: eks-openobserve-standard-4 permissions: contents: read actions: read # the no-blobs investigation reads run/job state from the Actions API outputs: # Compact Playwright stats (expected/unexpected/flaky/skipped) from the merged report.json, # consumed by report_to_openobserve to record per-test counts in ci_test_runs. stats: ${{ steps.report_stats.outputs.stats }} # Flaky + failed tests for the ci_test_results stream, consumed by # report_to_openobserve. JSON array: [{module,title,file,status}] where # status is "flaky"|"failed" (see .github/scripts/extract-flaky.js). "[]" if none. flaky: ${{ steps.report_flaky.outputs.flaky }} steps: - name: Clone the current repo uses: actions/checkout@v5 - name: Setup Node.js uses: actions/setup-node@v7 with: node-version: 24 cache: npm cache-dependency-path: tests/ui-testing/package-lock.json - name: Install project dependencies run: cd tests/ui-testing && npm ci - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v4 continue-on-error: true id: download_blobs with: path: tests/ui-testing/all-blob-reports pattern: blob-report-*-attempt-${{ github.run_attempt }} merge-multiple: true - name: Check if any artifacts were downloaded run: | if [ -d "tests/ui-testing/all-blob-reports" ] && [ "$(ls -A tests/ui-testing/all-blob-reports)" ]; then echo "✅ Artifacts downloaded successfully" ls -la tests/ui-testing/all-blob-reports else echo "⚠️ No artifacts found or download failed completely" # Don't fail here - let the Merge Reports step handle this fi - name: Merge Reports env: GH_TOKEN: ${{ github.token }} run: | cd tests/ui-testing # Check if directory exists and contains reports if [ ! -d "all-blob-reports" ] || [ -z "$(ls -A all-blob-reports 2>/dev/null)" ]; then echo "::warning::No blob reports found to merge." # Investigate: Check if all test jobs succeeded (meaning they skipped intentionally) # vs test jobs failed (meaning blob reporter might have failed) echo "::group::Investigating why no blob reports exist" JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs") # Count test matrix jobs by conclusion SUCCEEDED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "success")] | length') FAILED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "failure")] | length') TOTAL=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /"))] | length') echo "::notice::Test job results: $SUCCEEDED succeeded, $FAILED failed, $TOTAL total" if [ "$SUCCEEDED" -gt 0 ] && [ "$FAILED" -eq 0 ]; then echo "::notice::✅ All test jobs succeeded - they likely skipped because tests already passed" echo "::notice::This is VALID behavior for optimized reruns" echo "::notice::Creating empty report structure for downstream jobs" mkdir -p playwright-results/html-report # Add stats object so downstream report consumers get a well-formed report. # startTime is an ISO 8601 string (not a Unix timestamp number). START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z) echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json echo "

No tests executed - all shards skipped

All test shards exited early because tests had already passed in a previous attempt.

" > playwright-results/html-report/index.html echo "::endgroup::" exit 0 else echo "::error::❌ Some/all test jobs failed but no blob reports were found" echo "::error::This indicates a problem with test execution or blob reporter" echo "::error::Failed jobs: $FAILED, Succeeded jobs: $SUCCEEDED" echo "::endgroup::" exit 1 fi fi # Force the html + json reporters explicitly with their output paths instead of # relying on `unset CI` to flip playwright.config.js's `process.env.CI ? blob : html/json` # ternary. The runner exports CI=true and `unset CI` does not reliably reach the # merge subprocess, so the config would intermittently re-select the `blob` reporter — # which writes a merged blob-report/ and NOTHING to playwright-results/, making the # downstream `ls playwright-results/html-report/` fail with a cryptic exit 2. mkdir -p playwright-results # These env vars override playwright.config.js's reporter settings: # PLAYWRIGHT_HTML_OUTPUT_DIR - html reporter output directory # PLAYWRIGHT_HTML_OPEN=never - never auto-open the report (required for CI) # PLAYWRIGHT_JSON_OUTPUT_NAME - json reporter output file path PLAYWRIGHT_HTML_OUTPUT_DIR=playwright-results/html-report \ PLAYWRIGHT_HTML_OPEN=never \ PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-results/report.json \ npx playwright merge-reports --reporter html,json ./all-blob-reports echo "Contents of playwright-results:" ls -la playwright-results/ || true if [ ! -f playwright-results/html-report/index.html ] || [ ! -f playwright-results/report.json ]; then echo "::error::merge-reports did not produce the expected html-report/index.html and report.json" echo "::error::all-blob-reports contents:" ls -la all-blob-reports/ || true exit 1 fi # Diagnostics only — never fail the step on these (the guard above already # confirmed the outputs exist). echo "Contents of html-report:" ls -la playwright-results/html-report/ || true echo "Checking report.json:" ls -lh playwright-results/report.json || true - name: Extract report stats for OpenObserve id: report_stats if: always() run: | cd tests/ui-testing if [ -f playwright-results/report.json ]; then STATS=$(jq -c '.stats // {}' playwright-results/report.json 2>/dev/null || echo '{}') else STATS='{}' fi echo "stats=$STATS" >> "$GITHUB_OUTPUT" echo "report stats: $STATS" - name: Extract flaky tests for OpenObserve id: report_flaky if: always() run: | # Flaky + failed tests from the merged report, as one JSON array, for the # ci_test_results stream. Logic + schema live in extract-flaky.js (it # handles a missing/malformed report by printing [] non-fatally, and is # commented to explain status="flaky" vs "failed" and the sp.title choice). FLAKY=$(node .github/scripts/extract-flaky.js tests/ui-testing/playwright-results/report.json) echo "flaky=$FLAKY" >> "$GITHUB_OUTPUT" # Log a count only — avoid dumping test file paths/titles into public CI logs. echo "flaky/failed test count: $(printf '%s' "$FLAKY" | jq 'length' 2>/dev/null || echo '?')" # Read by playwright-auto-rerun.yml; failed_count -1 means "no advice", so a missing or broken report still writes the file. - name: Write rerun advice if: always() env: TESTED_OSS_SHA: ${{ github.sha }} run: | mkdir -p rerun-advice # Report file paths are relative to playwright.config.js testDir. jq -ce --arg sha "$TESTED_OSS_SHA" ' if (.suites | type) != "array" then error("not a Playwright report") else . end | [.suites[] | recurse(.suites[]?) | . as $s | (.specs // [])[] | . as $sp | (.tests // [])[] | select(.status == "unexpected") | ($sp.file // $s.file // "")] | {tested_oss_sha: $sha, failed_count: length, failed_files: (map(select(. != "") | "tests/ui-testing/playwright-tests/" + .) | unique)} ' tests/ui-testing/playwright-results/report.json > rerun-advice/rerun-advice.json 2>/dev/null \ || jq -nc --arg sha "$TESTED_OSS_SHA" '{tested_oss_sha: $sha, failed_count: -1, failed_files: []}' > rerun-advice/rerun-advice.json jq -c '{failed_count, failed_file_count: (.failed_files | length)}' rerun-advice/rerun-advice.json # runner.temp is emptied per job, so a reused runner cannot supply an earlier run's records. - name: Download failed-test lists id: download_last_runs if: ${{ always() && github.run_attempt == 1 && needs.ui_integration_tests.result != 'success' }} continue-on-error: true uses: actions/download-artifact@v4 with: path: ${{ runner.temp }}/last-runs pattern: last-run-*-attempt-${{ github.run_attempt }} # A shard that died before writing results adds nothing to the report, so the advice would miss its failure. - name: Check shard results match the merged report if: always() env: SHARDS_RESULT: ${{ needs.ui_integration_tests.result }} LAST_RUNS_OUTCOME: ${{ steps.download_last_runs.outcome }} MATRIX: ${{ needs.generate_matrix.outputs.matrix }} run: | ADVICE=rerun-advice/rerun-advice.json COMPUTED=$(cat "$ADVICE" 2>/dev/null || true) COUNT=$(jq '.failed_count' <<< "$COMPUTED" 2>/dev/null) && [ "$COUNT" -ge 0 ] 2>/dev/null || exit 0 # The bot reads advice only for a failed run. [ "$SHARDS_RESULT" != "success" ] || exit 0 # The file says "no advice" until every check below passes, so any early exit keeps the bot's old behaviour. jq -c '.failed_count = -1 | .failed_files = []' <<< "$COMPUTED" > "$ADVICE" if [ "$GITHUB_RUN_ATTEMPT" != "1" ]; then # The bot reads only attempt 1's advice, and a failed-jobs rerun leaves no record for shards that passed earlier. echo "Rerun advice is only written for the first attempt"; exit 0 fi if [ "$LAST_RUNS_OUTCOME" != "success" ]; then echo "::warning::Could not download the shard results; no rerun advice"; exit 0 fi if ! SHARDS=$(jq -r '.include | if type == "array" and length > 0 and all(.testfolder | type == "string") then .[].testfolder else error("not a shard matrix") end' <<< "$MATRIX" 2>/dev/null) \ || [ -z "$SHARDS" ]; then echo "::warning::Could not read the shard list from the matrix; no rerun advice"; exit 0 fi LISTED=0 FAILED_SHARDS="" while IFS= read -r SHARD; do RECORD="$RUNNER_TEMP/last-runs/last-run-$SHARD-attempt-$GITHUB_RUN_ATTEMPT" if ! STATUS=$(cat "$RECORD/job-status" 2>/dev/null); then echo "::warning::Shard $SHARD left no result record; no rerun advice"; exit 0 fi if [ "$STATUS" != "success" ]; then jq -e '.status == "failed" and (.failedTests | type == "array" and length > 0)' "$RECORD/.last-run.json" > /dev/null 2>&1 \ || { echo "::warning::Shard $SHARD failed without a complete failed-test list; no rerun advice"; exit 0; } FAILED_SHARDS="$FAILED_SHARDS $SHARD" fi if [ -f "$RECORD/.last-run.json" ]; then N=$(jq -e '.failedTests | if type == "array" then length else error("no failedTests array") end' "$RECORD/.last-run.json" 2>/dev/null) \ || { echo "::warning::Shard $SHARD left an unreadable failed-test list; no rerun advice"; exit 0; } LISTED=$((LISTED + N)) fi done <<< "$SHARDS" # Catches a failed shard whose blob report never reached the merged report. if [ "$LISTED" -ne "$COUNT" ]; then echo "::warning::Failed-test lists hold $LISTED tests but the merged report has $COUNT; no rerun advice"; exit 0 fi printf '%s\n' "$COMPUTED" > "$ADVICE" echo "rerun advice kept; failed shards checked:$FAILED_SHARDS" - name: Upload rerun advice if: always() continue-on-error: true uses: actions/upload-artifact@v4 with: name: rerun-advice-attempt-${{ github.run_attempt }} path: rerun-advice/rerun-advice.json retention-days: 7 - name: Upload merged report if: always() uses: actions/upload-artifact@v4 with: name: merged-report-${{ github.run_attempt }} path: tests/ui-testing/playwright-results retention-days: 1 if-no-files-found: warn # OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one # retry-aware run-summary doc (ci_test_runs, suite=ui) + per-test flaky/failed rows # (ci_test_results). The step is continue-on-error and the poster scripts never fail. report_openobserve: timeout-minutes: 10 name: Report to OpenObserve needs: [generate_matrix, ui_integration_tests, merge_reports] if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }} runs-on: labels: eks-openobserve-standard-4 permissions: contents: read actions: read # reads run/job timings from the Actions API steps: - name: Clone the current repo uses: actions/checkout@v5 - name: Report run metrics to OpenObserve continue-on-error: true env: GH_TOKEN: ${{ github.token }} O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }} O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }} O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }} # Track lightweight readiness separately from a successful fresh # build; skipped reuse-path builds are reported as null. RUNTIME_READY_JOB_NAME: resolve_binary TESTS: ${{ needs.ui_integration_tests.result }} MERGE_STATS: ${{ needs.merge_reports.outputs.stats }} MERGE_FLAKY: ${{ needs.merge_reports.outputs.flaky }} # Branch + PR for per-test rows so the dashboard needs no cross-stream JOIN. # head_ref is set on pull_request (the source branch); ref_name covers push + # merge_group (gh-readonly-queue/main/pr--). PR empty on merge_group/push. REPORT_BRANCH: ${{ github.head_ref || github.ref_name }} REPORT_PR: ${{ github.event.pull_request.number }} SMOKE_ACTIVE: ${{ needs.generate_matrix.outputs.smoke_active }} SHARDS_SELECTED: ${{ needs.generate_matrix.outputs.shards_selected }} run: | # This job runs only when tests actually ran (merge `if:` gates on !cancelled && # ui != skipped), so TESTS is success|failure here. if [ "$TESTS" = "failure" ]; then CONCLUSION=failure; else CONCLUSION=success; fi # Per-test counts from the merged report.json (tests_total includes skipped). if [ -n "${MERGE_STATS:-}" ] && [ "$MERGE_STATS" != "null" ] && [ "$MERGE_STATS" != "{}" ]; then EXTRA_JSON=$(printf '%s' "$MERGE_STATS" | jq -c '{tests_total:((.expected//0)+(.unexpected//0)+(.flaky//0)+(.skipped//0)), tests_passed:(.expected//0), tests_failed:(.unexpected//0), tests_flaky:(.flaky//0), tests_skipped:(.skipped//0)}' 2>/dev/null || echo "") fi # smoke_active/shards_selected let dashboards separate smoke rows from full runs, # since shards_total now varies by selection instead of always being the full set. SMOKE_JSON=$(jq -nc --arg a "${SMOKE_ACTIVE:-false}" --arg n "${SHARDS_SELECTED:-}" \ '{smoke_active:($a=="true"), shards_selected:(($n|tonumber?)//null)}') if [ -n "${EXTRA_JSON:-}" ]; then EXTRA_JSON=$(jq -nc --argjson a "$EXTRA_JSON" --argjson b "$SMOKE_JSON" '$a * $b') else EXTRA_JSON="$SMOKE_JSON" fi export EXTRA_JSON export CONCLUSION STREAM=ci_test_runs SUITE=ui bash .github/scripts/o2-run-report.sh # One row per flaky-or-failed test -> ci_test_results (status flaky|failed; suite/ # workflow hardcoded UI-only). o2-report.sh takes the stream as arg 1. Non-fatal. FLAKY="${MERGE_FLAKY:-[]}" if printf '%s' "$FLAKY" | jq -e 'type=="array" and length>0' >/dev/null 2>&1; then FLAKY_FILE=$(mktemp "${TMPDIR:-/tmp}/flaky.XXXXXX") trap 'rm -f "$FLAKY_FILE"' EXIT printf '%s' "$FLAKY" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg run_attempt "$GITHUB_RUN_ATTEMPT" --arg repo "$GITHUB_REPOSITORY" \ --arg branch "${REPORT_BRANCH:-}" --arg pr "${REPORT_PR:-}" \ '[.[] | {run_id:$run_id, run_attempt:($run_attempt|tonumber), repo:$repo, suite:"ui", workflow:"test", ingest_source:"live", status:.status, module:.module, test_title:.title, test_file:.file, branch:(if $branch=="" then null else $branch end), pr_number:(if $pr=="" then null else ($pr|tonumber) end)}]' \ > "$FLAKY_FILE" 2>/dev/null || echo '[]' > "$FLAKY_FILE" if [ "$(jq 'length' "$FLAKY_FILE" 2>/dev/null || echo 0)" -gt 0 ]; then bash .github/scripts/o2-report.sh ci_test_results "$FLAKY_FILE" fi fi playwright_summary: timeout-minutes: 5 runs-on: labels: eks-openobserve-standard-4 permissions: checks: write pull-requests: read needs: [ check_changes, resolve_binary, build_binary, generate_matrix, prior_pass, ui_integration_tests, ] if: always() && github.event_name != 'merge_group' && (github.event_name != 'schedule' || github.repository == 'openobserve/openobserve') && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false)) steps: - name: Check test results id: verdict env: ALREADY_PASSED: ${{ needs.prior_pass.result == 'success' && needs.prior_pass.outputs.already_passed == 'true' }} PRIOR_RUN_URL: ${{ needs.prior_pass.outputs.run_url }} run: | # 1. check_changes must have succeeded — if it failed, downstream jobs skipped # and we must not report a false success. if [ "${{ needs.check_changes.result }}" != "success" ]; then echo "check_changes job did not succeed: ${{ needs.check_changes.result }}" exit 1 fi # 2. Nothing Playwright-relevant changed → nothing ran → allow. if [ "${{ needs.check_changes.outputs.has_changes }}" != "true" ]; then echo "✅ No Playwright-relevant changes — suite not required." echo "::notice title=Playwright tests skipped::No Playwright-relevant changes — suite did not run." echo "### ⏭️ Playwright suite skipped — no Playwright-relevant changes in this PR, so no tests ran." >> "$GITHUB_STEP_SUMMARY" exit 0 fi # prior_pass matched the tested commit and every selected shard, so the earlier verdict stands. if [ "$ALREADY_PASSED" = "true" ]; then if [ "${{ needs.build_binary.result }}" = "failure" ]; then echo "build_binary failed on a commit that already passed: ${{ needs.build_binary.result }}" exit 1 fi echo "✅ Every selected shard already passed on this commit in $PRIOR_RUN_URL" exit 0 fi # 3. Suite was expected to run. The resolver must succeed, and either # an exact-SHA pair was reused or this run built a fresh pair. if [ "${{ needs.resolve_binary.result }}" != "success" ]; then echo "resolve_binary job did not succeed: ${{ needs.resolve_binary.result }}" exit 1 fi if [ "${{ needs.resolve_binary.outputs.reuse }}" != "true" ] && \ [ "${{ needs.build_binary.result }}" != "success" ]; then echo "No reusable artifact and build_binary did not succeed: ${{ needs.build_binary.result }}" exit 1 fi # 4. Runtime readiness alone is not enough; all selected shards must pass. if [ "${{ needs.ui_integration_tests.result }}" == "success" ]; then echo "All Playwright tests completed successfully" exit 0 else echo "Playwright tests failed:" echo " resolve_binary: ${{ needs.resolve_binary.result }} (reuse=${{ needs.resolve_binary.outputs.reuse }})" echo " build_binary: ${{ needs.build_binary.result }}" echo " ui_integration_tests: ${{ needs.ui_integration_tests.result }}" exit 1 fi - name: Complete the auto-merge placeholder if: ${{ !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false && github.actor != 'dependabot[bot]' && needs.check_changes.result == 'success' }} # github-script, not gh: the eks runners are not known to ship the gh CLI. uses: actions/github-script@v9 env: VERDICT: ${{ steps.verdict.outcome }} SMOKE: ${{ needs.check_changes.outputs.has_changes == 'true' && needs.generate_matrix.outputs.smoke_active == 'true' }} HAS_CHANGES: ${{ needs.check_changes.outputs.has_changes }} with: script: | if (process.env.SMOKE === 'true') { // A smoke verdict may stand in only once auto-merge is off, else a full run dropped from the queue strands the placeholder. const { data: pr } = await github.rest.pulls.get({ ...context.repo, pull_number: context.payload.pull_request.number }); if (pr.auto_merge) { core.info('Smoke run and auto-merge is on: leaving the placeholder to the full-matrix run.'); return; } } const conclusion = process.env.VERDICT === 'success' ? 'success' : 'failure'; const scope = process.env.HAS_CHANGES !== 'true' ? 'No Playwright changes' : process.env.SMOKE === 'true' ? 'Smoke' : 'Full matrix'; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const runs = await github.paginate(github.rest.checks.listForRef, { ...context.repo, ref: context.payload.pull_request.head.sha, check_name: 'playwright_summary', filter: 'all', per_page: 100, }); const placeholders = runs.filter((r) => r.external_id === 'playwright-automerge-placeholder'); if (!placeholders.length) core.info('No auto-merge placeholder on this SHA.'); for (const r of placeholders) { await github.rest.checks.update({ ...context.repo, check_run_id: r.id, status: 'completed', conclusion, details_url: runUrl, output: { title: `${scope}: ${conclusion}`, summary: `Completed by the ${scope} run ${runUrl}` }, }); core.info(`placeholder ${r.id} -> ${conclusion}`); }