name: Playwright UI Tests

on:
  # No `push: main` trigger: the merge_group (merge-queue) run already builds and
  # tests the exact commit that lands on main, so a post-merge push run would be a
  # full-suite duplicate. The merge_group run is now the trusted main-ward build and
  # also warms the shared Rust cache (see save-if on the rust-cache step below).
  pull_request:
    branches:
      - "**"
    # labeled/unlabeled so adding or removing 'ready-for-ci' starts or resets CI; drafts never run paid CI.
    # auto_merge_enabled: the one full-matrix run per PR, at merge time (see generate_matrix).
    types: [opened, reopened, synchronize, labeled, unlabeled, auto_merge_enabled, ready_for_review, converted_to_draft]
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.merge_group.head_ref || github.ref }}
  cancel-in-progress: true

env:
  COLUMNS: 150
  ZO_ROOT_USER_EMAIL: root@example.com
  ZO_ROOT_USER_PASSWORD: Complexpass#123
  ZO_BASE_URL: http://localhost:5080
  WS_ZO_BASE_URL: ws://localhost:5080
  ZO_BASE_URL_SC: http://localhost:5080
  ZO_BASE_URL_SC_UI: http://localhost:5080
  INGESTION_URL: http://localhost:5080
  ORGNAME: default
  ZO_WEB_URL: "http://localhost:5080"
  ZO_QUICK_MODE_NUM_FIELDS: 100
  ZO_QUICK_MODE_STRATEGY: first
  ZO_ALLOW_USER_DEFINED_SCHEMAS: true
  ZO_INGEST_ALLOWED_UPTO: 5
  ZO_SLO_BACKFILL_CHUNK_SECS: 86400 # matches the server default; per-shard override below
  ZO_FEATURE_QUERY_EXCLUDE_ALL: false
  ZO_USAGE_BATCH_SIZE: 200
  ZO_USAGE_PUBLISH_INTERVAL: 2
  ZO_USAGE_REPORTING_ENABLED: true
  ZO_MIN_AUTO_REFRESH_INTERVAL: 5
  ZO_STREAMING_ENABLED: true
  ZO_COLS_PER_RECORD_LIMIT: "80000"
  # SMTP was already enabled here, but with no host/port it fell back to the
  # server defaults (localhost:25) where nothing listens — so destinations could
  # be SAVED (the enabled-gate passes) while every send failed silently. The
  # Mailpit step below gives that flag something to talk to, which also makes the
  # delivery assertions in Alerts/dl-email-destinations.spec.js runnable.
  ZO_SMTP_ENABLED: true
  ZO_SMTP_HOST: localhost
  # 1026 is the DL relay, not Mailpit — it expands DL_ADDRESS and forwards
  # everything else through untouched, so non-email shards see no difference.
  ZO_SMTP_PORT: 1026
  ZO_SMTP_FROM_EMAIL: alerts@ci.openobserve.test
  # Read by tests/ui-testing/playwright-tests/utils/mail-sink.js
  MAILPIT_URL: http://localhost:8025
  # A distribution list that exists only inside the runner: one alias, three
  # member mailboxes. Lets the fan-out case run with no external mail provider,
  # no secrets and no egress.
  DL_ADDRESS: ci-dl@ci.openobserve.test
  DL_MEMBERS: ci-dl-m1@ci.openobserve.test,ci-dl-m2@ci.openobserve.test,ci-dl-m3@ci.openobserve.test
  ZO_FORMAT_STREAM_NAME_TO_LOWERCASE: false
  ZO_CREATE_ORG_THROUGH_INGESTION: true
  ZO_UTF8_VIEW_ENABLED: false
  ZO_ENABLE_CROSS_LINKING: true
  # Timechart (visualize) tab is hidden by default; enable it so visualize E2E tests can reach it.
  ZO_TIMECHART_ENABLED: true
  # Allow alert destinations to target localhost so the test validation stream
  # (http://localhost:5080/api/.../..._json) is not blocked by the SSRF guard.
  ZO_SSRF_ALLOW_LOOPBACK: true

jobs:
  check_changes:
    # Runs only on PRs carrying the 'ready-for-ci' label; the merge queue runs unit tests only (see docs/ci).
    if: ${{ github.event_name != 'merge_group' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false)) }}
    timeout-minutes: 5
    name: check_changes
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      has_changes: ${{ steps.check.outputs.has_changes }}
    steps:
      # Debounce: a push inside this window cancels the run (concurrency) before anything paid starts.
      - if: github.event_name == 'pull_request'
        run: sleep 60
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0

      # One shared classification for every suite (see .github/actions/classify-changes); a change to
      # this workflow file re-tests itself on top of it.
      - uses: ./.github/actions/classify-changes
        id: classify
      - id: check
        env:
          HAS_CHANGES: ${{ steps.classify.outputs.run_playwright }}
          CHANGED_WORKFLOWS: ${{ steps.classify.outputs.changed_workflows }}
        run: |
          if [ "$HAS_CHANGES" = "true" ] || echo " $CHANGED_WORKFLOWS " | grep -q ' playwright.yml '; then
            echo "has_changes=true" >> "$GITHUB_OUTPUT"
          else
            echo "has_changes=false" >> "$GITHUB_OUTPUT"
          fi

  resolve_binary:
    timeout-minutes: 5
    name: resolve_binary
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
    outputs:
      reuse: ${{ steps.resolve.outputs.reuse }}
      run_id: ${{ steps.resolve.outputs.run_id }}
      binary_artifact_name: ${{ steps.resolve.outputs.binary_artifact_name }}
      frontend_artifact_name: ${{ steps.resolve.outputs.frontend_artifact_name }}
    steps:
      - name: Find exact-SHA Playwright runtime
        id: resolve
        env:
          GH_TOKEN: ${{ github.token }}
          SOURCE_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          ARTIFACT_SHA: ${{ github.sha }}
          CURRENT_RUN_ID: ${{ github.run_id }}
        run: |
          set -euo pipefail
          binary_artifact="release-ci-binary-${ARTIFACT_SHA}"
          frontend_artifact="frontend-dist-${ARTIFACT_SHA}"
          selected_run_id="$CURRENT_RUN_ID"
          reuse=false

          # Scope candidates to this workflow and source head. ARTIFACT_SHA is
          # the exact tested merge/commit SHA, so a base-branch update cannot
          # reuse a binary built from an older synthetic merge commit.
          run_ids="$(
            gh api --method GET \
              "repos/${GITHUB_REPOSITORY}/actions/workflows/playwright.yml/runs?head_sha=${SOURCE_HEAD_SHA}&per_page=100" \
              --jq '.workflow_runs | sort_by(.created_at) | reverse | .[].id' \
              2>/dev/null || true
          )"
          while IFS= read -r run_id; do
            [ -n "$run_id" ] || continue
            artifact_names="$(
              gh api --method GET \
                "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100" \
                --jq '.artifacts[] | select(.expired == false) | .name' \
                2>/dev/null || true
            )"
            if grep -Fxq "$binary_artifact" <<< "$artifact_names" &&
               grep -Fxq "$frontend_artifact" <<< "$artifact_names"; then
              selected_run_id="$run_id"
              reuse=true
              break
            fi
          done <<< "$run_ids"

          {
            echo "reuse=$reuse"
            echo "run_id=$selected_run_id"
            echo "binary_artifact_name=$binary_artifact"
            echo "frontend_artifact_name=$frontend_artifact"
          } >> "$GITHUB_OUTPUT"
          if [ "$reuse" = "true" ]; then
            echo "Reusing exact-SHA Playwright runtime from run $selected_run_id"
          else
            echo "No reusable exact-SHA Playwright runtime found; build_binary will run"
          fi

  build_binary:
    timeout-minutes: 45
    name: build_binary
    needs: [check_changes, resolve_binary]
    if: >-
      always() &&
      needs.check_changes.result == 'success' &&
      needs.check_changes.outputs.has_changes == 'true' &&
      needs.resolve_binary.result == 'success' &&
      needs.resolve_binary.outputs.reuse != 'true'
    runs-on:
      labels: ubicloud-standard-16
    permissions:
      contents: read

    steps:
      - name: Remove unused tools
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /opt/ghc
          sudo rm -rf "/usr/local/share/boost"
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"

      - name: Clone the current repo
        uses: actions/checkout@v5

      - uses: ./.github/actions/github-auth
      - name: Setup Rust Toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly-2026-05-20
          targets: x86_64-unknown-linux-gnu
      - uses: Swatinem/rust-cache@v2
        with:
          cache-on-failure: true
          shared-key: playwright-release-ci
          # Save the shared cache on merge_group runs (the main-ward build now that
          # push:main is gone); keep the main-ref check for any non-PR fallback.
          save-if: ${{ github.event_name == 'merge_group' || github.ref == 'refs/heads/main' }}

      - name: Install Protoc
        run: |
          bash ./.github/protoc.sh

      - uses: actions/setup-node@v7
        with:
          node-version: 24

      - name: Lint frontend code
        run: cd web && npm ci && npm run lint:ci

      - name: Check for undefined CSS custom properties
        run: cd web && npm run lint:tokens

      - name: Lint styles (stylelint — bans legacy --o2-* tokens)
        run: cd web && npm run lint:styles

      - name: Check token-file purity (token files contain tokens only)
        run: cd web && npm run lint:token-purity

      - name: Check design-consistency ratchet (strict — hardcoded/arbitrary/raw-token, no slack)
        run: cd web && npm run lint:design:strict

      - name: Build binary and frontend
        env:
          NODE_OPTIONS: "--max-old-space-size=8192"
          GIT_HASH_OVERRIDE: ${{ github.sha }}
        run: |
          mkdir -p /tmp/dist
          bash -c "cd web && npm run build && cp -r ./dist/. /tmp/dist/" &
          NPM_PID=$!
          sed -i 's|folder = "../../web/dist/"|folder = "/tmp/dist/"|' src/web/src/lib.rs && grep -q 'folder = "/tmp/dist/"' src/web/src/lib.rs
          cargo build --profile release-ci --target x86_64-unknown-linux-gnu
          wait $NPM_PID
          test -f /tmp/dist/index.html

      - name: Upload binary artifact
        uses: actions/upload-artifact@v4
        with:
          name: release-ci-binary-${{ github.sha }}
          path: target/x86_64-unknown-linux-gnu/release-ci/openobserve
          retention-days: 1

      - name: Upload frontend artifact
        uses: actions/upload-artifact@v4
        with:
          name: frontend-dist-${{ github.sha }}
          path: /tmp/dist
          retention-days: 1
          if-no-files-found: error

  # Build the shard matrix from tests/ui-testing/ci-matrix/ci_matrix.json (single source of truth,
  # shared with ENT via build-ci-matrix.js). Runs under the same gate as build_binary so
  # ui_integration_tests always has a matrix whenever it is allowed to run.
  generate_matrix:
    timeout-minutes: 5
    name: generate_matrix
    needs: [check_changes]
    if: needs.check_changes.outputs.has_changes == 'true'
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      matrix: ${{ steps.gen.outputs.matrix }}
      smoke_active: ${{ steps.gen.outputs.smoke_active }}
      shards_selected: ${{ steps.gen.outputs.shards_selected }}
    steps:
      - uses: actions/checkout@v5
        with:
          # Full history only where the selection diff needs it; merge_group never diffs.
          fetch-depth: ${{ github.event_name == 'pull_request' && '0' || '1' }}
      - name: Selection contract tests
        run: node --test .github/scripts/build-ci-matrix.test.js
      - id: gen
        env:
          # Smoke selection runs on PR pushes; the merge queue no longer runs Playwright, so the
          # full matrix runs once per PR when auto-merge is enabled ("Merge when ready").
          # Escape hatches: the 'e2e-full' PR label forces the full matrix on every push; the
          # PW_SMOKE_DISABLED repo variable ('true') turns smoke off everywhere without a code change.
          IS_PR: ${{ github.event_name == 'pull_request' }}
          HAS_FULL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'e2e-full') || github.event.pull_request.auto_merge != null }}
          SMOKE_DISABLED: ${{ vars.PW_SMOKE_DISABLED }}
          BASE_REF: ${{ github.base_ref }}
        run: |
          SMOKE_DISABLED=$(echo "$SMOKE_DISABLED" | tr '[:upper:]' '[:lower:]')
          SELECT_ARGS=""
          if [ "$IS_PR" == "true" ] && [ "$HAS_FULL_LABEL" != "true" ] && [ "$SMOKE_DISABLED" != "true" ]; then
            # Three-dot: diff from the merge base, so commits that landed on the base branch
            # after this PR's merge commit was cut never leak into the selection.
            # --no-renames lists a moved file under BOTH paths, so the shard owning the old
            # location still runs; quotepath off keeps non-ASCII names matchable.
            if git -c core.quotepath=false diff --no-renames --name-only "origin/${BASE_REF:-main}...${{ github.sha }}" > /tmp/pw-changed-files.txt; then
              SELECT_ARGS="--select-for-changes /tmp/pw-changed-files.txt"
            else
              echo "::warning::changed-file diff failed — running the full matrix"
            fi
          fi
          # || true: under bash -e a die() would abort before the log below ever prints.
          MATRIX=$(node .github/scripts/build-ci-matrix.js tests/ui-testing/ci-matrix/ci_matrix.json $SELECT_ARGS 2> /tmp/pw-select-log.txt) || true
          cat /tmp/pw-select-log.txt >&2
          if [ -z "$MATRIX" ]; then echo "::error::matrix generation failed (see build-ci-matrix log above)"; exit 1; fi
          echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT"
          COUNT=$(echo "$MATRIX" | node -e "const m=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log('generated',m.include.length,'shards:', m.include.map(s=>s.testfolder).join(', ')); require('fs').appendFileSync(process.env.GITHUB_OUTPUT, 'shards_selected='+m.include.length+'\n')")
          echo "smoke_active=$([ -n "$SELECT_ARGS" ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
          # Durable record of what ran and why, visible from the PR checks page.
          {
            echo "### Playwright shard selection"
            echo "Mode: $([ -n "$SELECT_ARGS" ] && echo "smoke (changed-path selection)" || echo "full matrix")"
            echo '```'
            echo "$COUNT"
            cat /tmp/pw-select-log.txt
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

  ui_integration_tests:
    timeout-minutes: 45
    name: e2e / ${{ matrix.testfolder }}
    needs: [resolve_binary, build_binary, generate_matrix]
    if: >-
      always() &&
      !cancelled() &&
      needs.resolve_binary.result == 'success' &&
      needs.generate_matrix.result == 'success' &&
      (needs.resolve_binary.outputs.reuse == 'true' || needs.build_binary.result == 'success')
    runs-on:
      labels: eks-openobserve-standard-8
    permissions:
      contents: read
      actions: read
    # container:
    #   image: mcr.microsoft.com/playwright:v1.50.0-jammy
    #   options: --user root
    strategy:
      fail-fast: false
      # Matrix is generated from tests/ui-testing/ci-matrix/ci_matrix.json by the
      # generate_matrix job (the single source of truth shared with ENT). To add
      # or move a spec, edit that JSON file — do NOT hand-edit a matrix here.
      matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }}
    steps:
      - name: Kill background apt processes
        run: |
          sudo systemctl stop apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
          sudo systemctl stop apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
          sudo systemctl disable apt-daily.service apt-daily-upgrade.service unattended-upgrades.service 2>/dev/null || true
          sudo flock --wait 30 /var/lib/dpkg/lock-frontend true 2>/dev/null || true

      - name: Clone the current repo
        uses: actions/checkout@v5

      - name: Download binary artifact
        uses: actions/download-artifact@v5
        with:
          name: ${{ needs.resolve_binary.outputs.binary_artifact_name }}
          path: release-ci-binary
          github-token: ${{ github.token }}
          repository: ${{ github.repository }}
          run-id: ${{ needs.resolve_binary.outputs.run_id }}

      - name: Download frontend artifact
        uses: actions/download-artifact@v5
        with:
          name: ${{ needs.resolve_binary.outputs.frontend_artifact_name }}
          path: /tmp/dist
          github-token: ${{ github.token }}
          repository: ${{ github.repository }}
          run-id: ${{ needs.resolve_binary.outputs.run_id }}

      - name: Start mail sink (Mailpit)
        # A local SMTP server the tests can also READ. Runs as a static binary
        # rather than a service container: these shards run on a self-hosted EKS
        # runner where Docker-in-Docker is not guaranteed, and the app itself is
        # already launched the same way. ~2s, and it makes ZO_SMTP_* real.
        #
        # Gated to the one shard that needs it — every other shard would
        # otherwise pay a GitHub-releases download for nothing.
        if: matrix.testfolder == 'Alerts'
        env:
          # Pinned, not "latest": an upstream release must never change what this
          # shard does without a commit here.
          MAILPIT_VERSION: v1.21.8
        run: |
          set +e
          curl -fsSL -o mailpit.tar.gz \
            "https://github.com/axllent/mailpit/releases/download/${MAILPIT_VERSION}/mailpit-linux-amd64.tar.gz" \
            && tar -xzf mailpit.tar.gz mailpit && chmod +x ./mailpit
          if [ ! -x ./mailpit ]; then
            echo "::warning::Mailpit ${MAILPIT_VERSION} could not be fetched; email delivery assertions will skip"
            exit 0
          fi
          # Loopback only — this runner is shared, and delivered alert bodies
          # should not be readable from the rest of its network.
          ./mailpit --smtp 127.0.0.1:1025 --listen 127.0.0.1:8025 > mailpit.log 2>&1 &
          for i in $(seq 1 30); do
            curl -sf http://localhost:8025/api/v1/messages > /dev/null 2>&1 && { echo "Mailpit ready after ${i}s"; exit 0; }
            sleep 1
          done
          echo "::warning::Mailpit did not start; email delivery assertions will skip"

      - name: Start OpenObserve
        # Per-shard override of the workflow-level default. ZO_QUICK_MODE_ENABLED is
        # read once at server start and is global to the process, so a shard is the
        # smallest unit that can change it. Set via "quick_mode_enabled" in
        # tests/ui-testing/ci-matrix/ci_matrix.json; shards that omit it get "false".
        env:
          ZO_QUICK_MODE_ENABLED: ${{ matrix.quick_mode_enabled || 'false' }}
          # Same per-shard rule: how far back (hours) ingestion accepts a record.
          # The workflow default of 5 DROPS older rows while still answering 200,
          # so a suite that seeds history (SLO measures a rolling 7-day window)
          # would fail as "No data" with nothing in the artifacts to explain it.
          # Set via "ingest_allowed_upto" in ci-matrix/ci_matrix.json; shards that
          # omit it keep the workflow default.
          ZO_INGEST_ALLOWED_UPTO: ${{ matrix.ingest_allowed_upto || env.ZO_INGEST_ALLOWED_UPTO }}
          # Same per-shard rule: how much history one SLO backfill chunk covers.
          # The production default (86400 = 1 day) makes a 7-day-window test SLO
          # backfill in 7 sequential chunks under ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1
          # (SLO-Measurement's shard note above) — no test here asserts on
          # per-chunk timing or partial coverage, only on the final measured
          # state, so a shard-only larger chunk is free wall-clock, not a
          # behavior change. Set via "slo_backfill_chunk_secs" in
          # ci-matrix/ci_matrix.json; shards that omit it keep the server default.
          ZO_SLO_BACKFILL_CHUNK_SECS: ${{ matrix.slo_backfill_chunk_secs || env.ZO_SLO_BACKFILL_CHUNK_SECS }}
        run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 &

      - name: Wait for start
        run: |
          for i in $(seq 1 120); do
            if curl -sf http://localhost:5080/web/login > /dev/null 2>&1; then
              echo "Server ready after ${i}s"
              exit 0
            fi
            sleep 1
          done
          echo "Server failed to start in 120s"
          exit 1

      - name: Ensure we are getting a reply from the server
        run: curl http://localhost:5080/web/login

      - uses: actions/setup-node@v7
        with:
          node-version: 24

      - name: Start distribution-list relay
        # Mailpit is a sink: it accepts any address but stores ONE message, so it
        # can never show what a real distribution list does. This tiny relay sits
        # in front of it and expands the alias into per-member deliveries, the
        # same way alerts-content-templates.spec.js stands up an HTTP receiver to
        # capture what a webhook destination really sent.
        #
        # Gated to the one shard that needs it, matching the Mailpit step above.
        if: matrix.testfolder == 'Alerts'
        run: |
          if ! command -v node > /dev/null 2>&1; then
            echo "::warning::node not on PATH; DL fan-out case will skip"
            exit 0
          fi
          node tests/ui-testing/scripts/dl-relay.js > dl-relay.log 2>&1 &
          for i in $(seq 1 20); do
            (echo > /dev/tcp/127.0.0.1/1026) >/dev/null 2>&1 && { echo "DL relay ready after ${i}s"; exit 0; }
            sleep 1
          done
          echo "::warning::DL relay did not start; the fan-out case will skip"

      - name: Cache Playwright browser
        id: playwright-cache
        uses: actions/cache@v6
        with:
          path: ~/.cache/ms-playwright
          key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }}

      - name: Install dependencies and run ui-tests
        run: |
          touch .env
          echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env
          echo "ZO_ROOT_USER_PASSWORD=${ZO_ROOT_USER_PASSWORD}" >> .env
          echo "ZO_BASE_URL=${ZO_BASE_URL}" >> .env
          echo "WS_ZO_BASE_URL=${WS_ZO_BASE_URL}" >> .env
          echo "ZO_BASE_URL_SC=${ZO_BASE_URL_SC}" >> .env
          echo "ZO_BASE_URL_SC_UI=${ZO_BASE_URL_SC_UI}" >> .env
          echo "INGESTION_URL=${INGESTION_URL}" >> .env
          echo "ORGNAME=${ORGNAME}" >> .env
          echo "ZO_SMTP_ENABLED=${ZO_SMTP_ENABLED}" >> .env
          echo "MAILPIT_URL=${MAILPIT_URL}" >> .env
          echo "DL_ADDRESS=${DL_ADDRESS}" >> .env
          echo "DL_MEMBERS=${DL_MEMBERS}" >> .env
          mv .env tests/ui-testing
          cd tests/ui-testing && npm ci

          # Install apt system dependencies (always needed).
          # azure.archive.ubuntu.com intermittently times out fetching font/lib
          # packages, which makes `install-deps` exit 100 and fails the whole job.
          # Give apt its own per-fetch retries and wrap the call in a backoff loop.
          echo 'Acquire::Retries "5";' | sudo tee /etc/apt/apt.conf.d/99retries >/dev/null
          for attempt in 1 2 3; do
            if npx playwright install-deps chromium; then
              echo "install-deps succeeded on attempt ${attempt}"
              break
            fi
            if [ "${attempt}" = 3 ]; then
              echo "::error::playwright install-deps chromium failed after 3 attempts"
              exit 1
            fi
            echo "install-deps attempt ${attempt} failed; retrying in $((attempt * 20))s"
            sleep $((attempt * 20))
          done

          # Get revision from playwright's own metadata
          CHROMIUM_REV=$(node -pe "
            require('./node_modules/playwright-core/browsers.json')
              .browsers.find(b => b.name === 'chromium').revision
          ")

          install_browser() {
            local NAME="$1"
            local BINARY="$2"
            local ZIP_PATH="$3"
            local DIR="$HOME/.cache/ms-playwright/${NAME}-${CHROMIUM_REV}"
            if [ -f "${DIR}/INSTALLATION_COMPLETE" ]; then
              echo "${NAME} already installed (cache hit)"; return 0
            fi
            rm -rf "${DIR}"
            echo "Cache miss — downloading ${NAME} r${CHROMIUM_REV} via curl + system unzip"
            curl -fL --progress-bar \
              "https://cdn.playwright.dev/dbazure/download/playwright/builds/${ZIP_PATH}" \
              -o "/tmp/${NAME}.zip"
            mkdir -p "${DIR}"
            unzip -q "/tmp/${NAME}.zip" -d "${DIR}"
            rm -f "/tmp/${NAME}.zip"
            find "${DIR}" -name "${BINARY}" -type f -exec chmod 755 {} \;
            find "${DIR}" -name 'chrome_sandbox' -type f -exec chmod 4755 {} \; 2>/dev/null || true
            touch "${DIR}/INSTALLATION_COMPLETE"
            echo "${NAME} installed: $(ls ${DIR}/)"
          }

          install_browser "chromium" "chrome" \
            "chromium/${CHROMIUM_REV}/chromium-linux.zip"
          install_browser "chromium_headless_shell" "headless_shell" \
            "chromium/${CHROMIUM_REV}/chromium-headless-shell-linux.zip"

          # ffmpeg is required for video recording (video: retain-on-failure in playwright.config.js)
          # The custom install_browser uses CHROMIUM_REV which doesn't match ffmpeg's own revision,
          # so use the local playwright binary (pinned via package-lock.json) to install ffmpeg only.
          ./node_modules/.bin/playwright install ffmpeg

          # Get list of files to run using join function
          FILE_LIST="${{ join(matrix.run_files, ' ') }}"
          echo "DEBUG: FILE_LIST = $FILE_LIST"
          echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}"

          if [ -n "$FILE_LIST" ]; then
            # actual_folder (the real directory under playwright-tests/) comes straight
            # from the matrix now — the logical->directory mapping lives in ci-matrix/ci_matrix.json.
            ACTUAL_FOLDER="${{ matrix.actual_folder }}"
            echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER"

            # Build file paths
            FILE_PATHS=""
            for file in $FILE_LIST; do
              FILE_PATH="./playwright-tests/$ACTUAL_FOLDER/$file"
              echo "DEBUG: Will run file: $FILE_PATH"
              FILE_PATHS="$FILE_PATHS $FILE_PATH"
            done

            # Optional per-shard worker pin. Empty keeps playwright.config.js's
            # default; a shard whose specs contend for a shared server-side
            # resource sets "workers": 1 in ci_matrix.json, because fullyParallel
            # races separate spec FILES and mode:'serial' only orders tests
            # within one file.
            WORKERS_ARG=""
            if [ -n "${{ matrix.workers }}" ]; then
              WORKERS_ARG="--workers=${{ matrix.workers }}"
              echo "DEBUG: pinning workers to ${{ matrix.workers }}"
            fi

            echo "DEBUG: Final command: npx playwright test $FILE_PATHS $WORKERS_ARG"
            npx playwright test $FILE_PATHS $WORKERS_ARG
          else
            echo "No files specified to run for ${{ matrix.testfolder }} folder"
          fi

      - name: Upload blob report to GitHub Actions Artifacts
        if: ${{ !cancelled() }}
        uses: actions/upload-artifact@v4
        with:
          name: blob-report-${{ matrix.testfolder }}-attempt-${{ github.run_attempt }}
          path: tests/ui-testing/blob-report
          retention-days: 1

      - name: Check OpenObserve logs
        if: always()
        run: cat o2.log

  merge_reports:
    timeout-minutes: 20
    name: Merge Reports & Report to Kinora
    needs: [ui_integration_tests]
    if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
    # eks runner: `playwright merge-reports` silently stops mid-merge (exits 0, ~3 of N blobs, no
    # output) on the ubicloud and github-hosted runners — but works on the eks runners (same as ENT).
    # This job also uploads the merged run to the self-hosted Kinora dashboard (see the last two
    # steps) — folded in here to reuse the merge instead of re-downloading/re-merging in a 2nd job.
    runs-on:
      labels: eks-openobserve-standard-4
    permissions:
      contents: read
      actions: read          # the no-blobs investigation reads run/job state from the Actions API
      pull-requests: write   # upsert the single sticky Kinora comment on PRs
    outputs:
      # Compact Playwright stats (expected/unexpected/flaky/skipped) from the merged report.json,
      # consumed by report_to_openobserve to record per-test counts in ci_test_runs.
      stats: ${{ steps.report_stats.outputs.stats }}
      # Flaky + failed tests for the ci_test_results stream, consumed by
      # report_to_openobserve. JSON array: [{module,title,file,status}] where
      # status is "flaky"|"failed" (see .github/scripts/extract-flaky.js). "[]" if none.
      flaky: ${{ steps.report_flaky.outputs.flaky }}

    steps:
      - name: Clone the current repo
        uses: actions/checkout@v5

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: 24

      - name: Install project dependencies
        run: cd tests/ui-testing && npm ci

      - name: Download blob reports from GitHub Actions Artifacts
        uses: actions/download-artifact@v4
        continue-on-error: true
        id: download_blobs
        with:
          path: tests/ui-testing/all-blob-reports
          pattern: blob-report-*-attempt-${{ github.run_attempt }}
          merge-multiple: true

      - name: Check if any artifacts were downloaded
        run: |
          if [ -d "tests/ui-testing/all-blob-reports" ] && [ "$(ls -A tests/ui-testing/all-blob-reports)" ]; then
            echo "✅ Artifacts downloaded successfully"
            ls -la tests/ui-testing/all-blob-reports
          else
            echo "⚠️ No artifacts found or download failed completely"
            # Don't fail here - let the Merge Reports step handle this
          fi

      - name: Merge Reports
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          cd tests/ui-testing

          # Check if directory exists and contains reports
          if [ ! -d "all-blob-reports" ] || [ -z "$(ls -A all-blob-reports 2>/dev/null)" ]; then
            echo "::warning::No blob reports found to merge."

            # Investigate: Check if all test jobs succeeded (meaning they skipped intentionally)
            # vs test jobs failed (meaning blob reporter might have failed)
            echo "::group::Investigating why no blob reports exist"

            JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
              -H "Accept: application/vnd.github.v3+json" \
              "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs")

            # Count test matrix jobs by conclusion
            SUCCEEDED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "success")] | length')
            FAILED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "failure")] | length')
            TOTAL=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /"))] | length')

            echo "::notice::Test job results: $SUCCEEDED succeeded, $FAILED failed, $TOTAL total"

            if [ "$SUCCEEDED" -gt 0 ] && [ "$FAILED" -eq 0 ]; then
              echo "::notice::✅ All test jobs succeeded - they likely skipped because tests already passed"
              echo "::notice::This is VALID behavior for optimized reruns"
              echo "::notice::Creating empty report structure for downstream jobs"
              mkdir -p playwright-results/html-report
              # Add stats object so downstream report consumers get a well-formed report.
              # startTime is an ISO 8601 string (not a Unix timestamp number).
              START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
              echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json
              echo "<html><body><h1>No tests executed - all shards skipped</h1><p>All test shards exited early because tests had already passed in a previous attempt.</p></body></html>" > playwright-results/html-report/index.html
              echo "::endgroup::"
              exit 0
            else
              echo "::error::❌ Some/all test jobs failed but no blob reports were found"
              echo "::error::This indicates a problem with test execution or blob reporter"
              echo "::error::Failed jobs: $FAILED, Succeeded jobs: $SUCCEEDED"
              echo "::endgroup::"
              exit 1
            fi
          fi

          # Force the html + json reporters explicitly with their output paths instead of
          # relying on `unset CI` to flip playwright.config.js's `process.env.CI ? blob : html/json`
          # ternary. The runner exports CI=true and `unset CI` does not reliably reach the
          # merge subprocess, so the config would intermittently re-select the `blob` reporter —
          # which writes a merged blob-report/ and NOTHING to playwright-results/, making the
          # downstream `ls playwright-results/html-report/` fail with a cryptic exit 2.
          mkdir -p playwright-results
          # These env vars override playwright.config.js's reporter settings:
          #   PLAYWRIGHT_HTML_OUTPUT_DIR  - html reporter output directory
          #   PLAYWRIGHT_HTML_OPEN=never  - never auto-open the report (required for CI)
          #   PLAYWRIGHT_JSON_OUTPUT_NAME - json reporter output file path
          PLAYWRIGHT_HTML_OUTPUT_DIR=playwright-results/html-report \
          PLAYWRIGHT_HTML_OPEN=never \
          PLAYWRIGHT_JSON_OUTPUT_NAME=playwright-results/report.json \
            npx playwright merge-reports --reporter html,json ./all-blob-reports

          echo "Contents of playwright-results:"
          ls -la playwright-results/ || true
          if [ ! -f playwright-results/html-report/index.html ] || [ ! -f playwright-results/report.json ]; then
            echo "::error::merge-reports did not produce the expected html-report/index.html and report.json"
            echo "::error::all-blob-reports contents:"
            ls -la all-blob-reports/ || true
            exit 1
          fi
          # Diagnostics only — never fail the step on these (the guard above already
          # confirmed the outputs exist).
          echo "Contents of html-report:"
          ls -la playwright-results/html-report/ || true
          echo "Checking report.json:"
          ls -lh playwright-results/report.json || true

      - name: Extract report stats for OpenObserve
        id: report_stats
        if: always()
        run: |
          cd tests/ui-testing
          if [ -f playwright-results/report.json ]; then
            STATS=$(jq -c '.stats // {}' playwright-results/report.json 2>/dev/null || echo '{}')
          else
            STATS='{}'
          fi
          echo "stats=$STATS" >> "$GITHUB_OUTPUT"
          echo "report stats: $STATS"

      - name: Extract flaky tests for OpenObserve
        id: report_flaky
        if: always()
        run: |
          # Flaky + failed tests from the merged report, as one JSON array, for the
          # ci_test_results stream. Logic + schema live in extract-flaky.js (it
          # handles a missing/malformed report by printing [] non-fatally, and is
          # commented to explain status="flaky" vs "failed" and the sp.title choice).
          FLAKY=$(node .github/scripts/extract-flaky.js tests/ui-testing/playwright-results/report.json)
          echo "flaky=$FLAKY" >> "$GITHUB_OUTPUT"
          # Log a count only — avoid dumping test file paths/titles into public CI logs.
          echo "flaky/failed test count: $(printf '%s' "$FLAKY" | jq 'length' 2>/dev/null || echo '?')"

      - name: Upload merged report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: merged-report-${{ github.run_attempt }}
          path: tests/ui-testing/playwright-results
          retention-days: 1
          if-no-files-found: warn

      # Report to Kinora — reuses the report.json + trace.zips this job just merged (all on this
      # runner), so there is NO second artifact download or merge. Fully non-fatal.
      - name: Report to Kinora (upload merged run + traces)
        id: kinora
        if: ${{ vars.UPLOAD_TO_KINORA != 'false' }}
        continue-on-error: true
        env:
          KINORA_TOKEN: ${{ secrets.KINORA_TOKEN }}
          KINORA_URL: ${{ secrets.KINORA_URL }}
          KINORA_PROJECT: openobserve-e2e
        run: |
          cd tests/ui-testing
          if [ ! -f playwright-results/report.json ]; then
            echo "::notice::No merged report.json — skipping Kinora upload."
            exit 0
          fi
          echo "::group::Uploading to Kinora ($KINORA_PROJECT)"
          # Cap a hung upload at 5 min (SIGTERM, then SIGKILL after 30s) so a stalled
          # Kinora endpoint fails fast instead of burning the whole 20-min job budget.
          if OUT=$(timeout -k 30 300 npx --yes @kinora/cli upload playwright-results/report.json \
                     --project "$KINORA_PROJECT" \
                     --git-repo-url "https://github.com/${{ github.repository }}" 2>&1); then
            echo "$OUT"
          else
            echo "::warning::Kinora CLI upload failed (exit $?) — continuing (non-fatal)."
            echo "$OUT"; echo "::endgroup::"; exit 0
          fi
          echo "::endgroup::"
          # Match the CLI's "(run <uuid>)" form specifically so stray log text can't match.
          RUN_ID=$(printf '%s\n' "$OUT" | grep -oiE '\(run [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\)' | head -1 | grep -oiE '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}')
          if [ -n "$RUN_ID" ]; then
            RUN_URL="${KINORA_URL%/}/projects/$KINORA_PROJECT/runs/$RUN_ID"
            echo "run_url=$RUN_URL" >> "$GITHUB_OUTPUT"
            echo "::notice::Kinora run: $RUN_URL"
          else
            echo "::warning::Kinora upload did not return a run id (see log above)."
          fi

      - name: Upsert Kinora report comment on PR
        if: ${{ github.event_name == 'pull_request' && steps.kinora.outputs.run_url != '' }}
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          RUN_URL: ${{ steps.kinora.outputs.run_url }}
        run: |
          MARKER="<!-- kinora-report -->"
          # printf only interprets % in the format string; $RUN_URL is a %s arg, so it is safe.
          BODY=$(printf '%s\n### 📊 Kinora report\nPlaywright run uploaded to Kinora — **[View run in Kinora](%s)**\n\n<sub>Updated for `%s` · run attempt %s. This comment updates in place on every push (no new comments).</sub>' \
            "$MARKER" "$RUN_URL" "${HEAD_SHA:0:9}" "${{ github.run_attempt }}")
          CID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \
                  -q ".[] | select(.body | contains(\"$MARKER\")) | .id" | head -1)
          if [ -n "$CID" ]; then
            gh api -X PATCH "repos/$REPO/issues/comments/$CID" -f body="$BODY" >/dev/null && echo "Updated existing Kinora comment ($CID)."
          else
            gh api -X POST "repos/$REPO/issues/$PR_NUMBER/comments" -f body="$BODY" >/dev/null && echo "Posted new Kinora comment."
          fi

  # OpenObserve metrics — its own job so playwright_summary never depends on it. Emits one
  # retry-aware run-summary doc (ci_test_runs, suite=ui) + per-test flaky/failed rows
  # (ci_test_results). The step is continue-on-error and the poster scripts never fail.
  report_openobserve:
    timeout-minutes: 10
    name: Report to OpenObserve
    needs: [generate_matrix, ui_integration_tests, merge_reports]
    if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
    runs-on:
      labels: eks-openobserve-standard-4
    permissions:
      contents: read
      actions: read  # reads run/job timings from the Actions API
    steps:
      - name: Clone the current repo
        uses: actions/checkout@v5

      - name: Report run metrics to OpenObserve
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          O2_REPORTING_INGEST_BASE: ${{ secrets.O2_REPORTING_INGEST_BASE }}
          O2_REPORTING_AUTH: ${{ secrets.O2_REPORTING_AUTH }}
          O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
          # Track lightweight readiness separately from a successful fresh
          # build; skipped reuse-path builds are reported as null.
          RUNTIME_READY_JOB_NAME: resolve_binary
          TESTS: ${{ needs.ui_integration_tests.result }}
          MERGE_STATS: ${{ needs.merge_reports.outputs.stats }}
          MERGE_FLAKY: ${{ needs.merge_reports.outputs.flaky }}
          # Branch + PR for per-test rows so the dashboard needs no cross-stream JOIN.
          # head_ref is set on pull_request (the source branch); ref_name covers push +
          # merge_group (gh-readonly-queue/main/pr-<n>-<sha>). PR empty on merge_group/push.
          REPORT_BRANCH: ${{ github.head_ref || github.ref_name }}
          REPORT_PR: ${{ github.event.pull_request.number }}
          SMOKE_ACTIVE: ${{ needs.generate_matrix.outputs.smoke_active }}
          SHARDS_SELECTED: ${{ needs.generate_matrix.outputs.shards_selected }}
        run: |
          # This job runs only when tests actually ran (merge `if:` gates on !cancelled &&
          # ui != skipped), so TESTS is success|failure here.
          if [ "$TESTS" = "failure" ]; then CONCLUSION=failure; else CONCLUSION=success; fi
          # Per-test counts from the merged report.json (tests_total includes skipped).
          if [ -n "${MERGE_STATS:-}" ] && [ "$MERGE_STATS" != "null" ] && [ "$MERGE_STATS" != "{}" ]; then
            EXTRA_JSON=$(printf '%s' "$MERGE_STATS" | jq -c '{tests_total:((.expected//0)+(.unexpected//0)+(.flaky//0)+(.skipped//0)), tests_passed:(.expected//0), tests_failed:(.unexpected//0), tests_flaky:(.flaky//0), tests_skipped:(.skipped//0)}' 2>/dev/null || echo "")
          fi
          # smoke_active/shards_selected let dashboards separate smoke rows from full runs,
          # since shards_total now varies by selection instead of always being the full set.
          SMOKE_JSON=$(jq -nc --arg a "${SMOKE_ACTIVE:-false}" --arg n "${SHARDS_SELECTED:-}" \
            '{smoke_active:($a=="true"), shards_selected:(($n|tonumber?)//null)}')
          if [ -n "${EXTRA_JSON:-}" ]; then
            EXTRA_JSON=$(jq -nc --argjson a "$EXTRA_JSON" --argjson b "$SMOKE_JSON" '$a * $b')
          else
            EXTRA_JSON="$SMOKE_JSON"
          fi
          export EXTRA_JSON
          export CONCLUSION STREAM=ci_test_runs SUITE=ui
          bash .github/scripts/o2-run-report.sh
          # One row per flaky-or-failed test -> ci_test_results (status flaky|failed; suite/
          # workflow hardcoded UI-only). o2-report.sh takes the stream as arg 1. Non-fatal.
          FLAKY="${MERGE_FLAKY:-[]}"
          if printf '%s' "$FLAKY" | jq -e 'type=="array" and length>0' >/dev/null 2>&1; then
            FLAKY_FILE=$(mktemp "${TMPDIR:-/tmp}/flaky.XXXXXX")
            trap 'rm -f "$FLAKY_FILE"' EXIT
            printf '%s' "$FLAKY" | jq -c --arg run_id "$GITHUB_RUN_ID" --arg repo "$GITHUB_REPOSITORY" \
              --arg branch "${REPORT_BRANCH:-}" --arg pr "${REPORT_PR:-}" \
              '[.[] | {run_id:$run_id, repo:$repo, suite:"ui", workflow:"test", ingest_source:"live", status:.status, module:.module, test_title:.title, test_file:.file, branch:(if $branch=="" then null else $branch end), pr_number:(if $pr=="" then null else ($pr|tonumber) end)}]' \
              > "$FLAKY_FILE" 2>/dev/null || echo '[]' > "$FLAKY_FILE"
            if [ "$(jq 'length' "$FLAKY_FILE" 2>/dev/null || echo 0)" -gt 0 ]; then
              bash .github/scripts/o2-report.sh ci_test_results "$FLAKY_FILE"
            fi
          fi

  playwright_summary:
    timeout-minutes: 5
    runs-on:
      labels: eks-openobserve-standard-4
    permissions: {}
    needs:
      [
        check_changes,
        resolve_binary,
        build_binary,
        ui_integration_tests,
      ]
    if: always() && github.event_name != 'merge_group' && (github.event_name != 'pull_request' || (contains(github.event.pull_request.labels.*.name, 'ready-for-ci') && github.event.pull_request.draft == false))
    steps:
      - name: Check test results
        run: |
          # 1. check_changes must have succeeded — if it failed, downstream jobs skipped
          #    and we must not report a false success.
          if [ "${{ needs.check_changes.result }}" != "success" ]; then
            echo "check_changes job did not succeed: ${{ needs.check_changes.result }}"
            exit 1
          fi

          # 2. Nothing Playwright-relevant changed → nothing ran → allow.
          if [ "${{ needs.check_changes.outputs.has_changes }}" != "true" ]; then
            echo "✅ No Playwright-relevant changes — suite not required."
            exit 0
          fi

          # 3. Suite was expected to run. The resolver must succeed, and either
          #    an exact-SHA pair was reused or this run built a fresh pair.
          if [ "${{ needs.resolve_binary.result }}" != "success" ]; then
            echo "resolve_binary job did not succeed: ${{ needs.resolve_binary.result }}"
            exit 1
          fi
          if [ "${{ needs.resolve_binary.outputs.reuse }}" != "true" ] && \
             [ "${{ needs.build_binary.result }}" != "success" ]; then
            echo "No reusable artifact and build_binary did not succeed: ${{ needs.build_binary.result }}"
            exit 1
          fi

          # 4. Runtime readiness alone is not enough; all selected shards must pass.
          if [ "${{ needs.ui_integration_tests.result }}" == "success" ]; then
            echo "All Playwright tests completed successfully"
            exit 0
          else
            echo "Playwright tests failed:"
            echo "  resolve_binary: ${{ needs.resolve_binary.result }} (reuse=${{ needs.resolve_binary.outputs.reuse }})"
            echo "  build_binary: ${{ needs.build_binary.result }}"
            echo "  ui_integration_tests: ${{ needs.ui_integration_tests.result }}"
            exit 1
          fi