From 9bf7164e2b68ed5c277f0235dfbbce1a8268bd2a Mon Sep 17 00:00:00 2001
From: Shrinath Rao <shnath@openobserve.ai>
Date: Sat, 27 Jun 2026 22:59:44 +0530
Subject: [PATCH] ci: capture UI (Playwright) per-test counts (ci_test_runs)
 (#12915)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## What

Capture **UI (Playwright) per-test counts** in the `ci_test_runs`
OpenObserve stream — `tests_total/passed/failed/flaky/skipped` per run.

This was meant to land in #12914 but missed that squash-merge (pushed
just after), so it's its own PR now. Companion: o2-enterprise#2046.

## How
- `playwright.yml` merge job gains a `stats` output (from the merged
`report.json`).
- `report_to_openobserve` reads it and emits the test-count fields via a
small generic **`EXTRA_JSON`** merge hook added to `o2-run-report.sh`.
- Non-fatal as before; no-ops when secrets are unset.

API (pytest) counts are not included (would need `--junitxml` + XML
aggregation — separate change).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
---
 .github/scripts/o2-run-report.sh | 14 ++++++++++++++
 .github/workflows/playwright.yml | 26 +++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/o2-run-report.sh b/.github/scripts/o2-run-report.sh
index 862f181955..da6af0563c 100755
--- a/.github/scripts/o2-run-report.sh
+++ b/.github/scripts/o2-run-report.sh
@@ -8,6 +8,10 @@
 # shared o2-report.sh. Live re-runs append one doc per attempt; dashboards keep the latest
 # attempt per run_id (row_number() dedup), so retries become an attribute, not duplicate rows.
 #
+# Optional EXTRA_JSON (env): a JSON OBJECT the caller can pass to fold extra fields into the run
+# doc (e.g. test-case counts). Shallow-merged via jq `*` (EXTRA_JSON keys win on conflict). On any
+# parse/merge error the base payload is kept unchanged.
+#
 # Never fails the workflow: missing secrets or any API/ingest error only warns.
 set -uo pipefail
 : "${STREAM:?STREAM required}"
@@ -73,4 +77,14 @@ printf '%s' "$RUN_JSON" | jq \
      retry_wasted_sec: (($total-$final)|round)
    }' > "$PAYLOAD_FILE" 2>/dev/null || { echo "::warning::payload build failed"; exit 0; }
 
+# Optional EXTRA_JSON: shallow-merge caller-supplied fields into the doc (e.g. test counts).
+# On any merge error, keep the base payload (never fail the report).
+if [ -n "${EXTRA_JSON:-}" ] && printf '%s' "$EXTRA_JSON" | jq -e . >/dev/null 2>&1; then
+  if jq -s '.[0] * .[1]' "$PAYLOAD_FILE" <(printf '%s' "$EXTRA_JSON") > "$PAYLOAD_FILE.m" 2>/dev/null; then
+    mv "$PAYLOAD_FILE.m" "$PAYLOAD_FILE"
+  else
+    echo "::warning::EXTRA_JSON merge failed — keeping base payload"; rm -f "$PAYLOAD_FILE.m"
+  fi
+fi
+
 bash "$(dirname "$0")/o2-report.sh" "$STREAM" "$PAYLOAD_FILE"
diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
index e0c7f978ff..fc67c41999 100644
--- a/.github/workflows/playwright.yml
+++ b/.github/workflows/playwright.yml
@@ -888,6 +888,10 @@ jobs:
     permissions:
       contents: read
       actions: read
+    outputs:
+      # Compact Playwright stats (expected/unexpected/flaky/skipped) from the merged report.json,
+      # consumed by report_to_openobserve to record per-test counts in ci_test_runs.
+      stats: ${{ steps.report_stats.outputs.stats }}
 
     steps:
       - name: Clone the current repo
@@ -977,6 +981,19 @@ jobs:
           echo "Checking report.json:"
           ls -lh playwright-results/report.json
 
+      - name: Extract report stats for OpenObserve
+        id: report_stats
+        if: always()
+        run: |
+          cd tests/ui-testing
+          if [ -f playwright-results/report.json ]; then
+            STATS=$(jq -c '.stats // {}' playwright-results/report.json 2>/dev/null || echo '{}')
+          else
+            STATS='{}'
+          fi
+          echo "stats=$STATS" >> "$GITHUB_OUTPUT"
+          echo "report stats: $STATS"
+
       - name: Cache test failures to TestDino
         if: ${{ needs.ui_integration_tests.result == 'failure' }}
         env:
@@ -1143,7 +1160,7 @@ jobs:
   # suite=ui). Re-runs append one doc per attempt; the dashboard keeps the latest attempt per run_id.
   report_to_openobserve:
     name: Report run metrics to OpenObserve
-    needs: [check_changes, build_binary, ui_integration_tests]
+    needs: [check_changes, build_binary, ui_integration_tests, merge_and_upload_reports]
     if: always()
     runs-on: ubuntu-latest
     permissions:
@@ -1161,6 +1178,7 @@ jobs:
           O2_REPORTING_INSECURE: ${{ vars.O2_REPORTING_INSECURE }}
           BUILD: ${{ needs.build_binary.result }}
           TESTS: ${{ needs.ui_integration_tests.result }}
+          MERGE_STATS: ${{ needs.merge_and_upload_reports.outputs.stats }}
         run: |
           # No relevant changes → tests were skipped → no run to record.
           if [ "$TESTS" = "skipped" ] && [ "$BUILD" = "skipped" ]; then
@@ -1169,5 +1187,11 @@ jobs:
           if [ "$TESTS" = "failure" ] || [ "$BUILD" = "failure" ]; then CONCLUSION=failure
           elif [ "$TESTS" = "cancelled" ] || [ "$BUILD" = "cancelled" ]; then CONCLUSION=cancelled
           else CONCLUSION=success; fi
+          # Per-test counts from the merged report.json. tests_total is the FULL total (includes
+          # skipped, matching regression); the dashboard derives executed = passed+failed+flaky.
+          if [ -n "${MERGE_STATS:-}" ] && [ "$MERGE_STATS" != "null" ] && [ "$MERGE_STATS" != "{}" ]; then
+            EXTRA_JSON=$(printf '%s' "$MERGE_STATS" | jq -c '{tests_total:((.expected//0)+(.unexpected//0)+(.flaky//0)+(.skipped//0)), tests_passed:(.expected//0), tests_failed:(.unexpected//0), tests_flaky:(.flaky//0), tests_skipped:(.skipped//0)}' 2>/dev/null || echo "")
+            [ -n "$EXTRA_JSON" ] && export EXTRA_JSON
+          fi
           export CONCLUSION STREAM=ci_test_runs SUITE=ui
           bash .github/scripts/o2-run-report.sh
\ No newline at end of file