diff --git a/.github/scripts/build-ci-matrix.js b/.github/scripts/build-ci-matrix.js new file mode 100644 index 0000000000..fbb1de82b7 --- /dev/null +++ b/.github/scripts/build-ci-matrix.js @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/** + * build-ci-matrix.js — single source of truth for the Playwright UI shard matrix. + * + * Prints a GitHub Actions matrix object: {"include":[ {testfolder,actual_folder,browser,run_files}, ... ]} + * consumed by `strategy.matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }}`. + * + * Usage: + * node build-ci-matrix.js # OSS: base manifest verbatim + * node build-ci-matrix.js # ENT: base + enterprise overlay + * + * The base (OSS tests/ui-testing/ci-matrix/ci_matrix.json) is the ONLY place shared shards are + * listed. ENT never re-lists shared specs — its overlay (ci_matrix.ent.json) carries + * only the delta, so a spec added to OSS flows into ENT automatically: + * { "append": { "": ["extra.spec.js", ...] }, // add ENT-only specs to a shared shard + * "shards": [ {testfolder,actual_folder,browser,run_files} ], // whole ENT-only shards (Workflows, SDR…) + * "disabled": { "": [{file,reason}, ...] } } // ENT-only specs turned off (doc only) + * + * DISABLING A SPEC (JSON has no // comments): don't delete it — move it into a + * shard's "disabled": [{ "file": "x.spec.js", "reason": "why" }] array. Disabled + * entries are documentation only: this script never emits them, so they don't run, + * but the record + reason survive. "_comment" (or any _-prefixed key) is also ignored. + * + * This lives in OSS so ENT can reuse it from its tree-merged OSS checkout. + */ +const fs = require("fs"); + +function die(msg) { + process.stderr.write(`build-ci-matrix: ${msg}\n`); + process.exit(1); +} + +function readJson(path) { + try { + return JSON.parse(fs.readFileSync(path, "utf8")); + } catch (e) { + die(`cannot read/parse ${path}: ${e.message}`); + } +} + +const [basePath, overlayPath] = process.argv.slice(2); +if (!basePath) die("usage: build-ci-matrix.js [overlay.json]"); + +const base = readJson(basePath); +if (!Array.isArray(base)) die(`base ${basePath} must be a JSON array of shards`); + +// Deep-copy base so we never mutate the parsed source objects. +const include = base.map((s) => ({ ...s, run_files: [...(s.run_files || [])] })); +const byFolder = new Map(include.map((s) => [s.testfolder, s])); + +if (overlayPath) { + const overlay = readJson(overlayPath); + + // append: add enterprise-only specs to an EXISTING shared shard. + for (const [folder, specs] of Object.entries(overlay.append || {})) { + const shard = byFolder.get(folder); + if (!shard) { + die(`overlay append targets unknown folder "${folder}" — it must exist in the base manifest`); + } + for (const spec of specs) { + if (shard.run_files.includes(spec)) { + die(`overlay append: "${spec}" already in base shard "${folder}" — remove it from the overlay`); + } + shard.run_files.push(spec); + } + } + + // shards: whole enterprise-only shards (Workflows, SDR-Logs, SDR-Traces, …). + for (const shard of overlay.shards || []) { + if (byFolder.has(shard.testfolder)) { + die(`overlay shard "${shard.testfolder}" collides with a base shard — use "append" instead`); + } + const copy = { browser: "chrome", ...shard, run_files: [...(shard.run_files || [])] }; + include.push(copy); + byFolder.set(copy.testfolder, copy); + } + + // disabled: enterprise-only turned-off specs, recorded against a shard for docs only. + for (const [folder, entries] of Object.entries(overlay.disabled || {})) { + const shard = byFolder.get(folder); + if (!shard) die(`overlay disabled targets unknown folder "${folder}"`); + shard.disabled = [...(shard.disabled || []), ...entries]; + } +} + +// Sanity: unique testfolders, no empty shards, no dup specs, and no spec both +// active (run_files) and disabled in the same shard. +const seen = new Set(); +for (const s of include) { + if (!s.testfolder) die(`shard missing testfolder: ${JSON.stringify(s)}`); + if (seen.has(s.testfolder)) die(`duplicate testfolder "${s.testfolder}"`); + seen.add(s.testfolder); + if (!s.actual_folder) s.actual_folder = s.testfolder; + if (!s.run_files || s.run_files.length === 0) die(`shard "${s.testfolder}" has no run_files`); + if (new Set(s.run_files).size !== s.run_files.length) { + die(`shard "${s.testfolder}" has duplicate run_files`); + } + for (const d of s.disabled || []) { + if (s.run_files.includes(d.file)) { + die(`shard "${s.testfolder}": "${d.file}" is in both run_files and disabled`); + } + } +} + +// Emit ONLY the fields the CI matrix consumes — disabled/_comment/notes are stripped, +// so turned-off specs never reach `npx playwright test`. +const matrix = include.map((s) => ({ + testfolder: s.testfolder, + actual_folder: s.actual_folder, + browser: s.browser || "chrome", + run_files: s.run_files, +})); + +process.stdout.write(JSON.stringify({ include: matrix })); diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index f583d5fa9a..774e75f901 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -176,10 +176,36 @@ jobs: retention-days: 1 if-no-files-found: error + # Build the shard matrix from tests/ui-testing/ci-matrix/ci_matrix.json (single source of truth, + # shared with ENT via build-ci-matrix.js). Runs under the same gate as build_binary so + # ui_integration_tests always has a matrix whenever it is allowed to run. + generate_matrix: + timeout-minutes: 5 + name: generate_matrix + needs: [check_changes] + if: >- + (github.event_name == 'pull_request' && + contains(github.event.pull_request.labels.*.name, 'e2e')) || + (github.event_name != 'pull_request' && + needs.check_changes.outputs.has_changes == 'true') + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v5 + - id: gen + run: | + MATRIX=$(node .github/scripts/build-ci-matrix.js tests/ui-testing/ci-matrix/ci_matrix.json) + if [ -z "$MATRIX" ]; then echo "::error::matrix generation failed (build-ci-matrix produced no output)"; exit 1; fi + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$MATRIX" | node -e "const m=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log('generated',m.include.length,'shards:', m.include.map(s=>s.testfolder).join(', '))" + ui_integration_tests: timeout-minutes: 45 name: e2e / ${{ matrix.testfolder }} - needs: [build_binary] + needs: [build_binary, generate_matrix] if: ${{ !cancelled() && needs.build_binary.result == 'success' }} runs-on: labels: eks-openobserve-standard-8 @@ -191,328 +217,10 @@ jobs: # options: --user root strategy: fail-fast: false - matrix: - include: - - testfolder: "GeneralTests" - browser: "chrome" - run_files: - [ - "sanity.spec.js", - "changeOrg.spec.js", - "enrichment.spec.js", - "schema.spec.js", - "schemaload.spec.js", - "serviceAccount.spec.js", - "usersOrg.spec.js", - "org.spec.js", - "theme-management.spec.js", - "ingestion-config.spec.js", - "ingestionTokens.spec.js", - "landingPage.spec.js", - "languageTranslation.spec.js", - "model-pricing.spec.js", - "edition-features.spec.js", - "logsqueries.sql-comments.spec.js", - "iam-form-validation.spec.js", - "regex-patterns-form-validation.spec.js", - "cipher-keys-form-validation.spec.js", - "settings-form-validation.spec.js", - "ai-toolsets-form-validation.spec.js", - "shared-components-form-validation.spec.js", - "onboarding-form-validation.spec.js", - "rum-form-validation.spec.js", - "login-form-validation.spec.js", - ] - - testfolder: "Logs-Builder-Basic" - browser: "chrome" - run_files: - [ - "logsQueryBuilder-chart.spec.js", - "logsQueryBuilder-filters-basic.spec.js", - "pagination.spec.js", - "unflattened.spec.js", - ] - - testfolder: "Logs-Builder-Advanced" - browser: "chrome" - run_files: - [ - "logsQueryBuilder-editor.spec.js", - "logsQueryBuilder-filters-advanced.spec.js", - "logsquickmode.spec.js", - "logshistogram.spec.js", - "logsQuickPick.spec.js", - ] - - testfolder: "Logs-Core" - browser: "chrome" - run_files: - [ - "logspage.spec.js", - "logstable.spec.js", - "logsqueries.spec.js", - "join.spec.js", - "shareLink.spec.js", - "logsAnalyzeDimensions.spec.js", - "monaco-query-prefill.spec.js", - "logsqueries.matchall.spec.js", - "ftsDefaultColumn.spec.js", - "logsVisualizePersistence.spec.js", - ] - - testfolder: "Logs-Features" - browser: "chrome" - run_files: [ - "logs-autocomplete-suggestions.spec.js", - # "searchJobInspector.spec.js", # TODO: update these tests for new correlation feature - "logs-sql-autocomplete.spec.js", - "logsqueries.cte.spec.js", - "logsDownloads.spec.js", - "secondsPrecisionAdded.spec.js", - "searchpartition.spec.js", - "indexquery.spec.js", - "region.spec.js", - ] - - testfolder: "Alerts" - browser: "chrome" - run_files: [ - # "alerts-e2e-flow.spec.js", - "alerts-ui-operations.spec.js", - "alerts-import.spec.js", - # "alerts-advanced.spec.js", - # "alerts-scheduled-features.spec.js", - "alerts-metrics-notification.spec.js", - "alerts-destinations-prebuilt.spec.js", - "alerts-vrl-encoding.spec.js", - "alerts-history.spec.js", - "alerts-template-prebuilt-guard.spec.js", - "alerts-form-validation.spec.js", - "anomaly-detection-form-validation.spec.js", - # Alerts 4.0 (multi-alerts) e2e — PR #13547 - "alerts-priority-tags.spec.js", - "alerts-multialert-api.spec.js", - "alerts-multialert-ui.spec.js", - "alerts-multialert-regression.spec.js", - "alerts-multialert-firing.spec.js", - "alerts-multialert-transitions.spec.js", - ] - - testfolder: "Dashboards-Core" - browser: "chrome" - run_files: - [ - "dashboard.spec.js", - "dashboard2.spec.js", - "dashboardtype.spec.js", - "dashboard-folder.spec.js", - "dashboard-import.spec.js", - "maxquery.spec.js", - "dashboard-chartJson.spec.js", - "dashboard-promql-suggestions.spec.js", - "dashboard-sql-autocomplete.spec.js", - "dashboard-share-link.spec.js", - "crossLinking.spec.js", - "dashboard-multi-sql.spec.js", - "crossLinkMultiStream.spec.js", - "dashboards-form-validation.spec.js", - ] - - testfolder: "Dashboards-Isolated" - browser: "chrome" - run_files: ["dashboard-favorites.spec.js"] - - testfolder: "Dashboards-Settings" - browser: "chrome" - run_files: - [ - "dashboard-filter.spec.js", - "dashboard-general-setting.spec.js", - "dashboard-tabs-setting.spec.js", - "dashboard-transpose.spec.js", - "dashboard-join.spec.js", - "dashboard-nested-functions.spec.js", - "dashboard-legends-copy.spec.js", - "dashboard-raw-query.spec.js", - ] - # Dashboard panel tests are grouped by the feature they exercise so - # each shard stays well under the ephemeral-runner endurance window. - # A single combined 16-spec shard ran ~40 min on CI and got reclaimed - # mid-run ("runner lost communication"). Add new specs to the shard of - # the feature they cover; the Tables shard also owns the pagination and - # pivot-table specs (previously their own single-spec shards). - - testfolder: "Dashboards-Charts" - browser: "chrome" - run_files: - [ - "custom-charts.spec.js", - "dashboard-geoMap.spec.js", - "dashboard-maps.spec.js", - "dashboard-multi-y-axis.spec.js", - "dashboard-html-chart.spec.js", - "dashboard-sankey.spec.js", - "dashboard-pie-donut.spec.js", - "dashboard-metric-camelcase.spec.js", - "dashboard-create-alert.spec.js", - ] - - testfolder: "Dashboards-Visualize" - browser: "chrome" - run_files: - [ - "visualize.spec.js", - "visualize-vrl.spec.js", - ] - - testfolder: "Dashboards-Tables" - browser: "chrome" - run_files: - [ - "dashboard-table-chart.spec.js", - "dashboard-table-copy-cell.spec.js", - "dashboard-table-csv-download.spec.js", - "dashboard-table-column-formatting.spec.js", - "dashboard-table-column-filtering.spec.js", - "dashboard-table-pagination.spec.js", - "dashboard-pivot-table.spec.js", - ] - - testfolder: "Dashboards-Streaming" - browser: "chrome" - run_files: ["dashboard-streaming.spec.js"] - - testfolder: "Pipelines" - browser: "chrome" - run_files: [ - "pipeline-conditions.spec.js", - "pipeline-conditions-validation.spec.js", - "pipelines.spec.js", - "pipeline-dynamic.spec.js", - "pipeline-core.spec.js", - "pipeline-traces.spec.js", - "pipeline-metrics.spec.js", - "pipeline-backfill.spec.js", - "pipeline-history.spec.js", - "scheduled-pipeline-query-builder.spec.js", - # "remotepipeline.spec.js" - "pipelines-form-validation.spec.js", - ] - - testfolder: "Functions" - browser: "chrome" - run_files: - [ - "js-transform-type.spec.js", - "row-expansion.spec.js", - "enrichment-table-url.spec.js", - "functions-form-validation.spec.js", - "action-scripts-form-validation.spec.js", - ] - - testfolder: "Reports" - browser: "chrome" - run_files: - [ - "reportsScheduleNow.spec.js", - "reportsScheduleLater.spec.js", - "reportFolders.spec.js", - "reports-bulk-operations.spec.js", - "reports-form-validation.spec.js", - "reportCsvMediaType.spec.js", - ] - - testfolder: "Streams" - browser: "chrome" - run_files: - [ - "multiselect-stream.spec.js", - "streamname.spec.js", - "streaming.spec.js", - "stream-settings.spec.js", - "streams-form-validation.spec.js", - ] - - testfolder: "Traces" - browser: "chrome" - run_files: - [ - "service-catalog.spec.js", - "tracesSearch.spec.js", - "traceQueryEditor.spec.js", - "traceErrorFilter.spec.js", - "traceDetails.spec.js", - "traceAdvancedFiltering.spec.js", - "tracesAnalyzeDimensions.spec.js", - "traces-autocomplete-suggestions.spec.js", - ] - # The RUM ingest/token/sourcemaps routes are part of the standard - # (OSS) router (src/api/http/src/handler/http/router/mod.rs), so these specs run - # against the OSS binary built by this workflow. Ingestion-auth - # negative cases live in the API suite (tests/api-testing/tests/rum). - - testfolder: "RUM" - browser: "chrome" - run_files: - [ - "rum-cdn-dataflow.spec.js", - "rum-npm-dataflow.spec.js", - "rum-page-dataflow.spec.js", - "rum-onboarding-snippets.spec.js", - "sourcemap-ui.spec.js", - "sourcemap-upload-pretty.spec.js", - ] - # rum-token.spec.js RESETS the org RUM token; running it beside the - # dataflow specs (workers run files in parallel) would 401 their - # in-flight SDK beacons. Own shard = own isolated instance. - - testfolder: "RUM-Token" - browser: "chrome" - run_files: ["rum-token.spec.js"] - - testfolder: "Metrics" - browser: "chrome" - run_files: - [ - "metrics.spec.js", - "metrics-queries.spec.js", - "metrics-aggregations.spec.js", - "metrics-advanced.spec.js", - "metrics-config-tabs.spec.js", - "metrics-visualizations.spec.js", - "metrics-config.spec.js", - "metrics-promql-query-persistence.spec.js", - "metrics-table-column-order.spec.js", - "metrics-promql-builder.spec.js", - "promqlAutocomplete.spec.js", - "metrics-share-deep-link.spec.js", - ] - - testfolder: "Dashboards-Variables" - browser: "chrome" - run_files: - [ - "dashboard-variables-setting.spec.js", - "dashboard-variables-global.spec.js", - "dashboard-variables-tab-level.spec.js", - "dashboard-variables-panel-level.spec.js", - "dashboard-variables-dependency.spec.js", - "dashboard-variables-refresh.spec.js", - "dashboard-variables-url-sync.spec.js", - "dashboard-variables-creation-scopes.spec.js", - "dashboard-variables-default-values-chain.spec.js", - "dashboard-variables-custom-parents.spec.js", - "dashboard-variables-stream-field.spec.js", - "dashboard-mustache-variables.spec.js", - ] - - testfolder: "Dashboards-Panel-Level-DateTime-Config" - browser: "chrome" - run_files: - [ - "dashboard-panel-time-config-behavior.spec.js", - "dashboard-panel-time-url-priority.spec.js", - "dashboard-panel-time-advanced-edge-cases.spec.js", - "dashboard-panel-time-apply-behavior.spec.js", - "dashboard-panel-time-variables-behavior.spec.js", - ] - - testfolder: "Dashboard-Config-Settings" - browser: "chrome" - run_files: - [ - "dashboard-series-color-multiwindow.spec.js", - "dashboard-config-advanced.spec.js", - "dashboard-config-axis.spec.js", - "dashboard-config-gauge-maps.spec.js", - "dashboard-config-general.spec.js", - "dashboard-config-legends.spec.js", - "dashboard-config-line-style.spec.js", - "dashboard-config-panel-time.spec.js", - "dashboard-config-table.spec.js", - "dashboard-config-trellis.spec.js", - "dashboard-config-drilldown.spec.js", - "dashboard-config-promql.spec.js", - "dashboard-config-markline.spec.js", - ] + # Matrix is generated from tests/ui-testing/ci-matrix/ci_matrix.json by the + # generate_matrix job (the single source of truth shared with ENT). To add + # or move a spec, edit that JSON file — do NOT hand-edit a matrix here. + matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }} steps: - name: Kill background apt processes run: | @@ -629,22 +337,9 @@ jobs: echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}" if [ -n "$FILE_LIST" ]; then - # Map logical folder names to actual directories - case "${{ matrix.testfolder }}" in - "Logs-Builder-Basic"|"Logs-Builder-Advanced"|"Logs-Core"|"Logs-Features") - ACTUAL_FOLDER="Logs" - ;; - "Dashboards-Core"|"Dashboards-Settings"|"Dashboards-Charts"|"Dashboards-Visualize"|"Dashboards-Tables"|"Dashboards-Streaming"|"Dashboards-Variables"|"Dashboards-Panel-Level-DateTime-Config"|"Dashboard-Config-Settings"|"Dashboards-Isolated") - ACTUAL_FOLDER="Dashboards" - ;; - "RUM-Token") - ACTUAL_FOLDER="RUM" - ;; - *) - ACTUAL_FOLDER="${{ matrix.testfolder }}" - ;; - esac - + # actual_folder (the real directory under playwright-tests/) comes straight + # from the matrix now — the logical->directory mapping lives in ci-matrix/ci_matrix.json. + ACTUAL_FOLDER="${{ matrix.actual_folder }}" echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER" # Build file paths diff --git a/.github/workflows/playwright_regression.yml b/.github/workflows/playwright_regression.yml index 78f7daeb13..af17a89f07 100644 --- a/.github/workflows/playwright_regression.yml +++ b/.github/workflows/playwright_regression.yml @@ -167,76 +167,38 @@ jobs: retention-days: 1 if-no-files-found: error + generate_matrix: + timeout-minutes: 5 + name: generate_matrix + needs: [check_main_changed] + if: ${{ needs.check_main_changed.outputs.should_run == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + matrix: ${{ steps.gen.outputs.matrix }} + steps: + - uses: actions/checkout@v5 + - id: gen + run: | + MATRIX=$(node .github/scripts/build-ci-matrix.js tests/ui-testing/ci-matrix/ci_matrix_regression.json) + if [ -z "$MATRIX" ]; then echo "::error::matrix generation failed (build-ci-matrix produced no output)"; exit 1; fi + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "$MATRIX" | node -e "const m=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log('generated',m.include.length,'shards:', m.include.map(s=>s.testfolder).join(', '))" + ui_integration_tests: timeout-minutes: 45 name: e2e / ${{ matrix.testfolder }} - needs: [build_binary] + needs: [build_binary, generate_matrix] runs-on: labels: eks-openobserve-standard-8 permissions: contents: read strategy: fail-fast: false - matrix: - include: - - testfolder: "Logs-Regression" - run_files: - [ - "logs-regression.spec.js", - "logs-bugs.spec.js", - "logs-9754.spec.js", - "logs-9044-7354.spec.js", - ] - - testfolder: "Alerts-Regression" - run_files: - [ - "alerts-regression.spec.js", - "alerts-stream-switching.spec.js", - "alerts-bugs.spec.js", - ] - - testfolder: "Dashboard-Regression" - run_files: - [ - "dashboard-regression.spec.js", - ] - - testfolder: "Streams-Regression" - run_files: - [ - "streams-regression.spec.js", - ] - - testfolder: "Pipelines-Regression" - run_files: - [ - "enrichment-regression.spec.js", - "pipeline-regression.spec.js", - ] - - testfolder: "Traces-Regression" - run_files: - [ - "traces-regression.spec.js", - "traces-bugs.spec.js", - ] - - testfolder: "Metrics-Regression" - run_files: - [ - "metrics-regression.spec.js", - ] - - testfolder: "GeneralTests-Regression" - run_files: - [ - "ui-regression.spec.js", - "landing-regression.spec.js", - ] - - testfolder: "DataSources-Regression" - run_files: - [ - "datasources-regression.spec.js", - ] - - testfolder: "Reports-Regression" - run_files: - [ - "reports-regression-bugs.spec.js", - ] + # Matrix from tests/ui-testing/ci-matrix/ci_matrix_regression.json (single source of + # truth, shared with ENT via build-ci-matrix.js). Edit that JSON, not this file. + matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }} steps: - name: Kill background apt processes @@ -326,23 +288,10 @@ jobs: echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}" if [ -n "$FILE_LIST" ]; then - # Derive feature subfolder from testfolder (maps testfolder name → directory under RegressionSet/) - case "${{ matrix.testfolder }}" in - "Dashboard-Regression") FEATURE="Dashboards" ;; - "Pipelines-Regression") FEATURE="Pipelines" ;; - "Alerts-Regression") FEATURE="Alerts" ;; - "Logs-Regression") FEATURE="Logs" ;; - "Traces-Regression") FEATURE="Traces" ;; - "Metrics-Regression") FEATURE="Metrics" ;; - "Streams-Regression") FEATURE="Streams" ;; - "Reports-Regression") FEATURE="Reports" ;; - "DataSources-Regression") FEATURE="DataSources" ;; - "GeneralTests-Regression") FEATURE="General" ;; - *) FEATURE="RegressionSet" ;; - esac - ACTUAL_FOLDER="RegressionSet/$FEATURE" + # actual_folder (e.g. RegressionSet/Logs) comes from the matrix now — + # the testfolder->directory mapping lives in ci_matrix_regression.json. + ACTUAL_FOLDER="${{ matrix.actual_folder }}" echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER" - echo "DEBUG: FEATURE = $FEATURE" FILE_PATHS="" for file in $FILE_LIST; do diff --git a/tests/ui-testing/ci-matrix/README.md b/tests/ui-testing/ci-matrix/README.md new file mode 100644 index 0000000000..6402c55516 --- /dev/null +++ b/tests/ui-testing/ci-matrix/README.md @@ -0,0 +1,76 @@ +# Playwright CI shard matrices — single source of truth + +## Manifest index + +Each Playwright workflow builds its matrix from one of these JSON files (via a +`generate_matrix` job). Shared workflows use an OSS **base** + an ENT **overlay** +(`*.ent.json`, in the enterprise repo); ENT-only workflows use a standalone manifest. + +| Manifest | Drives workflow | Kind | +|---|---|---| +| `ci_matrix.json` (+ ENT `ci_matrix.ent.json`) | `playwright.yml` (PR gate) | shared base + overlay | +| `ci_matrix_regression.json` (+ ENT `ci_matrix_regression.ent.json`) | `playwright_regression.yml` | shared base + overlay | +| `ci_matrix_cloud.json` *(ENT repo)* | `playwright_alpha1.yml` | ENT-only standalone | +| `ci_matrix_env.json` *(ENT repo)* | `playwright_env.yml` | ENT-only standalone | +| `ci_matrix_env_scheduled.json` *(ENT repo)* | `playwright_env_scheduled.yml` | ENT-only standalone | +| `ci_matrix_firefox.json` *(ENT repo)* | `playwright-firefox-ondemand.yml` | ENT-only standalone | + +Base manifests + `build-ci-matrix.js` live in OSS; overlays and ENT-only manifests live +in `o2-enterprise/tests/ui-testing/ci-matrix/`. The merge script is shared (ENT reuses it +from its OSS checkout). + +--- + +## The shared PR-gate matrix (below refers to `ci_matrix.json`) + +`ci_matrix.json` (this directory) is the **only** place the Playwright UI shard list +lives. Both the OSS and Enterprise `playwright.yml` workflows build their test matrix +from it at run time via `.github/scripts/build-ci-matrix.js`, so a spec added here runs +in **both** repos automatically — no more hand-syncing two workflow files. + +## Adding / moving a spec + +- **A spec both OSS and ENT run:** edit `ci_matrix.json` only. Add the filename to the + `run_files` of the right shard (`testfolder`). Done — ENT picks it up on its next run. +- **An enterprise-only spec:** edit `o2-enterprise/tests/ui-testing/ci-matrix/ci_matrix.ent.json` + (the overlay), never this file. Two shapes: + - add it to an existing shared shard → `"append": { "": ["my.spec.js"] }` + - a whole new ENT-only shard → add an object to `"shards": [ … ]`. +- **A new shard:** add a new object to `ci_matrix.json` with `testfolder`, + `actual_folder`, `browser`, `run_files`. + +## Fields + +| field | meaning | +|-----------------|-------------------------------------------------------------------------| +| `testfolder` | shard label — becomes the job name `e2e / ` (must be unique) | +| `actual_folder` | real directory under `playwright-tests/` (e.g. `Logs-Core` → `Logs`) | +| `browser` | `chrome` | +| `run_files` | spec filenames run by this shard | +| `disabled` | *(optional)* specs intentionally turned off — see below | + +## Disabling a spec (JSON has no `//` comments) + +Every shard ships with a `"disabled": []` placeholder, so turning a spec off is a +fill-in-the-blank — don't delete the spec you want to remember, **move it into that +shard's `disabled` array** with a reason. `build-ci-matrix.js` never emits `disabled`, so +those specs don't run, but the record survives and is git-diffable: + +```json +{ + "testfolder": "Alerts", + "run_files": ["alerts-ui-operations.spec.js"], + "disabled": [ + { "file": "alerts-e2e-flow.spec.js", "reason": "flaky; pending rewrite" } + ] +} +``` + +A spec cannot be in both `run_files` and `disabled` — the build fails if it is. Any +`_comment` (or `_`-prefixed) key is also ignored, for free-form notes. + +**Enterprise-only disabled specs** go in the overlay's `disabled` map, keyed by shard: +`"disabled": { "Alerts": [ { "file": "…", "reason": "…" } ] }`. + +The ENT overlay only ever carries the **delta** from OSS. It must not re-list any spec +already in `ci_matrix.json`; `build-ci-matrix.js` fails the run if it does. diff --git a/tests/ui-testing/ci-matrix/ci_matrix.json b/tests/ui-testing/ci-matrix/ci_matrix.json new file mode 100644 index 0000000000..3a7d1e6547 --- /dev/null +++ b/tests/ui-testing/ci-matrix/ci_matrix.json @@ -0,0 +1,414 @@ +[ + { + "testfolder": "GeneralTests", + "actual_folder": "GeneralTests", + "browser": "chrome", + "run_files": [ + "sanity.spec.js", + "changeOrg.spec.js", + "enrichment.spec.js", + "schema.spec.js", + "schemaload.spec.js", + "serviceAccount.spec.js", + "usersOrg.spec.js", + "org.spec.js", + "theme-management.spec.js", + "ingestion-config.spec.js", + "ingestionTokens.spec.js", + "landingPage.spec.js", + "languageTranslation.spec.js", + "model-pricing.spec.js", + "edition-features.spec.js", + "logsqueries.sql-comments.spec.js", + "iam-form-validation.spec.js", + "regex-patterns-form-validation.spec.js", + "cipher-keys-form-validation.spec.js", + "settings-form-validation.spec.js", + "ai-toolsets-form-validation.spec.js", + "shared-components-form-validation.spec.js", + "onboarding-form-validation.spec.js", + "rum-form-validation.spec.js", + "login-form-validation.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Logs-Builder-Basic", + "actual_folder": "Logs", + "browser": "chrome", + "run_files": [ + "logsQueryBuilder-chart.spec.js", + "logsQueryBuilder-filters-basic.spec.js", + "pagination.spec.js", + "unflattened.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Logs-Builder-Advanced", + "actual_folder": "Logs", + "browser": "chrome", + "run_files": [ + "logsQueryBuilder-editor.spec.js", + "logsQueryBuilder-filters-advanced.spec.js", + "logsquickmode.spec.js", + "logshistogram.spec.js", + "logsQuickPick.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Logs-Core", + "actual_folder": "Logs", + "browser": "chrome", + "run_files": [ + "logspage.spec.js", + "logstable.spec.js", + "logsqueries.spec.js", + "join.spec.js", + "shareLink.spec.js", + "logsAnalyzeDimensions.spec.js", + "monaco-query-prefill.spec.js", + "logsqueries.matchall.spec.js", + "ftsDefaultColumn.spec.js", + "logsVisualizePersistence.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Logs-Features", + "actual_folder": "Logs", + "browser": "chrome", + "run_files": [ + "logs-autocomplete-suggestions.spec.js", + "logs-sql-autocomplete.spec.js", + "logsqueries.cte.spec.js", + "logsDownloads.spec.js", + "secondsPrecisionAdded.spec.js", + "searchpartition.spec.js", + "indexquery.spec.js", + "region.spec.js" + ], + "disabled": [ + { + "file": "searchJobInspector.spec.js", + "reason": "TODO: update for new correlation feature" + } + ] + }, + { + "testfolder": "Alerts", + "actual_folder": "Alerts", + "browser": "chrome", + "run_files": [ + "alerts-ui-operations.spec.js", + "alerts-import.spec.js", + "alerts-metrics-notification.spec.js", + "alerts-destinations-prebuilt.spec.js", + "alerts-vrl-encoding.spec.js", + "alerts-history.spec.js", + "alerts-template-prebuilt-guard.spec.js", + "alerts-form-validation.spec.js", + "anomaly-detection-form-validation.spec.js", + "alerts-priority-tags.spec.js", + "alerts-multialert-api.spec.js", + "alerts-multialert-ui.spec.js", + "alerts-multialert-regression.spec.js", + "alerts-multialert-firing.spec.js", + "alerts-multialert-transitions.spec.js" + ], + "disabled": [ + { + "file": "alerts-e2e-flow.spec.js", + "reason": "" + }, + { + "file": "alerts-advanced.spec.js", + "reason": "" + }, + { + "file": "alerts-scheduled-features.spec.js", + "reason": "" + } + ] + }, + { + "testfolder": "Dashboards-Core", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard.spec.js", + "dashboard2.spec.js", + "dashboardtype.spec.js", + "dashboard-folder.spec.js", + "dashboard-import.spec.js", + "maxquery.spec.js", + "dashboard-chartJson.spec.js", + "dashboard-promql-suggestions.spec.js", + "dashboard-sql-autocomplete.spec.js", + "dashboard-share-link.spec.js", + "crossLinking.spec.js", + "dashboard-multi-sql.spec.js", + "crossLinkMultiStream.spec.js", + "dashboards-form-validation.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Isolated", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-favorites.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Settings", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-filter.spec.js", + "dashboard-general-setting.spec.js", + "dashboard-tabs-setting.spec.js", + "dashboard-transpose.spec.js", + "dashboard-join.spec.js", + "dashboard-nested-functions.spec.js", + "dashboard-legends-copy.spec.js", + "dashboard-raw-query.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Charts", + "_comment": "Dashboard panel tests are grouped by feature so each shard stays under the ephemeral-runner endurance window. A single combined 16-spec shard ran ~40 min and got reclaimed mid-run ('runner lost communication'). Add new specs to the shard of the feature they cover.", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "custom-charts.spec.js", + "dashboard-geoMap.spec.js", + "dashboard-maps.spec.js", + "dashboard-multi-y-axis.spec.js", + "dashboard-html-chart.spec.js", + "dashboard-sankey.spec.js", + "dashboard-pie-donut.spec.js", + "dashboard-metric-camelcase.spec.js", + "dashboard-create-alert.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Visualize", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "visualize.spec.js", + "visualize-vrl.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Tables", + "_comment": "Owns the pagination and pivot-table specs (previously their own single-spec shards) \u2014 keep them here.", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-table-chart.spec.js", + "dashboard-table-copy-cell.spec.js", + "dashboard-table-csv-download.spec.js", + "dashboard-table-column-formatting.spec.js", + "dashboard-table-column-filtering.spec.js", + "dashboard-table-pagination.spec.js", + "dashboard-pivot-table.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Streaming", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-streaming.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Pipelines", + "actual_folder": "Pipelines", + "browser": "chrome", + "run_files": [ + "pipeline-conditions.spec.js", + "pipeline-conditions-validation.spec.js", + "pipelines.spec.js", + "pipeline-dynamic.spec.js", + "pipeline-core.spec.js", + "pipeline-traces.spec.js", + "pipeline-metrics.spec.js", + "pipeline-backfill.spec.js", + "pipeline-history.spec.js", + "scheduled-pipeline-query-builder.spec.js", + "pipelines-form-validation.spec.js" + ], + "disabled": [ + { + "file": "remotepipeline.spec.js", + "reason": "" + } + ] + }, + { + "testfolder": "Functions", + "actual_folder": "Functions", + "browser": "chrome", + "run_files": [ + "js-transform-type.spec.js", + "row-expansion.spec.js", + "enrichment-table-url.spec.js", + "functions-form-validation.spec.js", + "action-scripts-form-validation.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Reports", + "actual_folder": "Reports", + "browser": "chrome", + "run_files": [ + "reportsScheduleNow.spec.js", + "reportsScheduleLater.spec.js", + "reportFolders.spec.js", + "reports-bulk-operations.spec.js", + "reports-form-validation.spec.js", + "reportCsvMediaType.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Streams", + "actual_folder": "Streams", + "browser": "chrome", + "run_files": [ + "multiselect-stream.spec.js", + "streamname.spec.js", + "streaming.spec.js", + "stream-settings.spec.js", + "streams-form-validation.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Traces", + "actual_folder": "Traces", + "browser": "chrome", + "run_files": [ + "service-catalog.spec.js", + "tracesSearch.spec.js", + "traceQueryEditor.spec.js", + "traceErrorFilter.spec.js", + "traceDetails.spec.js", + "traceAdvancedFiltering.spec.js", + "tracesAnalyzeDimensions.spec.js", + "traces-autocomplete-suggestions.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "RUM", + "_comment": "The RUM ingest/token/sourcemaps routes are part of the standard (OSS) router (src/api/http/src/handler/http/router/mod.rs), so these specs run against the OSS binary. Ingestion-auth negative cases live in the API suite (tests/api-testing/tests/rum).", + "actual_folder": "RUM", + "browser": "chrome", + "run_files": [ + "rum-cdn-dataflow.spec.js", + "rum-npm-dataflow.spec.js", + "rum-page-dataflow.spec.js", + "rum-onboarding-snippets.spec.js", + "sourcemap-ui.spec.js", + "sourcemap-upload-pretty.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "RUM-Token", + "_comment": "rum-token.spec.js RESETS the org RUM token; running it beside the dataflow specs (workers run files in parallel) would 401 their in-flight SDK beacons. Own shard = own isolated instance \u2014 do NOT merge back into RUM.", + "actual_folder": "RUM", + "browser": "chrome", + "run_files": [ + "rum-token.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Metrics", + "actual_folder": "Metrics", + "browser": "chrome", + "run_files": [ + "metrics.spec.js", + "metrics-queries.spec.js", + "metrics-aggregations.spec.js", + "metrics-advanced.spec.js", + "metrics-config-tabs.spec.js", + "metrics-visualizations.spec.js", + "metrics-config.spec.js", + "metrics-promql-query-persistence.spec.js", + "metrics-table-column-order.spec.js", + "metrics-promql-builder.spec.js", + "promqlAutocomplete.spec.js", + "metrics-share-deep-link.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Variables", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-variables-setting.spec.js", + "dashboard-variables-global.spec.js", + "dashboard-variables-tab-level.spec.js", + "dashboard-variables-panel-level.spec.js", + "dashboard-variables-dependency.spec.js", + "dashboard-variables-refresh.spec.js", + "dashboard-variables-url-sync.spec.js", + "dashboard-variables-creation-scopes.spec.js", + "dashboard-variables-default-values-chain.spec.js", + "dashboard-variables-custom-parents.spec.js", + "dashboard-variables-stream-field.spec.js", + "dashboard-mustache-variables.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboards-Panel-Level-DateTime-Config", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-panel-time-config-behavior.spec.js", + "dashboard-panel-time-url-priority.spec.js", + "dashboard-panel-time-advanced-edge-cases.spec.js", + "dashboard-panel-time-apply-behavior.spec.js", + "dashboard-panel-time-variables-behavior.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboard-Config-Settings", + "actual_folder": "Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-series-color-multiwindow.spec.js", + "dashboard-config-advanced.spec.js", + "dashboard-config-axis.spec.js", + "dashboard-config-gauge-maps.spec.js", + "dashboard-config-general.spec.js", + "dashboard-config-legends.spec.js", + "dashboard-config-line-style.spec.js", + "dashboard-config-panel-time.spec.js", + "dashboard-config-table.spec.js", + "dashboard-config-trellis.spec.js", + "dashboard-config-drilldown.spec.js", + "dashboard-config-promql.spec.js", + "dashboard-config-markline.spec.js" + ], + "disabled": [] + } +] diff --git a/tests/ui-testing/ci-matrix/ci_matrix_regression.json b/tests/ui-testing/ci-matrix/ci_matrix_regression.json new file mode 100644 index 0000000000..86808f89f6 --- /dev/null +++ b/tests/ui-testing/ci-matrix/ci_matrix_regression.json @@ -0,0 +1,100 @@ +[ + { + "testfolder": "Logs-Regression", + "actual_folder": "RegressionSet/Logs", + "browser": "chrome", + "run_files": [ + "logs-regression.spec.js", + "logs-bugs.spec.js", + "logs-9754.spec.js", + "logs-9044-7354.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Alerts-Regression", + "actual_folder": "RegressionSet/Alerts", + "browser": "chrome", + "run_files": [ + "alerts-regression.spec.js", + "alerts-stream-switching.spec.js", + "alerts-bugs.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Dashboard-Regression", + "actual_folder": "RegressionSet/Dashboards", + "browser": "chrome", + "run_files": [ + "dashboard-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Streams-Regression", + "actual_folder": "RegressionSet/Streams", + "browser": "chrome", + "run_files": [ + "streams-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Pipelines-Regression", + "actual_folder": "RegressionSet/Pipelines", + "browser": "chrome", + "run_files": [ + "enrichment-regression.spec.js", + "pipeline-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Traces-Regression", + "actual_folder": "RegressionSet/Traces", + "browser": "chrome", + "run_files": [ + "traces-regression.spec.js", + "traces-bugs.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Metrics-Regression", + "actual_folder": "RegressionSet/Metrics", + "browser": "chrome", + "run_files": [ + "metrics-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "GeneralTests-Regression", + "actual_folder": "RegressionSet/General", + "browser": "chrome", + "run_files": [ + "ui-regression.spec.js", + "landing-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "DataSources-Regression", + "actual_folder": "RegressionSet/DataSources", + "browser": "chrome", + "run_files": [ + "datasources-regression.spec.js" + ], + "disabled": [] + }, + { + "testfolder": "Reports-Regression", + "actual_folder": "RegressionSet/Reports", + "browser": "chrome", + "run_files": [ + "reports-regression-bugs.spec.js" + ], + "disabled": [] + } +]