mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-05 08:09:22 +00:00
Shrinath RaoandClaude Opus 4.7
511bb5ef67
## Summary - Added `playwright_regression.yml` with one-shard-per-spec parallelism (11 regression specs) - Removed `RegressionSet` shard from main `playwright.yml` (tests now run via dedicated regression workflow) - Added `Cache test failures to TestDino` step to regression workflow (matching main playwright.yml) - Uses `ubicloud-standard-8` runners for all regression jobs ## Schedule - Auto-triggers every 4 hours starting from 9 AM IST (3:30, 7:30, 11:30, 15:30, 19:30, 23:30 UTC) - Manual trigger via `workflow_dispatch` ## Test Plan - [ ] Confirm `playwright_regression.yml` appears in Actions tab - [ ] Trigger a manual run to verify all 11 regression specs run in parallel - [ ] Verify `playwright.yml` no longer includes RegressionSet shard --------- Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>
433 lines
15 KiB
YAML
433 lines
15 KiB
YAML
name: Playwright Regression Tests
|
|
|
|
on:
|
|
schedule:
|
|
- cron: '30 3,7,11,15,19,23 * * *' # Every 4 hrs from 9 AM IST (3:30, 7:30, 11:30, 15:30, 19:30, 23:30 UTC)
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: false
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
env:
|
|
COLUMNS: 150
|
|
ZO_ROOT_USER_EMAIL: root@example.com
|
|
ZO_ROOT_USER_PASSWORD: Complexpass#123
|
|
ZO_BASE_URL: http://localhost:5080
|
|
WS_ZO_BASE_URL: ws://localhost:5080
|
|
ZO_BASE_URL_SC: http://localhost:5080
|
|
ZO_BASE_URL_SC_UI: http://localhost:5080
|
|
INGESTION_URL: http://localhost:5080
|
|
ORGNAME: default
|
|
ZO_WEB_URL: "http://localhost:5080"
|
|
ZO_QUICK_MODE_NUM_FIELDS: 100
|
|
ZO_QUICK_MODE_STRATEGY: first
|
|
ZO_ALLOW_USER_DEFINED_SCHEMAS: true
|
|
ZO_INGEST_ALLOWED_UPTO: 5
|
|
ZO_FEATURE_QUERY_EXCLUDE_ALL: false
|
|
ZO_USAGE_BATCH_SIZE: 200
|
|
ZO_USAGE_PUBLISH_INTERVAL: 2
|
|
ZO_USAGE_REPORTING_ENABLED: true
|
|
ZO_MIN_AUTO_REFRESH_INTERVAL: 5
|
|
ZO_STREAMING_ENABLED: true
|
|
ZO_COLS_PER_RECORD_LIMIT: "80000"
|
|
ZO_SMTP_ENABLED: true
|
|
ZO_FORMAT_STREAM_NAME_TO_LOWERCASE: false
|
|
ZO_CREATE_ORG_THROUGH_INGESTION: true
|
|
ZO_UTF8_VIEW_ENABLED: false
|
|
ZO_ENABLE_CROSS_LINKING: true
|
|
ZO_SSRF_ALLOW_LOOPBACK: true
|
|
UPLOAD_TO_TESTDINO: ${{ vars.UPLOAD_TO_TESTDINO || 'false' }}
|
|
|
|
jobs:
|
|
build_binary:
|
|
name: build_binary
|
|
runs-on:
|
|
labels: ubicloud-standard-16
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- name: Remove unused tools
|
|
run: |
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo rm -rf /opt/ghc
|
|
sudo rm -rf "/usr/local/share/boost"
|
|
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
|
|
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Setup Rust Toolchain
|
|
uses: dtolnay/rust-toolchain@master
|
|
with:
|
|
toolchain: nightly-2025-11-11
|
|
targets: x86_64-unknown-linux-gnu
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
cache-on-failure: true
|
|
shared-key: playwright-release-ci
|
|
save-if: ${{ github.ref == 'refs/heads/main' }}
|
|
|
|
- name: Install Protoc
|
|
run: |
|
|
bash ./.github/protoc.sh
|
|
|
|
- uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 22
|
|
|
|
- name: Lint frontend code
|
|
run: cd web && npm ci && npm run lint:ci
|
|
|
|
- name: Build binary and frontend
|
|
env:
|
|
NODE_OPTIONS: "--max-old-space-size=8192"
|
|
GIT_HASH_OVERRIDE: ${{ github.sha }}
|
|
run: |
|
|
bash -c "cd web && npm run build && cp -r ./dist /tmp/dist" &
|
|
sed -i 's|folder = "web/dist/"|folder = "/tmp/dist/"|' src/handler/http/router/ui.rs
|
|
cargo build --profile release-ci --target x86_64-unknown-linux-gnu
|
|
wait
|
|
|
|
- name: Upload binary artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: release-ci-binary
|
|
path: target/x86_64-unknown-linux-gnu/release-ci/openobserve
|
|
retention-days: 1
|
|
|
|
- name: Upload frontend artifact
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: frontend-dist
|
|
path: /tmp/dist
|
|
retention-days: 1
|
|
if-no-files-found: error
|
|
|
|
ui_integration_tests:
|
|
name: e2e / ${{ matrix.testfolder }}
|
|
needs: [build_binary]
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- testfolder: "Logs-Regression"
|
|
run_files:
|
|
[
|
|
"logs-regression.spec.js",
|
|
"logs-regression-bugs.spec.js",
|
|
"logs-9754.spec.js",
|
|
]
|
|
- testfolder: "Alerts-Regression"
|
|
run_files:
|
|
[
|
|
"alerts-stream-switching-regression.spec.js",
|
|
]
|
|
- testfolder: "Dashboard-Regression"
|
|
run_files:
|
|
[
|
|
"dashboard-regression.spec.js",
|
|
]
|
|
- testfolder: "Streams-Regression"
|
|
run_files:
|
|
[
|
|
"streams-regression.spec.js",
|
|
]
|
|
- testfolder: "Pipelines-Regression"
|
|
run_files:
|
|
[
|
|
"enrichment-regression.spec.js",
|
|
"pipeline-regression.spec.js",
|
|
]
|
|
- testfolder: "Traces-Regression"
|
|
run_files:
|
|
[
|
|
"traces-regression.spec.js",
|
|
]
|
|
- testfolder: "Metrics-Regression"
|
|
run_files:
|
|
[
|
|
"metrics-regression.spec.js",
|
|
]
|
|
- testfolder: "GeneralTests-Regression"
|
|
run_files:
|
|
[
|
|
"ui-regression.spec.js",
|
|
]
|
|
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Download binary artifact
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: release-ci-binary
|
|
path: release-ci-binary
|
|
|
|
- name: Download frontend artifact
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
name: frontend-dist
|
|
path: /tmp/dist
|
|
|
|
- name: Start OpenObserve
|
|
run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 &
|
|
|
|
- name: Wait for start
|
|
run: |
|
|
for i in $(seq 1 120); do
|
|
if curl -sf http://localhost:5080/web/login > /dev/null 2>&1; then
|
|
echo "Server ready after ${i}s"
|
|
exit 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
echo "Server failed to start in 120s"
|
|
exit 1
|
|
|
|
- name: Ensure we are getting a reply from the server
|
|
run: curl http://localhost:5080/web/login
|
|
|
|
- uses: actions/setup-node@v5
|
|
with:
|
|
node-version: 22
|
|
|
|
- name: Install dependencies and run ui-tests
|
|
run: |
|
|
touch .env
|
|
echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env
|
|
echo "ZO_ROOT_USER_PASSWORD=${ZO_ROOT_USER_PASSWORD}" >> .env
|
|
echo "ZO_BASE_URL=${ZO_BASE_URL}" >> .env
|
|
echo "WS_ZO_BASE_URL=${WS_ZO_BASE_URL}" >> .env
|
|
echo "ZO_BASE_URL_SC=${ZO_BASE_URL_SC}" >> .env
|
|
echo "ZO_BASE_URL_SC_UI=${ZO_BASE_URL_SC_UI}" >> .env
|
|
echo "INGESTION_URL=${INGESTION_URL}" >> .env
|
|
echo "ORGNAME=${ORGNAME}" >> .env
|
|
echo "ZO_SMTP_ENABLED=${ZO_SMTP_ENABLED}" >> .env
|
|
mv .env tests/ui-testing
|
|
cd tests/ui-testing && npm ci
|
|
npx playwright install --with-deps chromium
|
|
|
|
# Get list of files to run using join function
|
|
FILE_LIST="${{ join(matrix.run_files, ' ') }}"
|
|
echo "DEBUG: FILE_LIST = $FILE_LIST"
|
|
echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}"
|
|
|
|
if [ -n "$FILE_LIST" ]; then
|
|
ACTUAL_FOLDER="RegressionSet"
|
|
|
|
echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER"
|
|
|
|
# Build file paths
|
|
FILE_PATHS=""
|
|
for file in $FILE_LIST; do
|
|
FILE_PATH="./playwright-tests/$ACTUAL_FOLDER/$file"
|
|
echo "DEBUG: Will run file: $FILE_PATH"
|
|
FILE_PATHS="$FILE_PATHS $FILE_PATH"
|
|
done
|
|
|
|
echo "DEBUG: Final command: npx playwright test $FILE_PATHS"
|
|
npx playwright test $FILE_PATHS
|
|
else
|
|
echo "No files specified to run for ${{ matrix.testfolder }} folder"
|
|
fi
|
|
|
|
- name: Upload blob report to GitHub Actions Artifacts
|
|
if: ${{ !cancelled() }}
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: blob-report-${{ matrix.testfolder }}-${{ github.run_attempt }}
|
|
path: tests/ui-testing/blob-report
|
|
retention-days: 1
|
|
|
|
- name: Check OpenObserve logs
|
|
if: always()
|
|
run: cat o2.log
|
|
|
|
merge_and_upload_reports:
|
|
name: Merge Reports and Upload to TestDino
|
|
needs: [ui_integration_tests]
|
|
if: ${{ !cancelled() && needs.ui_integration_tests.result != 'skipped' }}
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions:
|
|
contents: read
|
|
|
|
steps:
|
|
- name: Clone the current repo
|
|
uses: actions/checkout@v5
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@v5
|
|
with:
|
|
node-version: '22'
|
|
|
|
- name: Install project dependencies
|
|
run: cd tests/ui-testing && npm ci
|
|
|
|
- name: Download blob reports from GitHub Actions Artifacts
|
|
uses: actions/download-artifact@v4
|
|
continue-on-error: true
|
|
with:
|
|
path: tests/ui-testing/all-blob-reports
|
|
pattern: blob-report-*-${{ github.run_attempt }}
|
|
merge-multiple: true
|
|
|
|
- name: Merge Reports
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
|
|
if [ ! -d "all-blob-reports" ] || [ -z "$(ls -A all-blob-reports 2>/dev/null)" ]; then
|
|
echo "::warning::No blob reports found to merge."
|
|
|
|
echo "::group::Investigating why no blob reports exist"
|
|
|
|
JOBS_RESPONSE=$(curl -s -H "Authorization: token $GH_TOKEN" \
|
|
-H "Accept: application/vnd.github.v3+json" \
|
|
"https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/attempts/${{ github.run_attempt }}/jobs")
|
|
|
|
SUCCEEDED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "success")] | length')
|
|
FAILED=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /")) | select(.conclusion == "failure")] | length')
|
|
TOTAL=$(echo "$JOBS_RESPONSE" | jq -r '[.jobs[] | select(.name | startswith("e2e /"))] | length')
|
|
|
|
echo "::notice::Test job results: $SUCCEEDED succeeded, $FAILED failed, $TOTAL total"
|
|
|
|
if [ "$SUCCEEDED" -gt 0 ] && [ "$FAILED" -eq 0 ]; then
|
|
echo "::notice::✅ All test jobs succeeded - they likely skipped because tests already passed"
|
|
echo "::notice::Creating empty report structure for downstream jobs"
|
|
mkdir -p playwright-results/html-report
|
|
START_TIME=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)
|
|
echo '{"config":{},"suites":[],"errors":[],"stats":{"startTime":"'"$START_TIME"'","duration":0,"expected":0,"unexpected":0,"flaky":0,"skipped":0}}' > playwright-results/report.json
|
|
echo "<html><body><h1>No tests executed - all shards skipped</h1><p>All test shards exited early because tests had already passed in a previous attempt.</p></body></html>" > playwright-results/html-report/index.html
|
|
echo "::endgroup::"
|
|
exit 0
|
|
else
|
|
echo "::error::❌ Some/all test jobs failed but no blob reports were found"
|
|
echo "::error::This indicates a problem with test execution or blob reporter"
|
|
echo "::error::Failed jobs: $FAILED, Succeeded jobs: $SUCCEEDED"
|
|
echo "::endgroup::"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
unset CI
|
|
npx playwright merge-reports --config playwright.config.js ./all-blob-reports
|
|
|
|
echo "Contents of playwright-results:"
|
|
ls -la playwright-results/
|
|
echo "Contents of html-report:"
|
|
ls -la playwright-results/html-report/
|
|
echo "Checking report.json:"
|
|
ls -lh playwright-results/report.json
|
|
|
|
- name: Cache test failures to TestDino
|
|
if: ${{ needs.ui_integration_tests.result == 'failure' }}
|
|
env:
|
|
TESTDINO_TOKEN: ${{ secrets.TESTDINO_API_TOKEN }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
echo "::group::Caching test failure metadata to TestDino"
|
|
if [ -f "playwright-results/report.json" ]; then
|
|
echo "::notice::JSON report found at playwright-results/report.json"
|
|
ls -lh playwright-results/report.json
|
|
|
|
mkdir -p ../../playwright-results
|
|
cp playwright-results/report.json ../../playwright-results/
|
|
|
|
cd ../../playwright-results
|
|
BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//_/g')
|
|
CACHE_ID="gh_openobserve_${BRANCH_NAME}_${{ github.run_id }}"
|
|
echo "::notice::Using cache ID: $CACHE_ID"
|
|
|
|
if CACHE_OUTPUT=$(npx tdpw cache --cache-id "$CACHE_ID" 2>&1); then
|
|
SUCCESS_MSG=$(echo "$CACHE_OUTPUT" | grep "Cache data submitted successfully" || echo "")
|
|
if [ -n "$SUCCESS_MSG" ]; then
|
|
echo "::notice::✅ Successfully cached test failure metadata to TestDino cloud"
|
|
else
|
|
echo "::notice::✅ Cache command completed"
|
|
fi
|
|
else
|
|
echo "::warning::⚠️ Failed to cache test metadata to TestDino"
|
|
echo "::warning::Rerun optimization may not work properly"
|
|
echo "$CACHE_OUTPUT"
|
|
fi
|
|
else
|
|
echo "::notice::Skipping cache - no report.json found (tests may have been skipped due to rerun optimization)"
|
|
fi
|
|
echo "::endgroup::"
|
|
|
|
- name: Upload to TestDino
|
|
if: ${{ needs.ui_integration_tests.result == 'failure' && env.UPLOAD_TO_TESTDINO == 'true' }}
|
|
run: |
|
|
cd tests/ui-testing
|
|
|
|
if [ ! -f "playwright-results/report.json" ]; then
|
|
echo "::notice::Skipping TestDino upload - no report.json found (tests may have been skipped due to rerun optimization)"
|
|
exit 0
|
|
fi
|
|
|
|
START_TIME=$(jq -r '.stats.startTime // empty' playwright-results/report.json)
|
|
if [ -n "$START_TIME" ] && [[ "$START_TIME" =~ ^[0-9]+$ ]]; then
|
|
echo "Converting startTime from Unix timestamp to ISO 8601 string"
|
|
jq '.stats.startTime = (.stats.startTime / 1000 | strftime("%Y-%m-%dT%H:%M:%S.000Z"))' playwright-results/report.json > playwright-results/report.json.tmp
|
|
mv playwright-results/report.json.tmp playwright-results/report.json
|
|
fi
|
|
|
|
npx --yes tdpw@latest upload playwright-results \
|
|
--json-report playwright-results/report.json \
|
|
--html-report playwright-results/html-report \
|
|
--upload-html \
|
|
--verbose \
|
|
--token="${{ secrets.TESTDINO_API_TOKEN }}"
|
|
|
|
playwright_summary:
|
|
runs-on:
|
|
labels: ubicloud-standard-8
|
|
permissions: {}
|
|
needs: [build_binary, ui_integration_tests, merge_and_upload_reports]
|
|
if: always()
|
|
steps:
|
|
- name: Check test results
|
|
run: |
|
|
BUILD_OK=false
|
|
UI_OK=false
|
|
MERGE_OK=false
|
|
|
|
if [ "${{ needs.build_binary.result }}" == "success" ] || [ "${{ needs.build_binary.result }}" == "skipped" ]; then
|
|
BUILD_OK=true
|
|
fi
|
|
|
|
if [ "${{ needs.ui_integration_tests.result }}" == "success" ] || [ "${{ needs.ui_integration_tests.result }}" == "skipped" ]; then
|
|
UI_OK=true
|
|
fi
|
|
|
|
if [ "${{ needs.merge_and_upload_reports.result }}" == "success" ] || [ "${{ needs.merge_and_upload_reports.result }}" == "skipped" ] || [ "${{ needs.merge_and_upload_reports.result }}" == "failure" ]; then
|
|
MERGE_OK=true
|
|
fi
|
|
|
|
if [ "$BUILD_OK" == "true" ] && [ "$UI_OK" == "true" ] && [ "$MERGE_OK" == "true" ]; then
|
|
echo "All regression tests completed successfully"
|
|
exit 0
|
|
else
|
|
echo "Regression tests failed:"
|
|
echo " build_binary: ${{ needs.build_binary.result }}"
|
|
echo " ui_integration_tests: ${{ needs.ui_integration_tests.result }}"
|
|
echo " merge_and_upload_reports: ${{ needs.merge_and_upload_reports.result }}"
|
|
exit 1
|
|
fi
|