mirror of
https://github.com/openobserve/openobserve.git
synced 2026-11-04 08:09:27 +00:00
## What Adds a small **log-only** Playwright reporter that prints a LOUD, unmissable banner to stdout **the moment any test starts a retry attempt**, plus an end-of-shard summary listing every test that was retried. ## Why In CI the reporter is `blob`, which writes nothing to stdout. So when a failed test is retried (`retries: process.env.CI ? 3 : 0` in `playwright.config.js`) there is **no signal in the GitHub Actions logs that a retry is even happening** — a test silently re-runs up to 3 more times and the log just shows the next `Test Started` line. This makes flaky-vs-real failures hard to spot when scanning CI output. The retry is internal to `npx playwright test`, so the workflow YAML can't see it — it has to be emitted from the Playwright reporter layer. This is that layer. ## What it looks like ``` 🔁🔁🔁══════════════════════════════════════════════════════════🔁🔁🔁 🔁 R E T R Y # 1 · attempt 2 of 4 (test previously failed — re-running) 🔁 ▶ chromium › logsqueries.cte.spec.js › should query with simple TotalCols CTE 🔁 ▶ Logs/logsqueries.cte.spec.js 🔁🔁🔁══════════════════════════════════════════════════════════🔁🔁🔁 ``` …and once per shard at the end: ``` 🔁 RETRY SUMMARY — N test(s) were retried in this shard: 🔁 • [2 retries] chromium › … › should query … (Logs/logsqueries.cte.spec.js) ``` ## Scope / safety - Added to the **CI** reporter list only, alongside `blob` (multiple reporters are fine). It only `console.log`s — never sends anything, never fails the run. - **ENT gets this for free**: the ENT Playwright workflow checks out and runs the OSS `tests/ui-testing` tree, so this one OSS change makes retries loud in both OSS and ENT CI. There is no ENT test tree to change. - Verified locally with a forced-failure probe (`CI=true`, `retries: 2`): the per-retry banner fires on attempt 2 and 3, and the summary lists the retried test. ## Files - `tests/ui-testing/playwright-tests/utils/retry-banner-reporter.js` — new reporter - `tests/ui-testing/playwright.config.js` — wire it into the CI reporter array
127 lines
4.7 KiB
JavaScript
127 lines
4.7 KiB
JavaScript
// @ts-check
|
|
const { defineConfig, devices } = require('@playwright/test');
|
|
const dotenv = require('dotenv');
|
|
// Load environment variables from .env file
|
|
try {
|
|
dotenv.config();
|
|
console.log('✅ Environment variables loaded from .env file');
|
|
} catch (error) {
|
|
console.warn('⚠️ dotenv not available, using system environment variables');
|
|
}
|
|
|
|
// Check if essential environment variables are set
|
|
if (!process.env.ZO_BASE_URL || !process.env.ZO_ROOT_USER_EMAIL || !process.env.ZO_ROOT_USER_PASSWORD) {
|
|
console.warn('⚠️ Essential environment variables not found. Make sure to set ZO_BASE_URL, ZO_ROOT_USER_EMAIL, and ZO_ROOT_USER_PASSWORD');
|
|
}
|
|
|
|
/**
|
|
* @see https://playwright.dev/docs/test-configuration
|
|
*/
|
|
module.exports = defineConfig({
|
|
testDir: './playwright-tests',
|
|
globalTimeout: process.env.CI ? 40 * 60 * 1000 : undefined,
|
|
/* Output directory for test artifacts */
|
|
outputDir: './test-results',
|
|
/* Exclude archived tests from all test runs */
|
|
testIgnore: ['**/test-archives/**', '**/*_old.js'],
|
|
/* Global setup and teardown */
|
|
globalSetup: './playwright-tests/utils/global-setup.js',
|
|
globalTeardown: './playwright-tests/utils/global-teardown.js',
|
|
/* Run tests in files in parallel */
|
|
fullyParallel: true,
|
|
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
|
forbidOnly: !!process.env.CI,
|
|
/* Retry failed tests: 3 times on CI, 0 times locally */
|
|
retries: process.env.CI ? 3 : 0,
|
|
/* Opt out of parallel tests on CI. */
|
|
workers: process.env.CI ? 5 : 5,
|
|
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
|
|
reporter: process.env.CI
|
|
? [
|
|
['blob', { outputDir: 'blob-report' }], // Use blob reporter in CI - JSON created during merge
|
|
// Prints a LOUD banner to stdout whenever a test is retried (blob writes nothing to
|
|
// the log, so retries are otherwise invisible in the CI output). Log-only, never fails.
|
|
['./playwright-tests/utils/retry-banner-reporter.js'],
|
|
]
|
|
: [
|
|
['html', { outputFolder: 'playwright-results/html-report', open: 'never' }], // HTML reporter
|
|
['json', { outputFile: 'playwright-results/report.json' }] // JSON reporter for downstream report consumers
|
|
],
|
|
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
|
|
use: {
|
|
/* Base URL to use in actions like `await page.goto('/')`. */
|
|
baseURL: process.env["ZO_BASE_URL"],
|
|
|
|
/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
|
|
trace: 'on-first-retry',
|
|
|
|
/* Navigation and action timeouts for CI stability */
|
|
navigationTimeout: process.env.CI ? 90000 : 30000,
|
|
actionTimeout: process.env.CI ? 45000 : 15000,
|
|
|
|
/* Capture screenshots and videos on failure for CI debugging */
|
|
screenshot: process.env.CI ? 'only-on-failure' : 'off',
|
|
video: process.env.CI ? 'retain-on-failure' : 'off',
|
|
|
|
/* Slow down execution for Pipeline tests on deployed environments to avoid sync issues */
|
|
slowMo: (process.env.SLOW_MO_TESTS === 'true' && process.env.TEST_SHARD === 'Pipelines') ? 1000 : 0,
|
|
},
|
|
/* Test timeout: 3 minutes locally, 5 minutes in CI */
|
|
timeout: process.env.CI ? 5 * 60 * 1000 : 3 * 60 * 1000,
|
|
|
|
/* Expect timeout for assertions - increased for deployed envs */
|
|
expect: {
|
|
timeout: process.env.CI ? 30000 : 10000,
|
|
},
|
|
|
|
/* Configure projects for major browsers */
|
|
projects: [
|
|
{
|
|
name: 'chromium',
|
|
use: {
|
|
...devices['Desktop Chrome'],
|
|
viewport: { width: 1500, height: 1024 },
|
|
permissions: ['clipboard-read', 'clipboard-write'],
|
|
launchOptions: {
|
|
args: process.env.CI ? ['--no-sandbox', '--disable-setuid-sandbox'] : [],
|
|
},
|
|
},
|
|
},
|
|
// {
|
|
// name: 'webkit', // Safari/WebKit configuration
|
|
// use: {
|
|
// ...devices['Desktop Safari'], // Uses Playwright's pre-defined device settings for Safari
|
|
// viewport: { width: 1500, height: 1024 },
|
|
// },
|
|
// },
|
|
// }
|
|
|
|
/* Test against mobile viewports. */
|
|
// {
|
|
// name: 'Mobile Chrome',
|
|
// use: { ...devices['Pixel 5'] },
|
|
// },
|
|
// {
|
|
// name: 'Mobile Safari',
|
|
// use: { ...devices['iPhone 12'] },
|
|
// },
|
|
|
|
/* Test against branded browsers. */
|
|
// {
|
|
// name: 'Microsoft Edge',
|
|
// use: { ...devices['Desktop Edge'], channel: 'msedge' },
|
|
// },
|
|
// {
|
|
// name: 'Google Chrome',
|
|
// use: { ...devices['Desktop Chrome'], channel: 'chrome' },
|
|
// }
|
|
|
|
],
|
|
|
|
/* Run your local dev server before starting the tests */
|
|
// webServer: {
|
|
// command: 'npm run start',
|
|
// url: 'http://127.0.0.1:3000',
|
|
// reuseExistingServer: !process.env.CI,
|
|
// },
|
|
}); |