diff --git a/.github/actions/allure-report-generate/action.yml b/.github/actions/allure-report-generate/action.yml index 07120c4c8a..7f7fa9e7a1 100644 --- a/.github/actions/allure-report-generate/action.yml +++ b/.github/actions/allure-report-generate/action.yml @@ -147,6 +147,8 @@ runs: echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT echo "report-json-url=${REPORT_URL%/index.html}/data/suites.json" >> $GITHUB_OUTPUT + echo "[Allure Report](${REPORT_URL})" >> ${GITHUB_STEP_SUMMARY} + - name: Release lock if: always() shell: bash -euxo pipefail {0} diff --git a/scripts/pr-comment-test-report.js b/scripts/pr-comment-test-report.js index 287a1ea8e5..3a7bba0daa 100644 --- a/scripts/pr-comment-test-report.js +++ b/scripts/pr-comment-test-report.js @@ -36,11 +36,9 @@ module.exports = async ({ github, context, fetch, report }) => { // Marker to find the comment in the subsequent runs const startMarker = `` // Let users know that the comment is updated automatically - const autoupdateNotice = `
The comment gets automatically updated with the latest test results :recycle:
` + const autoupdateNotice = `
The comment gets automatically updated with the latest test results
${context.payload.pull_request.head.sha} at ${new Date().toISOString()} :recycle:
` // GitHub bot id taken from (https://api.github.com/users/github-actions[bot]) const githubActionsBotId = 41898282 - // The latest commit in the PR URL - const commitUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/pull/${context.payload.number}/commits/${context.payload.pull_request.head.sha}` // Commend body itself let commentBody = `${startMarker}\n` @@ -74,7 +72,6 @@ module.exports = async ({ github, context, fetch, report }) => { let flakyTestsCount = 0 const pgVersions = new Set() - const buildTypes = new Set() for (const parentSuite of suites.children) { for (const suite of parentSuite.children) { @@ -92,28 +89,29 @@ module.exports = async ({ github, context, fetch, report }) => { } pgVersions.add(pgVersion) - buildTypes.add(buildType) // Removing build type and PostgreSQL version from the test name to make it shorter const testName = test.name.replace(new RegExp(`${buildType}-pg${pgVersion}-?`), "").replace("[]", "") test.pytestName = `${parentSuite.name.replace(".", "/")}/${suite.name}.py::${testName}` + test.pgVersion = pgVersion + test.buildType = buildType if (test.status === "passed") { - passedTests[pgVersion][buildType].push(test) + passedTests[pgVersion][testName].push(test) passedTestsCount += 1 } else if (test.status === "failed" || test.status === "broken") { - failedTests[pgVersion][buildType].push(test) + failedTests[pgVersion][testName].push(test) failedTestsCount += 1 } else if (test.status === "skipped") { - skippedTests[pgVersion][buildType].push(test) + skippedTests[pgVersion][testName].push(test) skippedTestsCount += 1 } if (test.retriesCount > 0) { - retriedTests[pgVersion][buildType].push(test) + retriedTests[pgVersion][testName].push(test) if (test.retriesStatusChange) { - flakyTests[pgVersion][buildType].push(test) + flakyTests[pgVersion][testName].push(test) flakyTestsCount += 1 } } @@ -122,39 +120,44 @@ module.exports = async ({ github, context, fetch, report }) => { } const totalTestsCount = failedTestsCount + passedTestsCount + skippedTestsCount - commentBody += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}) for ${commitUrl})\n___\n` + commentBody += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}))\n___\n` - // Print test resuls from the newest to the oldest PostgreSQL version for release and debug builds. + // Print test resuls from the newest to the oldest Postgres version for release and debug builds. for (const pgVersion of Array.from(pgVersions).sort().reverse()) { - for (const buildType of Array.from(buildTypes).sort().reverse()) { - if (failedTests[pgVersion][buildType].length > 0) { - commentBody += `#### PostgreSQL ${pgVersion} (${buildType} build)\n\n` - commentBody += `Failed tests:\n` - for (const test of failedTests[pgVersion][buildType]) { + if (Object.keys(failedTests[pgVersion]).length > 0) { + commentBody += `#### Failures on Posgres ${pgVersion}\n\n` + for (const [testName, tests] of Object.entries(failedTests[pgVersion])) { + const links = [] + for (const test of tests) { const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}` - - commentBody += `- [\`${test.pytestName}\`](${allureLink})` - if (test.retriesCount > 0) { - commentBody += ` (ran [${test.retriesCount + 1} times](${allureLink}/retries))` - } - commentBody += "\n" + links.push(`[${test.buildType}](${allureLink})`) } - commentBody += "\n" + commentBody += `- \`${testName}\`: ${links.join(", ")}\n` } + + const testsToRerun = Object.values(failedTests[pgVersion]).map(x => x[0].name) + const command = `DEFAULT_PG_VERSION=${pgVersion} scripts/pytest -k "${testsToRerun.join(" or ")}"` + + commentBody += "```\n" + commentBody += `# Run failed on Postgres ${pgVersion} tests locally:\n` + commentBody += `${command}\n` + commentBody += "```\n" } } if (flakyTestsCount > 0) { - commentBody += "
\nFlaky tests\n\n" + commentBody += `
\nFlaky tests (${flakyTestsCount})\n\n` for (const pgVersion of Array.from(pgVersions).sort().reverse()) { - for (const buildType of Array.from(buildTypes).sort().reverse()) { - if (flakyTests[pgVersion][buildType].length > 0) { - commentBody += `#### PostgreSQL ${pgVersion} (${buildType} build)\n\n` - for (const test of flakyTests[pgVersion][buildType]) { + if (Object.keys(flakyTests[pgVersion]).length > 0) { + commentBody += `#### Postgres ${pgVersion}\n\n` + for (const [testName, tests] of Object.entries(flakyTests[pgVersion])) { + const links = [] + for (const test of tests) { + const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}/retries` const status = test.status === "passed" ? ":white_check_mark:" : ":x:" - commentBody += `- ${status} [\`${test.pytestName}\`](${reportUrl}#suites/${test.parentUid}/${test.uid}/retries)\n` + links.push(`[${status} ${test.buildType}](${allureLink})`) } - commentBody += "\n" + commentBody += `- \`${testName}\`: ${links.join(", ")}\n` } } }