mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-05 04:22:56 +00:00
ed845b644b
## Problem Postgres submodule can be changed unintentionally, and these changes are easy to miss during the review. Adding a check that should prevent this from happening, the check fails `build-neon` job with the following message: ``` Expected postgres-v14 rev to be at '1414141414141414141414141414141414141414', but it is at '1144aee1661c79eec65e784a8dad8bd450d9df79' Expected postgres-v15 rev to be at '1515151515151515151515151515151515151515', but it is at '1984832c740a7fa0e468bb720f40c525b652835d' Please update vendors/revisions.json if these changes are intentional. ``` This is an alternative approach to https://github.com/neondatabase/neon/pull/4603 ## Summary of changes - Add `vendor/revisions.json` file with expected revisions - Add built-time check (to `build-neon` job) that Postgres submodules match revisions from `vendor/revisions.json` - A couple of small improvements for logs from https://github.com/neondatabase/neon/pull/4603 - Fixed GitHub autocomment for no tests was run case --------- Co-authored-by: Joonas Koivunen <joonas@neon.tech>
321 lines
12 KiB
JavaScript
Executable File
321 lines
12 KiB
JavaScript
Executable File
#! /usr/bin/env node
|
|
|
|
//
|
|
// The script parses Allure reports and posts a comment with a summary of the test results to the PR or to the latest commit in the branch.
|
|
//
|
|
// The comment is updated on each run with the latest results.
|
|
//
|
|
// It is designed to be used with actions/github-script from GitHub Workflows:
|
|
// - uses: actions/github-script@v6
|
|
// with:
|
|
// script: |
|
|
// const script = require("./scripts/comment-test-report.js")
|
|
// await script({
|
|
// github,
|
|
// context,
|
|
// fetch,
|
|
// report: {
|
|
// reportUrl: "...",
|
|
// reportJsonUrl: "...",
|
|
// },
|
|
// })
|
|
//
|
|
|
|
// Equivalent of Python's defaultdict.
|
|
//
|
|
// const dm = new DefaultMap(() => new DefaultMap(() => []))
|
|
// dm["firstKey"]["secondKey"].push("value")
|
|
//
|
|
class DefaultMap extends Map {
|
|
constructor(getDefaultValue) {
|
|
return new Proxy({}, {
|
|
get: (target, name) => name in target ? target[name] : (target[name] = getDefaultValue(name))
|
|
})
|
|
}
|
|
}
|
|
|
|
const parseReportJson = async ({ reportJsonUrl, fetch }) => {
|
|
const suites = await (await fetch(reportJsonUrl)).json()
|
|
|
|
// Allure distinguishes "failed" (with an assertion error) and "broken" (with any other error) tests.
|
|
// For this report it's ok to treat them in the same way (as failed).
|
|
const failedTests = new DefaultMap(() => new DefaultMap(() => []))
|
|
const passedTests = new DefaultMap(() => new DefaultMap(() => []))
|
|
const skippedTests = new DefaultMap(() => new DefaultMap(() => []))
|
|
const retriedTests = new DefaultMap(() => new DefaultMap(() => []))
|
|
const flakyTests = new DefaultMap(() => new DefaultMap(() => []))
|
|
|
|
let failedTestsCount = 0
|
|
let passedTestsCount = 0
|
|
let skippedTestsCount = 0
|
|
let flakyTestsCount = 0
|
|
|
|
const pgVersions = new Set()
|
|
|
|
for (const parentSuite of suites.children) {
|
|
for (const suite of parentSuite.children) {
|
|
for (const test of suite.children) {
|
|
let buildType, pgVersion
|
|
const match = test.name.match(/[\[-](?<buildType>debug|release)-pg(?<pgVersion>\d+)[-\]]/)?.groups
|
|
if (match) {
|
|
({ buildType, pgVersion } = match)
|
|
} else {
|
|
// It's ok, we embed BUILD_TYPE and Postgres Version into the test name only for regress suite and do not for other suites (like performance).
|
|
console.info(`Cannot get BUILD_TYPE and Postgres Version from test name: "${test.name}", defaulting to "release" and "14"`)
|
|
|
|
buildType = "release"
|
|
pgVersion = "14"
|
|
}
|
|
|
|
pgVersions.add(pgVersion)
|
|
|
|
// Removing build type and PostgreSQL version from the test name to make it shorter
|
|
const testName = test.name.replace(new RegExp(`${buildType}-pg${pgVersion}-?`), "").replace("[]", "")
|
|
test.pytestName = `${parentSuite.name.replace(".", "/")}/${suite.name}.py::${testName}`
|
|
test.pgVersion = pgVersion
|
|
test.buildType = buildType
|
|
|
|
if (test.status === "passed") {
|
|
passedTests[pgVersion][testName].push(test)
|
|
passedTestsCount += 1
|
|
} else if (test.status === "failed" || test.status === "broken") {
|
|
failedTests[pgVersion][testName].push(test)
|
|
failedTestsCount += 1
|
|
} else if (test.status === "skipped") {
|
|
skippedTests[pgVersion][testName].push(test)
|
|
skippedTestsCount += 1
|
|
}
|
|
|
|
if (test.retriesCount > 0) {
|
|
retriedTests[pgVersion][testName].push(test)
|
|
|
|
if (test.retriesStatusChange) {
|
|
flakyTests[pgVersion][testName].push(test)
|
|
flakyTestsCount += 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
failedTests,
|
|
failedTestsCount,
|
|
passedTests,
|
|
passedTestsCount,
|
|
skippedTests,
|
|
skippedTestsCount,
|
|
flakyTests,
|
|
flakyTestsCount,
|
|
retriedTests,
|
|
pgVersions,
|
|
}
|
|
}
|
|
|
|
const reportSummary = async (params) => {
|
|
const {
|
|
failedTests,
|
|
failedTestsCount,
|
|
passedTests,
|
|
passedTestsCount,
|
|
skippedTests,
|
|
skippedTestsCount,
|
|
flakyTests,
|
|
flakyTestsCount,
|
|
retriedTests,
|
|
pgVersions,
|
|
reportUrl,
|
|
} = params
|
|
|
|
let summary = ""
|
|
|
|
const totalTestsCount = failedTestsCount + passedTestsCount + skippedTestsCount
|
|
summary += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}))\n___\n`
|
|
|
|
// Print test resuls from the newest to the oldest Postgres version for release and debug builds.
|
|
for (const pgVersion of Array.from(pgVersions).sort().reverse()) {
|
|
if (Object.keys(failedTests[pgVersion]).length > 0) {
|
|
summary += `#### Failures on Posgres ${pgVersion}\n\n`
|
|
for (const [testName, tests] of Object.entries(failedTests[pgVersion])) {
|
|
const links = []
|
|
for (const test of tests) {
|
|
const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}`
|
|
links.push(`[${test.buildType}](${allureLink})`)
|
|
}
|
|
summary += `- \`${testName}\`: ${links.join(", ")}\n`
|
|
}
|
|
}
|
|
}
|
|
|
|
if (failedTestsCount > 0) {
|
|
const testsToRerun = []
|
|
for (const pgVersion of Object.keys(failedTests)) {
|
|
for (const testName of Object.keys(failedTests[pgVersion])) {
|
|
testsToRerun.push(...failedTests[pgVersion][testName].map(test => test.name))
|
|
}
|
|
}
|
|
const command = `scripts/pytest -vv -n $(nproc) -k "${testsToRerun.join(" or ")}"`
|
|
|
|
summary += "```\n"
|
|
summary += `# Run all failed tests locally:\n`
|
|
summary += `${command}\n`
|
|
summary += "```\n"
|
|
}
|
|
|
|
if (flakyTestsCount > 0) {
|
|
summary += `<details>\n<summary>Flaky tests (${flakyTestsCount})</summary>\n\n`
|
|
for (const pgVersion of Array.from(pgVersions).sort().reverse()) {
|
|
if (Object.keys(flakyTests[pgVersion]).length > 0) {
|
|
summary += `#### Postgres ${pgVersion}\n\n`
|
|
for (const [testName, tests] of Object.entries(flakyTests[pgVersion])) {
|
|
const links = []
|
|
for (const test of tests) {
|
|
const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}/retries`
|
|
links.push(`[${test.buildType}](${allureLink})`)
|
|
}
|
|
summary += `- \`${testName}\`: ${links.join(", ")}\n`
|
|
}
|
|
}
|
|
}
|
|
summary += "\n</details>\n"
|
|
}
|
|
|
|
return summary
|
|
}
|
|
|
|
module.exports = async ({ github, context, fetch, report }) => {
|
|
// Marker to find the comment in the subsequent runs
|
|
const startMarker = `<!--AUTOMATIC COMMENT START #${context.payload.number}-->`
|
|
// If we run the script in the PR or in the branch (main/release/...)
|
|
const isPullRequest = !!context.payload.pull_request
|
|
// Latest commit in PR or in the branch
|
|
const commitSha = isPullRequest ? context.payload.pull_request.head.sha : context.sha
|
|
// Let users know that the comment is updated automatically
|
|
const autoupdateNotice = `<div align="right"><sub>The comment gets automatically updated with the latest test results<br>${commitSha} at ${new Date().toISOString()} :recycle:</sub></div>`
|
|
// GitHub bot id taken from (https://api.github.com/users/github-actions[bot])
|
|
const githubActionsBotId = 41898282
|
|
// Commend body itself
|
|
let commentBody = `${startMarker}\n`
|
|
|
|
// Common parameters for GitHub API requests
|
|
const ownerRepoParams = {
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
}
|
|
|
|
const {reportUrl, reportJsonUrl} = report
|
|
|
|
if (reportUrl && reportJsonUrl) {
|
|
try {
|
|
const parsed = await parseReportJson({ reportJsonUrl, fetch })
|
|
commentBody += await reportSummary({ ...parsed, reportUrl })
|
|
} catch (error) {
|
|
commentBody += `### [full report](${reportUrl})\n___\n`
|
|
commentBody += `#### Failed to create a summary for the test run: \n`
|
|
commentBody += "```\n"
|
|
commentBody += `${error.stack}\n`
|
|
commentBody += "```\n"
|
|
commentBody += "\nTo reproduce and debug the error locally run:\n"
|
|
commentBody += "```\n"
|
|
commentBody += `scripts/comment-test-report.js ${reportJsonUrl}`
|
|
commentBody += "\n```\n"
|
|
}
|
|
} else {
|
|
commentBody += `#### No tests were run or test report is not available\n`
|
|
}
|
|
|
|
let createCommentFn, listCommentsFn, updateCommentFn, issueNumberOrSha
|
|
if (isPullRequest) {
|
|
createCommentFn = github.rest.issues.createComment
|
|
listCommentsFn = github.rest.issues.listComments
|
|
updateCommentFn = github.rest.issues.updateComment
|
|
issueNumberOrSha = {
|
|
issue_number: context.payload.number,
|
|
}
|
|
} else {
|
|
updateCommentFn = github.rest.repos.updateCommitComment
|
|
listCommentsFn = github.rest.repos.listCommentsForCommit
|
|
createCommentFn = github.rest.repos.createCommitComment
|
|
issueNumberOrSha = {
|
|
commit_sha: commitSha,
|
|
}
|
|
}
|
|
|
|
const { data: comments } = await listCommentsFn({
|
|
...issueNumberOrSha,
|
|
...ownerRepoParams,
|
|
})
|
|
|
|
const comment = comments.find(comment => comment.user.id === githubActionsBotId && comment.body.startsWith(startMarker))
|
|
if (comment) {
|
|
await updateCommentFn({
|
|
comment_id: comment.id,
|
|
body: commentBody,
|
|
...ownerRepoParams,
|
|
})
|
|
} else {
|
|
await createCommentFn({
|
|
body: commentBody,
|
|
...issueNumberOrSha,
|
|
...ownerRepoParams,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Equivalent of Python's `if __name__ == "__main__":`
|
|
// https://nodejs.org/docs/latest/api/modules.html#accessing-the-main-module
|
|
if (require.main === module) {
|
|
// Poor man's argument parsing: we expect the third argument is a JSON URL (0: node binary, 1: this script, 2: JSON url)
|
|
if (process.argv.length !== 3) {
|
|
console.error(`Unexpected number of arguments\nUsage: node ${process.argv[1]} <jsonUrl>`)
|
|
process.exit(1)
|
|
}
|
|
const jsonUrl = process.argv[2]
|
|
|
|
try {
|
|
new URL(jsonUrl)
|
|
} catch (error) {
|
|
console.error(`Invalid URL: ${jsonUrl}\nUsage: node ${process.argv[1]} <jsonUrl>`)
|
|
process.exit(1)
|
|
}
|
|
|
|
const htmlUrl = jsonUrl.replace("/data/suites.json", "/index.html")
|
|
|
|
const githubMock = {
|
|
rest: {
|
|
issues: {
|
|
createComment: console.log,
|
|
listComments: async () => ({ data: [] }),
|
|
updateComment: console.log
|
|
},
|
|
repos: {
|
|
createCommitComment: console.log,
|
|
listCommentsForCommit: async () => ({ data: [] }),
|
|
updateCommitComment: console.log
|
|
}
|
|
}
|
|
}
|
|
|
|
const contextMock = {
|
|
repo: {
|
|
owner: 'testOwner',
|
|
repo: 'testRepo'
|
|
},
|
|
payload: {
|
|
number: 42,
|
|
pull_request: null,
|
|
},
|
|
sha: '0000000000000000000000000000000000000000',
|
|
}
|
|
|
|
module.exports({
|
|
github: githubMock,
|
|
context: contextMock,
|
|
fetch: fetch,
|
|
report: {
|
|
reportUrl: htmlUrl,
|
|
reportJsonUrl: jsonUrl,
|
|
}
|
|
})
|
|
}
|