neon/scripts/comment-test-report.js

#! /usr/bin/env node

//
// The script parses Allure reports and posts a comment with a summary of the test results to the PR or to the latest commit in the branch.
//
// The comment is updated on each run with the latest results.
//
// It is designed to be used with actions/github-script from GitHub Workflows:
// - uses: actions/github-script@v6
//   with:
//     script: |
//       const script = require("./scripts/comment-test-report.js")
//       await script({
//         github,
//         context,
//         fetch,
//         report: {
//           reportUrl: "...",
//           reportJsonUrl: "...",
//         },
//       })
//

// Equivalent of Python's defaultdict.
//
// const dm = new DefaultMap(() => new DefaultMap(() => []))
// dm["firstKey"]["secondKey"].push("value")
//
class DefaultMap extends Map {
    constructor(getDefaultValue) {
        return new Proxy({}, {
            get: (target, name) => name in target ? target[name] : (target[name] = getDefaultValue(name))
        })
    }
}

const parseReportJson = async ({ reportJsonUrl, fetch }) => {
    const suites = await (await fetch(reportJsonUrl)).json()

    // Allure distinguishes "failed" (with an assertion error) and "broken" (with any other error) tests.
    // For this report it's ok to treat them in the same way (as failed).
    const failedTests = new DefaultMap(() => new DefaultMap(() => []))
    const passedTests = new DefaultMap(() => new DefaultMap(() => []))
    const skippedTests = new DefaultMap(() => new DefaultMap(() => []))
    const retriedTests = new DefaultMap(() => new DefaultMap(() => []))
    const flakyTests = new DefaultMap(() => new DefaultMap(() => []))

    let failedTestsCount = 0
    let passedTestsCount = 0
    let skippedTestsCount = 0
    let flakyTestsCount = 0

    const pgVersions = new Set()

    for (const parentSuite of suites.children) {
        for (const suite of parentSuite.children) {
            for (const test of suite.children) {
                let buildType, pgVersion
                const match = test.name.match(/[\[-](?<buildType>debug|release)-pg(?<pgVersion>\d+)[-\]]/)?.groups
                if (match) {
                    ({ buildType, pgVersion } = match)
                } else {
                    // It's ok, we embed BUILD_TYPE and Postgres Version into the test name only for regress suite and do not for other suites (like performance).
                    console.info(`Cannot get BUILD_TYPE and Postgres Version from test name: "${test.name}", defaulting to "release" and "14"`)

                    buildType = "release"
                    pgVersion = "14"
                }

                pgVersions.add(pgVersion)

                // Removing build type and PostgreSQL version from the test name to make it shorter
                const testName = test.name.replace(new RegExp(`${buildType}-pg${pgVersion}-?`), "").replace("[]", "")
                test.pytestName = `${parentSuite.name.replace(".", "/")}/${suite.name}.py::${testName}`
                test.pgVersion = pgVersion
                test.buildType = buildType

                if (test.status === "passed") {
                    passedTests[pgVersion][testName].push(test)
                    passedTestsCount += 1
                } else if (test.status === "failed" || test.status === "broken") {
                    failedTests[pgVersion][testName].push(test)
                    failedTestsCount += 1
                } else if (test.status === "skipped") {
                    skippedTests[pgVersion][testName].push(test)
                    skippedTestsCount += 1
                }

                if (test.retriesCount > 0) {
                    retriedTests[pgVersion][testName].push(test)

                    if (test.retriesStatusChange) {
                        flakyTests[pgVersion][testName].push(test)
                        flakyTestsCount += 1
                    }
                }
            }
        }
    }

    return {
        failedTests,
        failedTestsCount,
        passedTests,
        passedTestsCount,
        skippedTests,
        skippedTestsCount,
        flakyTests,
        flakyTestsCount,
        retriedTests,
        pgVersions,
    }
}

const reportSummary = async (params) => {
    const {
        failedTests,
        failedTestsCount,
        passedTests,
        passedTestsCount,
        skippedTests,
        skippedTestsCount,
        flakyTests,
        flakyTestsCount,
        retriedTests,
        pgVersions,
        reportUrl,
    } = params

    let summary = ""

    const totalTestsCount = failedTestsCount + passedTestsCount + skippedTestsCount
    summary += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}))\n___\n`

    // Print test resuls from the newest to the oldest Postgres version for release and debug builds.
    for (const pgVersion of Array.from(pgVersions).sort().reverse()) {
        if (Object.keys(failedTests[pgVersion]).length > 0) {
            summary += `#### Failures on Posgres ${pgVersion}\n\n`
            for (const [testName, tests] of Object.entries(failedTests[pgVersion])) {
                const links = []
                for (const test of tests) {
                    const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}`
                    links.push(`[${test.buildType}](${allureLink})`)
                }
                summary += `- \`${testName}\`: ${links.join(", ")}\n`
            }

            const testsToRerun = Object.values(failedTests[pgVersion]).map(x => x[0].name)
            const command = `DEFAULT_PG_VERSION=${pgVersion} scripts/pytest -k "${testsToRerun.join(" or ")}"`

            summary += "```\n"
            summary += `# Run failed on Postgres ${pgVersion} tests locally:\n`
            summary += `${command}\n`
            summary += "```\n"
        }
    }

    if (flakyTestsCount > 0) {
        summary += `<details>\n<summary>Flaky tests (${flakyTestsCount})</summary>\n\n`
        for (const pgVersion of Array.from(pgVersions).sort().reverse()) {
            if (Object.keys(flakyTests[pgVersion]).length > 0) {
                summary += `#### Postgres ${pgVersion}\n\n`
                for (const [testName, tests] of Object.entries(flakyTests[pgVersion])) {
                    const links = []
                    for (const test of tests) {
                        const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}/retries`
                        const status = test.status === "passed" ? ":white_check_mark:" : ":x:"
                        links.push(`[${status} ${test.buildType}](${allureLink})`)
                    }
                    summary += `- \`${testName}\`: ${links.join(", ")}\n`
                }
            }
        }
        summary += "\n</details>\n"
    }

    return summary
}

module.exports = async ({ github, context, fetch, report }) => {
    // Marker to find the comment in the subsequent runs
    const startMarker = `<!--AUTOMATIC COMMENT START #${context.payload.number}-->`
    // If we run the script in the PR or in the branch (main/release/...)
    const isPullRequest = !!context.payload.pull_request
    // Latest commit in PR or in the branch
    const commitSha = isPullRequest ? context.payload.pull_request.head.sha : context.sha
    // Let users know that the comment is updated automatically
    const autoupdateNotice = `<div align="right"><sub>The comment gets automatically updated with the latest test results<br>${commitSha} at ${new Date().toISOString()} :recycle:</sub></div>`
    // GitHub bot id taken from (https://api.github.com/users/github-actions[bot])
    const githubActionsBotId = 41898282
    // Commend body itself
    let commentBody = `${startMarker}\n`

    // Common parameters for GitHub API requests
    const ownerRepoParams = {
        owner: context.repo.owner,
        repo: context.repo.repo,
    }

    const {reportUrl, reportJsonUrl} = report

    if (!reportUrl || !reportJsonUrl) {
        commentBody += `#### No tests were run or test report is not available\n`
        commentBody += autoupdateNotice
        return
    }

    try {
        const parsed = await parseReportJson({ reportJsonUrl, fetch })
        commentBody += await reportSummary({ ...parsed, reportUrl })
    } catch (error) {
        commentBody += `### [full report](${reportUrl})\n___\n`
        commentBody += `#### Failed to create a summary for the test run: \n`
        commentBody += "```\n"
        commentBody += `${error.stack}\n`
        commentBody += "```\n"
        commentBody += "\nTo reproduce and debug the error locally run:\n"
        commentBody += "```\n"
        commentBody += `scripts/comment-test-report.js ${reportJsonUrl}`
        commentBody += "\n```\n"
    }

    commentBody += autoupdateNotice

    let createCommentFn, listCommentsFn, updateCommentFn, issueNumberOrSha
    if (isPullRequest) {
        createCommentFn  = github.rest.issues.createComment
        listCommentsFn   = github.rest.issues.listComments
        updateCommentFn  = github.rest.issues.updateComment
        issueNumberOrSha = {
            issue_number: context.payload.number,
        }
    } else {
        updateCommentFn  = github.rest.repos.updateCommitComment
        listCommentsFn   = github.rest.repos.listCommentsForCommit
        createCommentFn  = github.rest.repos.createCommitComment
        issueNumberOrSha = {
            commit_sha: commitSha,
        }
    }

    const { data: comments } = await listCommentsFn({
        ...issueNumberOrSha,
        ...ownerRepoParams,
    })

    const comment = comments.find(comment => comment.user.id === githubActionsBotId && comment.body.startsWith(startMarker))
    if (comment) {
        await updateCommentFn({
            comment_id: comment.id,
            body: commentBody,
            ...ownerRepoParams,
        })
    } else {
        await createCommentFn({
            body: commentBody,
            ...issueNumberOrSha,
            ...ownerRepoParams,
        })
    }
}

// Equivalent of Python's `if __name__ == "__main__":`
// https://nodejs.org/docs/latest/api/modules.html#accessing-the-main-module
if (require.main === module) {
    // Poor man's argument parsing: we expect the third argument is a JSON URL (0: node binary, 1: this script, 2: JSON url)
    if (process.argv.length !== 3) {
        console.error(`Unexpected number of arguments\nUsage: node ${process.argv[1]} <jsonUrl>`)
        process.exit(1)
    }
    const jsonUrl = process.argv[2]

    try {
        new URL(jsonUrl)
    } catch (error) {
        console.error(`Invalid URL: ${jsonUrl}\nUsage: node ${process.argv[1]} <jsonUrl>`)
        process.exit(1)
    }

    const htmlUrl = jsonUrl.replace("/data/suites.json", "/index.html")

    const githubMock = {
        rest: {
            issues: {
                createComment: console.log,
                listComments: async () => ({ data: [] }),
                updateComment: console.log
            },
            repos: {
                createCommitComment: console.log,
                listCommentsForCommit: async () => ({ data: [] }),
                updateCommitComment: console.log
            }
        }
    }

    const contextMock = {
        repo: {
            owner: 'testOwner',
            repo: 'testRepo'
        },
        payload: {
            number: 42,
            pull_request: null,
        },
        sha: '0000000000000000000000000000000000000000',
    }

    module.exports({
        github: githubMock,
        context: contextMock,
        fetch: fetch,
        report: {
            reportUrl: htmlUrl,
            reportJsonUrl: jsonUrl,
        }
    })
}