From ca73a1845997f5577e1bf10bb9acf1b0daad209a Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Fri, 30 Oct 2026 14:01:41 +0100 Subject: [PATCH] feat: run all PostgreSQL live tests in isolated advisory CI with failure and skip summaries --- .github/scripts/live-tests-summary.mjs | 62 ++++++++++++++++++ .github/scripts/live-tests-summary.test.mjs | 38 ++++++++++++ .github/workflows/ci.yml | 69 +++++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 .github/scripts/live-tests-summary.mjs create mode 100644 .github/scripts/live-tests-summary.test.mjs diff --git a/.github/scripts/live-tests-summary.mjs b/.github/scripts/live-tests-summary.mjs new file mode 100644 index 000000000..91b7cb4ce --- /dev/null +++ b/.github/scripts/live-tests-summary.mjs @@ -0,0 +1,62 @@ +import { appendFileSync, existsSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export function summarizeLiveTests(input, outcome) { + const tests = new Map(); + const failedPackages = new Set(); + for (const line of input.split("\n")) { + let event; + try { + event = JSON.parse(line); + } catch { + continue; // Compiler diagnostics can be plain text alongside go test JSON. + } + if (!event || typeof event !== "object") continue; + if (event.Action === "fail" && typeof event.Package === "string" && !event.Test) { + failedPackages.add(event.Package); + } + if (typeof event.Test !== "string" || !event.Test.startsWith("TestLive") || event.Test.includes("/")) continue; + if (!["pass", "fail", "skip"].includes(event.Action)) continue; + tests.set(`${event.Package}/${event.Test}`, event.Action); + } + const names = (action) => [...tests].filter(([, result]) => result === action).map(([name]) => name).sort(); + const passed = names("pass"); + const failed = names("fail"); + const skipped = names("skip"); + const list = (heading, entries) => [ + `### ${heading}`, + "", + ...(entries.length ? entries.slice(0, 200).map((name) => `- \`${name.replaceAll("`", "\\`")}\``) : ["None recorded."]), + ...(entries.length > 200 ? ["", "Showing the first 200 entries; see the complete output artifact."] : []), + "", + ]; + return { + needsAttention: outcome !== "success" || tests.size === 0 || failed.length > 0 || skipped.length > 0 || failedPackages.size > 0, + markdown: [ + "## PostgreSQL catch-all (advisory)", + "", + `Test step outcome: **${outcome || "not started"}**. This job is outside the required CI Status aggregate while #1078 is triaged.`, + "", + "| Top-level TestLive results | Count |", + "| --- | ---: |", + `| Passed | ${passed.length} |`, + `| Failed | ${failed.length} |`, + `| Skipped | ${skipped.length} |`, + "", + ...(tests.size === 0 ? ["No completed TestLive results were recorded. Check setup, compilation and timeouts.", ""] : []), + ...list("Failed packages (including setup or build failures)", [...failedPackages].sort()), + ...list("Failed tests", failed), + ...list("Skipped tests", skipped), + "Full JSON events and compiler output are retained in the postgres-live-advisory artifact. Subtests are included there; counts above include each top-level test once.", + "", + ].join("\n"), + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const file = process.argv[2]; + const report = summarizeLiveTests(file && existsSync(file) ? readFileSync(file, "utf8") : "", process.env.LIVE_TEST_OUTCOME); + if (process.env.GITHUB_STEP_SUMMARY) appendFileSync(process.env.GITHUB_STEP_SUMMARY, report.markdown); + else process.stdout.write(report.markdown); + if (report.needsAttention) console.log("::warning::Advisory PostgreSQL coverage needs triage. See the job summary and complete output artifact."); +} diff --git a/.github/scripts/live-tests-summary.test.mjs b/.github/scripts/live-tests-summary.test.mjs new file mode 100644 index 000000000..e9cea53d9 --- /dev/null +++ b/.github/scripts/live-tests-summary.test.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { summarizeLiveTests } from "./live-tests-summary.mjs"; + +const events = (...rows) => rows.map((row) => JSON.stringify(row)).join("\n"); + +test("reports failures, skips and failed packages without counting subtests twice", () => { + const report = summarizeLiveTests(events( + { Action: "pass", Package: "app", Test: "TestLivePassed/child" }, + { Action: "pass", Package: "app", Test: "TestLivePassed" }, + { Action: "fail", Package: "repository", Test: "TestLiveFailed/child" }, + { Action: "fail", Package: "repository", Test: "TestLiveFailed" }, + { Action: "fail", Package: "repository" }, + { Action: "skip", Package: "app", Test: "TestLiveSkipped" }, + { Action: "pass", Package: "app", Test: "TestOther" }, + ), "failure"); + assert.equal(report.needsAttention, true); + assert.match(report.markdown, /\| Passed \| 1 \|/); + assert.match(report.markdown, /\| Failed \| 1 \|/); + assert.match(report.markdown, /\| Skipped \| 1 \|/); + assert.match(report.markdown, /`repository\/TestLiveFailed`/); + assert.match(report.markdown, /`app\/TestLiveSkipped`/); + assert.doesNotMatch(report.markdown, /\/child/); +}); + +test("empty, interrupted and compile-only output cannot look like passing coverage", () => { + for (const input of ["", "# app\ncompiler error", 'null\n{"Action":"fail","Package":"app"}']) { + const report = summarizeLiveTests(input, "failure"); + assert.equal(report.needsAttention, true); + assert.match(report.markdown, /No completed TestLive results/); + } + assert.equal(summarizeLiveTests(events({ Action: "pass", Package: "app", Test: "TestLiveOK" }), "cancelled").needsAttention, true); +}); + +test("successful coverage is clean only when completed tests contain no skips", () => { + assert.equal(summarizeLiveTests(events({ Action: "pass", Package: "app", Test: "TestLiveOK" }), "success").needsAttention, false); + assert.equal(summarizeLiveTests(events({ Action: "skip", Package: "app", Test: "TestLiveSkipped" }), "success").needsAttention, true); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d49ec3d31..2a6af9bdc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: filters: | go-services: - '.github/workflows/ci.yml' + - '.github/scripts/live-tests-summary*.mjs' - 'go.mod' - 'go.sum' - 'internal/**' @@ -212,6 +213,10 @@ jobs: if: matrix.suite == 'lint' run: make lint + - name: Check live-test reporting + if: matrix.suite == 'lint' + run: node --test .github/scripts/live-tests-summary.test.mjs + - name: Verify protobuf files are up to date if: matrix.suite == 'lint' run: make check-proto @@ -644,6 +649,70 @@ jobs: flags: go fail_ci_if_error: false + # Keep advisory coverage separate until #1078's failure inventory is resolved. + go-live-advisory: + name: PostgreSQL catch-all (advisory) + needs: changes + if: needs.changes.outputs.go-services == 'true' + runs-on: ubuntu-latest + timeout-minutes: 45 + continue-on-error: true + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: warmbly + POSTGRES_PASSWORD: warmbly + POSTGRES_DB: warmbly_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U warmbly -d warmbly_test" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + - name: Prepare isolated regression database + env: + PRIMARY_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + run: go run ./cmd/migrate + - name: Run every TestLive test + id: live + timeout-minutes: 35 + env: + WARMBLY_TEST_DB: postgres://warmbly:warmbly@127.0.0.1:5432/warmbly_test?sslmode=disable + WARMBLY_TEST_REDIS: redis://127.0.0.1:6379/15 + run: | + set -o pipefail + go test -json -race -p 1 -timeout 10m -count=1 -run '^TestLive' ./... 2>&1 | tee live-tests.jsonl + - name: Summarize advisory results + if: always() + env: + LIVE_TEST_OUTCOME: ${{ steps.live.outcome }} + run: node .github/scripts/live-tests-summary.mjs live-tests.jsonl + - name: Retain complete live-test output + if: always() + uses: actions/upload-artifact@v4 + with: + name: postgres-live-advisory-${{ github.run_attempt }} + path: live-tests.jsonl + if-no-files-found: warn + retention-days: 14 + web-ci: name: Web CI needs: changes