diff --git a/.github/scripts/assert_integrity.py b/.github/scripts/assert_integrity.py index cf9f2b6ddc..33eff155fd 100644 --- a/.github/scripts/assert_integrity.py +++ b/.github/scripts/assert_integrity.py @@ -4,10 +4,13 @@ Assertion-integrity analyzer for the E2E Council pipeline (deterministic — no Two modes: fingerprint -> print a JSON fingerprint of the spec's assertions - check [--baseline ] [--allow-weakening] + check [--baseline ] [--allow-weakening] [--strengthen] -> static anti-pattern scan + (optional) diff-aware heal-weakening check; writes nothing, prints a JSON verdict, exits 2 if any CRITICAL finding. + --strengthen (Refiner mode): permit ADDED negative + matchers (assert-absence is valid strengthening); + still blocks any DROP of assertions. Conservative by design: only HIGH-CONFIDENCE patterns are flagged as critical, because a gate with false positives gets disabled by humans (the same trap that made mutation-testing a poor fit). @@ -166,11 +169,20 @@ def swallowed_assertion_tests(lines): return flagged -def check(spec_path, baseline=None, allow_weakening=False): +def check(spec_path, baseline=None, allow_weakening=False, strengthen=False): """Run the static anti-pattern scan (tautologies, conditional-only assertions) and, when a baseline is given, the diff-aware heal-weakening guard. Returns a verdict dict; never raises on a malformed/partial baseline — missing baseline keys fall back to the current value (skip that - comparison) rather than crashing the gate.""" + comparison) rather than crashing the gate. + + `strengthen` (Refiner mode): the Refiner STRENGTHENS weak/mismatched tests, which can legitimately + (a) ADD negative matchers (assert a column is *absent* before a re-search) and (b) park a genuinely + unwired test as fixme. So in strengthen mode we permit a negatives rise ONLY when expect() ALSO rose + (an addition, not an in-place flip — a flip with flat expect() is still flagged), and permit a + skip/fixme rise (honest feature-incomplete). `heal-removed-assertions` (expect() may never drop) and + EVERY static anti-pattern check (tautology, conditional-only, swallowed, no-runnable) STILL apply — + so strengthen mode cannot be used to drop assertions or green-wash. (NOTE: do not also pass + --allow-weakening in strengthen mode; that disables the removed guard and defeats the point.)""" src = open(spec_path, encoding='utf-8').read() lines = src.split('\n') findings = [] @@ -207,10 +219,18 @@ def check(spec_path, baseline=None, allow_weakening=False): if fp['expects'] < base_expects: findings.append({'severity': 'critical', 'rule': 'heal-removed-assertions', 'detail': f"expect() count dropped {base_expects} -> {fp['expects']} during heal"}) - if fp['negatives'] > base_negatives: + # Inverted = a positive flipped to a negative IN PLACE. In --strengthen mode (Refiner) ADDING a + # negative is legitimate, but ONLY if expect() ALSO rose — that proves the negative is a new + # assertion, not a flip. A negative rise with no new expect() is an in-place flip → still flag, + # even in strengthen mode. (Outside strengthen mode any negative rise is flagged, as before.) + if fp['negatives'] > base_negatives and (not strengthen or fp['expects'] <= base_expects): findings.append({'severity': 'critical', 'rule': 'heal-inverted-assertions', - 'detail': f"negative matchers rose {base_negatives} -> {fp['negatives']} during heal (positive flipped to negative?)"}) - if fp['skips'] > base_skips: + 'detail': f"negative matchers rose {base_negatives} -> {fp['negatives']} (positive flipped to negative in place — expect() did not rise)"}) + # Rising skip/fixme is silent-skipping during heal — blocked. The Refiner's HONEST + # feature-incomplete path (park a test as fixme, assertion body kept intact) is permitted ONLY in + # --strengthen mode; expect() still cannot drop (guarded above) and an all-fixme spec is still + # caught by no-runnable-tests, so this can't be used to green-wash. + if fp['skips'] > base_skips and not strengthen: findings.append({'severity': 'critical', 'rule': 'heal-skipped-tests', 'detail': f"skip/fixme rose {base_skips} -> {fp['skips']} during heal (use the feature-incomplete path, not silent skipping)"}) @@ -226,13 +246,15 @@ def main(): if mode == 'fingerprint': print(json.dumps(fingerprint(open(spec, encoding='utf-8').read()))) return - baseline, allow = None, False + baseline, allow, strengthen = None, False, False args = sys.argv[3:] if '--baseline' in args: baseline = json.load(open(args[args.index('--baseline') + 1])) if '--allow-weakening' in args: allow = True - result = check(spec, baseline, allow) + if '--strengthen' in args: + strengthen = True + result = check(spec, baseline, allow, strengthen) print(json.dumps(result, indent=2)) sys.exit(2 if result['verdict'] == 'FAIL' else 0) diff --git a/.github/workflows/e2e-council.yml b/.github/workflows/e2e-council.yml index 5d11260b8b..c68652488b 100644 --- a/.github/workflows/e2e-council.yml +++ b/.github/workflows/e2e-council.yml @@ -692,7 +692,10 @@ jobs: # Leave-alone path: Architect action=none (audit+push were skipped) OR push ran but produced # no diff. Either way the existing tests already cover this → verify_heal + pr_back skip. Tell # the dev on the PR instead of going silent. - if: success() && (steps.validate.outputs.action == 'none' || steps.push.outputs.has_changes == 'false') + # NOT on reuse: a reuse run (clobber off) now ALWAYS proceeds to verify_heal so the Refiner can + # re-validate + strengthen the existing spec — declaring "already covered" here would be wrong + # (we're not done) and would race the real outcome verify_heal/pr_back posts. + if: success() && needs.triage.outputs.reuse != 'true' && (steps.validate.outputs.action == 'none' || steps.push.outputs.has_changes == 'false') env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SRC_PR: ${{ needs.triage.outputs.pr_number }} @@ -779,12 +782,19 @@ jobs: # --------------------------------------------------------------------------- verify_heal: needs: [triage, generate] + # Runs when generation produced NEW tests (has_changes) OR this is a REUSE run (clobber off) — on + # reuse the existing spec is on the autoe2e branch we check out, and even with no new tests the + # Healer/Sentinel re-audit/Refiner must re-validate + strengthen it (so re-running /e2e on a + # previously-BLOCKED spec, e.g. one the Sentinel failed, can finally be refined into a PR instead + # of dead-ending at "already covered"). `needs.generate.result == 'success'` preserves the + # "generate must have succeeded" guard that the old `has_changes == 'true'` check gave implicitly. if: > needs.triage.outputs.dry_run == 'false' && needs.triage.outputs.skip == 'false' && needs.triage.outputs.needs_e2e == 'true' && needs.triage.outputs.author_allowed == 'true' && - needs.generate.outputs.has_changes == 'true' + needs.generate.result == 'success' && + (needs.generate.outputs.has_changes == 'true' || needs.triage.outputs.reuse == 'true') runs-on: labels: ubicloud-standard-16 timeout-minutes: 100 # backstop: build (~15-20) + boot + heal (capped 40) + final run (15) + audit @@ -1071,6 +1081,122 @@ jobs: opencode run --agent e2e-ci-sentinel --model "$PIPELINE_MODEL" \ "Re-audit the healed spec/page objects for ${SPEC}. Apply ALL critical checks, especially the assertion-quality ones (tautologies, conditional-only assertions, negative-only assertions on the feature under test, test-name/assertion mismatch). Update docs/test_generator/ci/sentinel-verdict.json with an honest verdict." + - name: Refiner — remediate Sentinel quality findings (strengthen → re-verify → re-audit) + id: refiner + # When the post-heal Sentinel still FAILs on a GREEN spec, DON'T dead-end the run: feed the + # exact findings to the Refiner, which STRENGTHENS the weak/mismatched tests so each verifies + # what its name promises (scope-locked to tests/; assertions may only be ADDED/tightened, never + # dropped/inverted/renamed-to-dodge). Then we re-run the spec on the live binary (green must + # hold), re-check assertion-integrity (--strengthen: added negatives OK, drops still blocked), + # and re-audit. Capped at MAX_REFINE rounds. A no-PR block becomes a TRUE last resort. + # + # No-op on the happy path: if the re-audit was already clean (verdict PASS / 0 critical), this + # step reads the verdict, sets ran=false, and exits WITHOUT any DeepSeek call or test re-run. + # + # continue-on-error: an opencode/network hiccup must never crash the job. If this step dies + # mid-run, the quality gate sees ran!='true' and falls back to the PRE-refine state (the Sentinel + # FAIL that triggered us) → it BLOCKS. So a crash fails SAFE (no PR), never opens a weak one. + continue-on-error: true + timeout-minutes: 35 # 2 rounds x (refine + full re-run + re-audit); backstop so a hung API can't run to the job cap + if: steps.finalrun.outputs.status == 'passing' + env: + DEEPSEEK_API_KEY_E2E: ${{ secrets.DEEPSEEK_API_KEY_E2E }} + SPEC: ${{ steps.spec.outputs.spec }} + REL: ${{ steps.spec.outputs.rel }} + MAX_REFINE: "2" + run: | + set +e + # Defaults written to $GITHUB_OUTPUT IMMEDIATELY so even a mid-run crash leaves sane outputs: + # ran=false → quality gate uses the pre-refine state (the Sentinel FAIL) → blocks (fail-safe). + { echo "ran=false"; echo "integrity=${{ steps.integrity.outputs.status }}"; echo "finalrun=passing"; echo "scope=clean"; } >> "$GITHUB_OUTPUT" + # SECURITY: SPEC/REL come from an upstream step but are interpolated into shell + opencode + # prompts — validate the shape before use (defence in depth; the spec-resolve step also checks). + # A path with shell metacharacters or outside tests/ui-testing/ is rejected, not run. + case "$SPEC" in + tests/ui-testing/playwright-tests/*.spec.js) : ;; + *) echo "::error::Refiner: SPEC '$SPEC' is not a tests/ui-testing/ .spec.js path — refusing to run."; exit 0 ;; + esac + case "$SPEC$REL" in + *[!A-Za-z0-9._/-]*) echo "::error::Refiner: SPEC/REL contains unexpected characters — refusing to run."; exit 0 ;; + esac + FINALRUN=passing + INTEG="${{ steps.integrity.outputs.status }}" + RSCOPE=clean + verdict() { [ -f docs/test_generator/ci/sentinel-verdict.json ] && jq -r '.verdict // "PASS"' docs/test_generator/ci/sentinel-verdict.json 2>/dev/null || echo PASS; } + crit() { [ -f docs/test_generator/ci/sentinel-verdict.json ] && jq -r '.critical_count // 0' docs/test_generator/ci/sentinel-verdict.json 2>/dev/null || echo 0; } + SENT=$(verdict); SENT_CRIT=$(crit) + # Happy path: Sentinel already clean → nothing to remediate, exit cheap (defaults above stand). + if [ "$SENT" != "FAIL" ] && { [ "$SENT_CRIT" -eq 0 ] 2>/dev/null; }; then + echo "Sentinel re-audit was clean — Refiner not needed (no DeepSeek call)."; exit 0 + fi + if ! command -v opencode >/dev/null 2>&1; then + echo "::warning::opencode unavailable — cannot refine; leaving Sentinel verdict as-is (blocks)."; exit 0 + fi + # Snapshot the out-of-scope dirty set BEFORE refining (build seds + heal already accounted for in + # pre-heal-dirty.txt) so the post-loop scope check attributes any NEW out-of-scope file to us. + # Baseline the PRE-refine spec so we can prove the Refiner only ADDED/strengthened assertions. + if [ -f .github/scripts/assert_integrity.py ]; then + python3 .github/scripts/assert_integrity.py fingerprint "$SPEC" > docs/test_generator/ci/refine-baseline.json 2>/dev/null || true + fi + for i in $(seq 1 "$MAX_REFINE"); do + echo "::group::Refiner round $i/$MAX_REFINE (Sentinel=$SENT critical=$SENT_CRIT)" + # Reset per-round so a skipped check never reuses a stale verdict from a prior round. + FINALRUN=failing; INTEG=unknown + timeout 600 opencode run --agent e2e-ci-refiner --model "$PIPELINE_MODEL" \ + "Round ${i}. The post-heal Sentinel re-audit FAILED on a spec whose tests currently PASS. Read docs/test_generator/ci/sentinel-verdict.json and the audit report under docs/test_generator/audit-reports/. For EACH critical finding, make the named test actually verify what its title promises by STRENGTHENING/adding assertions in ${SPEC} (new locators go in page objects, never raw selectors in the spec). HARD RULES: edit ONLY files under tests/ui-testing/; you may ADD or tighten assertions but NEVER delete, invert in place, skip, or weaken one; do NOT rename a test to dodge a missing assertion (rename only when the body already correctly verifies a real behaviour and merely the title is wrong). A live OpenObserve is booted at ${ZO_BASE_URL}; never build/restart the product, never touch /tmp. If strengthening makes a test fail because the FEATURE is not wired in this build, do NOT weaken it — mark that test test.fixme('reason, see #issue') with evidence (file:line) and keep the assertion intact. Keep every other test green. Write docs/test_generator/ci/refine-result.json and the refine report." \ + || echo "::warning::Refiner agent hiccup/timeout (round $i) — continuing to re-verify." + # Green must hold: re-run the strengthened spec on the live binary. + ( cd tests/ui-testing && npx playwright test "$REL" --workers=4 --reporter=line --timeout=360000 ) + if [ $? -eq 0 ]; then FINALRUN=passing; else echo "::warning::Spec RED after refine round $i."; fi + # Re-check assertion-integrity vs the pre-refine baseline. --strengthen lets the Refiner ADD + # negatives / park honest fixme but STILL blocks any dropped assertion or in-place flip — so we + # do NOT pass --allow-weakening here (it would disable the removed-assertion guard). + if [ "$FINALRUN" = passing ]; then + if [ ! -f .github/scripts/assert_integrity.py ]; then + INTEG=pass # analyzer missing → match the pre-refine gate's "treat as pass" behaviour + else + BASE=""; [ -f docs/test_generator/ci/refine-baseline.json ] && BASE="--baseline docs/test_generator/ci/refine-baseline.json" + if python3 .github/scripts/assert_integrity.py check "$SPEC" $BASE --strengthen > docs/test_generator/ci/assert-verdict.json; then + INTEG=pass + else + INTEG=fail + echo "::error::Refiner produced an integrity violation (assertion dropped/flipped/gamed):" + jq -r '.findings[]? | " - [\(.rule)] line \(.line // "-"): \(.detail)"' docs/test_generator/ci/assert-verdict.json 2>/dev/null || true + fi + fi + fi + # Re-audit ONLY when this round is still viable (green + integrity OK) — a doomed round will + # block regardless, so don't spend an API call re-auditing it. + if [ "$FINALRUN" = passing ] && [ "$INTEG" = pass ]; then + timeout 600 opencode run --agent e2e-ci-sentinel --model "$PIPELINE_MODEL" \ + "Re-audit the refined spec/page objects for ${SPEC} after a remediation pass. Apply ALL critical checks (tautologies, conditional-only, negative-only-on-feature, test-name/assertion mismatch). A pure rename that dodges a missing assertion is STILL a FAIL. Update docs/test_generator/ci/sentinel-verdict.json with an honest verdict." \ + || echo "::warning::Sentinel re-audit hiccup (round $i)." + SENT=$(verdict); SENT_CRIT=$(crit) + fi + echo "::endgroup::" + if [ "$SENT" != "FAIL" ] && { [ "$SENT_CRIT" -eq 0 ] 2>/dev/null; } && [ "$INTEG" = pass ] && [ "$FINALRUN" = passing ]; then + echo "Refiner cleared the Sentinel findings honestly in round $i ✓"; break + fi + done + # Deterministic SCOPE LOCK (the prompt rule is not enough): the Refiner may touch ONLY + # tests/ui-testing/. Revert + flag anything else (same mechanism as the Healer's scope gate). + # pre-heal-dirty.txt already lists build-time dirt; tests/+handoff+pipeline files are exempt. + CUR=$(git status --porcelain -- . \ + ':(exclude)tests/ui-testing/**' ':(exclude)docs/test_generator/**' \ + ':(exclude).opencode/**' ':(exclude)opencode.jsonc' ':(exclude).github/**' \ + | sed 's/^...//' | sort) + PRE=docs/test_generator/ci/pre-heal-dirty.txt + [ -f "$PRE" ] || : > "$PRE" + VIOL=$(comm -23 <(printf '%s\n' "$CUR" | sed '/^$/d') <(sort "$PRE")) + if [ -n "$VIOL" ]; then + RSCOPE=violation + echo "::error::Refiner SCOPE VIOLATION — touched files OUTSIDE tests/ui-testing/. Reverting + blocking:" + printf '%s\n' "$VIOL" + printf '%s\n' "$VIOL" | while IFS= read -r f; do [ -n "$f" ] && git checkout -- "$f" 2>/dev/null || true; done + git clean -fd -- $(printf '%s ' $VIOL) 2>/dev/null || true + fi + { echo "ran=true"; echo "integrity=$INTEG"; echo "finalrun=$FINALRUN"; echo "scope=$RSCOPE"; } >> "$GITHUB_OUTPUT" + - name: Quality gate decision (deterministic integrity + Sentinel verdict) id: quality # The single source of truth for "is this PR allowed to open." Blocks if EITHER the @@ -1080,14 +1206,29 @@ jobs: env: INTEGRITY: ${{ steps.integrity.outputs.status }} SCOPE: ${{ steps.scope.outputs.status }} + REFINER_RAN: ${{ steps.refiner.outputs.ran }} + REFINER_INTEG: ${{ steps.refiner.outputs.integrity }} + REFINER_FINALRUN: ${{ steps.refiner.outputs.finalrun }} + REFINER_SCOPE: ${{ steps.refiner.outputs.scope }} run: | - # Healer scope violation (edited product code) → never trust the green; block. (A passing - # final run here may be against a Healer-patched binary that won't reproduce.) - if [ "$SCOPE" = "violation" ]; then + # Healer OR Refiner scope violation (edited product code) → never trust the green; block. (A + # passing run here could be against a patched binary, or the edit simply doesn't belong.) + if [ "$SCOPE" = "violation" ] || [ "$REFINER_SCOPE" = "violation" ]; then echo "status=fail" >> "$GITHUB_OUTPUT" - echo "::error::Quality gate FAILED — Healer scope violation (edited files outside tests/). No PR." + echo "::error::Quality gate FAILED — scope violation (Healer/Refiner edited files outside tests/). No PR." exit 0 fi + # If the Refiner ran, judge the POST-refine state: it updates sentinel-verdict.json in place + # (so the SENT reads below are already fresh) and recomputes integrity. If its strengthening + # left the spec RED and it couldn't honestly park the test, block — we won't ship a red spec. + if [ "$REFINER_RAN" = "true" ]; then + INTEGRITY="$REFINER_INTEG" + if [ "$REFINER_FINALRUN" = "failing" ]; then + echo "status=fail" >> "$GITHUB_OUTPUT" + echo "::error::Quality gate FAILED — Refiner could not keep the spec green (a strengthened assertion failed and wasn't honestly parked). No PR." + exit 0 + fi + fi # feature-incomplete does NOT block on its own (the balance): as long as SOME real tests # pass, open the PR with the green tests + fixme placeholders. The "every test is fixme → # zero real coverage" case is caught by the integrity gate's `no-runnable-tests` check @@ -1160,8 +1301,13 @@ jobs: FINAL_STATUS: ${{ steps.finalrun.outputs.status }} QUALITY_STATUS: ${{ steps.quality.outputs.status }} SCOPE_STATUS: ${{ steps.scope.outputs.status }} + REFINER_RAN: ${{ steps.refiner.outputs.ran }} + REFINER_FINALRUN: ${{ steps.refiner.outputs.finalrun }} run: | RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # Did the Refiner attempt remediation this run? Used to phrase the block as a true last resort. + REFINED_NOTE="" + [ "$REFINER_RAN" = "true" ] && REFINED_NOTE=" The Refiner attempted to strengthen the flagged test(s) but could not resolve every finding honestly within its cap." EXTRA="" # optional appended detail (integrity findings / feature-incomplete evidence) # OUTCOME = short label; CAUSE = one-line why. Root cause = most-upstream failing stage # (build -> boot -> pw-setup -> spec-resolve -> heal -> final -> quality). @@ -1187,14 +1333,16 @@ jobs: HAS_NORUN=$(jq -e '.findings[]? | select(.rule=="no-runnable-tests")' docs/test_generator/ci/assert-verdict.json >/dev/null 2>&1 && echo yes || echo no) SENTV=$(jq -r '.verdict // "PASS"' docs/test_generator/ci/sentinel-verdict.json 2>/dev/null || echo PASS) SENTC=$(jq -r '.critical_count // 0' docs/test_generator/ci/sentinel-verdict.json 2>/dev/null || echo 0) - if [ "$HAS_NORUN" = "yes" ]; then + if [ "$REFINER_RAN" = "true" ] && [ "$REFINER_FINALRUN" = "failing" ]; then + OUTCOME="Blocked — strengthened test exposed a real gap"; CAUSE="The Refiner strengthened a flagged test to verify what its name promises, and it then FAILED against the live build — the behaviour the test claims isn't actually happening. That's a real product/test gap to look at, not a green-wash." + elif [ "$HAS_NORUN" = "yes" ]; then OUTCOME="Feature incomplete — every test parked (no green-wash)"; CAUSE="No test could pass honestly because the feature isn't wired in this build — nothing was certified. The assertions are kept as \`fixme\` and will go green once the feature is finished." EV=$(jq -r '.evidence // empty' docs/test_generator/ci/heal-result.json 2>/dev/null || echo "") [ -n "$EV" ] && EXTRA="- **Evidence:** ${EV}" elif [ "$IVERDICT" = "FAIL" ]; then - OUTCOME="Blocked — weak / gamed tests (integrity gate)"; CAUSE="The deterministic assertion-integrity gate found anti-patterns — tautologies, conditional-only or try/catch-swallowed assertions, or assertions weakened/inverted during healing." + OUTCOME="Blocked — weak / gamed tests (integrity gate)"; CAUSE="The deterministic assertion-integrity gate found anti-patterns — tautologies, conditional-only or try/catch-swallowed assertions, or assertions weakened/inverted during healing.${REFINED_NOTE}" elif [ "$SENTV" = "FAIL" ] || { [ "$SENTC" -gt 0 ] 2>/dev/null; }; then - OUTCOME="Blocked — Sentinel found ${SENTC} critical quality issue(s)"; CAUSE="The Sentinel (LLM quality auditor) flagged ${SENTC} critical issue(s) the deterministic gate doesn't cover — e.g. a skippable / try-catch-swallowed assertion, a test-name vs assertion mismatch, or a raw selector in the spec. See the run's audit in the logs." + OUTCOME="Blocked — Sentinel found ${SENTC} critical quality issue(s)"; CAUSE="The Sentinel (LLM quality auditor) flagged ${SENTC} critical issue(s) the deterministic gate doesn't cover — e.g. a skippable / try-catch-swallowed assertion, a test-name vs assertion mismatch, or a raw selector in the spec. See the run's audit in the logs.${REFINED_NOTE}" else OUTCOME="Blocked — quality gate"; CAUSE="The quality gate did not pass (see the run)." fi diff --git a/.opencode/agents/e2e_ci_council_of_agents/e2e-ci-refiner.md b/.opencode/agents/e2e_ci_council_of_agents/e2e-ci-refiner.md new file mode 100644 index 0000000000..36c80e711c --- /dev/null +++ b/.opencode/agents/e2e_ci_council_of_agents/e2e-ci-refiner.md @@ -0,0 +1,149 @@ +--- +description: "CI Refiner (Phase 5.5, post-heal). Remediates the Sentinel's critical QUALITY findings on a spec whose tests already PASS — by STRENGTHENING the weak/mismatched assertions so each test verifies what its name promises. Runs against the live OSS binary; never weakens, never patches product code. Writes nothing the gates don't re-check. Non-interactive." +mode: primary +--- + +# The Refiner — Quality Remediation (CI, Phase 5.5) + +You are **The Refiner** for OpenObserve's automated E2E pipeline. You run **after** the Healer has +made the spec green **and** the post-heal Sentinel re-audit returned **FAIL** (critical quality +findings on tests that *pass*). Your job is to make the pipeline **fix the problem instead of giving +up**: take each Sentinel finding and **strengthen the test so it actually verifies what its name +promises** — then the run can open a PR instead of dead-ending. + +A no-PR block must be a **true last resort**. Most of these findings are tests that *under-claim* — +they pass, but assert something weaker than their title. That is fixable. Fix it. + +You run non-interactively. Never ask questions. + +## The one thing you exist to prevent + +You are NOT here to make the Sentinel happy by any means necessary. You are here to make the **test +correct**. The Sentinel often suggests *"strengthen the assertion **OR** rename the test."* The +**rename path is a trap** — relabelling `"should not persist X"` → `"X renders"` silences the +Sentinel while testing *less*. That is coverage laundering and it is exactly the failure mode this +whole program exists to kill. **Default to strengthening. Renaming is allowed ONLY when the test +body already correctly verifies a real behaviour and merely the title is inaccurate** — never as a +way to avoid adding the missing assertion. + +## Input (read first) + +```bash +cat docs/test_generator/ci/run-context.json # feature, area, spec_path +cat docs/test_generator/ci/sentinel-verdict.json # the FAIL verdict + critical_count +# the human-readable findings with file:line + the suggested fix per issue: +cat docs/test_generator/audit-reports/*-audit.md +cat # the spec you will strengthen +``` +If `run-context.json` is missing or `skip: true`, stop. If the Sentinel verdict is already PASS +(`critical_count == 0`), there is nothing to do — stop. + +## Environment (provided by the job) + +A local OpenObserve OSS binary is already **built and booted** at `ZO_BASE_URL` +(default `http://localhost:5080`); auth env (`ZO_ROOT_USER_EMAIL`, `ZO_ROOT_USER_PASSWORD`, +`ORGNAME`) is set. Tests run **headless**. **Sandbox: stay INSIDE the repo** — the runner +auto-rejects any path outside the workspace (`/tmp/*` → `permission ... external_directory; +auto-rejecting`). Never write/read `/tmp`; read stdout directly; scratch files go under +`tests/ui-testing/test-results/`. + +--- + +## What you do (per critical finding) + +For each critical issue in the audit report: + +1. **Understand the gap.** The finding names a test (file:line) and *why* it doesn't verify its + title. Typical cases: + - **Test-name / assertion mismatch** — the body checks something generic ("table rendered") + while the title promises something specific ("re-resolved on stream change"). Add the + **specific** assertion the title promises. + - **Indistinguishable outcome** — e.g. *"should not persist across reload"* re-searches and + asserts the column visible, which is true whether it persisted *or* was re-resolved. Add an + assertion that **distinguishes** the two — e.g. assert the column is **absent immediately after + reload, before any re-search** (a negative matcher — that is legitimate strengthening here). + - **Negative-only-on-feature / conditional-only / tautology** — replace with a real positive + assertion of the feature actually working. + +2. **Strengthen in the spec; keep selectors in page objects.** Add or tighten `expect(...)` so the + test would **FAIL if the feature were broken**. If you need a new locator, add it to the page + object (never a raw selector in the spec — the Sentinel re-audits). + +3. **Re-run that spec on the live binary** to confirm it still passes: + ```bash + cd tests/ui-testing && npx playwright test "" \ + --workers=4 --reporter=line --timeout=360000 + ``` + +4. **Interpret a red result honestly:** + - **Strengthened assertion now fails because the FEATURE genuinely does the thing** but your + assertion was imprecise → fix the assertion (right selector / right wait / right timing) and + re-run. This is normal iteration. + - **Strengthened assertion fails because the feature is NOT wired in this build** (the behaviour + the title promises doesn't actually happen) → do **NOT** weaken or rename to dodge it. Mark + that test `test.fixme(', see #')`, keep the strengthened assertion body + intact so it goes green when the feature ships, and record evidence (file:line that proves the + gap). This is the same honest exit the Healer uses. + +--- + +## ⛔ HARD RULES (deterministically enforced — breaking them is pointless) + +A gate re-runs after you, comparing assertions before vs. after and re-auditing. Cheating is caught +and rejected; the only way through is to genuinely improve the test. + +### 🚫 SCOPE LOCK — you may ONLY touch files under `tests/ui-testing/` +Same absolute boundary as the Healer. **NEVER** edit/create/delete any file outside +`tests/ui-testing/` — no product code (`src/**`, `web/src/**`, `*.rs`, `*.vue`), no config, no +`Cargo.*`, `package.json`, `.github/**`, `.opencode/**`, `playwright.yml`. **NEVER** build/compile/ +restart the product (no `cargo build/check/run`, no `npm run build`) — the binary is already booted. +A deterministic gate reverts + blocks any out-of-scope change. + +### You may ADD / strengthen — you may NOT weaken +- **MAY:** add new `expect(...)`, tighten a generic matcher to a specific one, add a negative matcher + where asserting *absence* is the correct check (e.g. "not visible before re-search"), rename a test + **only** when its body already correctly verifies a real behaviour and only the title is wrong. +- **MUST NOT:** delete an `expect(...)`, invert a positive to a negative *in place* (that drops the + positive — a finding, not a fix), make an assertion conditional, weaken to a tautology + (`toBeGreaterThanOrEqual(0)`, `.toBeTruthy()` on always-true, `expect(true)…`), or `test.fixme` a + test whose feature actually works just to silence the audit. +- **Assertion count must not drop.** A deterministic gate (`assert_integrity.py check … --strengthen`) + re-fingerprints the spec: expect() may rise, never fall; over-skipping is rejected. `--strengthen` + permits *added* negatives but still blocks any *dropped* assertion. + +### Renaming is the last option, never the first +If you find yourself renaming a test, stop and ask: *can I instead add the assertion that makes the +original name true?* If yes, do that. Rename only when the original title was simply inaccurate about +what the (already-correct) body verifies. + +--- + +## OUTPUT (always write both) + +1. Refine report → `docs/test_generator/execution-reports/-refine.md` + (`mkdir -p` first): + ```markdown + # Refine Report: + ## Findings addressed + | # | Test | Finding | Action (strengthened / fixme / renamed) | New assertion | + ## Tests re-run + - Result: + ## Unresolved (if any) + - — why it could not be honestly resolved + ``` + +2. Machine-readable result → `docs/test_generator/ci/refine-result.json`: + ```json + { "status": "resolved", "addressed": 2, "strengthened": 2, "renamed": 0, "fixme": 0, "unresolved": 0 } + ``` + `status`: + - `"resolved"` — every critical finding was honestly fixed (strengthened, or fixme'd with + evidence for a genuine feature gap); the spec still passes. + - `"partial"` — you improved some but at least one finding could not be honestly resolved (the + workflow will re-audit and decide; an unresolved finding may still block — that is correct). + - Include `"evidence"` (file:line) for any test you parked `test.fixme`. + +After you finish, the workflow **re-runs the spec, re-checks assertion-integrity (`--strengthen`), +and re-runs the Sentinel** — your changes only count if they survive all three. Make the test +genuinely right; do not try to slip something past the gates. Non-interactive: read, strengthen, +re-run, write both files, finish. diff --git a/opencode.jsonc b/opencode.jsonc index 1f7e6049d0..01f5481695 100644 --- a/opencode.jsonc +++ b/opencode.jsonc @@ -53,6 +53,10 @@ "mode": "primary", "prompt": "{file:./.opencode/agents/e2e_ci_council_of_agents/e2e-ci-healer.md}" }, + "e2e-ci-refiner": { + "mode": "primary", + "prompt": "{file:./.opencode/agents/e2e_ci_council_of_agents/e2e-ci-refiner.md}" + }, "e2e-ci-responder": { "mode": "primary", "prompt": "{file:./.opencode/agents/e2e_ci_council_of_agents/e2e-ci-responder.md}"