openobserve/.github/scripts/assert_integrity.py

#!/usr/bin/env python3
"""
Assertion-integrity analyzer for the E2E Council pipeline (deterministic — no LLM).

Two modes:
  fingerprint <spec.js>                         -> print a JSON fingerprint of the spec's assertions
  check <spec.js> [--baseline <fingerprint.json>] [--allow-weakening] [--strengthen]
                                                -> static anti-pattern scan + (optional) diff-aware
                                                   heal-weakening check; writes nothing, prints a JSON
                                                   verdict, exits 2 if any CRITICAL finding.
                                                   --strengthen (Refiner mode): permit ADDED negative
                                                   matchers (assert-absence is valid strengthening);
                                                   still blocks any DROP of assertions.

Conservative by design: only HIGH-CONFIDENCE patterns are flagged as critical, because a gate with
false positives gets disabled by humans (the same trap that made mutation-testing a poor fit).
"""
import sys, re, json

NEG_MATCHERS = re.compile(r'\.not\.|\.toBeHidden\(|\.toBeFalsy\(')
TAUTOLOGIES = [
    (re.compile(r'toBeGreaterThanOrEqual\(\s*0\s*\)'), 'count >= 0 is always true'),
    (re.compile(r'toBeGreaterThan\(\s*-1\s*\)'), 'count > -1 is always true'),
    (re.compile(r'toBeLessThan\(\s*Infinity\s*\)'), '< Infinity is always true'),
    (re.compile(r'expect\(\s*true\s*\)\.toBe(Truthy)?\('), 'expect(true) asserts nothing'),
    (re.compile(r'expect\(\s*1\s*\)\.toBe\(\s*1\s*\)'), 'expect(1).toBe(1) asserts nothing'),
    (re.compile(r'expect\(\s*false\s*\)\.toBe(Falsy)?\('), 'expect(false) asserts nothing'),
]
EXPECT = re.compile(r'\bexpect\(')
TEST_START = re.compile(r'\b(test|it)(\.skip|\.fixme|\.only)?\s*\(')
SKIP_FIXME = re.compile(r'\b(test|it)\.(skip|fixme)\s*\(|\btest\.skip\(\s*\)|\.fixme\(')


def strip_noise(line):
    """Blank out comments + string literals so matchers don't fire inside them (best-effort,
    line-local). Handles `//` line comments and single-line `/* */` blocks; multi-line `/* */`
    blocks are rare in generated specs and not tracked across lines."""
    line = re.sub(r'/\*.*?\*/', '', line)   # single-line block comment
    line = re.sub(r'//.*$', '', line)        # line comment
    line = re.sub(r'`[^`]*`', '``', line)
    line = re.sub(r"'[^']*'", "''", line)
    line = re.sub(r'"[^"]*"', '""', line)
    return line


def fingerprint(src):
    """Return counts that summarize a spec's assertion shape: total expect()s, negative matchers,
    skip/fixme, and test blocks. Compared before vs. after heal to detect weakening."""
    code = '\n'.join(strip_noise(l) for l in src.split('\n'))
    return {
        'expects': len(EXPECT.findall(code)),
        'negatives': len(NEG_MATCHERS.findall(code)),
        'skips': len(re.findall(r'\.(skip|fixme)\s*\(', code)),
        'tests': len(TEST_START.findall(code)),
    }


def find_test_blocks(lines):
    """Yield (name_line_idx, start_idx, end_idx) for each test()/it() block by matching the block's
    braces. Ends exactly when the brace depth returns to 0 (== 0, not <= 0, so a stray nested
    structure can't terminate the block early)."""
    blocks = []
    i, n = 0, len(lines)
    while i < n:
        if TEST_START.search(strip_noise(lines[i])):
            depth, j, started = 0, i, False
            while j < n:
                for ch in strip_noise(lines[j]):
                    if ch == '{':
                        depth += 1; started = True
                    elif ch == '}':
                        depth -= 1
                if started and depth == 0:
                    break
                j += 1
            blocks.append((i, i, j))
            i = j + 1
        else:
            i += 1
    return blocks


# `if (...)` / `else` that directly precedes a statement on the SAME line with NO opening brace —
# i.e. a braceless single-statement conditional like `if (x) await expect(y).toBeVisible();`.
BRACELESS_GUARD = re.compile(r'(?:\bif\s*\([^{}]*\)|\belse)\s*(?:await\s+)?$')


def conditional_only_tests(lines):
    """A test where EVERY expect() sits inside an if/else block (so the test can end asserting nothing).
    Scans `{`, `}` and `expect(` events in POSITIONAL order so a single-line `if(){expect}else{}` is
    judged with the correct brace stack (not after all braces on the line are processed)."""
    flagged = []
    for name_i, s, e in find_test_blocks(lines):
        total, conditional = 0, 0
        stack = []  # frame value = True if introduced by if/else
        for raw in lines[s:e + 1]:
            line = strip_noise(raw)
            events = sorted(
                [(m.start(), m.group()) for m in re.finditer(r'[{}]', line)] +
                [(m.start(), 'E') for m in EXPECT.finditer(line)]
            )
            for pos, kind in events:
                if kind == '{':
                    stack.append(bool(re.search(r'\b(if|else)\b', line[:pos])))
                elif kind == '}':
                    if stack:
                        stack.pop()
                else:  # an expect(
                    total += 1
                    # conditional if inside a braced if/else frame OR guarded by a braceless
                    # single-statement `if (...)` / `else` immediately before it on this line
                    if any(stack) or BRACELESS_GUARD.search(line[:pos]):
                        conditional += 1
        if total > 0 and conditional == total:
            flagged.append((name_i + 1, total))
    return flagged


# Assertion-ish calls whose failure should surface — if these sit in a try{} whose catch swallows
# (no throw), the test passes even when the assertion fails.
ASSERT_TOKEN = re.compile(r'\bexpect\s*\(|\.waitFor\s*\(|\.toBe(Visible|Hidden|Truthy|Falsy)\b|\.toContainText\b|\.toHaveText\b')


def _match_brace(text, open_idx):
    """Index of the '}' matching the '{' at open_idx (-1 if unbalanced)."""
    depth = 0
    for i in range(open_idx, len(text)):
        if text[i] == '{':
            depth += 1
        elif text[i] == '}':
            depth -= 1
            if depth == 0:
                return i
    return -1


def swallowed_assertion_tests(lines):
    """A test where an assertion sits inside a `try {}` whose matching `catch {}` does NOT re-throw —
    the catch swallows the failure (logs/returns), so the test passes even when the assertion fails.
    A `catch` that `throw`s, or a try/catch with no assertion inside the try (e.g. cleanup), is fine."""
    flagged = []
    for name_i, s, e in find_test_blocks(lines):
        block = '\n'.join(strip_noise(l) for l in lines[s:e + 1])
        for m in re.finditer(r'\bcatch\b\s*(\([^)]*\))?\s*\{', block):
            co = block.index('{', m.start())
            cc = _match_brace(block, co)
            if cc < 0:
                continue
            if re.search(r'\bthrow\b', block[co + 1:cc]):
                continue  # catch re-throws → the failure surfaces → fine
            pre = block[:m.start()]
            tclose = pre.rstrip().rfind('}')
            if tclose < 0:
                continue
            depth, topen = 0, -1
            for i in range(tclose, -1, -1):
                if pre[i] == '}':
                    depth += 1
                elif pre[i] == '{':
                    depth -= 1
                    if depth == 0:
                        topen = i
                        break
            if topen < 0 or not re.search(r'\btry\s*$', pre[:topen]):
                continue
            if ASSERT_TOKEN.search(pre[topen + 1:tclose]):
                flagged.append(name_i + 1)
                break
    return flagged


def check(spec_path, baseline=None, allow_weakening=False, strengthen=False):
    """Run the static anti-pattern scan (tautologies, conditional-only assertions) and, when a
    baseline is given, the diff-aware heal-weakening guard. Returns a verdict dict; never raises on
    a malformed/partial baseline — missing baseline keys fall back to the current value (skip that
    comparison) rather than crashing the gate.

    `strengthen` (Refiner mode): the Refiner STRENGTHENS weak/mismatched tests, which can legitimately
    (a) ADD negative matchers (assert a column is *absent* before a re-search) and (b) park a genuinely
    unwired test as fixme. So in strengthen mode we permit a negatives rise ONLY when expect() ALSO rose
    (an addition, not an in-place flip — a flip with flat expect() is still flagged), and permit a
    skip/fixme rise (honest feature-incomplete). `heal-removed-assertions` (expect() may never drop) and
    EVERY static anti-pattern check (tautology, conditional-only, swallowed, no-runnable) STILL apply —
    so strengthen mode cannot be used to drop assertions or green-wash. (NOTE: do not also pass
    --allow-weakening in strengthen mode; that disables the removed guard and defeats the point.)"""
    src = open(spec_path, encoding='utf-8').read()
    lines = src.split('\n')
    findings = []

    # --- static anti-patterns ---
    for idx, raw in enumerate(lines, 1):
        line = strip_noise(raw)
        for pat, why in TAUTOLOGIES:
            if pat.search(line):
                findings.append({'severity': 'critical', 'rule': 'tautology', 'line': idx, 'detail': why})

    for line_no, total in conditional_only_tests(lines):
        findings.append({'severity': 'critical', 'rule': 'conditional-only-assertions', 'line': line_no,
                         'detail': f'all {total} expect() in this test are inside if/else — the test can pass asserting nothing'})

    for line_no in swallowed_assertion_tests(lines):
        findings.append({'severity': 'critical', 'rule': 'swallowed-assertion', 'line': line_no,
                         'detail': 'an assertion sits in a try{} whose catch swallows the failure (no throw) — the test passes even when the assertion fails'})

    # No real coverage: a spec with tests where EVERY one is skip/fixme is a green run that ran
    # nothing (e.g. "fixme everything" to dodge a gap). A genuinely feature-incomplete spec should
    # be blocked anyway — so flag it here too. (Applies even with --allow-weakening: this is static.)
    fp0 = fingerprint(src)
    if fp0['tests'] > 0 and fp0['skips'] >= fp0['tests']:
        findings.append({'severity': 'critical', 'rule': 'no-runnable-tests',
                         'detail': f"every test is skip/fixme ({fp0['skips']} skip/fixme vs {fp0['tests']} tests) — the spec asserts nothing at runtime"})

    # --- diff-aware heal-weakening guard ---
    fp = fingerprint(src)
    if isinstance(baseline, dict) and not allow_weakening:
        base_expects = baseline.get('expects', fp['expects'])
        base_negatives = baseline.get('negatives', fp['negatives'])
        base_skips = baseline.get('skips', fp['skips'])
        if fp['expects'] < base_expects:
            findings.append({'severity': 'critical', 'rule': 'heal-removed-assertions',
                             'detail': f"expect() count dropped {base_expects} -> {fp['expects']} during heal"})
        # Inverted = a positive flipped to a negative IN PLACE. In --strengthen mode (Refiner) ADDING a
        # negative is legitimate, but ONLY if expect() ALSO rose — that proves the negative is a new
        # assertion, not a flip. A negative rise with no new expect() is an in-place flip → still flag,
        # even in strengthen mode. (Outside strengthen mode any negative rise is flagged, as before.)
        if fp['negatives'] > base_negatives and (not strengthen or fp['expects'] <= base_expects):
            findings.append({'severity': 'critical', 'rule': 'heal-inverted-assertions',
                             'detail': f"negative matchers rose {base_negatives} -> {fp['negatives']} (positive flipped to negative in place — expect() did not rise)"})
        # Rising skip/fixme is silent-skipping during heal — blocked. The Refiner's HONEST
        # feature-incomplete path (park a test as fixme, assertion body kept intact) is permitted ONLY in
        # --strengthen mode; expect() still cannot drop (guarded above) and an all-fixme spec is still
        # caught by no-runnable-tests, so this can't be used to green-wash.
        if fp['skips'] > base_skips and not strengthen:
            findings.append({'severity': 'critical', 'rule': 'heal-skipped-tests',
                             'detail': f"skip/fixme rose {base_skips} -> {fp['skips']} during heal (use the feature-incomplete path, not silent skipping)"})

    critical = [f for f in findings if f['severity'] == 'critical']
    return {'verdict': 'FAIL' if critical else 'PASS', 'critical_count': len(critical),
            'fingerprint': fp, 'findings': findings}


def main():
    if len(sys.argv) < 3 or sys.argv[1] not in ('fingerprint', 'check'):
        sys.stderr.write(__doc__); sys.exit(64)
    mode, spec = sys.argv[1], sys.argv[2]
    if mode == 'fingerprint':
        print(json.dumps(fingerprint(open(spec, encoding='utf-8').read())))
        return
    baseline, allow, strengthen = None, False, False
    args = sys.argv[3:]
    if '--baseline' in args:
        baseline = json.load(open(args[args.index('--baseline') + 1]))
    if '--allow-weakening' in args:
        allow = True
    if '--strengthen' in args:
        strengthen = True
    result = check(spec, baseline, allow, strengthen)
    print(json.dumps(result, indent=2))
    sys.exit(2 if result['verdict'] == 'FAIL' else 0)


if __name__ == '__main__':
    main()