Files
orca/config/scripts/static-analysis/ratchets/test-quality/check-flaky-test-pattern-ratchet.mjs
Neil 6bb2e13c35 refactor(tooling): organize remaining scripts by domain owners (#38407)
* refactor(tooling): group remaining scripts by concrete domain owners

* fix(tooling): preserve relocated launcher resource roots
2026-11-05 00:29:16 -08:00

152 lines
5.2 KiB
JavaScript

import { execFileSync } from 'node:child_process'
import { readFileSync } from 'node:fs'
import { join } from 'node:path'
import process from 'node:process'
import { pathToFileURL } from 'node:url'
import ts from 'typescript-api'
import { baseRuleId, scanAstGrep } from '../../structural-scan/ast-grep-scan.mjs'
import { resolvePullRequestDiffBase } from '../../../git/pr-diff-base/git-pull-request-diff-base.mjs'
import { listTrackedFiles, runCountRatchet } from '../../ratchet-baseline/ratchet-baseline.mjs'
// Count each flaky-test rule per file so fixing one pattern cannot pay for adding another.
const BASELINE_PATH = 'config/flaky-test-pattern-baseline.txt'
const RULE_FILTER = '^test-'
const FLAKY_RULE = new RegExp(RULE_FILTER)
const HEADER = [
'# Flaky-test pattern findings per rule and file (config/ast-grep/rules/flaky-tests).',
'# This is a RATCHET: counts may only go DOWN. Run `pnpm lint:ast-grep:flaky-tests` to list them.',
'# Prune after a fix: pnpm check:flaky-test-pattern-ratchet --prune'
]
export function countFindings(matches, isTracked = () => true) {
const counts = new Map()
for (const match of matches) {
const file = match.file.split('\\').join('/')
if (!isTracked(file)) {
continue
}
const key = `${baseRuleId(match.ruleId)}:${file}`
counts.set(key, (counts.get(key) ?? 0) + 1)
}
return counts
}
/** `matches` may come from an unfiltered scan another gate shares; only `test-*` rules count. */
export function collectFlakyTestPatternCounts(root = process.cwd(), matches = undefined) {
// Tracked files only, so untracked scratch never changes the baseline.
const tracked = new Set(listTrackedFiles(root, []))
const flaky = (matches ?? scanAstGrep(root, { filter: RULE_FILTER })).filter((match) =>
FLAKY_RULE.test(match.ruleId)
)
return countFindings(flaky, (file) => tracked.has(file))
}
function git(root, args, options = {}) {
return execFileSync('git', args, {
cwd: root,
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'ignore'],
...options
})
}
function maskStaticImportPaths(file, bytes) {
const text = bytes.toString('utf8')
if (!Buffer.from(text).equals(bytes)) {
return null
}
const source = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true)
if (source.parseDiagnostics.length > 0) {
return null
}
let masked = ''
let offset = 0
for (const statement of source.statements) {
if (ts.isImportDeclaration(statement) && ts.isStringLiteral(statement.moduleSpecifier)) {
const specifier = statement.moduleSpecifier
masked += `${text.slice(offset, specifier.getStart(source) + 1)}<import-path>`
offset = specifier.end - 1
}
}
return masked + text.slice(offset)
}
function changesOnlyStaticImportPaths(root, base, file) {
if (!/\.[cm]?[jt]sx?$/.test(file)) {
return false
}
try {
const current = readFileSync(join(root, file))
const previous = git(root, ['show', `${base}:${file}`], { encoding: 'buffer' })
const maskedPrevious = maskStaticImportPaths(file, previous)
return maskedPrevious !== null && maskedPrevious === maskStaticImportPaths(file, current)
} catch {
return false
}
}
// Scope enforcement to changed files so unrelated merges cannot invalidate every PR's baseline.
export function listChangedFiles(root = process.cwd()) {
if (process.env.ORCA_FLAKY_TEST_RATCHET_ALL === '1') {
return null
}
let base
try {
const requested = process.env.ORCA_CODE_QUALITY_BASE || 'origin/main'
const mergeBase = git(root, ['merge-base', 'HEAD', requested]).trim()
base = resolvePullRequestDiffBase(root, mergeBase)
} catch {
try {
base = resolvePullRequestDiffBase(root, undefined)
} catch {
return null
}
}
if (!base) {
return null
}
const entries = git(root, ['diff', '--name-status', '--no-renames', '-z', base]).split('\0')
const changed = new Set()
for (let index = 0; index + 1 < entries.length; index += 2) {
const status = entries[index]
const file = entries[index + 1]
if (status !== 'M' || !changesOnlyStaticImportPaths(root, base, file)) {
changed.add(file)
}
}
return changed
}
export function changedFileScope(changed) {
if (!changed) {
return undefined
}
return (key) => changed.has(key.slice(key.indexOf(':') + 1))
}
export function main(root = process.cwd(), arg = undefined, matches = undefined) {
return runCountRatchet(
{
name: 'flaky-test-pattern',
baselinePath: BASELINE_PATH,
header: HEADER,
collect: (dir) => collectFlakyTestPatternCounts(dir, matches),
scope: (dir) => changedFileScope(listChangedFiles(dir)),
unit: 'flaky-test finding(s)',
fix: [
'Wait on the condition (expect.poll, vi.waitFor, locator assertions) instead of sleeping or',
'polling, use fake timers or fixed timestamps instead of the wall clock, put',
'`// quarantine: #<issue> <reason>` above a skip, and a reason comment above a timeout bump.',
'A deliberate exception takes `// ast-grep-ignore: <rule-id> -- <reason>` above the line.'
]
},
root,
arg
)
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
process.exit(main(process.cwd(), process.argv[2]))
}