mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
* refactor(tooling): group remaining scripts by concrete domain owners * fix(tooling): preserve relocated launcher resource roots
152 lines
5.2 KiB
JavaScript
152 lines
5.2 KiB
JavaScript
import { execFileSync } from 'node:child_process'
|
|
import { readFileSync } from 'node:fs'
|
|
import { join } from 'node:path'
|
|
import process from 'node:process'
|
|
import { pathToFileURL } from 'node:url'
|
|
import ts from 'typescript-api'
|
|
import { baseRuleId, scanAstGrep } from '../../structural-scan/ast-grep-scan.mjs'
|
|
import { resolvePullRequestDiffBase } from '../../../git/pr-diff-base/git-pull-request-diff-base.mjs'
|
|
import { listTrackedFiles, runCountRatchet } from '../../ratchet-baseline/ratchet-baseline.mjs'
|
|
|
|
// Count each flaky-test rule per file so fixing one pattern cannot pay for adding another.
|
|
|
|
const BASELINE_PATH = 'config/flaky-test-pattern-baseline.txt'
|
|
const RULE_FILTER = '^test-'
|
|
const FLAKY_RULE = new RegExp(RULE_FILTER)
|
|
const HEADER = [
|
|
'# Flaky-test pattern findings per rule and file (config/ast-grep/rules/flaky-tests).',
|
|
'# This is a RATCHET: counts may only go DOWN. Run `pnpm lint:ast-grep:flaky-tests` to list them.',
|
|
'# Prune after a fix: pnpm check:flaky-test-pattern-ratchet --prune'
|
|
]
|
|
|
|
export function countFindings(matches, isTracked = () => true) {
|
|
const counts = new Map()
|
|
for (const match of matches) {
|
|
const file = match.file.split('\\').join('/')
|
|
if (!isTracked(file)) {
|
|
continue
|
|
}
|
|
const key = `${baseRuleId(match.ruleId)}:${file}`
|
|
counts.set(key, (counts.get(key) ?? 0) + 1)
|
|
}
|
|
return counts
|
|
}
|
|
|
|
/** `matches` may come from an unfiltered scan another gate shares; only `test-*` rules count. */
|
|
export function collectFlakyTestPatternCounts(root = process.cwd(), matches = undefined) {
|
|
// Tracked files only, so untracked scratch never changes the baseline.
|
|
const tracked = new Set(listTrackedFiles(root, []))
|
|
const flaky = (matches ?? scanAstGrep(root, { filter: RULE_FILTER })).filter((match) =>
|
|
FLAKY_RULE.test(match.ruleId)
|
|
)
|
|
return countFindings(flaky, (file) => tracked.has(file))
|
|
}
|
|
|
|
function git(root, args, options = {}) {
|
|
return execFileSync('git', args, {
|
|
cwd: root,
|
|
encoding: 'utf8',
|
|
stdio: ['ignore', 'pipe', 'ignore'],
|
|
...options
|
|
})
|
|
}
|
|
|
|
function maskStaticImportPaths(file, bytes) {
|
|
const text = bytes.toString('utf8')
|
|
if (!Buffer.from(text).equals(bytes)) {
|
|
return null
|
|
}
|
|
const source = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, true)
|
|
if (source.parseDiagnostics.length > 0) {
|
|
return null
|
|
}
|
|
let masked = ''
|
|
let offset = 0
|
|
for (const statement of source.statements) {
|
|
if (ts.isImportDeclaration(statement) && ts.isStringLiteral(statement.moduleSpecifier)) {
|
|
const specifier = statement.moduleSpecifier
|
|
masked += `${text.slice(offset, specifier.getStart(source) + 1)}<import-path>`
|
|
offset = specifier.end - 1
|
|
}
|
|
}
|
|
return masked + text.slice(offset)
|
|
}
|
|
|
|
function changesOnlyStaticImportPaths(root, base, file) {
|
|
if (!/\.[cm]?[jt]sx?$/.test(file)) {
|
|
return false
|
|
}
|
|
try {
|
|
const current = readFileSync(join(root, file))
|
|
const previous = git(root, ['show', `${base}:${file}`], { encoding: 'buffer' })
|
|
const maskedPrevious = maskStaticImportPaths(file, previous)
|
|
return maskedPrevious !== null && maskedPrevious === maskStaticImportPaths(file, current)
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// Scope enforcement to changed files so unrelated merges cannot invalidate every PR's baseline.
|
|
export function listChangedFiles(root = process.cwd()) {
|
|
if (process.env.ORCA_FLAKY_TEST_RATCHET_ALL === '1') {
|
|
return null
|
|
}
|
|
let base
|
|
try {
|
|
const requested = process.env.ORCA_CODE_QUALITY_BASE || 'origin/main'
|
|
const mergeBase = git(root, ['merge-base', 'HEAD', requested]).trim()
|
|
base = resolvePullRequestDiffBase(root, mergeBase)
|
|
} catch {
|
|
try {
|
|
base = resolvePullRequestDiffBase(root, undefined)
|
|
} catch {
|
|
return null
|
|
}
|
|
}
|
|
if (!base) {
|
|
return null
|
|
}
|
|
const entries = git(root, ['diff', '--name-status', '--no-renames', '-z', base]).split('\0')
|
|
const changed = new Set()
|
|
for (let index = 0; index + 1 < entries.length; index += 2) {
|
|
const status = entries[index]
|
|
const file = entries[index + 1]
|
|
if (status !== 'M' || !changesOnlyStaticImportPaths(root, base, file)) {
|
|
changed.add(file)
|
|
}
|
|
}
|
|
return changed
|
|
}
|
|
|
|
export function changedFileScope(changed) {
|
|
if (!changed) {
|
|
return undefined
|
|
}
|
|
return (key) => changed.has(key.slice(key.indexOf(':') + 1))
|
|
}
|
|
|
|
export function main(root = process.cwd(), arg = undefined, matches = undefined) {
|
|
return runCountRatchet(
|
|
{
|
|
name: 'flaky-test-pattern',
|
|
baselinePath: BASELINE_PATH,
|
|
header: HEADER,
|
|
collect: (dir) => collectFlakyTestPatternCounts(dir, matches),
|
|
scope: (dir) => changedFileScope(listChangedFiles(dir)),
|
|
unit: 'flaky-test finding(s)',
|
|
fix: [
|
|
'Wait on the condition (expect.poll, vi.waitFor, locator assertions) instead of sleeping or',
|
|
'polling, use fake timers or fixed timestamps instead of the wall clock, put',
|
|
'`// quarantine: #<issue> <reason>` above a skip, and a reason comment above a timeout bump.',
|
|
'A deliberate exception takes `// ast-grep-ignore: <rule-id> -- <reason>` above the line.'
|
|
]
|
|
},
|
|
root,
|
|
arg
|
|
)
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
process.exit(main(process.cwd(), process.argv[2]))
|
|
}
|