orca/config/scripts/benchmark-preflight-static-analysis.mjs

import { readPinnedBunVersion } from './pinned-bun-executable.mjs'
import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import path from 'node:path'
import { performance } from 'node:perf_hooks'
import { fileURLToPath } from 'node:url'
import { measurePreflightCommand } from './preflight-benchmark-command.mjs'
import ts from 'typescript-api'
import { buildCounterbalancedSchedule } from './counterbalanced-benchmark-schedule.mjs'
import { summarizeBenchmarkSamples } from './benchmark-sample-summary.mjs'
import { resolveOxlintInvocation } from './oxlint-cli-invocation.mjs'
import { resolveOxcCliInvocation } from './oxc-cli-invocation.mjs'

const root = fileURLToPath(new URL('../..', import.meta.url))
const output = path.join(root, '.build/preflight-benchmark')
const baseline = path.join(output, 'baseline')
mkdirSync(output, { recursive: true })
const report = {
  sourceSha: process.env.GITHUB_SHA ?? null,
  host: { platform: process.platform, arch: process.arch, node: process.version },
  measurements: [],
  comparisons: {}
}
const save = () =>
  writeFileSync(path.join(output, 'static-analysis.json'), `${JSON.stringify(report, null, 2)}\n`)
const command = (program, args, label) =>
  measurePreflightCommand({
    program,
    args,
    label,
    root,
    output,
    env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' },
    timeoutMs: 180_000
  })
async function requireSuccess(program, args, label) {
  const result = await command(program, args, label)
  if (result.code !== 0 || result.signal || result.timedOut) {
    throw new Error(`${label}: ${JSON.stringify(result)}`)
  }
  return result
}
const bun = process.env.ORCA_BENCHMARK_BUN ?? 'bun'
report.host.bun = (await requireSuccess(bun, ['--version'], 'bun-version')).stdout.trim()
if (report.host.bun !== readPinnedBunVersion(root)) {
  throw new Error(`Pin Bun ${readPinnedBunVersion(root)}`)
}
await requireSuccess(process.execPath, ['config/scripts/sync-anti-slop-plugin.mjs'], 'sync-plugin')
rmSync(baseline, { recursive: true, force: true })
cpSync(path.join(root, '.anti-slop-plugin'), path.join(baseline, 'src'), { recursive: true })
// Reverse the committed patch so both arms use the same pinned upstream package.
await requireSuccess(
  'git',
  [
    'apply',
    '--reverse',
    '--directory=.build/preflight-benchmark/baseline',
    'config/patches/oxlint-plugin-anti-slop@0.1.2.patch'
  ],
  'reverse-patch'
)
const configText = readFileSync(path.join(root, 'config/oxlint-anti-slop.json'), 'utf8')
const { config, error } = ts.parseConfigFileTextToJson('config/oxlint-anti-slop.json', configText)
if (error) {
  throw new Error('Invalid anti-slop configuration')
}
for (const [arm, plugin] of [
  ['baseline', path.join(baseline, 'src/index.ts')],
  ['patched', path.join(root, '.anti-slop-plugin/index.ts')]
]) {
  writeFileSync(
    path.join(output, `${arm}.json`),
    JSON.stringify({ ...config, jsPlugins: [{ name: 'anti-slop', specifier: plugin }] })
  )
}
const oxlint = resolveOxlintInvocation(root)
const tsx = resolveOxcCliInvocation('tsx', 'tsx', root)
for (const [arm, directory] of [
  ['baseline', path.join(baseline, 'src')],
  ['patched', path.join(root, '.anti-slop-plugin')]
]) {
  for (const rule of ['no-object-parameters', 'no-unknown-type-aliases']) {
    const fixture = path.join(directory, 'rules', `${rule}.test.ts`)
    await requireSuccess(tsx.command, [...tsx.prefixArgs, fixture], `fixtures-${arm}-${rule}-node`)
  }
}
// RuleTester uses V8 raw transfer and rejects Bun; exercise Bun through the real CLI.
const fixturePath = path.join(output, 'anti-slop-invalid-fixture.ts')
writeFileSync(
  fixturePath,
  'type InvalidAlias = unknown; type ObjectAlias = object; export function consume(value: ObjectAlias): void {}\n'
)
let fixtureDiagnostics
for (const [runtime, flags] of [
  [process.execPath, []],
  [bun, []],
  [bun, ['--smol']]
]) {
  const result = await command(
    runtime,
    [
      ...flags,
      ...oxlint.prefixArgs,
      '--config',
      path.join(output, 'patched.json'),
      fixturePath,
      '--format=json'
    ],
    `cli-fixture-${path.basename(runtime)}-${flags.length}`
  )
  const findings = JSON.parse(result.stdout).diagnostics
  if (
    result.code !== 1 ||
    !findings.some((finding) => finding.code?.includes('no-object-parameters')) ||
    !findings.some((finding) => finding.code?.includes('no-unknown-type-aliases'))
  ) {
    throw new Error(`Invalid fixture did not produce expected findings: ${JSON.stringify(result)}`)
  }
  const normalized = JSON.stringify(findings)
  if (fixtureDiagnostics !== undefined && fixtureDiagnostics !== normalized) {
    throw new Error('Runtime fixture diagnostics differ')
  }
  fixtureDiagnostics = normalized
}
const lintArgs = (arm) => [
  ...oxlint.prefixArgs,
  '--config',
  path.join(output, `${arm}.json`),
  'src',
  'config',
  'tests',
  'mobile',
  '--deny-warnings',
  '--format=json'
]
for (const [comparison, first, second] of [
  ['anti-slop-patch', 'baseline', 'patched'],
  ['anti-slop-runtime', 'node', 'bun'],
  ['anti-slop-smol', 'node', 'bun-smol'],
  ['import-guard-runtime', 'node', 'bun']
]) {
  let diagnostics
  for (const [pair, arms] of buildCounterbalancedSchedule(4, first, second).entries()) {
    for (const arm of arms) {
      const program = arm.startsWith('bun') ? bun : process.execPath
      const args =
        comparison === 'import-guard-runtime'
          ? ['config/scripts/check-process-host-imports.mjs']
          : lintArgs(comparison === 'anti-slop-patch' ? arm : 'patched')
      const result = await requireSuccess(
        program,
        arm === 'bun-smol' ? ['--smol', ...args] : args,
        `${comparison}-${pair}-${arm}`
      )
      if (comparison !== 'import-guard-runtime') {
        const findings = JSON.stringify(JSON.parse(result.stdout).diagnostics)
        if (diagnostics !== undefined && diagnostics !== findings) {
          throw new Error(`Diagnostics changed: ${comparison}`)
        }
        diagnostics = findings
      }
      report.measurements.push({ comparison, pair, arm, ...result })
      save()
      console.log(
        JSON.stringify({
          comparison,
          pair,
          arm,
          durationMs: result.durationMs,
          peakRssKiB: result.peakRssKiB
        })
      )
    }
  }
  report.comparisons[comparison] = Object.fromEntries(
    [first, second].map((arm) => [
      arm,
      summarizeBenchmarkSamples(
        report.measurements
          .filter((sample) => sample.comparison === comparison && sample.arm === arm)
          .map((sample) => sample.durationMs)
      )
    ])
  )
  save()
}
const guardProbeDirectory = mkdtempSync(path.join(root, 'src/main/preflight-import-probe-'))
const guardProbe = path.join(guardProbeDirectory, 'forbidden-process-import.ts')
try {
  writeFileSync(
    guardProbe,
    "import { spawn } from 'node:child_process'\nexport const forbiddenSpawn = spawn\n"
  )
  for (const runtime of [process.execPath, bun]) {
    const result = await command(
      runtime,
      ['config/scripts/check-process-host-imports.mjs'],
      `guard-invalid-${path.basename(runtime)}`
    )
    if (
      result.code !== 1 ||
      !`${result.stdout}${result.stderr}`.includes('forbidden-process-import.ts')
    ) {
      throw new Error(`Import guard did not reject its fixture: ${JSON.stringify(result)}`)
    }
  }
} finally {
  rmSync(guardProbeDirectory, { recursive: true, force: true })
}
for (const [pair, arms] of buildCounterbalancedSchedule(4, 'baseline', 'optimized').entries()) {
  for (const arm of arms) {
    const runtime = arm === 'optimized' ? bun : process.execPath
    const start = performance.now()
    const processes = await Promise.all([
      requireSuccess(
        process.execPath,
        [...oxlint.prefixArgs, '--format=json'],
        `early-${pair}-${arm}-root-lint`
      ),
      requireSuccess(
        runtime,
        arm === 'optimized' ? ['--smol', ...lintArgs('patched')] : lintArgs('baseline'),
        `early-${pair}-${arm}-anti-slop`
      ),
      requireSuccess(
        runtime,
        ['config/scripts/check-process-host-imports.mjs'],
        `early-${pair}-${arm}-import-guard`
      )
    ])
    const durationMs = performance.now() - start
    report.measurements.push({ comparison: 'early-phase', pair, arm, durationMs, processes })
    save()
    console.log(JSON.stringify({ comparison: 'early-phase', pair, arm, durationMs }))
  }
}
report.comparisons['early-phase'] = Object.fromEntries(
  ['baseline', 'optimized'].map((arm) => [
    arm,
    summarizeBenchmarkSamples(
      report.measurements
        .filter((sample) => sample.comparison === 'early-phase' && sample.arm === arm)
        .map((sample) => sample.durationMs)
    )
  ])
)
save()
const summary = `Static analysis comparisons on ${process.platform}/${process.arch}\n\n\`\`\`json\n${JSON.stringify(report.comparisons, null, 2)}\n\`\`\`\n\nBoth upstream rule suites pass under Node; real CLI fixtures agree under Node and Bun. Full-scan diagnostics agree between the compared arms.\n`
writeFileSync(path.join(output, 'static-analysis.md'), summary)
if (process.env.GITHUB_STEP_SUMMARY) {
  writeFileSync(process.env.GITHUB_STEP_SUMMARY, summary, { flag: 'a' })
}