orca/config/scripts/benchmark-windows-security-reuse.mjs

import assert from 'node:assert/strict'
import {
  chmodSync,
  copyFileSync,
  existsSync,
  lstatSync,
  mkdirSync,
  mkdtempSync,
  readFileSync,
  realpathSync,
  rmSync,
  utimesSync,
  writeFileSync
} from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join, resolve } from 'node:path'
import { performance } from 'node:perf_hooks'
import { pathToFileURL } from 'node:url'
import { runProcess, runProcessSync } from '@orca/process-host'
import { NODE_RUNTIME_ASSETS, NODE_RUNTIME_PIN } from '../../src/shared/node-runtime-pin.ts'
import {
  buildWindowsSecurityAddon,
  probeWindowsSecurityAddon
} from './build-windows-security-addon.mjs'
import { assertNoHeaderDownloads, treeDigest } from './benchmark-windows-native-preparation.mjs'
import { ensurePinnedNodeExecutable, preparePinnedNodeDir } from './pinned-node-downloads.mjs'
import { sha256Of } from './orcad-prebuild-slot-contents.mjs'
import { hash } from './windows-native-benchmark-observer.mjs'

const ROOT = resolve(import.meta.dirname, '../..')
const { PE_MACHINE, readPeMachine } = createRequire(import.meta.url)('./windows-pe-machine.cjs')
const invalidCandidate = Buffer.from('orca-windows-security-qualification-invalid')
const preparedPath = (root) =>
  join(root, 'native/windows-security/build/Release/orca_windows_security.node')

export function assertSecurityComparisonHost({
  platform = process.platform,
  arch = process.arch,
  version = process.versions.node,
  electron = process.versions.electron,
  sourceSha = process.env.SOURCE_SHA,
  runner = process.env.BENCHMARK_RUNNER
} = {}) {
  assert.equal(platform, 'win32', 'Security comparison requires Windows')
  assert(!electron, 'Security comparison requires plain Node')
  assert.equal(
    { 'windows-2022': 'x64', 'windows-11-arm': 'arm64' }[runner],
    arch,
    'Runner architecture mismatch'
  )
  assert(['x64', 'arm64'].includes(arch), 'Unsupported architecture')
  assert.equal(version, NODE_RUNTIME_PIN.version, 'Build Node differs from pin')
  assert(/^[0-9a-f]{40}$/.test(sourceSha ?? ''), 'SOURCE_SHA must be exact')
}

export function assertSecurityCommand(result) {
  assert(
    result.code === 0 && !result.signal && !result.timedOut && !result.outputTruncated,
    'Command failed or produced incomplete output'
  )
}

export function probeSecurityArtifact(
  path,
  runtimePath,
  arch,
  run = runProcessSync,
  runtimeAssets = NODE_RUNTIME_ASSETS
) {
  const machine = readPeMachine(path)
  assert.equal(machine, PE_MACHINE[arch], 'Security artifact architecture mismatch')
  const sha256 = sha256Of(path)
  assert.equal(
    sha256Of(runtimePath),
    runtimeAssets[`win32-${arch}`]?.executableSha256,
    'Runtime hash mismatch'
  )
  const result = probeWindowsSecurityAddon(runtimePath, resolve(path), arch, run)
  assertSecurityCommand(result)
  assert.equal(result.stdout, 'windows-security-ready', 'Security operation failed')
  assert.equal(sha256Of(path), sha256, 'Security artifact changed during probe')
  return { sha256, machine, currentUserSid: 'passed' }
}

export async function withInvalidSecurityCandidate(candidate, backupDir, expectedHash, action) {
  const stat = lstatSync(candidate)
  assert(stat.isFile() && !stat.isSymbolicLink(), 'Candidate must be an owned regular file')
  assert.equal(sha256Of(candidate), expectedHash, 'Prepared candidate changed before backup')
  const backup = join(backupDir, 'prepared-security-backup.node')
  writeFileSync(backup, readFileSync(candidate), { flag: 'wx', mode: 0o600 })
  assert.equal(sha256Of(backup), expectedHash, 'Candidate backup mismatch')
  try {
    writeFileSync(candidate, invalidCandidate)
    assert.equal(sha256Of(candidate), hash(invalidCandidate), 'Invalidation failed')
    return await action()
  } finally {
    copyFileSync(backup, candidate)
    chmodSync(candidate, stat.mode)
    assert.equal(sha256Of(candidate), expectedHash, 'Candidate restoration failed')
    utimesSync(candidate, stat.atimeMs / 1000, stat.mtimeMs / 1000)
  }
}

export function runSecurityComparisonSample({
  root = ROOT,
  variant,
  arch = process.arch,
  runtimePath,
  output,
  receiptFile,
  run = runProcessSync,
  runtimeAssets = NODE_RUNTIME_ASSETS
}) {
  assert(['control', 'candidate', 'fallback'].includes(variant), 'Unknown sample')
  const receipt = { variant, qualification: 'incomplete', stage: 'initial-validation', calls: [] }
  const save = () => writeFileSync(receiptFile, `${JSON.stringify(receipt, null, 2)}\n`)
  const observed = (spec) => {
    const phase = spec.args?.includes('rebuild')
      ? 'compiler'
      : spec.args?.includes('--check-only')
        ? 'preparation-probe'
        : 'security-probe'
    const started = performance.now()
    const call = { phase, complete: false }
    receipt.stage = phase
    receipt.calls.push(call)
    try {
      const result = run({
        ...spec,
        stdio: 'pipe',
        timeoutMs: spec.timeoutMs ?? 480_000,
        maxOutputBytes: 64 * 1024 * 1024
      })
      Object.assign(call, {
        code: result.code,
        signal: result.signal,
        timedOut: result.timedOut,
        outputTruncated: result.outputTruncated,
        stdoutSha256: hash(result.stdout),
        stderrSha256: hash(result.stderr)
      })
      assertSecurityCommand(result)
      assertNoHeaderDownloads(result)
      call.complete = true
      return result
    } finally {
      call.durationMs = performance.now() - started
      save()
    }
  }
  try {
    assert.equal(arch, process.arch, 'Sample architecture differs from host')
    assert.equal(
      sha256Of(runtimePath),
      runtimeAssets[`win32-${arch}`]?.executableSha256,
      'Runtime hash mismatch'
    )
    receipt.stage = 'build'
    const start = performance.now()
    buildWindowsSecurityAddon(arch, output, {
      root,
      run: observed,
      enabled: variant !== 'control',
      runtimePath,
      runtimeAssets
    })
    receipt.buildIncludingReuseProbesMs = performance.now() - start
    receipt.staged = probeSecurityArtifact(output, runtimePath, arch, observed, runtimeAssets)
    receipt.compilerInvocationCount = receipt.calls.filter(
      (call) => call.phase === 'compiler'
    ).length
    receipt.stage = 'validate-observations'
    receipt.compilerCountMeaning =
      'Actual node-gyp rebuild subprocess calls; each uses the builder fresh work directory.'
    assert.equal(
      receipt.compilerInvocationCount,
      variant === 'candidate' ? 0 : 1,
      'Unexpected compiler invocation count'
    )
    assert(
      receipt.calls.every((call) => call.complete),
      'A rejected/incomplete command was hidden by fallback'
    )
    receipt.qualification = 'passed'
    return receipt
  } catch {
    receipt.qualification = 'failed'
    receipt.failureStage = receipt.stage
    throw new Error('Windows security sample did not qualify')
  } finally {
    receipt.compilerInvocationCount = receipt.calls.filter(
      (call) => call.phase === 'compiler'
    ).length
    save()
  }
}

export async function benchmarkWindowsSecurityReuse(
  root = ROOT,
  {
    run = runProcess,
    runSync = runProcessSync,
    ensureRuntime = ensurePinnedNodeExecutable,
    prepareHeaders = preparePinnedNodeDir,
    runtimeAssets = NODE_RUNTIME_ASSETS,
    hashFile = sha256Of
  } = {}
) {
  assertSecurityComparisonHost()
  const arch = process.arch
  const target = `win32-${arch}`
  const output = join(root, '.build/windows-native-benchmark/security')
  mkdirSync(output, { recursive: true })
  const temporary = mkdtempSync(
    join(process.env.RUNNER_TEMP || tmpdir(), 'orca-security-comparison-')
  )
  const report = {
    schemaVersion: 1,
    qualification: 'incomplete',
    startedAt: new Date().toISOString(),
    phases: [],
    samples: [],
    scope:
      'One same-run warm control/candidate pair and one invalid-candidate fallback; no CI occupancy or cache-hit-rate claim.',
    sourceSha: process.env.SOURCE_SHA,
    arch,
    runner: process.env.BENCHMARK_RUNNER,
    runId: process.env.GITHUB_RUN_ID,
    runAttempt: process.env.GITHUB_RUN_ATTEMPT,
    workflowSha: process.env.GITHUB_WORKFLOW_SHA,
    preparation:
      'Source-built in this run; no native cache restore. Headers and runtime preparation precede all measured samples.'
  }
  const save = () =>
    writeFileSync(join(output, 'receipt.json'), `${JSON.stringify(report, null, 2)}\n`)
  let environment = { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
  const command = async (label, program, args) => {
    report.stage = label
    const start = performance.now()
    const result = await run({
      program,
      args,
      cwd: root,
      env: environment,
      timeoutMs: 1_200_000,
      maxOutputBytes: 64 * 1024 * 1024,
      terminationBarrier: true
    })
    report.phases.push({
      label,
      durationMs: performance.now() - start,
      code: result.code,
      signal: result.signal,
      timedOut: result.timedOut,
      outputTruncated: result.outputTruncated,
      stdoutSha256: hash(result.stdout),
      stderrSha256: hash(result.stderr)
    })
    save()
    assertSecurityCommand(result)
    return result
  }
  const sourceCheck = async () => {
    const head = await command('source-head', 'git', ['rev-parse', 'HEAD'])
    assert.equal(head.stdout.trim(), process.env.SOURCE_SHA, 'Source commit mismatch')
    await command('source-clean', 'git', ['diff', '--exit-code', 'HEAD', '--'])
    const untracked = await command('source-untracked', 'git', [
      'ls-files',
      '--others',
      '--exclude-standard'
    ])
    assert.equal(untracked.stdout, '', 'Untracked source prevents exact comparison')
  }
  try {
    await sourceCheck()
    report.stage = 'verify-build-node'
    assert.equal(
      hashFile(process.execPath),
      runtimeAssets[target].executableSha256,
      'Build Node hash mismatch'
    )
    const sourceFiles = (
      await command('security-source-files', 'git', ['ls-files', '--', 'native/windows-security'])
    ).stdout
      .trim()
      .split(/\r?\n/)
    const identityFiles = [
      ...sourceFiles,
      'pnpm-lock.yaml',
      'pnpm-workspace.yaml',
      'package.json',
      '.github/actions/prepare-native-runtime/action.yml',
      'src/shared/node-runtime-pin.ts',
      'config/scripts/build-windows-security-addon.mjs',
      'config/scripts/ensure-native-runtime.mjs',
      'config/scripts/benchmark-windows-security-reuse.mjs'
    ]
    report.sourceHashes = Object.fromEntries(
      identityFiles.map((path) => [path, sha256Of(join(root, path))])
    )
    report.sourceInputsSha256 = hash(JSON.stringify(report.sourceHashes))
    report.tree = (await command('source-tree', 'git', ['rev-parse', 'HEAD^{tree}'])).stdout.trim()
    // This mode owns the candidate because its preparation creates it in a fresh checkout.
    const candidate = preparedPath(root)
    report.stage = 'verify-fresh-checkout'
    assert(!existsSync(candidate), 'Security comparison requires a fresh unprepared checkout')
    report.stage = 'prepare-pinned-runtime-and-headers'
    const preparationStart = performance.now()
    const runtimePath = await ensureRuntime({ target })
    const nodeDir = await prepareHeaders({ target, workDir: join(temporary, 'headers') })
    environment = {
      ...environment,
      npm_config_nodedir: nodeDir,
      npm_package_config_node_gyp_nodedir: nodeDir
    }
    report.downloadAndHeaderPreparationMs = performance.now() - preparationStart
    report.runtime = {
      version: NODE_RUNTIME_PIN.version,
      sha256: sha256Of(runtimePath),
      buildNodeSha256: hashFile(process.execPath)
    }
    report.headers = treeDigest(nodeDir)
    await command('prepare-node-artifacts', process.execPath, [
      join(root, 'config/scripts/initialize-windows-native-toolchain.mjs')
    ])
    await command('verify-prepared-node', process.execPath, [
      join(root, 'config/scripts/ensure-native-runtime.mjs'),
      '--check-only'
    ])
    report.stage = 'verify-prepared-candidate-ownership'
    assert.equal(
      realpathSync(candidate).toLowerCase(),
      preparedPath(realpathSync(root)).toLowerCase(),
      'Candidate escapes owned checkout'
    )
    assert(
      lstatSync(candidate).isFile() && !lstatSync(candidate).isSymbolicLink(),
      'Prepared candidate must be regular'
    )
    report.prepared = probeSecurityArtifact(candidate, runtimePath, arch, runSync, runtimeAssets)
    const sample = async (variant) => {
      await sourceCheck()
      const receiptFile = join(output, `${variant}.json`)
      const result = await command(`${variant}-full-build-and-probe`, process.execPath, [
        import.meta.filename,
        `--sample=${variant}`,
        runtimePath,
        join(output, `${variant}.node`),
        receiptFile
      ])
      assertNoHeaderDownloads(result)
      const receipt = JSON.parse(readFileSync(receiptFile, 'utf8'))
      assert.equal(receipt.qualification, 'passed', 'Sample did not qualify')
      assert.deepEqual(
        treeDigest(nodeDir),
        report.headers,
        'Measured build changed prepared headers'
      )
      if (variant === 'candidate') {
        assert.equal(
          receipt.staged.sha256,
          report.prepared.sha256,
          'Reuse did not copy prepared bytes'
        )
      }
      report.samples.push({ ...receipt, fullCommandMs: report.phases.at(-1).durationMs })
      save()
      await sourceCheck()
    }
    // Warm compiler discovery and local build inputs before measuring the pair.
    await command('warm-compiler-before-measurement', process.execPath, [
      join(root, 'config/scripts/build-windows-security-addon.mjs'),
      `--arch=${arch}`
    ])
    await sample('control')
    await sample('candidate')
    assert.equal(
      sha256Of(candidate),
      report.prepared.sha256,
      'Prepared candidate changed during pair'
    )
    report.stage = 'invalidate-owned-candidate'
    await withInvalidSecurityCandidate(candidate, output, report.prepared.sha256, async () => {
      report.invalidCandidateSha256 = sha256Of(candidate)
      await sample('fallback')
    })
    report.restoredPreparedSha256 = sha256Of(candidate)
    assert.equal(
      report.restoredPreparedSha256,
      report.prepared.sha256,
      'Prepared bytes were not restored'
    )
    await sourceCheck()
    const [control, candidateSample] = report.samples
    report.comparison = {
      controlMs: control.fullCommandMs,
      candidateMs: candidateSample.fullCommandMs,
      candidateMinusControlMs: candidateSample.fullCommandMs - control.fullCommandMs,
      limitation:
        'One fixed-order same-run pair per architecture; descriptive only, not a significance or job-time estimate.'
    }
    report.qualification = 'passed'
  } catch {
    report.qualification = 'failed'
    report.failureStage = report.stage
    throw new Error('Windows security comparison did not qualify')
  } finally {
    report.finishedAt = new Date().toISOString()
    report.restored = report.prepared
      ? existsSync(preparedPath(root)) && sha256Of(preparedPath(root)) === report.prepared.sha256
      : null
    save()
    rmSync(temporary, { recursive: true, force: true })
  }
  return report
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  try {
    const variant = process.argv
      .find((arg) => arg.startsWith('--sample='))
      ?.slice('--sample='.length)
    if (variant) {
      assertSecurityComparisonHost()
      runSecurityComparisonSample({
        variant,
        runtimePath: process.argv[3],
        output: process.argv[4],
        receiptFile: process.argv[5]
      })
    } else {
      await benchmarkWindowsSecurityReuse()
    }
  } catch {
    console.error('[windows-security-comparison] failed; inspect the redacted receipt')
    process.exitCode = 1
  }
}