Pin each 4a arm to a full SHA and prove its build came from it

Each --arm now names name=<checkout>@<full sha>. The harness refuses an arm whose HEAD
differs, whose product sources are dirty, whose renderer build-commit stamp is not that
HEAD, or whose sources changed after the build, and records that provenance in every
result. A cached seed from another seed-arm commit is refused, and report-only refuses to
mix results from different pins, seeds or recipes. Tables name full SHAs and disclose that
writer BEGIN-to-COMMIT spans are not observed.
This commit is contained in:
Brennan Benson
2026-11-09 14:38:17 -08:00
parent d724950c7e
commit 8e06fb56d2
4 changed files with 228 additions and 24 deletions
@@ -0,0 +1,175 @@
// Which commit a 4a arm's built app came from, proved from the checkout and its existing build output, never a product hook.
// The build already inlines `git rev-parse HEAD` as ORCA_BUILD_COMMIT (config/scripts/build-identity), which the renderer
// passes as the skills install `sourceCommit`; that literal is the artifact's own record of the commit it was built from.
import { execFileSync } from 'node:child_process'
import { createHash } from 'node:crypto'
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
import path from 'node:path'
import type { Manifest } from './native-chat-4a-fixture'
import type { ArmResult } from './native-chat-4a-report'
const FULL_SHA = /^[0-9a-f]{40}$/
const BUILD_COMMIT_STAMP = /sourceCommit\s*:\s*["'`]([0-9a-f]{40})["'`]/g
const MAIN_BUNDLE = path.join('out', 'main', 'index.js')
const RENDERER_OUT = path.join('out', 'renderer')
// Inputs to `electron-vite build`; the harness itself (config/scripts/performance) is not product source.
const PRODUCT_SOURCES = [
'src',
'electron.vite.config.ts',
'package.json',
'pnpm-lock.yaml',
'config/build-plugins',
'config/dark-build-switches.cjs',
'config/scripts/build-identity'
]
export type ArmSpec = { name: string; root: string; expectedSha: string }
export type ArmProvenance = {
expectedSha: string
/** Full checkout HEAD; equal to expectedSha or the arm was refused. */
sha: string
headCommittedAt: string
productSources: string
/** The commit the built renderer names; equal to sha or the arm was refused. */
builtCommit: string
stampFiles: string
mainBundleSha256: string
/** Oldest of the main bundle and stamped renderer files. */
artifactBuiltAt: string
newestSourceFile: string
newestSourceModifiedAt: string
}
/** `name=<checkout>@<full sha>`: every arm names the exact commit it must run. */
export function parseArmSpec(spec: string): ArmSpec {
const equals = spec.indexOf('=')
const at = spec.lastIndexOf('@')
const name = spec.slice(0, Math.max(0, equals))
const root = spec.slice(equals + 1, at)
const expectedSha = spec.slice(at + 1).toLowerCase()
if (!name || equals === -1 || at <= equals || !FULL_SHA.test(expectedSha)) {
throw new Error(`--arm takes name=<checkout>@<full 40-character sha>, got ${spec}`)
}
return { name, root, expectedSha }
}
const git = (root: string, ...args: string[]): string =>
execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 ** 2 })
function rendererScripts(dir: string): string[] {
return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
const file = path.join(dir, entry.name)
if (entry.isDirectory()) {
return rendererScripts(file)
}
return entry.name.endsWith('.js') ? [file] : []
})
}
/** Refuse an arm whose HEAD is not its pin, whose product sources are dirty, or whose build is not from that HEAD. */
export function verifyArmProvenance(spec: ArmSpec): ArmProvenance {
const { name, root, expectedSha } = spec
const refuse = (reason: string): never => {
throw new Error(`${name}: ${root} ${reason}`)
}
const sha = git(root, 'rev-parse', 'HEAD').trim()
if (sha !== expectedSha) {
refuse(`is at ${sha}, not the pinned ${expectedSha}`)
}
const dirty = git(root, 'status', '--porcelain=v1', '--', ...PRODUCT_SOURCES)
.split('\n')
.filter(Boolean)
if (dirty.length > 0) {
refuse(`has uncommitted product sources: ${dirty.slice(0, 5).join('; ')}`)
}
if (!existsSync(path.join(root, MAIN_BUNDLE)) || !existsSync(path.join(root, RENDERER_OUT))) {
refuse(`has no ${MAIN_BUNDLE} or ${RENDERER_OUT}; build that checkout first`)
}
const stamps = new Map<string, string[]>()
for (const file of rendererScripts(path.join(root, RENDERER_OUT))) {
for (const match of readFileSync(file, 'utf8').matchAll(BUILD_COMMIT_STAMP)) {
stamps.set(match[1]!, [...(stamps.get(match[1]!) ?? []), path.relative(root, file)])
}
}
if (stamps.size === 0) {
refuse(`has a renderer build with no build-commit stamp; rebuild it from this checkout`)
}
if (stamps.size > 1 || !stamps.has(sha)) {
refuse(`was built from ${[...stamps.keys()].join(', ')}, not HEAD ${sha}; rebuild it`)
}
const stampFiles = [...new Set(stamps.get(sha))]
const artifacts = [MAIN_BUNDLE, ...stampFiles].map((file) => ({
file,
modifiedAt: statSync(path.join(root, file)).mtimeMs
}))
const oldestArtifact = Math.min(...artifacts.map((artifact) => artifact.modifiedAt))
// A checkout or edit after the build rewrites a tracked file later than the output, whatever HEAD the stamp names.
let newest = { file: '', modifiedAt: 0 }
for (const file of git(root, 'ls-files', '-z', '--', ...PRODUCT_SOURCES).split('\0')) {
const absolute = path.join(root, file)
if (file && existsSync(absolute)) {
const modifiedAt = statSync(absolute).mtimeMs
newest = modifiedAt > newest.modifiedAt ? { file, modifiedAt } : newest
}
}
if (newest.modifiedAt > oldestArtifact) {
refuse(`has ${newest.file} modified after its build output; rebuild it`)
}
return {
expectedSha,
sha,
headCommittedAt: git(root, 'show', '-s', '--format=%cI', sha).trim(),
productSources: PRODUCT_SOURCES.join(' '),
builtCommit: sha,
stampFiles: stampFiles.join(' '),
mainBundleSha256: createHash('sha256')
.update(readFileSync(path.join(root, MAIN_BUNDLE)))
.digest('hex'),
artifactBuiltAt: new Date(oldestArtifact).toISOString(),
newestSourceFile: newest.file,
newestSourceModifiedAt: new Date(newest.modifiedAt).toISOString()
}
}
/** One run only: one result per arm, each from its pin, all on one seed and recipe; anything else is refused. */
export function assertOneRun(results: ArmResult[], pins: readonly ArmSpec[]): void {
const refuse = (reason: string): never => {
throw new Error(`report-only: ${reason}; re-run into a fresh --out instead of mixing arms`)
}
for (const result of results) {
const p: Partial<ArmResult['provenance']> = result.provenance ?? {}
if (p.expectedSha !== result.sha || p.builtCommit !== result.sha) {
refuse(`${result.arm} ${result.sha} carries no pin and build provenance matching its sha`)
}
}
const names = results.map((result) => result.arm)
if (new Set(names).size !== names.length) {
refuse(`more than one result for an arm (${names.join(', ')})`)
}
for (const field of ['seedSha', 'recipe', 'largeSourceRecipe']) {
const seen = new Set(results.map((result) => String(result.fixture[field])))
if (seen.size !== 1 || seen.has('undefined')) {
refuse(`arms disagree on or lack fixture ${field} (${[...seen].join(', ')})`)
}
}
for (const pin of pins) {
const result = results.find((each) => each.arm === pin.name)
if (result?.sha !== pin.expectedSha) {
refuse(`${pin.name} is ${result?.sha ?? 'missing'}, not the pinned ${pin.expectedSha}`)
}
}
if (pins.length > 0 && results.length !== pins.length) {
refuse(`found arms ${names.join(', ')}, pinned ${pins.map((pin) => pin.name).join(', ')}`)
}
}
/** A cached seed is reused by recipe alone, so one written by another commit of the seed arm is refused. */
export function seededBy(arm: { name: string; sha: string }, seeded: Manifest): Manifest {
if (seeded.seedSha !== arm.sha) {
throw new Error(
`a cached seed was written by ${seeded.seedSha}, not ${arm.name} ${arm.sha}; use a fresh --out`
)
}
return seeded
}
@@ -21,6 +21,11 @@ import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electr
import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown'
import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile'
import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override'
import {
verifyArmProvenance,
type ArmProvenance,
type ArmSpec
} from './native-chat-4a-arm-provenance'
import { installVisibilityProbe } from './native-chat-4a-probes'
const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..')
@@ -59,6 +64,7 @@ export type Arm = {
name: string
root: string
sha: string
provenance: ArmProvenance
electronPath: string
protocol: ArmProtocol
}
@@ -75,14 +81,11 @@ export type Launch = {
readyEpochMs: number
}
const git = (root: string, ...args: string[]): string =>
execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
export async function resolveArm(name: string, root: string, scratch: string): Promise<Arm> {
const absolute = path.resolve(root)
if (!existsSync(path.join(absolute, 'out', 'main', 'index.js'))) {
throw new Error(`${name}: ${absolute} has no out/main/index.js; build that checkout first`)
}
/** A pinned, clean checkout whose build output names its HEAD; anything else is refused before any launch. */
export async function resolveArm(spec: ArmSpec, scratch: string): Promise<Arm> {
const { name } = spec
const absolute = path.resolve(spec.root)
const provenance = verifyArmProvenance({ ...spec, root: absolute })
const electronPath: unknown = createRequire(path.join(absolute, 'package.json'))('electron')
if (typeof electronPath !== 'string') {
throw new Error(`${name}: the electron package in ${absolute} names no executable`)
@@ -99,7 +102,7 @@ export async function resolveArm(name: string, root: string, scratch: string): P
]
]
)
return { name, root: absolute, sha: git(absolute, 'rev-parse', 'HEAD'), electronPath, protocol }
return { name, root: absolute, sha: provenance.sha, provenance, electronPath, protocol }
}
/** Compile `exports` ([names, checkout-relative file]) from a checkout's own source and load them. */
@@ -1,7 +1,7 @@
// The 4a "no slower than main" comparison: one profile seeded once, every arm measured on its own copy, one JSON + one table.
// Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run
// Full run: ... --arm main=<built main checkout> --arm candidate=<built candidate checkout> --profile full --out <dir> [--large-source]
// Re-table: ... --out <dir> --report-only
// Full run: ... --arm main=<built main checkout>@<full sha> --arm candidate=<built candidate checkout>@<full sha> --profile full --out <dir> [--large-source]
// Re-table: ... --out <dir> --report-only [--arm main=@<full sha> --arm candidate=@<full sha>]
import {
existsSync,
mkdirSync,
@@ -14,6 +14,7 @@ import os from 'node:os'
import path from 'node:path'
import { parseArgs } from 'node:util'
import type { Arm } from './native-chat-4a-arm'
import { assertOneRun, parseArmSpec, seededBy } from './native-chat-4a-arm-provenance'
import type { Manifest } from './native-chat-4a-fixture'
import type { RunContext } from './native-chat-4a-run-context'
import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report'
@@ -194,6 +195,7 @@ async function runArm(input: {
return {
arm: input.arm.name,
sha: input.arm.sha,
provenance: input.arm.provenance,
startedAt,
finishedAt: new Date().toISOString(),
rows,
@@ -202,6 +204,8 @@ async function runArm(input: {
worktreeIds: input.manifest.workspace.worktreeIds.join(' '),
seedArm: input.manifest.seedArm,
seedSha: input.manifest.seedSha,
recipe: input.manifest.recipe,
largeSourceRecipe: input.largeSource?.manifest.recipe ?? 'none',
largeItems: input.manifest.largeItems,
storageBytes: input.manifest.storageBytes,
seedMs: input.manifest.seedMs
@@ -221,6 +225,7 @@ async function main(): Promise<void> {
.filter((file) => /^arm-.*\.json$/.test(file))
.map((file): ArmResult => JSON.parse(readFileSync(path.join(out, file), 'utf8')))
.sort((a, b) => a.startedAt.localeCompare(b.startedAt))
assertOneRun(results, values.arm.map(parseArmSpec))
writeComparison(out, workload, results, log)
return
}
@@ -237,15 +242,16 @@ async function main(): Promise<void> {
const scratch = path.join(out, 'scratch')
mkdirSync(scratch, { recursive: true })
const arms: Arm[] = []
for (const spec of values.arm) {
const [name, root] = spec.split('=')
if (!name || !root) {
throw new Error(`--arm takes name=checkout, got ${spec}`)
for (const spec of values.arm.map(parseArmSpec)) {
if (!spec.root) {
throw new Error(`--arm ${spec.name} names no checkout`)
}
arms.push(await resolveArm(name, root, scratch))
arms.push(await resolveArm(spec, scratch))
}
if (arms.length === 0) {
throw new Error('Pass at least one --arm name=<built checkout>; main first, then the candidate')
throw new Error(
'Pass at least one --arm name=<built checkout>@<full sha>; main first, then the candidate'
)
}
const stubCommand = writeProviderStub(path.join(out, 'stub'))
const repoFile = path.join(out, 'repo-path.txt')
@@ -265,13 +271,16 @@ async function main(): Promise<void> {
toolOutputBytes,
log
})
const manifest = await seed(workload.history.toolOutputBytes, workload)
const manifest = seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload))
const large =
values['large-source'] && workload.conversionSourceBytes > 0
? await seed(conversionToolOutputBytes(workload), {
...workload,
history: { ...workload.history, fillerChats: 5, mediumTurns: 0 }
})
? seededBy(
seedArm,
await seed(conversionToolOutputBytes(workload), {
...workload,
history: { ...workload.history, fillerChats: 5, mediumTurns: 0 }
})
)
: null
const results: ArmResult[] = []
for (const arm of arms) {
@@ -1,7 +1,8 @@
// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's head SHA.
// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's pinned full SHA.
import { writeFileSync } from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import type { ArmProvenance } from './native-chat-4a-arm-provenance'
import type { HostWork } from './native-chat-4a-probes'
import type { Workload } from './native-chat-4a-workloads'
@@ -32,6 +33,7 @@ export type Row = {
export type ArmResult = {
arm: string
sha: string
provenance: ArmProvenance
startedAt: string
finishedAt: string
rows: Row[]
@@ -90,6 +92,14 @@ function delta(base: Stat | null, next: Stat | null): string {
return `${change >= 0 ? '+' : ''}${ms(change)}${percent}`
}
/** One line per arm: the full pinned SHA and the build output that proved it. */
const provenanceLine = ({ arm, provenance: p }: ArmResult): string =>
`- ${arm}: \`${p.sha}\` (pinned \`${p.expectedSha}\`, committed ${p.headCommittedAt}); clean product sources; build stamp \`${p.builtCommit}\` in ${p.stampFiles}; out/main/index.js sha256 \`${p.mainBundleSha256}\`; built ${p.artifactBuiltAt}, after the newest source ${p.newestSourceFile} (${p.newestSourceModifiedAt}).`
// No product hook or timing field exposes it, and SQLite state records no transaction start, so it is disclosed, not derived.
const WRITER_SPAN_DISCLOSURE =
'Not measured: writer BEGIN → COMMIT and grouped-transaction spans. Neither arm exposes them without a product hook, and on-disk SQLite state records commits but not when a transaction began; host spans above include storage but are not transaction durations.'
const caseCell = (row: Row): string =>
row.endToEndOnly ? `${row.variant} (end-to-end only: ${row.endToEndOnly})` : row.variant
@@ -113,6 +123,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
const lines = [
`4a comparison: ${base.arm} \`${base.sha.slice(0, 11)}\` vs ${next.arm} \`${next.sha.slice(0, 11)}\` (p50 / p95 / max ms, nearest rank).`,
'',
provenanceLine(base),
provenanceLine(next),
'',
`| Family | Operation | Case | ${base.arm} | ${next.arm} | Δ p95 | Host span p95 ${base.arm} → ${next.arm} | Off-main CPU p95 ${base.arm} → ${next.arm} | Shape |`,
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |'
]
@@ -134,7 +147,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
}
lines.push(
'',
'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.'
'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.',
'',
WRITER_SPAN_DISCLOSURE
)
return `${lines.join('\n')}\n`
}
@@ -143,6 +158,8 @@ export function armMarkdown(result: ArmResult): string {
const lines = [
`4a ${result.arm} \`${result.sha.slice(0, 11)}\` (p50 / p95 / max ms).`,
'',
provenanceLine(result),
'',
'| Family | Operation | Case | Primary | Host span | Shape |',
'| --- | --- | --- | --- | --- | --- |'
]