diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm-provenance.ts b/config/scripts/performance/native-chat/native-chat-4a-arm-provenance.ts new file mode 100644 index 00000000000..1cdf5c4cb75 --- /dev/null +++ b/config/scripts/performance/native-chat/native-chat-4a-arm-provenance.ts @@ -0,0 +1,175 @@ +// Which commit a 4a arm's built app came from, proved from the checkout and its existing build output, never a product hook. +// The build already inlines `git rev-parse HEAD` as ORCA_BUILD_COMMIT (config/scripts/build-identity), which the renderer +// passes as the skills install `sourceCommit`; that literal is the artifact's own record of the commit it was built from. +import { execFileSync } from 'node:child_process' +import { createHash } from 'node:crypto' +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' +import path from 'node:path' +import type { Manifest } from './native-chat-4a-fixture' +import type { ArmResult } from './native-chat-4a-report' + +const FULL_SHA = /^[0-9a-f]{40}$/ +const BUILD_COMMIT_STAMP = /sourceCommit\s*:\s*["'`]([0-9a-f]{40})["'`]/g +const MAIN_BUNDLE = path.join('out', 'main', 'index.js') +const RENDERER_OUT = path.join('out', 'renderer') +// Inputs to `electron-vite build`; the harness itself (config/scripts/performance) is not product source. +const PRODUCT_SOURCES = [ + 'src', + 'electron.vite.config.ts', + 'package.json', + 'pnpm-lock.yaml', + 'config/build-plugins', + 'config/dark-build-switches.cjs', + 'config/scripts/build-identity' +] + +export type ArmSpec = { name: string; root: string; expectedSha: string } + +export type ArmProvenance = { + expectedSha: string + /** Full checkout HEAD; equal to expectedSha or the arm was refused. */ + sha: string + headCommittedAt: string + productSources: string + /** The commit the built renderer names; equal to sha or the arm was refused. */ + builtCommit: string + stampFiles: string + mainBundleSha256: string + /** Oldest of the main bundle and stamped renderer files. */ + artifactBuiltAt: string + newestSourceFile: string + newestSourceModifiedAt: string +} + +/** `name=@`: every arm names the exact commit it must run. */ +export function parseArmSpec(spec: string): ArmSpec { + const equals = spec.indexOf('=') + const at = spec.lastIndexOf('@') + const name = spec.slice(0, Math.max(0, equals)) + const root = spec.slice(equals + 1, at) + const expectedSha = spec.slice(at + 1).toLowerCase() + if (!name || equals === -1 || at <= equals || !FULL_SHA.test(expectedSha)) { + throw new Error(`--arm takes name=@, got ${spec}`) + } + return { name, root, expectedSha } +} + +const git = (root: string, ...args: string[]): string => + execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 ** 2 }) + +function rendererScripts(dir: string): string[] { + return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const file = path.join(dir, entry.name) + if (entry.isDirectory()) { + return rendererScripts(file) + } + return entry.name.endsWith('.js') ? [file] : [] + }) +} + +/** Refuse an arm whose HEAD is not its pin, whose product sources are dirty, or whose build is not from that HEAD. */ +export function verifyArmProvenance(spec: ArmSpec): ArmProvenance { + const { name, root, expectedSha } = spec + const refuse = (reason: string): never => { + throw new Error(`${name}: ${root} ${reason}`) + } + const sha = git(root, 'rev-parse', 'HEAD').trim() + if (sha !== expectedSha) { + refuse(`is at ${sha}, not the pinned ${expectedSha}`) + } + const dirty = git(root, 'status', '--porcelain=v1', '--', ...PRODUCT_SOURCES) + .split('\n') + .filter(Boolean) + if (dirty.length > 0) { + refuse(`has uncommitted product sources: ${dirty.slice(0, 5).join('; ')}`) + } + if (!existsSync(path.join(root, MAIN_BUNDLE)) || !existsSync(path.join(root, RENDERER_OUT))) { + refuse(`has no ${MAIN_BUNDLE} or ${RENDERER_OUT}; build that checkout first`) + } + const stamps = new Map() + for (const file of rendererScripts(path.join(root, RENDERER_OUT))) { + for (const match of readFileSync(file, 'utf8').matchAll(BUILD_COMMIT_STAMP)) { + stamps.set(match[1]!, [...(stamps.get(match[1]!) ?? []), path.relative(root, file)]) + } + } + if (stamps.size === 0) { + refuse(`has a renderer build with no build-commit stamp; rebuild it from this checkout`) + } + if (stamps.size > 1 || !stamps.has(sha)) { + refuse(`was built from ${[...stamps.keys()].join(', ')}, not HEAD ${sha}; rebuild it`) + } + const stampFiles = [...new Set(stamps.get(sha))] + const artifacts = [MAIN_BUNDLE, ...stampFiles].map((file) => ({ + file, + modifiedAt: statSync(path.join(root, file)).mtimeMs + })) + const oldestArtifact = Math.min(...artifacts.map((artifact) => artifact.modifiedAt)) + // A checkout or edit after the build rewrites a tracked file later than the output, whatever HEAD the stamp names. + let newest = { file: '', modifiedAt: 0 } + for (const file of git(root, 'ls-files', '-z', '--', ...PRODUCT_SOURCES).split('\0')) { + const absolute = path.join(root, file) + if (file && existsSync(absolute)) { + const modifiedAt = statSync(absolute).mtimeMs + newest = modifiedAt > newest.modifiedAt ? { file, modifiedAt } : newest + } + } + if (newest.modifiedAt > oldestArtifact) { + refuse(`has ${newest.file} modified after its build output; rebuild it`) + } + return { + expectedSha, + sha, + headCommittedAt: git(root, 'show', '-s', '--format=%cI', sha).trim(), + productSources: PRODUCT_SOURCES.join(' '), + builtCommit: sha, + stampFiles: stampFiles.join(' '), + mainBundleSha256: createHash('sha256') + .update(readFileSync(path.join(root, MAIN_BUNDLE))) + .digest('hex'), + artifactBuiltAt: new Date(oldestArtifact).toISOString(), + newestSourceFile: newest.file, + newestSourceModifiedAt: new Date(newest.modifiedAt).toISOString() + } +} + +/** One run only: one result per arm, each from its pin, all on one seed and recipe; anything else is refused. */ +export function assertOneRun(results: ArmResult[], pins: readonly ArmSpec[]): void { + const refuse = (reason: string): never => { + throw new Error(`report-only: ${reason}; re-run into a fresh --out instead of mixing arms`) + } + for (const result of results) { + const p: Partial = result.provenance ?? {} + if (p.expectedSha !== result.sha || p.builtCommit !== result.sha) { + refuse(`${result.arm} ${result.sha} carries no pin and build provenance matching its sha`) + } + } + const names = results.map((result) => result.arm) + if (new Set(names).size !== names.length) { + refuse(`more than one result for an arm (${names.join(', ')})`) + } + for (const field of ['seedSha', 'recipe', 'largeSourceRecipe']) { + const seen = new Set(results.map((result) => String(result.fixture[field]))) + if (seen.size !== 1 || seen.has('undefined')) { + refuse(`arms disagree on or lack fixture ${field} (${[...seen].join(', ')})`) + } + } + for (const pin of pins) { + const result = results.find((each) => each.arm === pin.name) + if (result?.sha !== pin.expectedSha) { + refuse(`${pin.name} is ${result?.sha ?? 'missing'}, not the pinned ${pin.expectedSha}`) + } + } + if (pins.length > 0 && results.length !== pins.length) { + refuse(`found arms ${names.join(', ')}, pinned ${pins.map((pin) => pin.name).join(', ')}`) + } +} + +/** A cached seed is reused by recipe alone, so one written by another commit of the seed arm is refused. */ +export function seededBy(arm: { name: string; sha: string }, seeded: Manifest): Manifest { + if (seeded.seedSha !== arm.sha) { + throw new Error( + `a cached seed was written by ${seeded.seedSha}, not ${arm.name} ${arm.sha}; use a fresh --out` + ) + } + return seeded +} diff --git a/config/scripts/performance/native-chat/native-chat-4a-arm.ts b/config/scripts/performance/native-chat/native-chat-4a-arm.ts index 9ef72d8823a..3a7c549dd8a 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-arm.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-arm.ts @@ -21,6 +21,11 @@ import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electr import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown' import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile' import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override' +import { + verifyArmProvenance, + type ArmProvenance, + type ArmSpec +} from './native-chat-4a-arm-provenance' import { installVisibilityProbe } from './native-chat-4a-probes' const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..') @@ -59,6 +64,7 @@ export type Arm = { name: string root: string sha: string + provenance: ArmProvenance electronPath: string protocol: ArmProtocol } @@ -75,14 +81,11 @@ export type Launch = { readyEpochMs: number } -const git = (root: string, ...args: string[]): string => - execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim() - -export async function resolveArm(name: string, root: string, scratch: string): Promise { - const absolute = path.resolve(root) - if (!existsSync(path.join(absolute, 'out', 'main', 'index.js'))) { - throw new Error(`${name}: ${absolute} has no out/main/index.js; build that checkout first`) - } +/** A pinned, clean checkout whose build output names its HEAD; anything else is refused before any launch. */ +export async function resolveArm(spec: ArmSpec, scratch: string): Promise { + const { name } = spec + const absolute = path.resolve(spec.root) + const provenance = verifyArmProvenance({ ...spec, root: absolute }) const electronPath: unknown = createRequire(path.join(absolute, 'package.json'))('electron') if (typeof electronPath !== 'string') { throw new Error(`${name}: the electron package in ${absolute} names no executable`) @@ -99,7 +102,7 @@ export async function resolveArm(name: string, root: string, scratch: string): P ] ] ) - return { name, root: absolute, sha: git(absolute, 'rev-parse', 'HEAD'), electronPath, protocol } + return { name, root: absolute, sha: provenance.sha, provenance, electronPath, protocol } } /** Compile `exports` ([names, checkout-relative file]) from a checkout's own source and load them. */ diff --git a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts index 1122d1ce14d..cd879a64aea 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-comparison.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-comparison.ts @@ -1,7 +1,7 @@ // The 4a "no slower than main" comparison: one profile seeded once, every arm measured on its own copy, one JSON + one table. // Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run -// Full run: ... --arm main= --arm candidate= --profile full --out [--large-source] -// Re-table: ... --out --report-only +// Full run: ... --arm main=@ --arm candidate=@ --profile full --out [--large-source] +// Re-table: ... --out --report-only [--arm main=@ --arm candidate=@] import { existsSync, mkdirSync, @@ -14,6 +14,7 @@ import os from 'node:os' import path from 'node:path' import { parseArgs } from 'node:util' import type { Arm } from './native-chat-4a-arm' +import { assertOneRun, parseArmSpec, seededBy } from './native-chat-4a-arm-provenance' import type { Manifest } from './native-chat-4a-fixture' import type { RunContext } from './native-chat-4a-run-context' import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report' @@ -194,6 +195,7 @@ async function runArm(input: { return { arm: input.arm.name, sha: input.arm.sha, + provenance: input.arm.provenance, startedAt, finishedAt: new Date().toISOString(), rows, @@ -202,6 +204,8 @@ async function runArm(input: { worktreeIds: input.manifest.workspace.worktreeIds.join(' '), seedArm: input.manifest.seedArm, seedSha: input.manifest.seedSha, + recipe: input.manifest.recipe, + largeSourceRecipe: input.largeSource?.manifest.recipe ?? 'none', largeItems: input.manifest.largeItems, storageBytes: input.manifest.storageBytes, seedMs: input.manifest.seedMs @@ -221,6 +225,7 @@ async function main(): Promise { .filter((file) => /^arm-.*\.json$/.test(file)) .map((file): ArmResult => JSON.parse(readFileSync(path.join(out, file), 'utf8'))) .sort((a, b) => a.startedAt.localeCompare(b.startedAt)) + assertOneRun(results, values.arm.map(parseArmSpec)) writeComparison(out, workload, results, log) return } @@ -237,15 +242,16 @@ async function main(): Promise { const scratch = path.join(out, 'scratch') mkdirSync(scratch, { recursive: true }) const arms: Arm[] = [] - for (const spec of values.arm) { - const [name, root] = spec.split('=') - if (!name || !root) { - throw new Error(`--arm takes name=checkout, got ${spec}`) + for (const spec of values.arm.map(parseArmSpec)) { + if (!spec.root) { + throw new Error(`--arm ${spec.name} names no checkout`) } - arms.push(await resolveArm(name, root, scratch)) + arms.push(await resolveArm(spec, scratch)) } if (arms.length === 0) { - throw new Error('Pass at least one --arm name=; main first, then the candidate') + throw new Error( + 'Pass at least one --arm name=@; main first, then the candidate' + ) } const stubCommand = writeProviderStub(path.join(out, 'stub')) const repoFile = path.join(out, 'repo-path.txt') @@ -265,13 +271,16 @@ async function main(): Promise { toolOutputBytes, log }) - const manifest = await seed(workload.history.toolOutputBytes, workload) + const manifest = seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload)) const large = values['large-source'] && workload.conversionSourceBytes > 0 - ? await seed(conversionToolOutputBytes(workload), { - ...workload, - history: { ...workload.history, fillerChats: 5, mediumTurns: 0 } - }) + ? seededBy( + seedArm, + await seed(conversionToolOutputBytes(workload), { + ...workload, + history: { ...workload.history, fillerChats: 5, mediumTurns: 0 } + }) + ) : null const results: ArmResult[] = [] for (const arm of arms) { diff --git a/config/scripts/performance/native-chat/native-chat-4a-report.ts b/config/scripts/performance/native-chat/native-chat-4a-report.ts index f9efc93ac6d..4947bc5cfef 100644 --- a/config/scripts/performance/native-chat/native-chat-4a-report.ts +++ b/config/scripts/performance/native-chat/native-chat-4a-report.ts @@ -1,7 +1,8 @@ -// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's head SHA. +// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's pinned full SHA. import { writeFileSync } from 'node:fs' import os from 'node:os' import path from 'node:path' +import type { ArmProvenance } from './native-chat-4a-arm-provenance' import type { HostWork } from './native-chat-4a-probes' import type { Workload } from './native-chat-4a-workloads' @@ -32,6 +33,7 @@ export type Row = { export type ArmResult = { arm: string sha: string + provenance: ArmProvenance startedAt: string finishedAt: string rows: Row[] @@ -90,6 +92,14 @@ function delta(base: Stat | null, next: Stat | null): string { return `${change >= 0 ? '+' : ''}${ms(change)}${percent}` } +/** One line per arm: the full pinned SHA and the build output that proved it. */ +const provenanceLine = ({ arm, provenance: p }: ArmResult): string => + `- ${arm}: \`${p.sha}\` (pinned \`${p.expectedSha}\`, committed ${p.headCommittedAt}); clean product sources; build stamp \`${p.builtCommit}\` in ${p.stampFiles}; out/main/index.js sha256 \`${p.mainBundleSha256}\`; built ${p.artifactBuiltAt}, after the newest source ${p.newestSourceFile} (${p.newestSourceModifiedAt}).` + +// No product hook or timing field exposes it, and SQLite state records no transaction start, so it is disclosed, not derived. +const WRITER_SPAN_DISCLOSURE = + 'Not measured: writer BEGIN → COMMIT and grouped-transaction spans. Neither arm exposes them without a product hook, and on-disk SQLite state records commits but not when a transaction began; host spans above include storage but are not transaction durations.' + const caseCell = (row: Row): string => row.endToEndOnly ? `${row.variant} (end-to-end only: ${row.endToEndOnly})` : row.variant @@ -113,6 +123,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { const lines = [ `4a comparison: ${base.arm} \`${base.sha.slice(0, 11)}\` vs ${next.arm} \`${next.sha.slice(0, 11)}\` (p50 / p95 / max ms, nearest rank).`, '', + provenanceLine(base), + provenanceLine(next), + '', `| Family | Operation | Case | ${base.arm} | ${next.arm} | Δ p95 | Host span p95 ${base.arm} → ${next.arm} | Off-main CPU p95 ${base.arm} → ${next.arm} | Shape |`, '| --- | --- | --- | --- | --- | --- | --- | --- | --- |' ] @@ -134,7 +147,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string { } lines.push( '', - 'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.' + 'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.', + '', + WRITER_SPAN_DISCLOSURE ) return `${lines.join('\n')}\n` } @@ -143,6 +158,8 @@ export function armMarkdown(result: ArmResult): string { const lines = [ `4a ${result.arm} \`${result.sha.slice(0, 11)}\` (p50 / p95 / max ms).`, '', + provenanceLine(result), + '', '| Family | Operation | Case | Primary | Host span | Shape |', '| --- | --- | --- | --- | --- | --- |' ]