mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
Pin each 4a arm to a full SHA and prove its build came from it
Each --arm now names name=<checkout>@<full sha>. The harness refuses an arm whose HEAD differs, whose product sources are dirty, whose renderer build-commit stamp is not that HEAD, or whose sources changed after the build, and records that provenance in every result. A cached seed from another seed-arm commit is refused, and report-only refuses to mix results from different pins, seeds or recipes. Tables name full SHAs and disclose that writer BEGIN-to-COMMIT spans are not observed.
This commit is contained in:
@@ -0,0 +1,175 @@
|
||||
// Which commit a 4a arm's built app came from, proved from the checkout and its existing build output, never a product hook.
|
||||
// The build already inlines `git rev-parse HEAD` as ORCA_BUILD_COMMIT (config/scripts/build-identity), which the renderer
|
||||
// passes as the skills install `sourceCommit`; that literal is the artifact's own record of the commit it was built from.
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import type { Manifest } from './native-chat-4a-fixture'
|
||||
import type { ArmResult } from './native-chat-4a-report'
|
||||
|
||||
const FULL_SHA = /^[0-9a-f]{40}$/
|
||||
const BUILD_COMMIT_STAMP = /sourceCommit\s*:\s*["'`]([0-9a-f]{40})["'`]/g
|
||||
const MAIN_BUNDLE = path.join('out', 'main', 'index.js')
|
||||
const RENDERER_OUT = path.join('out', 'renderer')
|
||||
// Inputs to `electron-vite build`; the harness itself (config/scripts/performance) is not product source.
|
||||
const PRODUCT_SOURCES = [
|
||||
'src',
|
||||
'electron.vite.config.ts',
|
||||
'package.json',
|
||||
'pnpm-lock.yaml',
|
||||
'config/build-plugins',
|
||||
'config/dark-build-switches.cjs',
|
||||
'config/scripts/build-identity'
|
||||
]
|
||||
|
||||
export type ArmSpec = { name: string; root: string; expectedSha: string }
|
||||
|
||||
export type ArmProvenance = {
|
||||
expectedSha: string
|
||||
/** Full checkout HEAD; equal to expectedSha or the arm was refused. */
|
||||
sha: string
|
||||
headCommittedAt: string
|
||||
productSources: string
|
||||
/** The commit the built renderer names; equal to sha or the arm was refused. */
|
||||
builtCommit: string
|
||||
stampFiles: string
|
||||
mainBundleSha256: string
|
||||
/** Oldest of the main bundle and stamped renderer files. */
|
||||
artifactBuiltAt: string
|
||||
newestSourceFile: string
|
||||
newestSourceModifiedAt: string
|
||||
}
|
||||
|
||||
/** `name=<checkout>@<full sha>`: every arm names the exact commit it must run. */
|
||||
export function parseArmSpec(spec: string): ArmSpec {
|
||||
const equals = spec.indexOf('=')
|
||||
const at = spec.lastIndexOf('@')
|
||||
const name = spec.slice(0, Math.max(0, equals))
|
||||
const root = spec.slice(equals + 1, at)
|
||||
const expectedSha = spec.slice(at + 1).toLowerCase()
|
||||
if (!name || equals === -1 || at <= equals || !FULL_SHA.test(expectedSha)) {
|
||||
throw new Error(`--arm takes name=<checkout>@<full 40-character sha>, got ${spec}`)
|
||||
}
|
||||
return { name, root, expectedSha }
|
||||
}
|
||||
|
||||
const git = (root: string, ...args: string[]): string =>
|
||||
execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', maxBuffer: 256 * 1024 ** 2 })
|
||||
|
||||
function rendererScripts(dir: string): string[] {
|
||||
return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
|
||||
const file = path.join(dir, entry.name)
|
||||
if (entry.isDirectory()) {
|
||||
return rendererScripts(file)
|
||||
}
|
||||
return entry.name.endsWith('.js') ? [file] : []
|
||||
})
|
||||
}
|
||||
|
||||
/** Refuse an arm whose HEAD is not its pin, whose product sources are dirty, or whose build is not from that HEAD. */
|
||||
export function verifyArmProvenance(spec: ArmSpec): ArmProvenance {
|
||||
const { name, root, expectedSha } = spec
|
||||
const refuse = (reason: string): never => {
|
||||
throw new Error(`${name}: ${root} ${reason}`)
|
||||
}
|
||||
const sha = git(root, 'rev-parse', 'HEAD').trim()
|
||||
if (sha !== expectedSha) {
|
||||
refuse(`is at ${sha}, not the pinned ${expectedSha}`)
|
||||
}
|
||||
const dirty = git(root, 'status', '--porcelain=v1', '--', ...PRODUCT_SOURCES)
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
if (dirty.length > 0) {
|
||||
refuse(`has uncommitted product sources: ${dirty.slice(0, 5).join('; ')}`)
|
||||
}
|
||||
if (!existsSync(path.join(root, MAIN_BUNDLE)) || !existsSync(path.join(root, RENDERER_OUT))) {
|
||||
refuse(`has no ${MAIN_BUNDLE} or ${RENDERER_OUT}; build that checkout first`)
|
||||
}
|
||||
const stamps = new Map<string, string[]>()
|
||||
for (const file of rendererScripts(path.join(root, RENDERER_OUT))) {
|
||||
for (const match of readFileSync(file, 'utf8').matchAll(BUILD_COMMIT_STAMP)) {
|
||||
stamps.set(match[1]!, [...(stamps.get(match[1]!) ?? []), path.relative(root, file)])
|
||||
}
|
||||
}
|
||||
if (stamps.size === 0) {
|
||||
refuse(`has a renderer build with no build-commit stamp; rebuild it from this checkout`)
|
||||
}
|
||||
if (stamps.size > 1 || !stamps.has(sha)) {
|
||||
refuse(`was built from ${[...stamps.keys()].join(', ')}, not HEAD ${sha}; rebuild it`)
|
||||
}
|
||||
const stampFiles = [...new Set(stamps.get(sha))]
|
||||
const artifacts = [MAIN_BUNDLE, ...stampFiles].map((file) => ({
|
||||
file,
|
||||
modifiedAt: statSync(path.join(root, file)).mtimeMs
|
||||
}))
|
||||
const oldestArtifact = Math.min(...artifacts.map((artifact) => artifact.modifiedAt))
|
||||
// A checkout or edit after the build rewrites a tracked file later than the output, whatever HEAD the stamp names.
|
||||
let newest = { file: '', modifiedAt: 0 }
|
||||
for (const file of git(root, 'ls-files', '-z', '--', ...PRODUCT_SOURCES).split('\0')) {
|
||||
const absolute = path.join(root, file)
|
||||
if (file && existsSync(absolute)) {
|
||||
const modifiedAt = statSync(absolute).mtimeMs
|
||||
newest = modifiedAt > newest.modifiedAt ? { file, modifiedAt } : newest
|
||||
}
|
||||
}
|
||||
if (newest.modifiedAt > oldestArtifact) {
|
||||
refuse(`has ${newest.file} modified after its build output; rebuild it`)
|
||||
}
|
||||
return {
|
||||
expectedSha,
|
||||
sha,
|
||||
headCommittedAt: git(root, 'show', '-s', '--format=%cI', sha).trim(),
|
||||
productSources: PRODUCT_SOURCES.join(' '),
|
||||
builtCommit: sha,
|
||||
stampFiles: stampFiles.join(' '),
|
||||
mainBundleSha256: createHash('sha256')
|
||||
.update(readFileSync(path.join(root, MAIN_BUNDLE)))
|
||||
.digest('hex'),
|
||||
artifactBuiltAt: new Date(oldestArtifact).toISOString(),
|
||||
newestSourceFile: newest.file,
|
||||
newestSourceModifiedAt: new Date(newest.modifiedAt).toISOString()
|
||||
}
|
||||
}
|
||||
|
||||
/** One run only: one result per arm, each from its pin, all on one seed and recipe; anything else is refused. */
|
||||
export function assertOneRun(results: ArmResult[], pins: readonly ArmSpec[]): void {
|
||||
const refuse = (reason: string): never => {
|
||||
throw new Error(`report-only: ${reason}; re-run into a fresh --out instead of mixing arms`)
|
||||
}
|
||||
for (const result of results) {
|
||||
const p: Partial<ArmResult['provenance']> = result.provenance ?? {}
|
||||
if (p.expectedSha !== result.sha || p.builtCommit !== result.sha) {
|
||||
refuse(`${result.arm} ${result.sha} carries no pin and build provenance matching its sha`)
|
||||
}
|
||||
}
|
||||
const names = results.map((result) => result.arm)
|
||||
if (new Set(names).size !== names.length) {
|
||||
refuse(`more than one result for an arm (${names.join(', ')})`)
|
||||
}
|
||||
for (const field of ['seedSha', 'recipe', 'largeSourceRecipe']) {
|
||||
const seen = new Set(results.map((result) => String(result.fixture[field])))
|
||||
if (seen.size !== 1 || seen.has('undefined')) {
|
||||
refuse(`arms disagree on or lack fixture ${field} (${[...seen].join(', ')})`)
|
||||
}
|
||||
}
|
||||
for (const pin of pins) {
|
||||
const result = results.find((each) => each.arm === pin.name)
|
||||
if (result?.sha !== pin.expectedSha) {
|
||||
refuse(`${pin.name} is ${result?.sha ?? 'missing'}, not the pinned ${pin.expectedSha}`)
|
||||
}
|
||||
}
|
||||
if (pins.length > 0 && results.length !== pins.length) {
|
||||
refuse(`found arms ${names.join(', ')}, pinned ${pins.map((pin) => pin.name).join(', ')}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** A cached seed is reused by recipe alone, so one written by another commit of the seed arm is refused. */
|
||||
export function seededBy(arm: { name: string; sha: string }, seeded: Manifest): Manifest {
|
||||
if (seeded.seedSha !== arm.sha) {
|
||||
throw new Error(
|
||||
`a cached seed was written by ${seeded.seedSha}, not ${arm.name} ${arm.sha}; use a fresh --out`
|
||||
)
|
||||
}
|
||||
return seeded
|
||||
}
|
||||
@@ -21,6 +21,11 @@ import { retryTransientMainEvaluate } from '../../../../tests/e2e/helpers/electr
|
||||
import { closeElectronAppForE2E } from '../../../../tests/e2e/helpers/electron-process-shutdown'
|
||||
import { getE2ECompletedOnboardingProfile } from '../../../../tests/e2e/helpers/e2e-completed-onboarding-profile'
|
||||
import { buildFakeAgentCommandOverride } from '../../../../tests/e2e/helpers/fake-agent-command-override'
|
||||
import {
|
||||
verifyArmProvenance,
|
||||
type ArmProvenance,
|
||||
type ArmSpec
|
||||
} from './native-chat-4a-arm-provenance'
|
||||
import { installVisibilityProbe } from './native-chat-4a-probes'
|
||||
|
||||
const HARNESS_ROOT = path.resolve(import.meta.dirname, '../../../..')
|
||||
@@ -59,6 +64,7 @@ export type Arm = {
|
||||
name: string
|
||||
root: string
|
||||
sha: string
|
||||
provenance: ArmProvenance
|
||||
electronPath: string
|
||||
protocol: ArmProtocol
|
||||
}
|
||||
@@ -75,14 +81,11 @@ export type Launch = {
|
||||
readyEpochMs: number
|
||||
}
|
||||
|
||||
const git = (root: string, ...args: string[]): string =>
|
||||
execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim()
|
||||
|
||||
export async function resolveArm(name: string, root: string, scratch: string): Promise<Arm> {
|
||||
const absolute = path.resolve(root)
|
||||
if (!existsSync(path.join(absolute, 'out', 'main', 'index.js'))) {
|
||||
throw new Error(`${name}: ${absolute} has no out/main/index.js; build that checkout first`)
|
||||
}
|
||||
/** A pinned, clean checkout whose build output names its HEAD; anything else is refused before any launch. */
|
||||
export async function resolveArm(spec: ArmSpec, scratch: string): Promise<Arm> {
|
||||
const { name } = spec
|
||||
const absolute = path.resolve(spec.root)
|
||||
const provenance = verifyArmProvenance({ ...spec, root: absolute })
|
||||
const electronPath: unknown = createRequire(path.join(absolute, 'package.json'))('electron')
|
||||
if (typeof electronPath !== 'string') {
|
||||
throw new Error(`${name}: the electron package in ${absolute} names no executable`)
|
||||
@@ -99,7 +102,7 @@ export async function resolveArm(name: string, root: string, scratch: string): P
|
||||
]
|
||||
]
|
||||
)
|
||||
return { name, root: absolute, sha: git(absolute, 'rev-parse', 'HEAD'), electronPath, protocol }
|
||||
return { name, root: absolute, sha: provenance.sha, provenance, electronPath, protocol }
|
||||
}
|
||||
|
||||
/** Compile `exports` ([names, checkout-relative file]) from a checkout's own source and load them. */
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
// The 4a "no slower than main" comparison: one profile seeded once, every arm measured on its own copy, one JSON + one table.
|
||||
// Dry run (no app): pnpm exec tsx config/scripts/performance/native-chat/native-chat-4a-comparison.ts --profile full --dry-run
|
||||
// Full run: ... --arm main=<built main checkout> --arm candidate=<built candidate checkout> --profile full --out <dir> [--large-source]
|
||||
// Re-table: ... --out <dir> --report-only
|
||||
// Full run: ... --arm main=<built main checkout>@<full sha> --arm candidate=<built candidate checkout>@<full sha> --profile full --out <dir> [--large-source]
|
||||
// Re-table: ... --out <dir> --report-only [--arm main=@<full sha> --arm candidate=@<full sha>]
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
@@ -14,6 +14,7 @@ import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { parseArgs } from 'node:util'
|
||||
import type { Arm } from './native-chat-4a-arm'
|
||||
import { assertOneRun, parseArmSpec, seededBy } from './native-chat-4a-arm-provenance'
|
||||
import type { Manifest } from './native-chat-4a-fixture'
|
||||
import type { RunContext } from './native-chat-4a-run-context'
|
||||
import { armMarkdown, writeComparison, type ArmResult, type Row } from './native-chat-4a-report'
|
||||
@@ -194,6 +195,7 @@ async function runArm(input: {
|
||||
return {
|
||||
arm: input.arm.name,
|
||||
sha: input.arm.sha,
|
||||
provenance: input.arm.provenance,
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
rows,
|
||||
@@ -202,6 +204,8 @@ async function runArm(input: {
|
||||
worktreeIds: input.manifest.workspace.worktreeIds.join(' '),
|
||||
seedArm: input.manifest.seedArm,
|
||||
seedSha: input.manifest.seedSha,
|
||||
recipe: input.manifest.recipe,
|
||||
largeSourceRecipe: input.largeSource?.manifest.recipe ?? 'none',
|
||||
largeItems: input.manifest.largeItems,
|
||||
storageBytes: input.manifest.storageBytes,
|
||||
seedMs: input.manifest.seedMs
|
||||
@@ -221,6 +225,7 @@ async function main(): Promise<void> {
|
||||
.filter((file) => /^arm-.*\.json$/.test(file))
|
||||
.map((file): ArmResult => JSON.parse(readFileSync(path.join(out, file), 'utf8')))
|
||||
.sort((a, b) => a.startedAt.localeCompare(b.startedAt))
|
||||
assertOneRun(results, values.arm.map(parseArmSpec))
|
||||
writeComparison(out, workload, results, log)
|
||||
return
|
||||
}
|
||||
@@ -237,15 +242,16 @@ async function main(): Promise<void> {
|
||||
const scratch = path.join(out, 'scratch')
|
||||
mkdirSync(scratch, { recursive: true })
|
||||
const arms: Arm[] = []
|
||||
for (const spec of values.arm) {
|
||||
const [name, root] = spec.split('=')
|
||||
if (!name || !root) {
|
||||
throw new Error(`--arm takes name=checkout, got ${spec}`)
|
||||
for (const spec of values.arm.map(parseArmSpec)) {
|
||||
if (!spec.root) {
|
||||
throw new Error(`--arm ${spec.name} names no checkout`)
|
||||
}
|
||||
arms.push(await resolveArm(name, root, scratch))
|
||||
arms.push(await resolveArm(spec, scratch))
|
||||
}
|
||||
if (arms.length === 0) {
|
||||
throw new Error('Pass at least one --arm name=<built checkout>; main first, then the candidate')
|
||||
throw new Error(
|
||||
'Pass at least one --arm name=<built checkout>@<full sha>; main first, then the candidate'
|
||||
)
|
||||
}
|
||||
const stubCommand = writeProviderStub(path.join(out, 'stub'))
|
||||
const repoFile = path.join(out, 'repo-path.txt')
|
||||
@@ -265,13 +271,16 @@ async function main(): Promise<void> {
|
||||
toolOutputBytes,
|
||||
log
|
||||
})
|
||||
const manifest = await seed(workload.history.toolOutputBytes, workload)
|
||||
const manifest = seededBy(seedArm, await seed(workload.history.toolOutputBytes, workload))
|
||||
const large =
|
||||
values['large-source'] && workload.conversionSourceBytes > 0
|
||||
? await seed(conversionToolOutputBytes(workload), {
|
||||
...workload,
|
||||
history: { ...workload.history, fillerChats: 5, mediumTurns: 0 }
|
||||
})
|
||||
? seededBy(
|
||||
seedArm,
|
||||
await seed(conversionToolOutputBytes(workload), {
|
||||
...workload,
|
||||
history: { ...workload.history, fillerChats: 5, mediumTurns: 0 }
|
||||
})
|
||||
)
|
||||
: null
|
||||
const results: ArmResult[] = []
|
||||
for (const arm of arms) {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's head SHA.
|
||||
// 4a rows, percentiles and the PR-body table: main vs candidate per row, keyed by each arm's pinned full SHA.
|
||||
import { writeFileSync } from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
import type { ArmProvenance } from './native-chat-4a-arm-provenance'
|
||||
import type { HostWork } from './native-chat-4a-probes'
|
||||
import type { Workload } from './native-chat-4a-workloads'
|
||||
|
||||
@@ -32,6 +33,7 @@ export type Row = {
|
||||
export type ArmResult = {
|
||||
arm: string
|
||||
sha: string
|
||||
provenance: ArmProvenance
|
||||
startedAt: string
|
||||
finishedAt: string
|
||||
rows: Row[]
|
||||
@@ -90,6 +92,14 @@ function delta(base: Stat | null, next: Stat | null): string {
|
||||
return `${change >= 0 ? '+' : ''}${ms(change)}${percent}`
|
||||
}
|
||||
|
||||
/** One line per arm: the full pinned SHA and the build output that proved it. */
|
||||
const provenanceLine = ({ arm, provenance: p }: ArmResult): string =>
|
||||
`- ${arm}: \`${p.sha}\` (pinned \`${p.expectedSha}\`, committed ${p.headCommittedAt}); clean product sources; build stamp \`${p.builtCommit}\` in ${p.stampFiles}; out/main/index.js sha256 \`${p.mainBundleSha256}\`; built ${p.artifactBuiltAt}, after the newest source ${p.newestSourceFile} (${p.newestSourceModifiedAt}).`
|
||||
|
||||
// No product hook or timing field exposes it, and SQLite state records no transaction start, so it is disclosed, not derived.
|
||||
const WRITER_SPAN_DISCLOSURE =
|
||||
'Not measured: writer BEGIN → COMMIT and grouped-transaction spans. Neither arm exposes them without a product hook, and on-disk SQLite state records commits but not when a transaction began; host spans above include storage but are not transaction durations.'
|
||||
|
||||
const caseCell = (row: Row): string =>
|
||||
row.endToEndOnly ? `${row.variant} (end-to-end only: ${row.endToEndOnly})` : row.variant
|
||||
|
||||
@@ -113,6 +123,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
|
||||
const lines = [
|
||||
`4a comparison: ${base.arm} \`${base.sha.slice(0, 11)}\` vs ${next.arm} \`${next.sha.slice(0, 11)}\` (p50 / p95 / max ms, nearest rank).`,
|
||||
'',
|
||||
provenanceLine(base),
|
||||
provenanceLine(next),
|
||||
'',
|
||||
`| Family | Operation | Case | ${base.arm} | ${next.arm} | Δ p95 | Host span p95 ${base.arm} → ${next.arm} | Off-main CPU p95 ${base.arm} → ${next.arm} | Shape |`,
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |'
|
||||
]
|
||||
@@ -134,7 +147,9 @@ export function comparisonMarkdown(base: ArmResult, next: ArmResult): string {
|
||||
}
|
||||
lines.push(
|
||||
'',
|
||||
'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.'
|
||||
'Primary latency is action → visible result where the row has a rendered end point, else the RPC host span. Off-main CPU is process CPU minus main-thread event-loop active time (storage workers). Conversion rows are one-time import cost, reported separately from steady state; steady-state families start only after every chat is seen ready. Rows whose Shape names `observedDatabase` end at an outside read-only SQLite observation and carry its resolution. Per-row spans, WAL/file growth, event-loop lag and peak memory are in the JSON.',
|
||||
'',
|
||||
WRITER_SPAN_DISCLOSURE
|
||||
)
|
||||
return `${lines.join('\n')}\n`
|
||||
}
|
||||
@@ -143,6 +158,8 @@ export function armMarkdown(result: ArmResult): string {
|
||||
const lines = [
|
||||
`4a ${result.arm} \`${result.sha.slice(0, 11)}\` (p50 / p95 / max ms).`,
|
||||
'',
|
||||
provenanceLine(result),
|
||||
'',
|
||||
'| Family | Operation | Case | Primary | Host span | Shape |',
|
||||
'| --- | --- | --- | --- | --- | --- |'
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user