ci: refresh unit shard timings from five current 12-shard runs (held-out spread 10.2% → 3.8%) (#37393)

* ci: refresh unit shard timings from a 12-shard four-worker run

Replaces the October 25 five-shard eight-worker baseline with run 41403569437,
which includes the November 1 test-time fixes. On an independent run's
measurements the projected per-shard spread falls from 12.0% to 7.4%.

* ci: average unit shard timings over five runs

A single run's per-file noise left this PR's own run 21.8% apart. The importer
now accepts several complete runs and averages each file; held out, the
averaged baseline projects 3.8% against 10.2% for the October 25 baseline.

config/scripts/ci-shard-timings.md

@@ -46,7 +46,36 @@ age as specs change. Full CI runs on the existing runner classes are required to
 measure elapsed-time and occupancy improvements, including discovery overhead.
 No retries, assertions, coverage exclusions, runner classes or shard counts changed.
 
-## Current unit refresh: October 25, 2026
+## Current unit refresh: November 2, 2026
+
+The baseline averages each file over five complete, successful twelve-shard full runs on
+`blacksmith-4vcpu-ubuntu-2404` with four workers, Node 24.22.2 and Bun 1.4.4:
+[41403569437](https://github.com/stablyai/orca/actions/runs/41403569437),
+[41406098648](https://github.com/stablyai/orca/actions/runs/41406098648),
+[41406030210](https://github.com/stablyai/orca/actions/runs/41406030210),
+[41405874918](https://github.com/stablyai/orca/actions/runs/41405874918) and
+[41405380104](https://github.com/stablyai/orca/actions/runs/41405380104), all attempt 1 and
+all after the November 1 test-time fixes. The October 25 baseline came from five eight-worker
+shards, lacked 1,595 measured files and kept 374 deleted ones.
+
+One run is not enough: per-file durations vary between runs, and a baseline taken from
+41403569437 alone left PR run 41406120225 at 1,244–1,515 worker-seconds (21.8%). Assigning the
+held-out run 41406120225 with the averaged baseline and evaluating it with that run's own
+measurements projects 1,336–1,387 worker-seconds per shard (3.8%), against 1,297–1,430
+(10.2%) for the October 25 baseline. Leaving out each of six post-fix runs in turn, the
+average of the other five projects 1.9–8.6% spread, against 8.8–21.2% for the October 25
+baseline. Runs from before the fixes are not comparable, because their `pty-subprocess-*`
+files still carried 5-second waits.
+
+```sh
+node config/scripts/ci-unit-timing-import.mjs RUN_DIR... config/scripts/ci-shard-timings.json
+```
+
+Each directory holds one run's twelve `unit-shard-node-*` artifacts. Every run is validated
+as before, the runs must be distinct and share one runtime version, and a single directory
+keeps the previous single-run output.
+
+## Historical unit refresh: October 25, 2026
 
 The baseline imports all five successful unit shards from
 [run 40276310369, attempt 1](https://github.com/stablyai/orca/actions/runs/40276310369/attempts/1),

config/scripts/ci-unit-timing-import.mjs

@@ -59,15 +59,56 @@ export function importUnitTimingReports(reports) {
   }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
-  const [directory, output] = process.argv.slice(2)
-  if (!directory || !output) {
-    throw new Error('Usage: ci-unit-timing-import.mjs ARTIFACT_DIRECTORY BASELINE_JSON')
+/** Averages complete runs per file; one run's per-file noise otherwise skews whole shards. */
+export function importUnitTimingRuns(runs) {
+  const imported = runs.map(importUnitTimingReports)
+  if (imported.length === 1) {
+    return imported[0]
   }
-  const reports = readdirSync(directory, { recursive: true })
-    .filter((file) => file.endsWith('unit-timings.json'))
-    .map((file) => JSON.parse(readFileSync(join(directory, file), 'utf8')))
+  const runtime = (run) => JSON.stringify([run.nodeVersion, run.bunVersion, run.metric])
+  if (
+    new Set(imported.map(runtime)).size !== 1 ||
+    new Set(imported.map((run) => run.runId)).size !== imported.length
+  ) {
+    throw new Error('Unit timing runs must be distinct runs on one runtime version')
+  }
+  const samples = {}
+  for (const run of imported) {
+    for (const [file, duration] of Object.entries(run.timings)) {
+      ;(samples[file] ??= []).push(duration)
+    }
+  }
+  const [first] = imported
+  return {
+    runIds: imported.map((run) => run.runId),
+    sourceShas: imported.map((run) => run.sourceSha),
+    nodeVersion: first.nodeVersion,
+    ...(first.bunVersion ? { bunVersion: first.bunVersion } : {}),
+    metric: first.metric,
+    overheadMs: 0,
+    timings: Object.fromEntries(
+      Object.entries(samples)
+        .sort(([a], [b]) => compareIds(a, b))
+        .map(([file, durations]) => [
+          file,
+          Math.round(durations.reduce((sum, value) => sum + value, 0) / durations.length)
+        ])
+    )
+  }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const directories = process.argv.slice(2, -1)
+  const output = process.argv.at(-1)
+  if (!directories.length || !output) {
+    throw new Error('Usage: ci-unit-timing-import.mjs ARTIFACT_DIRECTORY... BASELINE_JSON')
+  }
+  const runs = directories.map((directory) =>
+    readdirSync(directory, { recursive: true })
+      .filter((file) => file.endsWith('unit-timings.json'))
+      .map((file) => JSON.parse(readFileSync(join(directory, file), 'utf8')))
+  )
   const baseline = JSON.parse(readFileSync(output, 'utf8'))
-  baseline.unit = importUnitTimingReports(reports)
+  baseline.unit = importUnitTimingRuns(runs)
   writeFileSync(output, `${JSON.stringify(baseline, null, 2)}\n`)
 }

config/scripts/ci-unit-timing-reporter.test.mjs

@@ -2,7 +2,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join, resolve } from 'node:path'
 import { describe, expect, it, vi } from 'vitest'
-import { importUnitTimingReports } from './ci-unit-timing-import.mjs'
+import { importUnitTimingReports, importUnitTimingRuns } from './ci-unit-timing-import.mjs'
 import UnitTimingReporter, {
   buildUnitPhaseReport,
   moduleDuration,
@@ -94,6 +94,23 @@ describe('unit worker timing evidence', () => {
     })
   })
 
+  it('averages each file over several complete runs and rejects repeated or mixed runs', () => {
+    const run = (runId, a, b) => [
+      { ...report(1, { a }), runId },
+      { ...report(2, { b }), runId }
+    ]
+    expect(importUnitTimingRuns([run('r1', 100, 300), run('r2', 200, 500)])).toMatchObject({
+      runIds: ['r1', 'r2'],
+      overheadMs: 0,
+      timings: { a: 150, b: 400 }
+    })
+    expect(() => importUnitTimingRuns([run('r1', 100, 300), run('r1', 100, 300)])).toThrow(
+      'distinct'
+    )
+    const otherNode = run('r2', 200, 500).map((entry) => ({ ...entry, nodeVersion: '26.6.0' }))
+    expect(() => importUnitTimingRuns([run('r1', 100, 300), otherNode])).toThrow('distinct')
+  })
+
   it('preserves Bun provenance and rejects mixing Bun and Node timing sets', () => {
     const first = { ...report(1, { a: 100 }), bunVersion: '1.4.2' }
     const second = { ...report(2, { b: 200 }), bunVersion: '1.4.2' }