mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 16:02:32 +00:00
Jinwoo Hong
a634bf9b49
* test(bench): count runtime-graph publications from main The build-provided `__orcaBenchmarkInstrumentation` is gone from the tree, so the typing bench could no longer report graph-publication counts at all. The renderer cannot supply them either: `window.api` is frozen by contextBridge, so `runtime.syncWindowGraph` is not wrappable. Count them where they land instead — main's `runtime:syncWindowGraph` invoke handler — behind ORCA_TYPING_BENCH_GRAPH_PROBE=1, and record the result in the bench report. Measured on an 870-worktree fixture: 21 publications over a 50 s metadata-only window versus ~1,205 with recurring OSC title/status traffic. The long-task fields ship unproven: an injected 250 ms renderer busy-wait produced zero entries even though `longtask` is in `supportedEntryTypes`, so their zeros mean "oracle unverified", not "no long task". The self-test knob exists to make that falsifiable, and the file says so; per-publication build time still needs a separate --cpu-profile run. * test(bench): optional CDP CPU throttle around the typing window * test(bench): report the throttle that ran and the long task the self-test caused Two ways the bench could misreport its own conditions. `cpuThrottleRate` was the requested rate, written into every report, but only two of the three scenarios wrapped their typing window in the throttle — a `--cpu-throttle 4` visible-split run claimed a 4x throttle it never applied. Recording the rate per scenario would have made the report honest; it would also have left one scenario silently ignoring the flag, and a fourth scenario would inherit the same omission. So both: every scenario now goes through one `measureTypingWindow` helper, and the value it returns is the rate the throttle actually applied. `writeBenchReport` takes that composite instead of a bare measurement, so a scenario cannot produce a report without saying what it ran under. Unthrottled runs are unchanged — rate 1 still opens no CDP session. `selfTestLongTaskMs` took the *earliest* long task starting before a cutoff captured after the busy-wait. The observer has been live since probe start, so any unrelated long task from fixture setup satisfied it — the field whose whole job is to prove the oracle is live was the easiest one to fake. The busy-wait now reports its own renderer-clock bounds and the matching entry is the one containing their midpoint: main-thread tasks never overlap, so at most one can, and it is the task the busy-wait ran in. That entry is then withheld from `longTasks`, `longestLongTasks`, and `longTasksAroundPublication`, which had been counting the oracle's injected 250 ms as workload. A zero still means "oracle unproven" — it now also means it honestly. * test(bench): stop the graph probe when the typing run throws * test(e2e): drain queued long-task records before the probe disconnects
58 lines
2.4 KiB
TypeScript
58 lines
2.4 KiB
TypeScript
import { describe, expect, it } from 'vitest'
|
|
import { partitionSelfTestLongTask, type LongTaskSample } from './runtime-graph-publication-probe'
|
|
|
|
// The observer is live from probe start, so setup work lands in the array before
|
|
// the injected busy-wait. Index 0 is the decoy a start-time cutoff would accept.
|
|
const setupTask: LongTaskSample = { startEpochMs: 1_000, durationMs: 120 }
|
|
const selfTestTask: LongTaskSample = { startEpochMs: 5_000, durationMs: 260 }
|
|
const workloadTask: LongTaskSample = { startEpochMs: 9_000, durationMs: 80 }
|
|
|
|
const selfTestWindow = { startEpochMs: 5_005, endEpochMs: 5_255 }
|
|
|
|
describe('partitionSelfTestLongTask', () => {
|
|
it('attributes the entry the busy-wait ran in, not an earlier one', () => {
|
|
const result = partitionSelfTestLongTask(
|
|
[setupTask, selfTestTask, workloadTask],
|
|
selfTestWindow
|
|
)
|
|
expect(result.selfTestLongTaskMs).toBe(260)
|
|
expect(result.workloadLongTasks).toEqual([setupTask, workloadTask])
|
|
})
|
|
|
|
it('leaves the oracle unproven when only unrelated tasks were observed', () => {
|
|
const result = partitionSelfTestLongTask([setupTask, workloadTask], selfTestWindow)
|
|
expect(result.selfTestLongTaskMs).toBe(0)
|
|
expect(result.workloadLongTasks).toEqual([setupTask, workloadTask])
|
|
})
|
|
|
|
it('leaves the oracle unproven when a task ends before the busy-wait starts', () => {
|
|
// Touches the window's lower edge but does not reach its midpoint.
|
|
const result = partitionSelfTestLongTask([{ startEpochMs: 4_900, durationMs: 110 }], {
|
|
startEpochMs: 5_000,
|
|
endEpochMs: 5_250
|
|
})
|
|
expect(result.selfTestLongTaskMs).toBe(0)
|
|
})
|
|
|
|
it('keeps every task when no self-test ran', () => {
|
|
const result = partitionSelfTestLongTask([setupTask, workloadTask], null)
|
|
expect(result.selfTestLongTaskMs).toBe(0)
|
|
expect(result.workloadLongTasks).toEqual([setupTask, workloadTask])
|
|
})
|
|
|
|
it('removes only the matched entry when another has identical values', () => {
|
|
const twin: LongTaskSample = { ...selfTestTask }
|
|
const result = partitionSelfTestLongTask([selfTestTask, twin], selfTestWindow)
|
|
expect(result.workloadLongTasks).toHaveLength(1)
|
|
expect(result.workloadLongTasks[0]).toBe(twin)
|
|
})
|
|
|
|
it('rounds the reported duration to one decimal', () => {
|
|
const result = partitionSelfTestLongTask(
|
|
[{ startEpochMs: 5_000, durationMs: 251.2649 }],
|
|
selfTestWindow
|
|
)
|
|
expect(result.selfTestLongTaskMs).toBe(251.3)
|
|
})
|
|
})
|