feat(terminal): weight-layer forensics for the bold-collapse bug (STA-4042)

Field instrumentation to name the writer behind regular-text-renders-bold:
- metric-weight-change crumbs at the writePaneMetricOptions funnel
  (prev/next/reason; weights never change in normal operation)
- terminal-weight-parity-mismatch audit on every visibility resume
- sentinel weightProbe capture fields: live options vs atlas captured
  config vs renderer-buffer bold census
- Cmd/Ctrl+Shift+click unconditional capture (no divergence gate, no
  recovery) for states the missing-ink detector cannot see
- patched addon-webgl ctx.font readback probe: detects failed font
  assignments that rasterize glyphs at a stale weight
This commit is contained in:
Brennan Benson
2026-08-27 14:22:56 -07:00
parent f400f8fd5f
commit e59140da7b
12 changed files with 840 additions and 191 deletions
File diff suppressed because one or more lines are too long
+3 -3
View File
@@ -18,7 +18,7 @@ patchedDependencies:
hash: af3b156143ed2ee9903b753145b63dd4a2ee72cadd7ef333ab0f4d5d3ebfc32c
path: config/patches/@xterm__addon-serialize@0.15.0-beta.287.patch
'@xterm/addon-webgl@0.20.0-beta.286':
hash: 6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258
hash: d4b9fdfc15ae1bddc7c3228af2e97339c9285ed0ddabe7e575f3b18444ef81b8
path: config/patches/@xterm__addon-webgl@0.20.0-beta.286.patch
'@xterm/xterm@6.1.0-beta.287':
hash: 46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7
@@ -229,7 +229,7 @@ importers:
version: 0.13.0-beta.287(@xterm/xterm@6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7))
'@xterm/addon-webgl':
specifier: 0.20.0-beta.286
version: 0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7))
version: 0.20.0-beta.286(patch_hash=d4b9fdfc15ae1bddc7c3228af2e97339c9285ed0ddabe7e575f3b18444ef81b8)(@xterm/xterm@6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7))
'@xterm/xterm':
specifier: 6.1.0-beta.287
version: 6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7)
@@ -9638,7 +9638,7 @@ snapshots:
dependencies:
'@xterm/xterm': 6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7)
'@xterm/addon-webgl@0.20.0-beta.286(patch_hash=6da7d7770b6427246f2a0d057d97da418040e498068b41d0c2d3c6b20bf49258)(@xterm/xterm@6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7))':
'@xterm/addon-webgl@0.20.0-beta.286(patch_hash=d4b9fdfc15ae1bddc7c3228af2e97339c9285ed0ddabe7e575f3b18444ef81b8)(@xterm/xterm@6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7))':
dependencies:
'@xterm/xterm': 6.1.0-beta.287(patch_hash=46796c152f3b73e28238f44499eaf5a867a863809bc7b470b159526a41e354f7)
@@ -9,9 +9,10 @@ import {
} from '../../../../shared/pty-delivery-diagnostics'
import {
TERMINAL_WEBGL_DIAGNOSTIC_BREADCRUMB,
recordTerminalWebglDiagnostic,
setTerminalWebglDiagnosticRecorder
} from '../../../../shared/terminal-webgl-diagnostics'
import { maybeStartTerminalRenderDesyncSentinel } from './terminal-render-desync-sentinel'
import { maybeStartTerminalRenderDesyncSentinel } from './terminal-render-desync-trigger'
const rendererDeliveryBreadcrumbs = createPtyDeliveryBreadcrumbRing()
@@ -44,6 +45,20 @@ setTerminalWebglDiagnosticRecorder((kind, detail) => {
// of any specific pane mounting first. No-op unless its localStorage flag is set.
maybeStartTerminalRenderDesyncSentinel()
// Sink for the patched @xterm/addon-webgl atlas font probe: the atlas cannot
// import Orca code, so it reports failed ctx.font assignments (the stuck-
// rasterizer arm of the bold-collapse family) through this global. Crumbs are
// coalesced upstream, so a rasterization storm cannot flood the report.
type AtlasFontProbeMismatch = { desired?: string; actual?: string }
;(globalThis as { __orcaAtlasFontProbe?: (mismatch: AtlasFontProbeMismatch) => void })[
'__orcaAtlasFontProbe'
] = (mismatch) => {
recordTerminalWebglDiagnostic('atlas-font-probe-mismatch', {
desired: mismatch?.desired ?? null,
actual: mismatch?.actual ?? null
})
}
export function getTerminalFreezeBreadcrumbs(): PtyDeliveryBreadcrumb[] {
return rendererDeliveryBreadcrumbs.snapshot()
}
@@ -0,0 +1,66 @@
import type { SentinelEvidence } from './terminal-render-desync-sentinel'
/**
* Durable persistence for render-desync captures, split from the sentinel so
* the detection loop, trigger gestures and storage lifecycles stay separately
* readable. Failure contract: a failed write must leave the live pane intact —
* recovering after a failed write would destroy the only evidence without
* producing a durable capture, so callers gate recovery on the returned
* directory.
*/
export async function persistCorruptEvidence(entry: SentinelEvidence): Promise<string | null> {
const pngDataUrl = entry.livePngDataUrl
const bufferText = entry.bufferText
try {
if (!pngDataUrl || bufferText == null) {
throw new Error('Render-desync evidence payload was released before persistence')
}
const persisted = await window.api.app.writeTerminalRenderDesyncEvidence({
captureId: entry.captureId,
phase: 'corrupt',
pngDataUrl,
metadata: {
paneKey: entry.paneKey,
when: entry.when,
divergence: entry.divergence,
paused: entry.paused,
trigger: entry.trigger,
rendererState: entry.rendererState,
weightProbe: entry.weightProbe,
bufferText
}
})
entry.persistedDirectory = persisted.directory
return persisted.directory
} catch (error) {
console.error('[terminal] could not persist render-desync evidence; leaving pane intact', error)
return null
} finally {
// Why: persistence owns a successful payload, while a failed write leaves
// the live pane intact; neither path should retain duplicate full-canvas data.
entry.livePngDataUrl = undefined
entry.bufferText = undefined
}
}
export async function persistHealedReference(
captureId: string,
canvas: HTMLCanvasElement
): Promise<void> {
try {
await window.api.app.writeTerminalRenderDesyncEvidence({
captureId,
phase: 'healed',
pngDataUrl: canvas.toDataURL(),
metadata: { when: Date.now() }
})
} catch (error) {
console.error('[terminal] could not persist healed render reference', error)
}
}
export function createCaptureId(paneKey: string): string {
const panePart = paneKey.replace(/[^a-zA-Z0-9_-]/g, '-')
const nonce = globalThis.crypto?.randomUUID?.() ?? Math.random().toString(36).slice(2)
return `${Date.now()}-${panePart}-${nonce}`
}
@@ -25,11 +25,14 @@ vi.mock('../../../../shared/terminal-webgl-diagnostics', () => ({
import {
getRenderDesyncEvidence,
maybeStartTerminalRenderDesyncSentinel,
RENDER_DESYNC_SENTINEL_FLAG,
sampleRenderDesyncOnce,
stopTerminalRenderDesyncSentinelForTesting
} from './terminal-render-desync-sentinel'
import {
maybeStartTerminalRenderDesyncSentinel,
RENDER_DESYNC_SENTINEL_FLAG,
stopTerminalRenderDesyncTriggerForTesting
} from './terminal-render-desync-trigger'
function fakePane(overrides: { paused?: boolean } = {}) {
const refreshRows = vi.fn()
@@ -94,6 +97,7 @@ describe('terminal-render-desync-sentinel', () => {
})
afterEach(() => {
stopTerminalRenderDesyncSentinelForTesting()
stopTerminalRenderDesyncTriggerForTesting()
vi.unstubAllGlobals()
})
@@ -123,7 +127,14 @@ describe('terminal-render-desync-sentinel', () => {
expect(writeTerminalRenderDesyncEvidence).toHaveBeenCalledWith(
expect.objectContaining({
phase: 'corrupt',
metadata: expect.objectContaining({ bufferText: expect.stringContaining('x') })
metadata: expect.objectContaining({
bufferText: expect.stringContaining('x'),
trigger: 'divergence',
weightProbe: expect.objectContaining({
totalTextCells: expect.any(Number),
boldTextCells: expect.any(Number)
})
})
})
)
expect(getRenderDesyncEvidence()).toHaveLength(1)
@@ -168,9 +179,11 @@ describe('terminal-render-desync-sentinel', () => {
sampleWith(divergenceOf(manyCells(0)), false, `m1:p${pane}`)
}
await vi.waitFor(() => expect(writeTerminalRenderDesyncEvidence).toHaveBeenCalledTimes(4))
// 4 capture recoveries + 1 budget-exhausted recovery; waiting on the reset
// count (not the write count) keeps this robust to persist microtask depth.
await vi.waitFor(() => expect(resetAndRefreshAllTerminalWebglAtlases).toHaveBeenCalledTimes(5))
expect(writeTerminalRenderDesyncEvidence).toHaveBeenCalledTimes(4)
expect(getRenderDesyncEvidence()).toHaveLength(4)
expect(resetAndRefreshAllTerminalWebglAtlases).toHaveBeenCalledTimes(5)
})
it('resets tracking for paused panes instead of sampling them', () => {
@@ -215,4 +228,36 @@ describe('terminal-render-desync-sentinel', () => {
vi.useRealTimers()
}
})
it('shift-modifier-click captures immediately and leaves the pane unrecovered', async () => {
const storage = new Map<string, string>([[RENDER_DESYNC_SENTINEL_FLAG, '1']])
vi.stubGlobal('localStorage', {
getItem: (k: string) => storage.get(k) ?? null,
setItem: (k: string, v: string) => storage.set(k, v)
})
maybeStartTerminalRenderDesyncSentinel()
const { pane } = fakePane()
;(
pane.terminal as { element: { contains: ReturnType<typeof vi.fn> } }
).element.contains.mockReturnValue(true)
forEachLivePaneForDesyncSentinel.mockImplementation(
(visit: (key: string, pane: unknown) => void) => visit('m1:p1', pane)
)
const listener = documentAddEventListener.mock.calls.at(-1)?.[1]
listener({ button: 0, metaKey: true, ctrlKey: true, shiftKey: true, target: new FakeNode() })
await vi.waitFor(() => expect(writeTerminalRenderDesyncEvidence).toHaveBeenCalledTimes(1))
expect(writeTerminalRenderDesyncEvidence).toHaveBeenCalledWith(
expect.objectContaining({
phase: 'corrupt',
metadata: expect.objectContaining({ trigger: 'manual' })
})
)
expect(recordTerminalWebglDiagnostic).toHaveBeenCalledWith(
'webgl-render-desync-manual-capture',
expect.objectContaining({ paneKey: 'm1:p1' })
)
// The captured state must stay on screen for further pokes: no recovery.
expect(resetAndRefreshAllTerminalWebglAtlases).not.toHaveBeenCalled()
})
})
@@ -3,6 +3,11 @@ import {
forEachLivePaneForDesyncSentinel,
resetAndRefreshAllTerminalWebglAtlases
} from '@/lib/pane-manager/pane-manager-registry'
import {
createCaptureId,
persistCorruptEvidence,
persistHealedReference
} from './terminal-render-desync-evidence-persistence'
import {
activeBuffer,
bufferSnapshot,
@@ -13,6 +18,10 @@ import {
type SentinelRendererState,
type SentinelRenderInternals
} from './terminal-render-desync-frame'
import {
readSentinelWeightProbe,
type SentinelWeightProbe
} from './terminal-render-desync-weight-probe'
/**
* Flag-gated render-desync sentinel for WebGL terminal panes.
@@ -25,12 +34,13 @@ import {
* failure. A confirmed trip writes the pixels, buffer, and atlas/model versions
* to local app data before running the shared-atlas recovery.
*
* Off by default; enabled via localStorage so a production build can arm it
* from DevTools without a settings-schema change:
* localStorage.setItem('orca:render-desync-sentinel', '1') // then reload
* Cmd/Ctrl+Shift+click (see terminal-render-desync-trigger.ts) captures the
* clicked pane immediately and unconditionally — no divergence gate and no
* recovery afterwards — for states the ink detector cannot see, e.g. the
* bold-collapse bug (STA-4042 family) where every cell has ink but regular
* text rasterized at the bold weight.
*/
export const RENDER_DESYNC_SENTINEL_FLAG = 'orca:render-desync-sentinel'
const SAMPLE_INTERVAL_MS = 250
const SAMPLE_BURST_MS = 10_000
// A real desync is pinned to fixed screen cells; scroll/frame lag moves around.
@@ -44,9 +54,11 @@ export type SentinelEvidence = {
captureId: string
paneKey: string
when: number
trigger: 'divergence' | 'manual'
divergence: { textCells: number; missing: number; missPct: number }
paused: boolean
rendererState: SentinelRendererState
weightProbe: SentinelWeightProbe
livePngDataUrl?: string
bufferText?: string
persistedDirectory?: string
@@ -63,7 +75,6 @@ const healedCaptureTimeoutIds = new Set<ReturnType<typeof setTimeout>>()
const evidence: SentinelEvidence[] = []
let burstIntervalId: ReturnType<typeof setInterval> | null = null
let burstTimeoutId: ReturnType<typeof setTimeout> | null = null
let clickListener: ((event: MouseEvent) => void) | null = null
let burstTerminal: unknown = null
export function getRenderDesyncEvidence(): SentinelEvidence[] {
@@ -132,73 +143,115 @@ export function sampleRenderDesyncOnce(
stopSampleBurst()
return
}
pendingPaneKeys.add(paneKey)
const entry: SentinelEvidence = {
captureId: createCaptureId(paneKey),
paneKey,
when: Date.now(),
divergence: {
textCells: divergence.textCells,
missing: divergence.missing,
missPct: divergence.missPct
},
paused: internals.isPaused,
rendererState: internals.rendererState,
livePngDataUrl: internals.canvas.toDataURL(),
bufferText: bufferSnapshot(buffer, internals.rows)
}
evidence.push(entry)
const entry = buildEvidenceEntry(paneKey, terminal, internals, divergence, 'divergence')
console.warn(
`[terminal] render desync detected on pane ${paneKey} ` +
`(${divergence.missing}/${divergence.textCells} cells, ${divergence.missPct.toFixed(1)}%) — persisting evidence`
)
void persistEvidenceThenRecover(entry, internals)
void persistEntry(entry, internals, { recover: true })
})
}
export function maybeStartTerminalRenderDesyncSentinel(): void {
if (clickListener != null) {
/**
* Unconditional capture of one pane, bypassing the divergence detector. No
* recovery afterwards: the caller wants the broken state left on screen to
* poke at, and recovery would also destroy any not-yet-understood layer state.
*/
export function captureRenderDesyncNow(paneKey: string, pane: unknown): void {
const terminal = (pane as SentinelPane).terminal
if (pendingPaneKeys.has(paneKey) || evidence.length >= MAX_EVIDENCE_ENTRIES) {
console.warn(`[terminal] manual desync capture skipped for ${paneKey}: budget or in flight`)
return
}
let enabled = false
const internals = reachRenderInternals(terminal)
if (!internals) {
console.warn(`[terminal] manual desync capture failed for ${paneKey}: no renderer internals`)
return
}
// Why tolerant: the manual gesture must produce a capture even when the
// canvas readback path fails — the PNG and probe fields are the payload.
let measured: ReturnType<typeof measureDivergence> = null
try {
enabled = globalThis.localStorage?.getItem(RENDER_DESYNC_SENTINEL_FLAG) === '1'
const buffer = activeBuffer(terminal)
measured = buffer && measureDivergence(internals, buffer)
} catch {
enabled = false
measured = null
}
if (!enabled) {
const divergence = measured ?? {
textCells: 0,
missing: 0,
missPct: 0,
missingCells: new Set<number>()
}
const entry = buildEvidenceEntry(paneKey, terminal, internals, divergence, 'manual')
recordTerminalWebglDiagnostic('webgl-render-desync-manual-capture', {
paneKey,
boldTextCells: entry.weightProbe.boldTextCells,
totalTextCells: entry.weightProbe.totalTextCells,
optionsFontWeight: entry.weightProbe.optionsFontWeight,
atlasConfigFontWeight: entry.weightProbe.atlasConfigFontWeight
})
console.warn(`[terminal] manual render-desync capture on pane ${paneKey} — persisting evidence`)
void persistEntry(entry, internals, { recover: false })
}
function buildEvidenceEntry(
paneKey: string,
terminal: unknown,
internals: SentinelRenderInternals,
divergence: { textCells: number; missing: number; missPct: number },
trigger: SentinelEvidence['trigger']
): SentinelEvidence {
pendingPaneKeys.add(paneKey)
const buffer = activeBuffer(terminal)
const entry: SentinelEvidence = {
captureId: createCaptureId(paneKey),
paneKey,
when: Date.now(),
trigger,
divergence: {
textCells: divergence.textCells,
missing: divergence.missing,
missPct: divergence.missPct
},
paused: internals.isPaused,
rendererState: internals.rendererState,
weightProbe: readSentinelWeightProbe(terminal, buffer, internals.rows, internals.cols),
livePngDataUrl: internals.canvas.toDataURL(),
bufferText: buffer ? bufferSnapshot(buffer, internals.rows) : ''
}
evidence.push(entry)
return entry
}
async function persistEntry(
entry: SentinelEvidence,
internals: SentinelRenderInternals,
{ recover }: { recover: boolean }
): Promise<void> {
const directory = await persistCorruptEvidence(entry)
if (directory == null) {
// Why: a failed write must leave the bad pixels intact; recovering here
// would destroy the only evidence without producing a durable capture.
pendingPaneKeys.delete(entry.paneKey)
return
}
clickListener = (event) => {
const isMac = navigator.userAgent.includes('Mac')
if (event.button !== 0 || (isMac ? !event.metaKey : !event.ctrlKey)) {
return
}
const target = event.target
if (!(target instanceof Node)) {
return
}
let clickedTerminal: unknown = null
forEachLivePaneForDesyncSentinel((_paneKey, pane) => {
const terminal = (pane as SentinelPane).terminal as { element?: HTMLElement }
if (terminal.element?.contains(target)) {
clickedTerminal = terminal
}
})
if (clickedTerminal) {
startSampleBurst(clickedTerminal)
}
if (!recover) {
pendingPaneKeys.delete(entry.paneKey)
return
}
document.addEventListener('mouseup', clickListener, true)
console.warn('[terminal] render-desync sentinel armed (10s post-link bursts)')
resetAndRefreshAllTerminalWebglAtlases('render-desync')
const timeoutId = setTimeout(() => {
healedCaptureTimeoutIds.delete(timeoutId)
void persistHealedReference(entry.captureId, internals.canvas).finally(() =>
pendingPaneKeys.delete(entry.paneKey)
)
}, SAMPLE_INTERVAL_MS)
healedCaptureTimeoutIds.add(timeoutId)
}
export function stopTerminalRenderDesyncSentinelForTesting(): void {
stopSampleBurst()
if (clickListener != null) {
document.removeEventListener('mouseup', clickListener, true)
clickListener = null
}
missingHistoryByPane.clear()
pendingPaneKeys.clear()
for (const timeoutId of healedCaptureTimeoutIds) {
@@ -208,13 +261,7 @@ export function stopTerminalRenderDesyncSentinelForTesting(): void {
evidence.length = 0
}
function createCaptureId(paneKey: string): string {
const panePart = paneKey.replace(/[^a-zA-Z0-9_-]/g, '-')
const nonce = globalThis.crypto?.randomUUID?.() ?? Math.random().toString(36).slice(2)
return `${Date.now()}-${panePart}-${nonce}`
}
function startSampleBurst(terminal: unknown): void {
export function startRenderDesyncSampleBurst(terminal: unknown): void {
stopSampleBurst()
burstTerminal = terminal
sampleRenderDesyncOnce()
@@ -235,58 +282,3 @@ function stopSampleBurst(): void {
missingHistoryByPane.clear()
releaseRenderDesyncReadback()
}
async function persistEvidenceThenRecover(
entry: SentinelEvidence,
internals: SentinelRenderInternals
): Promise<void> {
try {
const pngDataUrl = entry.livePngDataUrl
const bufferText = entry.bufferText
if (!pngDataUrl || bufferText == null) {
throw new Error('Render-desync evidence payload was released before persistence')
}
const persisted = await window.api.app.writeTerminalRenderDesyncEvidence({
captureId: entry.captureId,
phase: 'corrupt',
pngDataUrl,
metadata: {
paneKey: entry.paneKey,
when: entry.when,
divergence: entry.divergence,
paused: entry.paused,
rendererState: entry.rendererState,
bufferText
}
})
entry.persistedDirectory = persisted.directory
} catch (error) {
// Why: a failed write must leave the bad pixels intact; recovering here
// would destroy the only evidence without producing a durable capture.
console.error('[terminal] could not persist render-desync evidence; leaving pane intact', error)
pendingPaneKeys.delete(entry.paneKey)
return
} finally {
// Why: persistence owns a successful payload, while a failed write leaves
// the live pane intact; neither path should retain duplicate full-canvas data.
entry.livePngDataUrl = undefined
entry.bufferText = undefined
}
resetAndRefreshAllTerminalWebglAtlases('render-desync')
const timeoutId = setTimeout(() => {
healedCaptureTimeoutIds.delete(timeoutId)
void window.api.app
.writeTerminalRenderDesyncEvidence({
captureId: entry.captureId,
phase: 'healed',
pngDataUrl: internals.canvas.toDataURL(),
metadata: { when: Date.now() }
})
.catch((error) =>
console.error('[terminal] could not persist healed render reference', error)
)
.finally(() => pendingPaneKeys.delete(entry.paneKey))
}, SAMPLE_INTERVAL_MS)
healedCaptureTimeoutIds.add(timeoutId)
}
@@ -0,0 +1,76 @@
import { forEachLivePaneForDesyncSentinel } from '@/lib/pane-manager/pane-manager-registry'
import {
captureRenderDesyncNow,
startRenderDesyncSampleBurst
} from './terminal-render-desync-sentinel'
/**
* User gestures that arm the render-desync sentinel, split from the detection
* loop so each stays under its own roof:
* - Cmd/Ctrl+click on a terminal: 10s divergence-sampling burst (the ink
* detector, which can only see missing glyphs).
* - Cmd/Ctrl+Shift+click: immediate unconditional capture of the clicked
* pane, for states the ink detector is blind to (bold-collapse family).
*
* Off by default; enabled via localStorage so a production build can arm it
* from DevTools without a settings-schema change:
* localStorage.setItem('orca:render-desync-sentinel', '1') // then reload
*/
export const RENDER_DESYNC_SENTINEL_FLAG = 'orca:render-desync-sentinel'
type SentinelPane = {
terminal: unknown
}
let clickListener: ((event: MouseEvent) => void) | null = null
export function maybeStartTerminalRenderDesyncSentinel(): void {
if (clickListener != null) {
return
}
let enabled = false
try {
enabled = globalThis.localStorage?.getItem(RENDER_DESYNC_SENTINEL_FLAG) === '1'
} catch {
enabled = false
}
if (!enabled) {
return
}
clickListener = (event) => {
const isMac = navigator.userAgent.includes('Mac')
if (event.button !== 0 || (isMac ? !event.metaKey : !event.ctrlKey)) {
return
}
const target = event.target
if (!(target instanceof Node)) {
return
}
let clickedPaneKey: string | null = null
let clickedPane: unknown = null
forEachLivePaneForDesyncSentinel((paneKey, pane) => {
const terminal = (pane as SentinelPane).terminal as { element?: HTMLElement }
if (terminal.element?.contains(target)) {
clickedPaneKey = paneKey
clickedPane = pane
}
})
if (!clickedPane || clickedPaneKey == null) {
return
}
if (event.shiftKey) {
captureRenderDesyncNow(clickedPaneKey, clickedPane)
return
}
startRenderDesyncSampleBurst((clickedPane as SentinelPane).terminal)
}
document.addEventListener('mouseup', clickListener, true)
console.warn('[terminal] render-desync sentinel armed (10s post-link bursts + ⇧-capture)')
}
export function stopTerminalRenderDesyncTriggerForTesting(): void {
if (clickListener != null) {
document.removeEventListener('mouseup', clickListener, true)
clickListener = null
}
}
@@ -0,0 +1,160 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { setTerminalWebglDiagnosticRecorder } from '../../../../shared/terminal-webgl-diagnostics'
import type { ManagedPane } from '@/lib/pane-manager/pane-manager-types'
import { applyOrDeferPaneMetricOptions } from '@/lib/pane-manager/pane-metric-options-deferral'
import {
auditPaneWeightParity,
readSentinelWeightProbe
} from './terminal-render-desync-weight-probe'
function fakeTerminal(
overrides: {
fontWeight?: number | string
fontWeightBold?: number | string
atlasConfig?: Record<string, unknown>
fontProbe?: { count: number; desired: string; actual: string }
} = {}
) {
return {
options: {
fontWeight: overrides.fontWeight ?? 500,
fontWeightBold: overrides.fontWeightBold ?? 700
},
_core: {
_renderService: {
_renderer: {
value: {
_charAtlas: {
_config: overrides.atlasConfig ?? {
fontWeight: 500,
fontWeightBold: 700,
fontFamily: '"SF Mono", Menlo, monospace',
devicePixelRatio: 2
},
...(overrides.fontProbe
? {
fontProbeMismatchCount: overrides.fontProbe.count,
fontProbeLastMismatch: {
desired: overrides.fontProbe.desired,
actual: overrides.fontProbe.actual
}
}
: {})
}
}
}
}
}
}
}
function fakeBuffer(rows: { chars: string; bold: boolean }[][]) {
return {
cursorY: 999,
viewportY: 0,
getLine: (y: number) => {
const row = rows[y]
if (!row) {
return undefined
}
return {
getCell: (x: number) =>
row[x] && {
getChars: () => row[x].chars,
getWidth: () => 1,
isBold: () => (row[x].bold ? 1 : 0)
},
translateToString: () => row.map((c) => c.chars).join('')
}
}
}
}
describe('readSentinelWeightProbe', () => {
it('reads live options, captured atlas config, and a bold census', () => {
const terminal = fakeTerminal({
fontWeight: 500,
atlasConfig: {
fontWeight: 700,
fontWeightBold: 700,
fontFamily: 'Menlo',
devicePixelRatio: 2
},
fontProbe: { count: 3, desired: '500', actual: 'italic 700 28px Menlo' }
})
const buffer = fakeBuffer([
[
{ chars: 'a', bold: false },
{ chars: 'b', bold: true },
{ chars: ' ', bold: false }
],
[{ chars: 'c', bold: true }]
])
const probe = readSentinelWeightProbe(terminal, buffer, 2, 3)
expect(probe).toEqual({
optionsFontWeight: '500',
optionsFontWeightBold: '700',
atlasConfigFontWeight: '700',
atlasConfigFontWeightBold: '700',
atlasConfigFontFamily: 'Menlo',
atlasConfigDevicePixelRatio: 2,
boldTextCells: 2,
totalTextCells: 3,
fontProbeMismatches: 3,
fontProbeLastDesired: '500',
fontProbeLastActual: 'italic 700 28px Menlo'
})
})
it('degrades to nulls when renderer internals are unreachable', () => {
const probe = readSentinelWeightProbe({ options: {} }, null, 0, 0)
expect(probe.optionsFontWeight).toBeNull()
expect(probe.atlasConfigFontWeight).toBeNull()
expect(probe.fontProbeMismatches).toBeNull()
expect(probe.totalTextCells).toBe(0)
})
})
describe('auditPaneWeightParity', () => {
const recorder = vi.fn()
const settings = { terminalFontWeight: 500, terminalFontWeightBold: 700 }
beforeEach(() => {
recorder.mockClear()
setTerminalWebglDiagnosticRecorder(recorder)
})
function paneWith(options: Record<string, unknown>): ManagedPane {
return { id: 7, terminal: { options } } as unknown as ManagedPane
}
it('records a crumb for a pane whose live weights diverge from settings', () => {
auditPaneWeightParity([paneWith({ fontWeight: 700, fontWeightBold: 700 })], settings)
expect(recorder).toHaveBeenCalledWith('terminal-weight-parity-mismatch', {
paneId: 7,
liveFontWeight: '700',
liveFontWeightBold: '700',
expectedFontWeight: 500,
expectedFontWeightBold: 700
})
})
it('stays silent for healthy panes', () => {
auditPaneWeightParity([paneWith({ fontWeight: 500, fontWeightBold: 700 })], settings)
expect(recorder).not.toHaveBeenCalled()
})
it('skips panes with a pending metric deferral (legitimately stale options)', () => {
const pane = paneWith({ fontWeight: 700, fontWeightBold: 700 })
applyOrDeferPaneMetricOptions(pane, { fontWeight: 500 }, false)
auditPaneWeightParity([pane], settings)
expect(recorder).not.toHaveBeenCalled()
})
})
@@ -0,0 +1,171 @@
import { recordTerminalWebglDiagnostic } from '../../../../shared/terminal-webgl-diagnostics'
import { resolveTerminalFontWeights } from '../../../../shared/terminal-fonts'
import { hasDeferredPaneMetricOptions } from '@/lib/pane-manager/pane-metric-options-deferral'
import type { ManagedPane } from '@/lib/pane-manager/pane-manager-types'
import type { BufferLike } from './terminal-render-desync-frame'
/**
* Weight-layer forensics for the terminal bold-collapse bug (STA-4042 family):
* regular text rasterizing at the bold weight while the byte stream is clean.
* One capture of these fields discriminates the candidate layers:
* - live options poisoned -> optionsFontWeight is bold-tier
* - atlas captured a stale config -> atlasConfig* diverges from options
* - renderer buffer cells latched bold -> boldTextCells is viewport-wide
* - stuck rasterizer font (failed ctx.font assignment) -> fontProbe* set
* (fontProbe fields are maintained by the patched @xterm/addon-webgl atlas)
*/
export type SentinelWeightProbe = {
optionsFontWeight: string | null
optionsFontWeightBold: string | null
atlasConfigFontWeight: string | null
atlasConfigFontWeightBold: string | null
atlasConfigFontFamily: string | null
atlasConfigDevicePixelRatio: number | null
boldTextCells: number
totalTextCells: number
fontProbeMismatches: number | null
fontProbeLastDesired: string | null
fontProbeLastActual: string | null
}
type BoldReadableCell = {
getChars: () => string
getWidth: () => number
isBold?: () => number | boolean
}
export function readSentinelWeightProbe(
terminal: unknown,
buffer: BufferLike | null,
rows: number,
cols: number
): SentinelWeightProbe {
const term = terminal as {
options?: { fontWeight?: string | number; fontWeightBold?: string | number }
_core?: {
_renderService?: {
_renderer?: {
value?: {
_charAtlas?: {
_config?: {
fontWeight?: string | number
fontWeightBold?: string | number
fontFamily?: string
devicePixelRatio?: number
}
fontProbeMismatchCount?: number
fontProbeLastMismatch?: { desired?: string; actual?: string }
}
}
}
}
}
}
const atlas = term._core?._renderService?._renderer?.value?._charAtlas
const config = atlas?._config
const census = countBoldTextCells(buffer, rows, cols)
return {
optionsFontWeight: stringOrNull(term.options?.fontWeight),
optionsFontWeightBold: stringOrNull(term.options?.fontWeightBold),
atlasConfigFontWeight: stringOrNull(config?.fontWeight),
atlasConfigFontWeightBold: stringOrNull(config?.fontWeightBold),
atlasConfigFontFamily: stringOrNull(config?.fontFamily),
atlasConfigDevicePixelRatio:
typeof config?.devicePixelRatio === 'number' ? config.devicePixelRatio : null,
boldTextCells: census.bold,
totalTextCells: census.total,
fontProbeMismatches: atlas?.fontProbeMismatchCount ?? null,
fontProbeLastDesired: atlas?.fontProbeLastMismatch?.desired ?? null,
fontProbeLastActual: atlas?.fontProbeLastMismatch?.actual ?? null
}
}
function stringOrNull(value: string | number | undefined): string | null {
return value === undefined ? null : String(value)
}
/**
* Reveal-time audit: any pane whose live weight options diverge from the
* settings-resolved pair has been poisoned by some writer, known or unknown.
* Runs on the visibility-resume path so every "come back to a finished agent"
* reveal — the reported trigger for the bold-collapse bug — checks itself.
* Always on (not sentinel-gated): it reads two option fields per pane and
* records nothing when healthy, and the poisoning is rare enough that a field
* occurrence must never be missed because a flag was unset. Panes with a
* pending metric deferral are skipped — their live options are legitimately
* stale until the flush, which records its own weight-change crumb.
* Settings are passed in (not read from the store) so this stays importable
* from lean unit-test graphs.
*/
export function auditPaneWeightParity(
panes: Iterable<ManagedPane>,
settings: { terminalFontWeight?: number; terminalFontWeightBold?: number } | null | undefined
): void {
if (!settings) {
return
}
const expected = resolveTerminalFontWeights(
settings.terminalFontWeight,
settings.terminalFontWeightBold
)
for (const pane of panes) {
if (hasDeferredPaneMetricOptions(pane)) {
continue
}
const live = pane.terminal.options
const liveWeight = stringOrNull(live.fontWeight)
const liveBold = stringOrNull(live.fontWeightBold)
if (
liveWeight === String(expected.fontWeight) &&
liveBold === String(expected.fontWeightBold)
) {
continue
}
recordTerminalWebglDiagnostic('terminal-weight-parity-mismatch', {
paneId: pane.id,
liveFontWeight: liveWeight,
liveFontWeightBold: liveBold,
expectedFontWeight: expected.fontWeight,
expectedFontWeightBold: expected.fontWeightBold
})
}
}
/**
* Census of bold-attributed cells across the visible viewport, read from the
* renderer's own buffer. The daemon model is clean in every field occurrence,
* so a viewport-wide bold census here convicts renderer-side cell corruption;
* a clean census with bold pixels convicts the rasterization layer instead.
*/
function countBoldTextCells(
buffer: BufferLike | null,
rows: number,
cols: number
): { bold: number; total: number } {
let bold = 0
let total = 0
if (!buffer) {
return { bold, total }
}
for (let row = 0; row < rows; row++) {
const line = buffer.getLine(buffer.viewportY + row)
if (!line) {
continue
}
for (let column = 0; column < cols; column++) {
const cell = line.getCell(column) as BoldReadableCell | undefined
if (!cell) {
break
}
const chars = cell.getChars()
if (chars === '' || chars === ' ' || cell.getWidth() === 0) {
continue
}
total++
if (cell.isBold?.()) {
bold++
}
}
}
return { bold, total }
}
@@ -14,6 +14,8 @@ import {
resetTerminalLinkifierHoverState
} from '@/lib/pane-manager/terminal-linkifier-hover-reset'
import { focusActivePane } from './pane-helpers'
import { useAppStore } from '@/store'
import { auditPaneWeightParity } from './terminal-render-desync-weight-probe'
import { flushDeferredPaneMetricOptionsIfMeasurable } from '@/lib/pane-manager/pane-fit'
import { repairPaneWebglCanvasDprMismatch } from '@/lib/pane-manager/terminal-canvas-dpr-repair'
import { presentPaneViewport } from '@/lib/pane-manager/pane-webgl-renderer'
@@ -83,6 +85,7 @@ export function resumeTerminalVisibility({
// change that landed while this tab was hidden has no other repair point.
repairPaneWebglCanvasDprMismatch(pane)
}
auditPaneWeightParity(manager.getPanes(), useAppStore.getState().settings)
// Why: intra-worktree tab switches only toggle the overlay. Keeping
// synchronous drain and atlas rebuilds off this path avoids racing the
// overlay's delayed geometry fit. Still request hidden-output recovery:
@@ -103,6 +106,7 @@ export function resumeTerminalVisibility({
}
enforceTerminalViewportIntents(manager)
if (!shouldUseLightTabResume) {
auditPaneWeightParity(manager.getPanes(), useAppStore.getState().settings)
// Why: this clear wipes the glyph atlas shared with other same-config
// terminals; refresh after reset so rebuilt atlases repaint from xterm.
resetAndRefreshAllTerminalWebglAtlases('visibility-resume')
@@ -1,4 +1,5 @@
import { describe, expect, it } from 'vitest'
import { describe, expect, it, vi } from 'vitest'
import { setTerminalWebglDiagnosticRecorder } from '../../../../shared/terminal-webgl-diagnostics'
import type { ManagedPane, ManagedPaneInternal } from './pane-manager-types'
import { toPublicPane } from './pane-public-view'
import { canApplyPaneMetricOptions, canMeasurePaneForFit } from './pane-fit-measurability'
@@ -165,3 +166,72 @@ describe('canApplyPaneMetricOptions gating', () => {
expect(canMeasurePaneForFit(makeSizedPane({ width: 50, height: 600 }, 5))).toBe(false)
})
})
describe('metric weight-change forensics', () => {
// Field probe for the bold-collapse bug (STA-4042 family): weights never
// change in normal operation, so a transition crumb is either the poisoning
// or the healing write. These fail if the crumb is removed.
it('records prev/next/reason when a weight value actually changes', () => {
const recorder = vi.fn()
setTerminalWebglDiagnosticRecorder(recorder)
const pane = makePane()
pane.terminal.options.fontWeight = 500
applyOrDeferPaneMetricOptions(pane, { fontWeight: 700, fontWeightBold: 700 }, true)
expect(recorder).toHaveBeenCalledWith('metric-weight-change', {
paneId: 1,
key: 'fontWeight',
prev: '500',
next: '700',
reason: 'appearance'
})
expect(recorder).toHaveBeenCalledWith('metric-weight-change', {
paneId: 1,
key: 'fontWeightBold',
prev: null,
next: '700',
reason: 'appearance'
})
setTerminalWebglDiagnosticRecorder(null)
})
it('stays silent when the written weights equal the live values', () => {
const recorder = vi.fn()
setTerminalWebglDiagnosticRecorder(recorder)
const pane = makePane()
pane.terminal.options.fontWeight = 500
pane.terminal.options.fontWeightBold = 700
applyOrDeferPaneMetricOptions(
pane,
{ fontWeight: 500, fontWeightBold: 700, fontSize: 15 },
true
)
expect(recorder).not.toHaveBeenCalledWith('metric-weight-change', expect.anything())
setTerminalWebglDiagnosticRecorder(null)
})
it('labels a flushed deferral as the deferred-flush writer', () => {
const recorder = vi.fn()
setTerminalWebglDiagnosticRecorder(recorder)
const pane = makePane()
pane.terminal.options.fontWeight = 700
applyOrDeferPaneMetricOptions(pane, { fontWeight: 500 }, false)
expect(recorder).not.toHaveBeenCalledWith('metric-weight-change', expect.anything())
flushDeferredPaneMetricOptions(pane)
expect(recorder).toHaveBeenCalledWith(
'metric-weight-change',
expect.objectContaining({
key: 'fontWeight',
prev: '700',
next: '500',
reason: 'deferred-flush'
})
)
setTerminalWebglDiagnosticRecorder(null)
})
})
@@ -27,7 +27,8 @@ const deferredMetricOptions = new WeakMap<PaneTerminal, PaneMetricOptions>()
export function applyOrDeferPaneMetricOptions(
pane: ManagedPane,
options: PaneMetricOptions,
measurable: boolean
measurable: boolean,
reason = 'appearance'
): 'applied' | 'deferred' {
if (!measurable) {
// Latest wins: a newer settings change while hidden supersedes the pending one.
@@ -35,7 +36,7 @@ export function applyOrDeferPaneMetricOptions(
return 'deferred'
}
deferredMetricOptions.delete(pane.terminal)
writePaneMetricOptions(pane, options)
writePaneMetricOptions(pane, options, reason)
return 'applied'
}
@@ -46,7 +47,7 @@ export function flushDeferredPaneMetricOptions(pane: ManagedPane): boolean {
return false
}
deferredMetricOptions.delete(pane.terminal)
writePaneMetricOptions(pane, pending)
writePaneMetricOptions(pane, pending, 'deferred-flush')
recordTerminalWebglDiagnostic('metric-options-deferred-flush', { paneId: pane.id })
return true
}
@@ -93,7 +94,11 @@ export function overridePendingPaneMetricOptions(
deferredMetricOptions.set(pane.terminal, { ...pending, ...options })
}
function writePaneMetricOptions(pane: ManagedPane, options: PaneMetricOptions): void {
function writePaneMetricOptions(
pane: ManagedPane,
options: PaneMetricOptions,
reason: string
): void {
const target = pane.terminal.options
if (options.fontSize !== undefined) {
target.fontSize = options.fontSize
@@ -102,12 +107,44 @@ function writePaneMetricOptions(pane: ManagedPane, options: PaneMetricOptions):
target.fontFamily = options.fontFamily
}
if (options.fontWeight !== undefined) {
recordMetricWeightChange(pane, 'fontWeight', target.fontWeight, options.fontWeight, reason)
target.fontWeight = options.fontWeight as typeof target.fontWeight
}
if (options.fontWeightBold !== undefined) {
recordMetricWeightChange(
pane,
'fontWeightBold',
target.fontWeightBold,
options.fontWeightBold,
reason
)
target.fontWeightBold = options.fontWeightBold as typeof target.fontWeightBold
}
if (options.lineHeight !== undefined) {
target.lineHeight = options.lineHeight
}
}
/**
* Field forensics for the terminal bold-collapse bug (STA-4042 family): weights
* never change in normal operation, so any real transition is the poisoning or
* healing write. Recorded before the write so `prev` is the pre-write value.
*/
function recordMetricWeightChange(
pane: ManagedPane,
key: 'fontWeight' | 'fontWeightBold',
prev: string | number | undefined,
next: string | number,
reason: string
): void {
if (prev === next) {
return
}
recordTerminalWebglDiagnostic('metric-weight-change', {
paneId: pane.id,
key,
prev: prev === undefined ? null : String(prev),
next: String(next),
reason
})
}