Cut the hidden-restore flood feedback loop (A) + query carve-out on drops (B)

(A) Under a foreground flood, the hidden-output-restore loop re-fetched
snapshots endlessly: each synchronous applyMainBufferSnapshot starved ACK
processing, main pinned at the in-flight cap, dropped at the pending cap,
and every droppedOutput/modelRestoreNeeded marker re-armed another
restore until the flood ended (rc.7.perf DSR timeouts).
- Restore loop: a foreground live-chunk queue overflow now abandons the
  restore immediately (the stream is outrunning snapshot fetch+replay),
  with a 3-iteration hard cap + lifecycle warn as backstop.
- Re-arm gate: drop markers/sentinels and reconcile seq-gaps on a visible
  pane during its own in-flight/just-abandoned restore no longer re-arm;
  live bytes write through and ONE deferred repaint (2s after the last
  backpressure signal) heals the gap. Hidden-pane gate semantics are
  unchanged.
- Query salvage: discarding queued restore bytes (overflow/refetch) now
  extracts DSR/CPR/DA/OSC-color queries and replays them to xterm so
  replies still flow.

(B) Main-side: dropOversizedPendingPtyData carves reply-eliciting query
sequences out of the dropped buffer (and out of post-drop latched data,
bounded) and ships them on the droppedOutput sentinel, so DSR probes
survive bulk drops. Query scanning moved to
src/shared/terminal-reply-query-extraction.ts, shared verbatim with the
renderer's hidden-startup query extraction.

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinwoo-H
2026-07-03 22:45:02 -04:00
co-authored by Orca
parent 1ce7f1b8cd
commit 09b68cbb12
5 changed files with 650 additions and 197 deletions
+43
View File
@@ -7133,6 +7133,49 @@ describe('registerPtyHandlers', () => {
}
})
it('carves reply-eliciting queries out of a pending-cap bulk drop so probes survive', async () => {
vi.useFakeTimers()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
const mockProc = createMockProc()
spawnMock.mockReturnValue(mockProc.proc)
try {
registerPtyHandlers(mainWindow as never)
const spawn = (await handlers.get('pty:spawn')!(null, {
cols: 80,
rows: 24,
cwd: '/tmp'
})) as { id: string }
const ackData = getPtyAckDataListener()
mainWindow.webContents.send.mockClear()
// Saturate the in-flight window so everything after buffers in pendingData.
mockProc.emitData('x'.repeat(600 * 1024))
vi.advanceTimersByTime(2)
for (let index = 0; index < 32; index++) {
vi.advanceTimersByTime(1)
}
// Flood past the pending cap WITH an embedded DSR probe — the program
// that wrote it blocks on the reply (the bench DSR timeout).
mockProc.emitData(`${'y'.repeat(2 * 1024 * 1024)}\x1b[6n${'y'.repeat(1024 * 1024)}`)
// While latched, a later probe must also be carved out (bounded).
mockProc.emitData(`${'z'.repeat(32 * 1024)}\x1b[0c${'z'.repeat(32 * 1024)}`)
mainWindow.webContents.send.mockClear()
ackData(null, { id: spawn.id, charCount: 512 * 1024 })
vi.advanceTimersByTime(2)
expect(mainWindow.webContents.send).toHaveBeenCalledWith('pty:data', {
id: spawn.id,
data: '\x1b[6n\x1b[0c',
droppedOutput: true
})
} finally {
errorSpy.mockRestore()
vi.useRealTimers()
}
})
it('scales the pending-output cap with the scrollback setting', async () => {
vi.useFakeTimers()
const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {})
+37 -7
View File
@@ -18,6 +18,7 @@ import type { OrcaRuntimeService } from '../runtime/orca-runtime'
import type { Store } from '../persistence'
import type { GlobalSettings, TuiAgent } from '../../shared/types'
import { terminalOutputBacklogCapChars } from '../../shared/terminal-scrollback-policy'
import { extractHiddenStartupRendererQueryData } from '../../shared/terminal-reply-query-extraction'
import { recordCrashBreadcrumb } from '../crash-reporting/crash-breadcrumb-store'
import type { SleepingAgentLaunchConfig } from '../../shared/agent-session-resume'
import type { ProjectExecutionRuntimeResolution } from '../../shared/project-execution-runtime'
@@ -1797,6 +1798,24 @@ export function registerPtyHandlers(
const pendingDataDropWarnedPtys = new Set<string>()
// Why capped: the drop path guarantees O(1) memory per PTY; salvaged query
// bytes are tiny (a DSR probe is 4 chars) and anything past the cap means a
// pathological stream, where degrading to the plain sentinel is fine.
const DROPPED_QUERY_SALVAGE_MAX_CHARS = 4096
// Why: a bulk drop must not swallow reply-eliciting queries embedded in the
// flood (DSR 6n / CPR, DA1/DA2, DECRQM, OSC 10/11 probes). The program that
// wrote them blocks on the reply (the bench DSR timeout). Carve just the
// query bytes out and let them ride the droppedOutput sentinel — content is
// healed by the snapshot restore, so replies cannot double-fire.
function extractDroppedPtyQueryBytes(data: string): string {
if (!data.includes('\x1b')) {
return ''
}
const extracted = extractHiddenStartupRendererQueryData(data, '')
return extracted.statelessQueryData + extracted.statefulQueryData + extracted.oscColorQueryData
}
function dropOversizedPendingPtyData(id: string, pending: PendingPtyData): PendingPtyData {
const capChars = pendingDataCapChars()
if (pending.droppedOutput === true || pending.data.length <= capChars) {
@@ -1824,11 +1843,15 @@ export function registerPtyHandlers(
sendModelRestoreNeededMarker(id, 'pending-cap', runtime?.getPtyOutputSequence(id))
}
pendingDroppedChars += pending.data.length
// Why empty data, not a trimmed tail: a mid-stream gap would silently
// corrupt the pane. The droppedOutput sentinel routes the pane through
// Why no trimmed content tail: a mid-stream gap would silently corrupt
// the pane. The droppedOutput sentinel routes the pane through
// hidden-output restore, which repaints from the authoritative main-owned
// buffer and realigns with the live stream by sequence.
return { data: '', droppedOutput: true }
// buffer and realigns with the live stream by sequence. Only carved-out
// query bytes ride along so their replies survive the drop.
return {
data: extractDroppedPtyQueryBytes(pending.data).slice(0, DROPPED_QUERY_SALVAGE_MAX_CHARS),
droppedOutput: true
}
}
function appendPendingPtyData(
@@ -1841,8 +1864,14 @@ export function registerPtyHandlers(
): PendingPtyData {
// Why: once over the cap, stay dropped at O(1) memory until the renderer
// can receive again — the restore sentinel supersedes any interim bytes.
// Queries arriving while latched still get carved out (bounded) so their
// replies survive the whole drop episode, not just the first burst.
if (existing?.droppedOutput === true) {
return existing
if (existing.data.length >= DROPPED_QUERY_SALVAGE_MAX_CHARS) {
return existing
}
const salvaged = extractDroppedPtyQueryBytes(data)
return salvaged ? { ...existing, data: existing.data + salvaged } : existing
}
const nextContainsBackgroundOutput =
existing?.containsBackgroundOutput === true || containsBackgroundOutput
@@ -1916,8 +1945,9 @@ export function registerPtyHandlers(
updateProducerFlowControl(id)
// Why: the buffered bytes were dropped at the pending cap; tell the
// renderer so the pane repaints from the main-owned buffer snapshot
// instead of continuing a stream with a silent gap.
sendPtyDataToRenderer(id, { id, data: '', droppedOutput: true })
// instead of continuing a stream with a silent gap. data carries only
// the carved-out query bytes (see extractDroppedPtyQueryBytes).
sendPtyDataToRenderer(id, { id, data: pending.data, droppedOutput: true })
writes++
continue
}
@@ -6303,13 +6303,21 @@ describe('connectPanePty', () => {
async function connectHiddenPane(deps: ReturnType<typeof createDeps>): Promise<{
transport: MockTransport
pane: ReturnType<typeof createPane>
dataCallback: (data: string, meta?: { seq?: number; rawLength?: number }) => void
dataCallback: (
data: string,
meta?: { seq?: number; rawLength?: number; droppedOutput?: boolean }
) => void
binding: { syncProcessTracking: () => void; dispose: () => void }
}> {
const { connectPanePty } = await import('./pty-connection')
const transport = createMockTransport('pty-id')
const capturedDataCallback: {
current: ((data: string, meta?: { seq?: number; rawLength?: number }) => void) | null
current:
| ((
data: string,
meta?: { seq?: number; rawLength?: number; droppedOutput?: boolean }
) => void)
| null
} = { current: null }
transport.connect.mockImplementation(
async ({ callbacks }: { callbacks: ConnectCallbacks }) => {
@@ -6627,7 +6635,7 @@ describe('connectPanePty', () => {
expect(getMainBufferSnapshot).not.toHaveBeenCalled()
})
it('fetches a fresh snapshot when a marker lands while a restore is in flight', async () => {
it('gates marker re-arms during an in-flight foreground restore and repaints once after', async () => {
enableMainAuthority()
const deps = createDeps({ isVisibleRef: { current: true } })
await connectHiddenPane(deps)
@@ -6646,20 +6654,182 @@ describe('connectPanePty', () => {
}>()
getMainBufferSnapshot
.mockReturnValueOnce(firstSnapshot.promise)
.mockResolvedValue({ data: 'fresh snapshot\r\n', cols: 100, rows: 30, seq: 96 })
.mockResolvedValue({ data: 'post-flood repaint\r\n', cols: 100, rows: 30, seq: 96 })
const { _dispatchPtyModelRestoreNeededForTest } = await import('./pty-model-restore-channel')
_dispatchPtyModelRestoreNeededForTest({ id: 'pty-id', reason: 'pending-cap', markerSeq: 64 })
await flushAsyncTicks(4)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
// Second drop while the first snapshot is still being serialized — the
// in-flight snapshot may predate it, so a fresh one must follow.
_dispatchPtyModelRestoreNeededForTest({ id: 'pty-id', reason: 'pending-cap', markerSeq: 80 })
firstSnapshot.resolve({ data: 'stale snapshot\r\n', cols: 100, rows: 30, seq: 64 })
await flushAsyncTicks(20)
try {
// Why (rc.7.perf feedback loop): a second drop marker while the first
// snapshot is still serializing on a VISIBLE pane is this pane's own
// restore backpressure. Re-fetching per marker kept the loop alive for
// the whole flood — the marker must NOT schedule another fetch.
vi.useFakeTimers()
_dispatchPtyModelRestoreNeededForTest({
id: 'pty-id',
reason: 'pending-cap',
markerSeq: 80
})
firstSnapshot.resolve({ data: 'first snapshot\r\n', cols: 100, rows: 30, seq: 64 })
await flushAsyncTicks(20)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2)
// Flood quiet: the suppression window elapses and exactly ONE deferred
// repaint fetches a fresh snapshot to heal the dropped gap.
vi.advanceTimersByTime(2_100)
await flushAsyncTicks(20)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2)
} finally {
vi.useRealTimers()
}
})
describe('foreground flood restore feedback loop (rc.7.perf)', () => {
function writtenFloodData(pane: ReturnType<typeof createPane>): string {
return pane.terminal.write.mock.calls.map((call) => String(call[0])).join('')
}
async function startInFlightRestore(): Promise<{
pane: ReturnType<typeof createPane>
dataCallback: (
data: string,
meta?: { seq?: number; rawLength?: number; droppedOutput?: boolean }
) => void
getMainBufferSnapshot: ReturnType<typeof vi.fn>
resolveFirstSnapshot: (snapshot: {
data: string
cols: number
rows: number
seq: number
}) => void
}> {
enableMainAuthority()
const deps = createDeps({ isVisibleRef: { current: true } })
const { pane, dataCallback } = await connectHiddenPane(deps)
const transportOptions = createdTransportOptions.at(-1) as {
onPtySpawn?: (ptyId: string) => void
}
transportOptions.onPtySpawn?.('pty-id')
const getMainBufferSnapshot = window.api.pty.getMainBufferSnapshot as unknown as ReturnType<
typeof vi.fn
>
const firstSnapshot = createDeferred<{
data: string
cols: number
rows: number
seq: number
}>()
getMainBufferSnapshot
.mockReturnValueOnce(firstSnapshot.promise)
.mockResolvedValue({ data: 'repaint snapshot\r\n', cols: 100, rows: 30, seq: 5_000_000 })
const { _dispatchPtyModelRestoreNeededForTest } =
await import('./pty-model-restore-channel')
_dispatchPtyModelRestoreNeededForTest({
id: 'pty-id',
reason: 'pending-cap',
markerSeq: 64
})
await flushAsyncTicks(4)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
return {
pane,
dataCallback,
getMainBufferSnapshot,
resolveFirstSnapshot: (snapshot) => firstSnapshot.resolve(snapshot)
}
}
it('abandons the restore on queue overflow, writes the stream through, and repaints once', async () => {
const { pane, dataCallback, getMainBufferSnapshot, resolveFirstSnapshot } =
await startInFlightRestore()
// Flood while the snapshot is in flight: overflows the 512KB restore
// queue — the live stream is outrunning snapshot fetch+replay.
dataCallback('f'.repeat(300 * 1024), { seq: 300 * 1024 + 64, rawLength: 300 * 1024 })
dataCallback('g'.repeat(300 * 1024), { seq: 600 * 1024 + 64, rawLength: 300 * 1024 })
try {
vi.useFakeTimers()
resolveFirstSnapshot({ data: 'flood snapshot\r\n', cols: 100, rows: 30, seq: 64 })
await flushAsyncTicks(20)
// Cut 1: the overflow abandons the restore instead of re-fetching.
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
// Cut 2: drop sentinels and seq-gap chunks during the flood window
// must not re-arm restores — the post-gap bytes write through.
dataCallback('', { droppedOutput: true })
dataCallback('AFTER-FLOOD', { seq: 700 * 1024, rawLength: 11 })
await flushAsyncTicks(8)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
expect(writtenFloodData(pane)).toContain('AFTER-FLOOD')
// After the flood goes quiet: exactly ONE deferred repaint.
vi.advanceTimersByTime(2_100)
await flushAsyncTicks(20)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2)
vi.advanceTimersByTime(5_000)
await flushAsyncTicks(20)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(2)
} finally {
vi.useRealTimers()
}
})
it('salvages stateful queries out of an overflowing restore queue', async () => {
const { pane, dataCallback } = await startInFlightRestore()
// Queue 400KB, then a chunk that overflows the cap and carries a DSR
// probe. The content is discarded (snapshot owns it) but the query
// must reach xterm so its reply flows.
dataCallback('a'.repeat(400 * 1024), { seq: 400 * 1024, rawLength: 400 * 1024 })
dataCallback(`${'b'.repeat(200 * 1024)}\x1b[6n`, {
seq: 600 * 1024 + 4,
rawLength: 200 * 1024 + 4
})
await flushAsyncTicks(8)
const written = writtenFloodData(pane)
expect(written).toContain('\x1b[6n')
expect(written).not.toContain('aaaa')
expect(written).not.toContain('bbbb')
})
it('keeps the hidden-pane drop sentinel arming a reveal restore (gate unchanged)', async () => {
enableMainAuthority()
const isVisibleRef = { current: false }
const deps = createDeps({ isVisibleRef })
const { pane, dataCallback } = await connectHiddenPane(deps)
const transportOptions = createdTransportOptions.at(-1) as {
onPtySpawn?: (ptyId: string) => void
}
transportOptions.onPtySpawn?.('pty-id')
const getMainBufferSnapshot = window.api.pty.getMainBufferSnapshot as unknown as ReturnType<
typeof vi.fn
>
getMainBufferSnapshot.mockResolvedValue({
data: 'hidden reveal snapshot\r\n',
cols: 100,
rows: 30,
seq: 64
})
// Hidden pane: the sentinel latches restore-needed but must not fetch.
dataCallback('', { droppedOutput: true })
await flushAsyncTicks(8)
expect(getMainBufferSnapshot).not.toHaveBeenCalled()
// Reveal: the latched restore fetches exactly one snapshot.
isVisibleRef.current = true
const { requestTerminalBacklogRecovery } =
await import('@/lib/pane-manager/pane-terminal-output-scheduler')
requestTerminalBacklogRecovery(pane.terminal as never)
await flushAsyncTicks(20)
expect(getMainBufferSnapshot).toHaveBeenCalledTimes(1)
expect(writtenFloodData(pane)).toContain('hidden reveal snapshot')
})
})
describe('post-restore backlog reconciliation', () => {
@@ -14,6 +14,15 @@ import { getWorktreeMapFromState } from '@/store/selectors'
import { parseWorkspaceKey } from '../../../../shared/workspace-scope'
import { createTerminalZeroDimensionsMessage } from '../../../../shared/terminal-zero-dimensions-diagnostic'
import { parseTerminalOscColorQuery } from '../../../../shared/terminal-osc-color-reply'
import {
HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS,
containsCsiRendererQuery,
containsStatefulRendererQuery,
extractHiddenStartupRendererQueryData,
findCsiFinalByteIndex,
isStatefulRendererReplyCsiQuery,
isStatelessRendererReplyCsiQuery
} from '../../../../shared/terminal-reply-query-extraction'
import { serializeWithAbsoluteCursor } from '../../../../shared/terminal-serialize-absolute-cursor'
import type { PtyBufferSnapshot, PtyConnectResult } from './pty-transport'
import { createIpcPtyTransport } from './pty-transport'
@@ -205,6 +214,18 @@ const HIDDEN_OUTPUT_RESTORE_PENDING_CHARS = 512 * 1024
const HIDDEN_OUTPUT_RESTORE_DEFERRED_RETRY_MS = 50
const HIDDEN_OUTPUT_RESTORE_DEFERRED_RETRY_MAX = 3
const HIDDEN_OUTPUT_RESTORE_FOREGROUND_TIMEOUT_MS = 750
// Why (rc.7.perf DSR-timeout feedback loop): under a foreground flood the
// restore pipeline is its own bottleneck — each synchronous snapshot replay
// starves ACK processing, main pins at the in-flight cap, drops at the
// pending cap, and every drop marker re-armed another restore until the
// flood ended. Backpressure evidence opens this suppression window: drop
// markers inside it must not re-arm restores; live bytes write through and
// ONE deferred repaint (when the window closes) heals the visual gap.
const HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS = 2000
// Backstop for the same loop: a single in-flight restore task may re-iterate
// (fresh-snapshot marks, unmappable slices) only this many times before it
// abandons and lets live bytes flow.
const HIDDEN_OUTPUT_RESTORE_MAX_LOOP_ITERATIONS = 3
const TERMINAL_RENDERER_RISK_SCAN_TAIL_CHARS = 256
const SYNCHRONIZED_OUTPUT_START_SEQUENCE = '\x1b[?2026h'
const SYNCHRONIZED_OUTPUT_END_SEQUENCE = '\x1b[?2026l'
@@ -513,157 +534,6 @@ function containsHiddenStartupRendererQuery(data: string): boolean {
return containsCsiRendererQuery(data) || data.includes('\x1b]10;?') || data.includes('\x1b]11;?')
}
const HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS = 64
function extractHiddenStartupRendererQueryData(
data: string,
pending: string
): {
statelessQueryData: string
statefulQueryData: string
oscColorQueryData: string
pending: string
} {
const input = pending + data
let statelessQueryData = ''
let statefulQueryData = ''
let oscColorQueryData = ''
let offset = 0
while (offset < input.length) {
const candidateIndex = input.indexOf('\x1b', offset)
if (candidateIndex === -1) {
break
}
if (candidateIndex + 1 >= input.length) {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(candidateIndex)
}
}
if (input.startsWith('\x1b[', candidateIndex)) {
const finalByteIndex = findCsiFinalByteIndex(input, candidateIndex + 2)
if (finalByteIndex === -1) {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(
candidateIndex,
candidateIndex + HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS
)
}
}
const sequence = input.slice(candidateIndex, finalByteIndex + 1)
if (isStatelessRendererReplyCsiQuery(sequence)) {
statelessQueryData += sequence
} else if (isStatefulRendererReplyCsiQuery(sequence)) {
statefulQueryData += sequence
}
offset = finalByteIndex + 1
continue
}
if (input.startsWith('\x1b]', candidateIndex)) {
const query = parseTerminalOscColorQuery(input, candidateIndex)
if (query.kind === 'partial') {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(
candidateIndex,
candidateIndex + HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS
)
}
}
if (query.kind === 'none') {
offset = candidateIndex + 2
continue
}
oscColorQueryData += input.slice(candidateIndex, query.endIndex)
offset = query.endIndex
continue
}
if (parseTerminalOscColorQuery(input, candidateIndex).kind === 'partial') {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(candidateIndex)
}
}
{
offset = candidateIndex + 1
continue
}
}
return { statelessQueryData, statefulQueryData, oscColorQueryData, pending: '' }
}
function containsCsiRendererQuery(data: string): boolean {
let offset = data.indexOf('\x1b[')
while (offset !== -1) {
const finalByteIndex = findCsiFinalByteIndex(data, offset + 2)
if (finalByteIndex === -1) {
return false
}
const sequence = data.slice(offset, finalByteIndex + 1)
if (isStatelessRendererReplyCsiQuery(sequence) || isStatefulRendererReplyCsiQuery(sequence)) {
return true
}
offset = data.indexOf('\x1b[', finalByteIndex + 1)
}
return false
}
function containsStatefulRendererQuery(data: string): boolean {
let offset = data.indexOf('\x1b[')
while (offset !== -1) {
const finalByteIndex = findCsiFinalByteIndex(data, offset + 2)
if (finalByteIndex === -1) {
return false
}
const sequence = data.slice(offset, finalByteIndex + 1)
if (isStatefulRendererReplyCsiQuery(sequence)) {
return true
}
offset = data.indexOf('\x1b[', finalByteIndex + 1)
}
return false
}
function findCsiFinalByteIndex(data: string, offset: number): number {
for (let index = offset; index < data.length; index++) {
const code = data.charCodeAt(index)
if (code >= 0x40 && code <= 0x7e) {
return index
}
}
return -1
}
function isStatelessRendererReplyCsiQuery(sequence: string): boolean {
if (sequence.endsWith('c')) {
return true
}
return (
sequence === '\x1b[5n' ||
sequence === '\x1b[>q' ||
sequence === '\x1b[14t' ||
sequence === '\x1b[16t'
)
}
function isStatefulRendererReplyCsiQuery(sequence: string): boolean {
return sequence === '\x1b[6n' || (sequence.startsWith('\x1b[?') && sequence.endsWith('$p'))
}
let codexRestartNoticePresenceSource: Record<
string,
{ previousAccountLabel: string; nextAccountLabel: string }
@@ -1018,6 +888,7 @@ export function connectPanePty(
let unregisterDocumentVisibilityRecovery: (() => void) | null = null
let cleanupHiddenOutputRestoreDeferredRetry = (): void => {}
let cleanupHiddenOutputRestoreForegroundDeadline = (): void => {}
let cleanupHiddenOutputRestoreFloodRepaint = (): void => {}
let resetRendererOrderedSeqForPtyExit: (exitedPtyId: string) => void = () => {}
let cleanupStartupDraftPasteTimers = (): void => {}
let unregisterE2ePtyDataInjection = (): void => {}
@@ -4031,6 +3902,9 @@ export function connectPanePty(
// can reuse the pane object for a different session before visibility.
let hiddenOutputRestorePtyId: string | null = null
let hiddenOutputRestoreGeneration = 0
// Flood-backpressure suppression (HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS).
let hiddenOutputRestoreFloodSuppressedUntil = 0
let hiddenOutputRestoreFloodRepaintTimer: ReturnType<typeof setTimeout> | null = null
// Why: after a snapshot restore, main can still drain ACK-backlog chunks
// whose bytes the snapshot already covers — writing them unguarded
// duplicates visible output. Track the restored baseline seq (per PTY)
@@ -4158,6 +4032,57 @@ export function connectPanePty(
window.api.pty.setHiddenRendererPty?.(ptyId, hidden)
}
function isHiddenOutputRestoreFloodSuppressed(): boolean {
return Date.now() < hiddenOutputRestoreFloodSuppressedUntil
}
// True when a drop/gap signal on a visible pane is attributable to this
// pane's OWN restore backpressure (a restore is replaying right now, or
// one was just cut off for outrunning the stream). Such signals must not
// re-arm restores — that is the rc.7.perf feedback loop.
function isForegroundRestoreBackpressureContext(): boolean {
return (
shouldWritePtyOutputForeground(deps.isVisibleRef.current) &&
(hiddenOutputRestoreInFlight !== null || isHiddenOutputRestoreFloodSuppressed())
)
}
function clearHiddenOutputRestoreFloodRepaintTimer(): void {
if (hiddenOutputRestoreFloodRepaintTimer === null) {
return
}
clearTimeout(hiddenOutputRestoreFloodRepaintTimer)
hiddenOutputRestoreFloodRepaintTimer = null
}
cleanupHiddenOutputRestoreFloodRepaint = clearHiddenOutputRestoreFloodRepaintTimer
function resetHiddenOutputRestoreFloodSuppression(): void {
hiddenOutputRestoreFloodSuppressedUntil = 0
clearHiddenOutputRestoreFloodRepaintTimer()
}
// Extends the suppression window and (re)schedules the single deferred
// repaint for when the flood goes quiet. Every backpressure signal resets
// the timer, so it fires exactly once, SUPPRESS_MS after the last signal.
function noteHiddenOutputRestoreFloodBackpressure(): void {
hiddenOutputRestoreFloodSuppressedUntil = Date.now() + HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS
const ptyId = transport.getPtyId()
if (ptyId === null) {
return
}
clearHiddenOutputRestoreFloodRepaintTimer()
hiddenOutputRestoreFloodRepaintTimer = setTimeout(() => {
hiddenOutputRestoreFloodRepaintTimer = null
if (disposed || transport.getPtyId() !== ptyId) {
return
}
// Why one repaint: bytes were dropped during the flood, so the screen
// has a gap the live stream cannot heal. Now that the flood is quiet,
// a single snapshot restore repaints from main's authoritative buffer.
markHiddenOutputRestoreNeeded()
}, HIDDEN_OUTPUT_RESTORE_FLOOD_SUPPRESS_MS)
}
// Why: main reports dropped renderer-bound bytes (hidden gate / pending
// cap) out-of-band — routed per PTY by pty-model-restore-channel.ts.
function handleModelRestoreNeededMarker(): void {
@@ -4167,6 +4092,14 @@ export function connectPanePty(
// Why: dropped bytes invalidate every cross-chunk carry — a partial
// OSC-9999 prefix spanning the gap would corrupt the next live chunk.
transport.resetCrossChunkParserState?.()
// Why gated (rc.7.perf loop): on a visible pane these markers are the
// product of our own restore starving ACKs. Re-arming per marker kept
// the snapshot-fetch loop alive for the whole flood; defer to one
// post-flood repaint instead and let live bytes flow.
if (isForegroundRestoreBackpressureContext()) {
noteHiddenOutputRestoreFloodBackpressure()
return
}
// Why: parity with the hidden skip path — a marker landing while a
// restore is in flight means the in-flight snapshot may predate the
// drop, so a fresh snapshot must follow. Captured BEFORE the mark: on a
@@ -4650,6 +4583,28 @@ export function connectPanePty(
recordHiddenRendererSkip(data.length)
}
// Why: discarding queued flood bytes must never swallow terminal queries —
// a lost DSR/CPR (or color/DA) reply hangs the querying program (the bench
// DSR timeout). The discarded CONTENT is owned by the snapshot repaint;
// the queries are re-played to xterm immediately so replies still flow.
// Only called for bytes that are being thrown away, so replies cannot
// double-fire against a later queue drain.
function salvageRendererQueriesFromDiscardedRestoreData(data: string): void {
if (!data || !data.includes('\x1b')) {
return
}
const extracted = extractHiddenStartupRendererQueryData(data, '')
if (extracted.oscColorQueryData) {
sendTerminalOscColorQueryReplies(extracted.oscColorQueryData, pane.terminal, (reply) =>
transport.sendInput(reply)
)
}
const queryData = extracted.statelessQueryData + extracted.statefulQueryData
if (queryData) {
writePtyOutputToXterm(queryData, true, { hiddenStartupRendererQuery: true })
}
}
function queueLiveChunkDuringRestore(data: string, meta?: PtyDataMeta): void {
if (!data) {
return
@@ -4663,10 +4618,23 @@ export function connectPanePty(
}
hiddenOutputRestorePtyId = ptyId
hiddenOutputRestoreNeeded = true
if (hiddenOutputRestorePendingOverflow) {
// Why: the overflow latch means everything queued gets discarded at the
// next drain — queueing more only grows the discard. Salvage queries,
// drop the content.
salvageRendererQueriesFromDiscardedRestoreData(data)
armHiddenOutputRestoreForegroundDeadline()
return
}
if (hiddenOutputRestorePendingChars + data.length > HIDDEN_OUTPUT_RESTORE_PENDING_CHARS) {
const discardedChunks = hiddenOutputRestorePendingChunks
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
hiddenOutputRestorePendingOverflow = true
for (const chunk of discardedChunks) {
salvageRendererQueriesFromDiscardedRestoreData(chunk.data)
}
salvageRendererQueriesFromDiscardedRestoreData(data)
armHiddenOutputRestoreForegroundDeadline()
return
}
@@ -4852,26 +4820,33 @@ export function connectPanePty(
)
}
function drainPendingLiveChunksAfterSnapshot(snapshotSeq: number | undefined): boolean {
// 'drained' painted every queued live byte; 'overflow' means the queue
// blew its cap during this restore (the stream is outrunning snapshot
// fetch+replay); 'refetch' means offsets were unmappable and only a
// fresher snapshot can realign.
function drainPendingLiveChunksAfterSnapshot(
snapshotSeq: number | undefined
): 'drained' | 'overflow' | 'refetch' {
if (hiddenOutputRestorePendingOverflow) {
hiddenOutputRestorePendingOverflow = false
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
return false
discardPendingLiveChunksSalvagingQueries()
return 'overflow'
}
while (hiddenOutputRestorePendingChunks.length > 0) {
const chunks = hiddenOutputRestorePendingChunks
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
for (const chunk of chunks) {
for (const [index, chunk] of chunks.entries()) {
const data = getChunkDataAfterSnapshot(chunk, snapshotSeq)
if (data === null) {
// Why: renderer-only OSC stripping makes raw sequence offsets
// impossible to map onto cleaned text. Fetch a fresher main
// snapshot instead of risking duplicate visible output.
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
return false
for (const discarded of chunks.slice(index)) {
salvageRendererQueriesFromDiscardedRestoreData(discarded.data)
}
discardPendingLiveChunksSalvagingQueries()
return 'refetch'
}
// Why: drained chunks advance the post-restore continuity point so
// the live-chunk reconciliation neither re-drops them as duplicates
@@ -4886,12 +4861,20 @@ export function connectPanePty(
}
if (hiddenOutputRestorePendingOverflow) {
hiddenOutputRestorePendingOverflow = false
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
return false
discardPendingLiveChunksSalvagingQueries()
return 'overflow'
}
}
return true
return 'drained'
}
function discardPendingLiveChunksSalvagingQueries(): void {
const discarded = hiddenOutputRestorePendingChunks
hiddenOutputRestorePendingChunks = []
hiddenOutputRestorePendingChars = 0
for (const chunk of discarded) {
salvageRendererQueriesFromDiscardedRestoreData(chunk.data)
}
}
function clearPendingLiveChunksDuringRestore(): void {
@@ -4955,7 +4938,10 @@ export function connectPanePty(
}, HIDDEN_OUTPUT_RESTORE_FOREGROUND_TIMEOUT_MS)
}
function abandonHiddenOutputRestoreAndDrainPendingForeground(expectedPtyId: string): void {
function abandonHiddenOutputRestoreAndDrainPendingForeground(
expectedPtyId: string,
opts: { quiet?: boolean } = {}
): void {
if (transport.getPtyId() !== expectedPtyId || hiddenOutputRestorePtyId !== expectedPtyId) {
resetHiddenOutputRestoreIfPtyChanged()
return
@@ -4982,7 +4968,12 @@ export function connectPanePty(
clearHiddenOutputRestoreForegroundDeadlineTimer()
hiddenOutputRestoreDeferredRetryAttempts = 0
writeRestoreUnavailableWarning()
// Why quiet exists: flood cuts abandon deliberately and schedule a
// post-flood repaint — the "restore unavailable" warning would be
// misleading noise the repaint immediately wipes.
if (!opts.quiet) {
writeRestoreUnavailableWarning()
}
if (hadPendingOverflow) {
return
}
@@ -5083,6 +5074,8 @@ export function connectPanePty(
clearHiddenOutputRestoreState()
clearRestoredSnapshotBaseline()
clearPaneMode2031State()
// Why: flood-backpressure evidence is per PTY stream too.
resetHiddenOutputRestoreFloodSuppression()
discardTerminalOutput(pane.terminal)
}
}
@@ -5229,6 +5222,10 @@ export function connectPanePty(
hiddenOutputRestoreRetryDeferred = false
hiddenOutputRestoreInFlight = (async () => {
// Backstop for the rc.7.perf feedback loop: bound how many snapshot
// fetch+replay rounds one task may burn before it must yield to the
// live stream.
let restoreIterations = 0
while (!disposed) {
const currentPtyId = hiddenOutputRestorePtyId
if (currentPtyId === null) {
@@ -5282,6 +5279,7 @@ export function connectPanePty(
return
}
hiddenOutputRestoreDeferredRetryAttempts = 0
restoreIterations += 1
applyMainBufferSnapshot(snapshot)
// Why: everything at or before snapshot.seq is now painted; chunks
// still draining from main's ACK backlog below that point are
@@ -5289,7 +5287,8 @@ export function connectPanePty(
setRestoredSnapshotBaseline(currentPtyId, snapshot)
const needsFreshSnapshot = hiddenOutputRestoreFreshSnapshotNeeded
hiddenOutputRestoreFreshSnapshotNeeded = false
if (drainPendingLiveChunksAfterSnapshot(snapshot.seq) && !needsFreshSnapshot) {
const drainOutcome = drainPendingLiveChunksAfterSnapshot(snapshot.seq)
if (drainOutcome === 'drained' && !needsFreshSnapshot) {
hiddenOutputRestoreNeeded = false
hiddenOutputRestorePtyId = null
clearHiddenOutputRestoreForegroundDeadlineTimer()
@@ -5302,6 +5301,31 @@ export function connectPanePty(
hiddenOutputRestoreNeeded = true
return
}
if (drainOutcome === 'overflow') {
// Cut 1 of the rc.7.perf feedback loop: a FOREGROUND queue
// overflow means the live stream outruns snapshot fetch+replay.
// Re-fetching would starve ACK processing again and feed the
// drop/re-arm cycle — abandon now, let bytes write through, and
// heal with one repaint after the flood.
noteHiddenOutputRestoreFloodBackpressure()
abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { quiet: true })
return
}
if (restoreIterations >= HIDDEN_OUTPUT_RESTORE_MAX_LOOP_ITERATIONS) {
// Backstop: fresh-snapshot marks / unmappable slices re-looping
// this many times means the stream is winning the race.
warnTerminalLifecycleAnomaly('hidden output restore hit its iteration cap', {
tabId: deps.tabId,
worktreeId: deps.worktreeId,
leafId: pane.leafId,
paneId: pane.id,
ptyId: currentPtyId,
reason: drainOutcome
})
noteHiddenOutputRestoreFloodBackpressure()
abandonHiddenOutputRestoreAndDrainPendingForeground(currentPtyId, { quiet: true })
return
}
hiddenOutputRestoreNeeded = true
}
})()
@@ -5367,11 +5391,25 @@ export function connectPanePty(
observeStartupDraftPasteReadiness(data)
resetHiddenOutputRestoreIfPtyChanged()
if (meta?.droppedOutput === true) {
// Why: main dropped this PTY's buffered output at the pending cap
// (renderer was not receiving). The stream has a gap, so repaint the
// pane from the main-owned buffer snapshot instead of writing on.
markHiddenOutputRestoreNeeded()
return
// Why gated (rc.7.perf loop): a visible pane's cap-drop during its own
// restore is backpressure the restore itself caused — re-arming per
// sentinel kept the snapshot-fetch loop alive for the whole flood.
// Defer to one post-flood repaint; any carved-out query bytes riding
// the sentinel still flow through the normal write path below.
if (meta?.background !== true && isForegroundRestoreBackpressureContext()) {
noteHiddenOutputRestoreFloodBackpressure()
} else {
// Why: main dropped this PTY's buffered output at the pending cap
// (renderer was not receiving). The stream has a gap, so repaint the
// pane from the main-owned buffer snapshot instead of writing on.
markHiddenOutputRestoreNeeded()
if (data) {
// The sentinel can carry query bytes carved out of the bulk drop
// (extractDroppedPtyQueryBytes in main) — replies must still flow.
salvageRendererQueriesFromDiscardedRestoreData(data)
}
return
}
}
respondToTerminalPixelSizeQueries(data)
observeTerminalBracketedPasteModeOutput(pane.terminal, data)
@@ -5410,17 +5448,28 @@ export function connectPanePty(
return
}
if (reconciliation.action === 'force-fresh-restore') {
// Why: in-flight captured BEFORE the mark — on a visible pane the
// mark starts the restore synchronously and must not flag itself.
const restoreWasInFlight = hiddenOutputRestoreInFlight !== null
markHiddenOutputRestoreNeeded()
if (restoreWasInFlight) {
hiddenOutputRestoreFreshSnapshotNeeded = true
// Why gated (rc.7.perf loop): during a foreground flood the seq gaps
// come from our own backpressure drops — fetching a snapshot per gap
// IS the feedback loop. Retire the stale baseline, write the post-gap
// bytes through, and heal with one repaint after the flood.
if (foreground && isForegroundRestoreBackpressureContext()) {
noteHiddenOutputRestoreFloodBackpressure()
clearRestoredSnapshotBaseline()
// fall through with the ORIGINAL data/meta — post-gap bytes are new
} else {
// Why: in-flight captured BEFORE the mark — on a visible pane the
// mark starts the restore synchronously and must not flag itself.
const restoreWasInFlight = hiddenOutputRestoreInFlight !== null
markHiddenOutputRestoreNeeded()
if (restoreWasInFlight) {
hiddenOutputRestoreFreshSnapshotNeeded = true
}
return
}
return
} else {
data = reconciliation.data
meta = reconciliation.meta
}
data = reconciliation.data
meta = reconciliation.meta
// Why: a hidden Codex query can be split just before visibility changes;
// xterm needs the completed query, while other bytes still follow restore.
const pendingForegroundQuery = foreground
@@ -6349,6 +6398,7 @@ export function connectPanePty(
}
cleanupHiddenOutputRestoreDeferredRetry()
cleanupHiddenOutputRestoreForegroundDeadline()
cleanupHiddenOutputRestoreFloodRepaint()
unregisterBacklogRecovery?.()
unregisterBacklogRecovery = null
unregisterDocumentVisibilityRecovery?.()
@@ -0,0 +1,160 @@
// Terminal-output scanning for reply-eliciting query sequences (DSR/CPR,
// DA1/DA2, DECRQM, XTGETTCAP-adjacent CSI queries, OSC 10/11 color probes).
// Shared because both sides must salvage queries out of bytes they are about
// to drop: the renderer's hidden-output restore queue and main's pending-cap
// bulk drop. A swallowed query means the program that sent it waits forever
// for a reply (the bench DSR timeout).
import { parseTerminalOscColorQuery } from './terminal-osc-color-reply'
export const HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS = 64
export type ExtractedRendererQueryData = {
statelessQueryData: string
statefulQueryData: string
oscColorQueryData: string
pending: string
}
export function extractHiddenStartupRendererQueryData(
data: string,
pending: string
): ExtractedRendererQueryData {
const input = pending + data
let statelessQueryData = ''
let statefulQueryData = ''
let oscColorQueryData = ''
let offset = 0
while (offset < input.length) {
const candidateIndex = input.indexOf('\x1b', offset)
if (candidateIndex === -1) {
break
}
if (candidateIndex + 1 >= input.length) {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(candidateIndex)
}
}
if (input.startsWith('\x1b[', candidateIndex)) {
const finalByteIndex = findCsiFinalByteIndex(input, candidateIndex + 2)
if (finalByteIndex === -1) {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(
candidateIndex,
candidateIndex + HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS
)
}
}
const sequence = input.slice(candidateIndex, finalByteIndex + 1)
if (isStatelessRendererReplyCsiQuery(sequence)) {
statelessQueryData += sequence
} else if (isStatefulRendererReplyCsiQuery(sequence)) {
statefulQueryData += sequence
}
offset = finalByteIndex + 1
continue
}
if (input.startsWith('\x1b]', candidateIndex)) {
const query = parseTerminalOscColorQuery(input, candidateIndex)
if (query.kind === 'partial') {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(
candidateIndex,
candidateIndex + HIDDEN_STARTUP_RENDERER_QUERY_PENDING_CHARS
)
}
}
if (query.kind === 'none') {
offset = candidateIndex + 2
continue
}
oscColorQueryData += input.slice(candidateIndex, query.endIndex)
offset = query.endIndex
continue
}
if (parseTerminalOscColorQuery(input, candidateIndex).kind === 'partial') {
return {
statelessQueryData,
statefulQueryData,
oscColorQueryData,
pending: input.slice(candidateIndex)
}
}
{
offset = candidateIndex + 1
continue
}
}
return { statelessQueryData, statefulQueryData, oscColorQueryData, pending: '' }
}
export function containsCsiRendererQuery(data: string): boolean {
let offset = data.indexOf('\x1b[')
while (offset !== -1) {
const finalByteIndex = findCsiFinalByteIndex(data, offset + 2)
if (finalByteIndex === -1) {
return false
}
const sequence = data.slice(offset, finalByteIndex + 1)
if (isStatelessRendererReplyCsiQuery(sequence) || isStatefulRendererReplyCsiQuery(sequence)) {
return true
}
offset = data.indexOf('\x1b[', finalByteIndex + 1)
}
return false
}
export function containsStatefulRendererQuery(data: string): boolean {
let offset = data.indexOf('\x1b[')
while (offset !== -1) {
const finalByteIndex = findCsiFinalByteIndex(data, offset + 2)
if (finalByteIndex === -1) {
return false
}
const sequence = data.slice(offset, finalByteIndex + 1)
if (isStatefulRendererReplyCsiQuery(sequence)) {
return true
}
offset = data.indexOf('\x1b[', finalByteIndex + 1)
}
return false
}
export function findCsiFinalByteIndex(data: string, offset: number): number {
for (let index = offset; index < data.length; index++) {
const code = data.charCodeAt(index)
if (code >= 0x40 && code <= 0x7e) {
return index
}
}
return -1
}
export function isStatelessRendererReplyCsiQuery(sequence: string): boolean {
if (sequence.endsWith('c')) {
return true
}
return (
sequence === '\x1b[5n' ||
sequence === '\x1b[>q' ||
sequence === '\x1b[14t' ||
sequence === '\x1b[16t'
)
}
export function isStatefulRendererReplyCsiQuery(sequence: string): boolean {
return sequence === '\x1b[6n' || (sequence.startsWith('\x1b[?') && sequence.endsWith('$p'))
}