diff --git a/src/main/ripgrep/bundled-ripgrep-text-search.test.ts b/src/main/ripgrep/bundled-ripgrep-text-search.test.ts index e0ae49248f1..f1bd739f04f 100644 --- a/src/main/ripgrep/bundled-ripgrep-text-search.test.ts +++ b/src/main/ripgrep/bundled-ripgrep-text-search.test.ts @@ -35,3 +35,31 @@ it('runs the bundled binary, bounds results and releases process ownership once' await rm(rootPath, { recursive: true, force: true }) } }) + +// Real rg must serialize over 64 MiB before the later ordinary match reaches the executor. +it('skips an oversized record and keeps a later ordinary match', async () => { + const rootPath = await mkdtemp(join(tmpdir(), 'orca-search-record-continuation-')) + const release = vi.fn() + try { + await writeFile(join(rootPath, 'later-hit.txt'), `${'x '.repeat(1_300_000)}\nordinary x\n`) + const result = await runBundledRipgrepTextSearch({ + rootPath, + resultRootPath: rootPath, + options: { rootPath, query: 'x' }, + onSpawn: () => release + }) + expect(result).toMatchObject({ + totalMatches: 1, + truncated: true, + files: [ + { + filePath: join(rootPath, 'later-hit.txt'), + matches: [{ line: 2, column: 10, lineContent: 'ordinary x' }] + } + ] + }) + expect(release).toHaveBeenCalledOnce() + } finally { + await rm(rootPath, { recursive: true, force: true }) + } +}, 30_000) diff --git a/src/main/ripgrep/bundled-ripgrep-text-search.ts b/src/main/ripgrep/bundled-ripgrep-text-search.ts index ccea89f4c19..59f1e828721 100644 --- a/src/main/ripgrep/bundled-ripgrep-text-search.ts +++ b/src/main/ripgrep/bundled-ripgrep-text-search.ts @@ -56,7 +56,7 @@ export function runBundledRipgrepTextSearch({ const rgArgs = buildRgArgs(options.query, '.', options) const acc = createAccumulator() - const lines = new SearchSubprocessLineAccumulator() + const lines = new SearchSubprocessLineAccumulator(undefined, 'discard') const diagnostics = new RipgrepSearchDiagnostics( process.platform === 'win32' && !wslDistroForOutput ? WINDOWS_UNOPENABLE_PATH_OS_ERRORS @@ -134,8 +134,10 @@ export function runBundledRipgrepTextSearch({ releaseChild = onSpawn(nextChild) const handleStdoutData = (chunk: string): void => { - const withinLineLimit = lines.push(chunk, processLine) - if (!withinLineLimit || acc.totalMatches >= maxResults) { + if (!lines.push(chunk, processLine)) { + acc.truncated = true + } + if (acc.totalMatches >= maxResults) { acc.truncated = true child?.once('error', ignoreCompletedSearchTerminationError) resolveOnce() diff --git a/src/shared/search-subprocess-lines.test.ts b/src/shared/search-subprocess-lines.test.ts index 9d0d8618a9c..58d6e94848b 100644 --- a/src/shared/search-subprocess-lines.test.ts +++ b/src/shared/search-subprocess-lines.test.ts @@ -78,3 +78,46 @@ describe('SearchSubprocessLineAccumulator', () => { expect(() => new SearchSubprocessLineAccumulator(-1)).toThrow(RangeError) }) }) + +describe('oversized search record continuation', () => { + it.each(['string', 'buffer'])('keeps complete lines around an oversized %s record', (kind) => { + const parser = new SearchSubprocessLineAccumulator(4, 'discard') + const lines: string[] = [] + const batch = 'four\noversized\nnext\n' + expect( + parser.push(kind === 'buffer' ? Buffer.from(batch) : batch, (line) => lines.push(line)) + ).toBe(false) + expect(lines).toEqual(['four', 'next']) + expect(parser.retainedCapacityBytes()).toBeNull() + expect(parser.finish()).toBeNull() + }) + + it('discards a fragmented record through its newline before accepting UTF-8 again', () => { + const parser = new SearchSubprocessLineAccumulator(4, 'discard') + const lines: string[] = [] + expect(parser.push(Buffer.from('ab'), (line) => lines.push(line))).toBe(true) + expect(parser.push(Buffer.from('cde'), (line) => lines.push(line))).toBe(false) + expect(parser.retainedCapacityBytes()).toBeNull() + expect(parser.push('suffix', (line) => lines.push(line))).toBe(true) + expect(parser.push('\néé\n', (line) => lines.push(line))).toBe(true) + expect(lines).toEqual(['éé']) + expect(parser.finish()).toBeNull() + }) + + it.each(['clear', 'finish'] as const)('resets an unterminated discard on %s', (reset) => { + const parser = new SearchSubprocessLineAccumulator(4, 'discard') + const lines: string[] = [] + expect(parser.push('oversized', (line) => lines.push(line))).toBe(false) + parser[reset]() + expect(parser.push('four\n', (line) => lines.push(line))).toBe(true) + expect(lines).toEqual(['four']) + }) + + it('skips adjacent oversized records without changing the byte limit for later records', () => { + const parser = new SearchSubprocessLineAccumulator(4, 'discard') + const lines: string[] = [] + expect(parser.push('five5\nsixsix\n漢é\nfour\n', (line) => lines.push(line))).toBe(false) + expect(lines).toEqual(['four']) + expect(parser.retainedCapacityBytes()).toBeNull() + }) +}) diff --git a/src/shared/search-subprocess-lines.ts b/src/shared/search-subprocess-lines.ts index 54e1defa5c0..112b41a154a 100644 --- a/src/shared/search-subprocess-lines.ts +++ b/src/shared/search-subprocess-lines.ts @@ -4,18 +4,24 @@ const SEARCH_SUBPROCESS_INITIAL_LINE_BUFFER_BYTES = 4 * 1024 export class SearchSubprocessLineAccumulator { private buffer: Buffer | null = null private bytes = 0 + private discardingLine = false - constructor(private readonly maxLineBytes = SEARCH_SUBPROCESS_MAX_LINE_BYTES) { + constructor( + private readonly maxLineBytes = SEARCH_SUBPROCESS_MAX_LINE_BYTES, + private readonly oversizedLine: 'reject' | 'discard' = 'reject' + ) { if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 0) { throw new RangeError('Search line limit must be a non-negative safe integer') } } + /** False reports an oversized line, even when discard mode continues with later lines. */ push(rawChunk: Buffer | string, onLine: (line: string) => void): boolean { // Three bytes per UTF-16 code unit bounds UTF-8 size without re-encoding complete batches. if ( typeof rawChunk === 'string' && this.bytes === 0 && + !this.discardingLine && rawChunk.endsWith('\n') && rawChunk.length * 3 <= this.maxLineBytes ) { @@ -28,13 +34,31 @@ export class SearchSubprocessLineAccumulator { } const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk, 'utf8') let cursor = 0 + let overflowed = false while (cursor < chunk.length) { const newline = chunk.indexOf(0x0a, cursor) const end = newline === -1 ? chunk.length : newline const segmentBytes = end - cursor + if (this.discardingLine) { + if (newline === -1) { + return !overflowed + } + this.discardingLine = false + cursor = newline + 1 + continue + } if (this.bytes + segmentBytes > this.maxLineBytes) { this.clear() - return false + if (this.oversizedLine === 'reject') { + return false + } + overflowed = true + this.discardingLine = newline === -1 + if (newline === -1) { + return false + } + cursor = newline + 1 + continue } if (newline !== -1 && this.bytes === 0) { @@ -49,20 +73,24 @@ export class SearchSubprocessLineAccumulator { } if (newline === -1) { - return true + return !overflowed } cursor = newline + 1 } - return true + return !overflowed } finish(): string | null { + if (this.discardingLine) { + this.clear() + } return this.bytes > 0 ? this.takeLine() : null } clear(): void { this.buffer = null this.bytes = 0 + this.discardingLine = false } /** Capacity of the retained growable buffer, or null once it has been released. */