fix(search): keep later hits after an oversized record (#35336)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>

src/main/ripgrep/bundled-ripgrep-text-search.test.ts

@@ -35,3 +35,31 @@ it('runs the bundled binary, bounds results and releases process ownership once'
     await rm(rootPath, { recursive: true, force: true })
   }
 })
+
+// Real rg must serialize over 64 MiB before the later ordinary match reaches the executor.
+it('skips an oversized record and keeps a later ordinary match', async () => {
+  const rootPath = await mkdtemp(join(tmpdir(), 'orca-search-record-continuation-'))
+  const release = vi.fn()
+  try {
+    await writeFile(join(rootPath, 'later-hit.txt'), `${'x '.repeat(1_300_000)}\nordinary x\n`)
+    const result = await runBundledRipgrepTextSearch({
+      rootPath,
+      resultRootPath: rootPath,
+      options: { rootPath, query: 'x' },
+      onSpawn: () => release
+    })
+    expect(result).toMatchObject({
+      totalMatches: 1,
+      truncated: true,
+      files: [
+        {
+          filePath: join(rootPath, 'later-hit.txt'),
+          matches: [{ line: 2, column: 10, lineContent: 'ordinary x' }]
+        }
+      ]
+    })
+    expect(release).toHaveBeenCalledOnce()
+  } finally {
+    await rm(rootPath, { recursive: true, force: true })
+  }
+}, 30_000)

src/main/ripgrep/bundled-ripgrep-text-search.ts

@@ -56,7 +56,7 @@ export function runBundledRipgrepTextSearch({
     const rgArgs = buildRgArgs(options.query, '.', options)
 
     const acc = createAccumulator()
-    const lines = new SearchSubprocessLineAccumulator()
+    const lines = new SearchSubprocessLineAccumulator(undefined, 'discard')
     const diagnostics = new RipgrepSearchDiagnostics(
       process.platform === 'win32' && !wslDistroForOutput
         ? WINDOWS_UNOPENABLE_PATH_OS_ERRORS
@@ -134,8 +134,10 @@ export function runBundledRipgrepTextSearch({
     releaseChild = onSpawn(nextChild)
 
     const handleStdoutData = (chunk: string): void => {
-      const withinLineLimit = lines.push(chunk, processLine)
-      if (!withinLineLimit || acc.totalMatches >= maxResults) {
+      if (!lines.push(chunk, processLine)) {
+        acc.truncated = true
+      }
+      if (acc.totalMatches >= maxResults) {
         acc.truncated = true
         child?.once('error', ignoreCompletedSearchTerminationError)
         resolveOnce()

src/shared/search-subprocess-lines.test.ts

@@ -78,3 +78,46 @@ describe('SearchSubprocessLineAccumulator', () => {
     expect(() => new SearchSubprocessLineAccumulator(-1)).toThrow(RangeError)
   })
 })
+
+describe('oversized search record continuation', () => {
+  it.each(['string', 'buffer'])('keeps complete lines around an oversized %s record', (kind) => {
+    const parser = new SearchSubprocessLineAccumulator(4, 'discard')
+    const lines: string[] = []
+    const batch = 'four\noversized\nnext\n'
+    expect(
+      parser.push(kind === 'buffer' ? Buffer.from(batch) : batch, (line) => lines.push(line))
+    ).toBe(false)
+    expect(lines).toEqual(['four', 'next'])
+    expect(parser.retainedCapacityBytes()).toBeNull()
+    expect(parser.finish()).toBeNull()
+  })
+
+  it('discards a fragmented record through its newline before accepting UTF-8 again', () => {
+    const parser = new SearchSubprocessLineAccumulator(4, 'discard')
+    const lines: string[] = []
+    expect(parser.push(Buffer.from('ab'), (line) => lines.push(line))).toBe(true)
+    expect(parser.push(Buffer.from('cde'), (line) => lines.push(line))).toBe(false)
+    expect(parser.retainedCapacityBytes()).toBeNull()
+    expect(parser.push('suffix', (line) => lines.push(line))).toBe(true)
+    expect(parser.push('\néé\n', (line) => lines.push(line))).toBe(true)
+    expect(lines).toEqual(['éé'])
+    expect(parser.finish()).toBeNull()
+  })
+
+  it.each(['clear', 'finish'] as const)('resets an unterminated discard on %s', (reset) => {
+    const parser = new SearchSubprocessLineAccumulator(4, 'discard')
+    const lines: string[] = []
+    expect(parser.push('oversized', (line) => lines.push(line))).toBe(false)
+    parser[reset]()
+    expect(parser.push('four\n', (line) => lines.push(line))).toBe(true)
+    expect(lines).toEqual(['four'])
+  })
+
+  it('skips adjacent oversized records without changing the byte limit for later records', () => {
+    const parser = new SearchSubprocessLineAccumulator(4, 'discard')
+    const lines: string[] = []
+    expect(parser.push('five5\nsixsix\n漢é\nfour\n', (line) => lines.push(line))).toBe(false)
+    expect(lines).toEqual(['four'])
+    expect(parser.retainedCapacityBytes()).toBeNull()
+  })
+})

src/shared/search-subprocess-lines.ts

@@ -4,18 +4,24 @@ const SEARCH_SUBPROCESS_INITIAL_LINE_BUFFER_BYTES = 4 * 1024
 export class SearchSubprocessLineAccumulator {
   private buffer: Buffer | null = null
   private bytes = 0
+  private discardingLine = false
 
-  constructor(private readonly maxLineBytes = SEARCH_SUBPROCESS_MAX_LINE_BYTES) {
+  constructor(
+    private readonly maxLineBytes = SEARCH_SUBPROCESS_MAX_LINE_BYTES,
+    private readonly oversizedLine: 'reject' | 'discard' = 'reject'
+  ) {
     if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 0) {
       throw new RangeError('Search line limit must be a non-negative safe integer')
     }
   }
 
+  /** False reports an oversized line, even when discard mode continues with later lines. */
   push(rawChunk: Buffer | string, onLine: (line: string) => void): boolean {
     // Three bytes per UTF-16 code unit bounds UTF-8 size without re-encoding complete batches.
     if (
       typeof rawChunk === 'string' &&
       this.bytes === 0 &&
+      !this.discardingLine &&
       rawChunk.endsWith('\n') &&
       rawChunk.length * 3 <= this.maxLineBytes
     ) {
@@ -28,13 +34,31 @@ export class SearchSubprocessLineAccumulator {
     }
     const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk, 'utf8')
     let cursor = 0
+    let overflowed = false
     while (cursor < chunk.length) {
       const newline = chunk.indexOf(0x0a, cursor)
       const end = newline === -1 ? chunk.length : newline
       const segmentBytes = end - cursor
+      if (this.discardingLine) {
+        if (newline === -1) {
+          return !overflowed
+        }
+        this.discardingLine = false
+        cursor = newline + 1
+        continue
+      }
       if (this.bytes + segmentBytes > this.maxLineBytes) {
         this.clear()
-        return false
+        if (this.oversizedLine === 'reject') {
+          return false
+        }
+        overflowed = true
+        this.discardingLine = newline === -1
+        if (newline === -1) {
+          return false
+        }
+        cursor = newline + 1
+        continue
       }
 
       if (newline !== -1 && this.bytes === 0) {
@@ -49,20 +73,24 @@ export class SearchSubprocessLineAccumulator {
       }
 
       if (newline === -1) {
-        return true
+        return !overflowed
       }
       cursor = newline + 1
     }
-    return true
+    return !overflowed
   }
 
   finish(): string | null {
+    if (this.discardingLine) {
+      this.clear()
+    }
     return this.bytes > 0 ? this.takeLine() : null
   }
 
   clear(): void {
     this.buffer = null
     this.bytes = 0
+    this.discardingLine = false
   }
 
   /** Capacity of the retained growable buffer, or null once it has been released. */