From f0dfc5de7b8c00c833c36eb83edfedfe95dbaeea Mon Sep 17 00:00:00 2001 From: OrcaWin Date: Thu, 17 Sep 2026 20:31:37 -0700 Subject: [PATCH] fix(projects): release processed repository scan records (#21022) Co-authored-by: m4air --- .../nested-repo-processed-queue/README.md | 34 +++ .../nested-repo-processed-queue/fix.patch | 28 ++ .../nested-repo-processed-queue/reproduce.cjs | 286 ++++++++++++++++++ .../nested-repo-processed-queue/results.json | 74 +++++ .../nested-repo-discovery-queue.test.ts | 122 ++++++++ .../project-groups/nested-repo-discovery.ts | 10 +- 6 files changed, 552 insertions(+), 2 deletions(-) create mode 100644 docs/audits/nested-repo-processed-queue/README.md create mode 100644 docs/audits/nested-repo-processed-queue/fix.patch create mode 100644 docs/audits/nested-repo-processed-queue/reproduce.cjs create mode 100644 docs/audits/nested-repo-processed-queue/results.json create mode 100644 src/main/project-groups/nested-repo-discovery-queue.test.ts diff --git a/docs/audits/nested-repo-processed-queue/README.md b/docs/audits/nested-repo-processed-queue/README.md new file mode 100644 index 00000000000..c5ef32e0a2b --- /dev/null +++ b/docs/audits/nested-repo-processed-queue/README.md @@ -0,0 +1,34 @@ +# Release completed nested-repository scan records + +`scanNestedRepos` kept every consumed `TraversalFolder` in its breadth-first queue until the scan finished. Those records retained path segments and inherited parsed ignore rules after their directories had been processed. Releasing each consumed slot and occasionally compacting the empty prefix removes that temporary retention while preserving traversal order. + +## Run + +```sh +ORCA_BACKGROUND_LAUNCH=1 node docs/audits/nested-repo-processed-queue/reproduce.cjs +``` + +The runner uses the repository's process launcher to start a Node child with forced GC, a 256 MiB old-space limit and a 15-second timeout. It bundles the actual scan and ignore-rule parser. An observational hook records weak references and scalar queue counts. A finite injected filesystem pauses one directory read; no app window, PTY, SSH connection or native watcher starts. The unused local Git detector is a throwing stub, ensuring the injected filesystem owns every probe. + +The fixture has 96 branches, each with 64 distinct ignore rules and one child directory. It pauses the penultimate child read, leaving one pending directory. Four event-loop-separated GC rounds precede each observation. + +| Observation | Before | Slot-release control | Fixed | +| ------------------------------------------ | ------: | -------------------: | -----: | +| Completed child records surviving GC | 94 / 94 | 0 / 94 | 0 / 94 | +| Their inherited rule arrays surviving GC | 94 / 94 | 0 / 94 | 0 / 94 | +| Observed records surviving scan completion | 0 | 0 | 0 | +| Total directories visited | 193 | 193 | 193 | + +All variants visit the same directories in exactly the same order and return the same empty result. The baseline reverses only `fix.patch` in memory. The diagnostic control adds only consumed-slot clearing to that baseline, without compaction; it isolates the retaining path. The fixed variant executes the current queue implementation. `results.json` includes source hashes, exact queue counts, runtime provenance, process exit and timeout status. + +The narrow source regression suite exercises a wider traversal across repeated compaction, including Windows and SSH POSIX path forms, inherited ignore rules, breadth-first result order, maximum depth, repository caps, cancellation and optional timeout behavior. All 37 discovery, queue and scan-rule tests passed, along with Node typechecking and focused lint checks. Existing discovery tests cover local filesystem and symlink behavior. + +## Reuse and scope + +The change follows the consumed-slot release pattern in `ws-outbound-backpressure-queue.ts` and the amortized prefix compaction pattern in `runtime-rpc-call-queue.ts`. It introduces no new queue abstraction, traversal policy, RPC field or host boundary. + +The baseline source matches `v1.4.198`; the runner verifies this named-tag comparison after normalizing CRLF line endings to LF for Windows checkout portability. This is not an execution of the historical packaged application. + +The IPC route uses this scanner for local and SSH-backed folder selection, with filesystem operations delegated to the selected host. Runtime scan/import routes request a 15-second timeout; IPC forwards options, whose timeout defaults to null. Existing time checks happen between awaited operations and do not cancel a pending read. + +This fix releases completed work. It does not cap the active frontier, directory entry arrays, `.gitignore` size or directory breadth. The original implementation releases its records on scan completion. No heap-byte savings or field-incident attribution is claimed; no affected-host data was used. diff --git a/docs/audits/nested-repo-processed-queue/fix.patch b/docs/audits/nested-repo-processed-queue/fix.patch new file mode 100644 index 00000000000..b79d294abb4 --- /dev/null +++ b/docs/audits/nested-repo-processed-queue/fix.patch @@ -0,0 +1,28 @@ +diff --git a/src/main/project-groups/nested-repo-discovery.ts b/src/main/project-groups/nested-repo-discovery.ts +index 84fddfd116..1f25a85a26 100644 +--- a/src/main/project-groups/nested-repo-discovery.ts ++++ b/src/main/project-groups/nested-repo-discovery.ts +@@ -90,7 +90,7 @@ export async function scanNestedRepos(args: { + return buildResult('non_git_folder') + } + +- const foldersToTraverse: TraversalFolder[] = [ ++ const foldersToTraverse: (TraversalFolder | undefined)[] = [ + { path: args.path, depth: 0, segments: [], ignoreRules: [] } + ] + let nextFolderIndex = 0 +@@ -107,7 +107,13 @@ export async function scanNestedRepos(args: { + if (noteAbort()) { + break + } +- const currentFolder = foldersToTraverse[nextFolderIndex++] ++ const currentFolder = foldersToTraverse[nextFolderIndex++]! ++ // Release processed paths and inherited ignore rules before the next filesystem await. ++ foldersToTraverse[nextFolderIndex - 1] = undefined ++ if (nextFolderIndex >= 64 && nextFolderIndex * 2 >= foldersToTraverse.length) { ++ foldersToTraverse.splice(0, nextFolderIndex) ++ nextFolderIndex = 0 ++ } + if (currentFolder.depth > options.maxDepth) { + continue + } diff --git a/docs/audits/nested-repo-processed-queue/reproduce.cjs b/docs/audits/nested-repo-processed-queue/reproduce.cjs new file mode 100644 index 00000000000..17fd5874c35 --- /dev/null +++ b/docs/audits/nested-repo-processed-queue/reproduce.cjs @@ -0,0 +1,286 @@ +const assert = require('node:assert/strict') +const { createHash } = require('node:crypto') +const { readFileSync, mkdtempSync, rmSync } = require('node:fs') +const { tmpdir } = require('node:os') +const { join, resolve } = require('node:path') +const { build } = require('esbuild') +const { applyPatch, parsePatch, reversePatch } = require('diff') + +if (process.env.ORCA_BACKGROUND_LAUNCH !== '1') { + throw new Error('Run with ORCA_BACKGROUND_LAUNCH=1') +} +if (process.argv[2] === '--proof-child' && typeof global.gc !== 'function') { + throw new Error('Child proof requires --expose-gc') +} +const root = resolve(__dirname, '../../..') +const readSource = (path) => readFileSync(path, 'utf8').replace(/\r\n/g, '\n') +const sourcePath = join(root, 'src/main/project-groups/nested-repo-discovery.ts') +const original = readSource(sourcePath) +const patch = parsePatch(readSource(join(__dirname, 'fix.patch'))) +assert.equal(patch.length, 1) +const baseline = applyPatch(original, reversePatch(patch[0])) +assert.notEqual(baseline, false, 'Source changed; review fix.patch') +const hookPoint = ' if (currentFolder.depth > options.maxDepth) {' +assert.equal(original.split(hookPoint).length, 2) +assert.equal(baseline.split(hookPoint).length, 2) +const sha256 = (text) => createHash('sha256').update(text).digest('hex') +const scratch = mkdtempSync(join(tmpdir(), 'orca-nested-queue-proof-')) +const branchCount = 96 +const rulesPerBranch = 64 +const pauseLeaf = branchCount - 2 +const tick = () => new Promise((resolve) => setImmediate(resolve)) +async function gc() { + for (let round = 0; round < 4; round++) { + await tick() + global.gc() + } +} +async function run(mode) { + const output = join(scratch, `${mode}.cjs`) + let source = mode === 'after' ? original : baseline + if (mode === 'clear-consumed-slot') { + const dequeue = ' const currentFolder = foldersToTraverse[nextFolderIndex++]' + assert.equal(source.split(dequeue).length, 2) + source = source.replace( + dequeue, + `${dequeue}\n foldersToTraverse[nextFolderIndex - 1] = undefined` + ) + } + const observedSource = source.replace( + hookPoint, + ` globalThis.__orcaObserveNestedQueue(currentFolder, foldersToTraverse, nextFolderIndex)\n${ + hookPoint + }` + ) + await build({ + entryPoints: [sourcePath], + outfile: output, + bundle: true, + platform: 'node', + format: 'cjs', + logLevel: 'silent', + plugins: [ + { + name: 'observe-actual-nested-queue', + setup(build) { + build.onLoad({ filter: /nested-repo-discovery\.ts$/ }, () => ({ + contents: observedSource, + loader: 'ts', + resolveDir: join(root, 'src/main/project-groups') + })) + build.onResolve({ filter: /^\.\.\/git\/repo$/ }, () => ({ + path: 'inert-git', + namespace: 'proof' + })) + build.onLoad({ filter: /.*/, namespace: 'proof' }, () => ({ + contents: + 'export function isGitRepo() { throw new Error("fixture must use injected filesystem") }', + loader: 'js' + })) + } + } + ] + }) + const { scanNestedRepos } = require(output) + const references = [] + const visits = [] + let pausedState + globalThis.__orcaObserveNestedQueue = (current, queue, head) => { + references.push({ + path: current.path, + record: new WeakRef(current), + inheritedRules: new WeakRef(current.ignoreRules) + }) + if (current.path === `/fixture/b${String(pauseLeaf).padStart(3, '0')}/leaf`) { + pausedState = { + allocatedSlots: queue.length, + consumedSlots: head, + pendingSlots: queue.length - head, + occupiedConsumedSlots: queue.slice(0, head).filter(Boolean).length + } + } + } + let release + const gate = new Promise((resolve) => { + release = resolve + }) + let markPaused + const paused = new Promise((resolve) => { + markPaused = resolve + }) + const resultPromise = scanNestedRepos({ + path: '/fixture', + options: { maxDepth: 3 }, + filesystem: { + async readDirectory(path) { + visits.push(path) + if (path === '/fixture') { + return Array.from({ length: branchCount }, (_, index) => ({ + name: `b${String(index).padStart(3, '0')}`, + isDirectory: true + })) + } + if (!path.endsWith('/leaf')) { + return [ + { name: '.gitignore', isDirectory: false }, + { name: 'leaf', isDirectory: true } + ] + } + if (path === `/fixture/b${String(pauseLeaf).padStart(3, '0')}/leaf`) { + markPaused() + await gate + } + return [] + }, + async readTextFile(path) { + return Array.from( + { length: rulesPerBranch }, + (_, index) => `${path.replaceAll('/', '_')}_unused_${index}` + ).join('\n') + }, + joinPath: (parent, name) => `${parent}/${name}`, + basename: (path) => path.split('/').at(-1), + hasGitMarker: () => false, + isSelectedPathGitRepo: () => false + } + }) + await paused + await gc() + const completedLeaves = references.filter( + ({ path }) => + path.endsWith('/leaf') && path !== `/fixture/b${String(pauseLeaf).padStart(3, '0')}/leaf` + ) + const retained = { + completedLeaves: completedLeaves.length, + retainedCompletedRecords: completedLeaves.filter(({ record }) => record.deref()).length, + retainedCompletedRuleArrays: completedLeaves.filter(({ inheritedRules }) => + inheritedRules.deref() + ).length + } + release() + const result = await resultPromise + delete globalThis.__orcaObserveNestedQueue + await gc() + const afterCompletion = references.filter(({ record }) => record.deref()).length + assert.equal(result.repos.length, 0) + assert.equal(result.stopped, false) + assert.equal(result.timedOut, false) + assert.equal(result.timeoutMs, null) + assert.equal(visits.length, branchCount * 2 + 1) + assert.equal(new Set(visits).size, visits.length) + assert.equal(pausedState.pendingSlots, 1) + assert.equal(retained.completedLeaves, pauseLeaf) + assert.equal(afterCompletion, 0) + delete require.cache[require.resolve(output)] + return { + mode, + pausedState, + retained, + afterCompletion, + totalVisited: visits.length, + visitedOrder: visits + } +} +async function main() { + try { + if (process.argv[2] !== '--proof-child') { + const runnerPath = join(scratch, 'run-process.cjs') + await build({ + entryPoints: [join(root, 'src/shared/child-process/run-process.ts')], + outfile: runnerPath, + bundle: true, + platform: 'node', + format: 'cjs', + logLevel: 'silent' + }) + const { runProcess } = require(runnerPath) + const child = await runProcess({ + program: process.execPath, + args: ['--expose-gc', '--max-old-space-size=256', __filename, '--proof-child'], + cwd: root, + env: process.env, + timeoutMs: 15_000, + maxOutputBytes: 1024 * 1024 + }) + assert.equal(child.timedOut, false, 'Proof timed out') + assert.equal(child.code, 0, child.stderr || child.stdout) + const recorded = JSON.parse(child.stdout) + const historical = await runProcess({ + program: 'git', + args: ['show', 'v1.4.198:src/main/project-groups/nested-repo-discovery.ts'], + cwd: root, + timeoutMs: 5_000, + maxOutputBytes: 256 * 1024 + }) + assert.equal(historical.timedOut, false) + assert.equal(historical.code, 0) + const historicalHash = sha256(historical.stdout.replace(/\r\n/g, '\n')) + assert.equal(historicalHash, recorded.sourceHashes.before) + console.log( + JSON.stringify( + { + ...recorded, + historicalSource: { ref: 'v1.4.198', sha256: historicalHash, equalsBaseline: true }, + process: { + exitCode: child.code, + timedOut: child.timedOut, + timeoutMs: 15_000, + oldSpaceMiB: 256 + } + }, + null, + 2 + ) + ) + delete require.cache[require.resolve(runnerPath)] + return + } + const before = await run('before') + const control = await run('clear-consumed-slot') + const after = await run('after') + assert.equal(before.retained.retainedCompletedRecords, pauseLeaf) + assert.equal(before.retained.retainedCompletedRuleArrays, pauseLeaf) + for (const phase of [control, after]) { + assert.equal(phase.retained.retainedCompletedRecords, 0) + assert.equal(phase.retained.retainedCompletedRuleArrays, 0) + assert.equal(phase.pausedState.occupiedConsumedSlots, 0) + assert.deepEqual(before.visitedOrder, phase.visitedOrder) + } + assert.ok(after.pausedState.allocatedSlots <= 64) + for (const phase of [before, control, after]) { + delete phase.visitedOrder + } + console.log( + JSON.stringify({ + description: + 'Actual nested-repo scan with observational dequeue hook; before reverses fix.patch and control clears only consumed slots.', + sourceHashes: { + normalization: 'UTF-8 source with CRLF line endings normalized to LF', + before: sha256(baseline), + after: sha256(original), + rules: sha256( + readSource(join(root, 'src/main/project-groups/nested-repo-scan-rules.ts')) + ), + regression: sha256( + readSource(join(root, 'src/main/project-groups/nested-repo-discovery-queue.test.ts')) + ), + runner: sha256(readSource(__filename)) + }, + nodeVersion: process.version, + branchCount, + rulesPerBranch, + before, + control, + after, + passed: true + }) + ) + } finally { + delete globalThis.__orcaObserveNestedQueue + rmSync(scratch, { recursive: true, force: true }) + } +} +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/docs/audits/nested-repo-processed-queue/results.json b/docs/audits/nested-repo-processed-queue/results.json new file mode 100644 index 00000000000..0fca2b45cee --- /dev/null +++ b/docs/audits/nested-repo-processed-queue/results.json @@ -0,0 +1,74 @@ +{ + "description": "Actual nested-repo scan with observational dequeue hook; before reverses fix.patch and control clears only consumed slots.", + "sourceHashes": { + "normalization": "UTF-8 source with CRLF line endings normalized to LF", + "before": "0a0952888db648a77776becfa9fcfc783d0b905ce096e70311d532ea2675065e", + "after": "8517a2bc2220e5fb3e96f063e1714911235ab040ee21487ca537d5c34d3cc81d", + "rules": "4613599bf5382edd86ae33bc84548f247018f26acbf2f7db072d847fa5533660", + "regression": "5e343d4da0627458ebd15c5c619b861c07388830b2a65aef76c9fb0dcd9d4c8d", + "runner": "2bafd8e2de95a86bcefdef484f63002cb0f59428b5359a3b9b9f16b8942ac11f" + }, + "nodeVersion": "v26.6.0", + "branchCount": 96, + "rulesPerBranch": 64, + "before": { + "mode": "before", + "pausedState": { + "allocatedSlots": 193, + "consumedSlots": 192, + "pendingSlots": 1, + "occupiedConsumedSlots": 192 + }, + "retained": { + "completedLeaves": 94, + "retainedCompletedRecords": 94, + "retainedCompletedRuleArrays": 94 + }, + "afterCompletion": 0, + "totalVisited": 193 + }, + "control": { + "mode": "clear-consumed-slot", + "pausedState": { + "allocatedSlots": 193, + "consumedSlots": 192, + "pendingSlots": 1, + "occupiedConsumedSlots": 0 + }, + "retained": { + "completedLeaves": 94, + "retainedCompletedRecords": 0, + "retainedCompletedRuleArrays": 0 + }, + "afterCompletion": 0, + "totalVisited": 193 + }, + "after": { + "mode": "after", + "pausedState": { + "allocatedSlots": 34, + "consumedSlots": 33, + "pendingSlots": 1, + "occupiedConsumedSlots": 0 + }, + "retained": { + "completedLeaves": 94, + "retainedCompletedRecords": 0, + "retainedCompletedRuleArrays": 0 + }, + "afterCompletion": 0, + "totalVisited": 193 + }, + "passed": true, + "historicalSource": { + "ref": "v1.4.198", + "sha256": "0a0952888db648a77776becfa9fcfc783d0b905ce096e70311d532ea2675065e", + "equalsBaseline": true + }, + "process": { + "exitCode": 0, + "timedOut": false, + "timeoutMs": 15000, + "oldSpaceMiB": 256 + } +} diff --git a/src/main/project-groups/nested-repo-discovery-queue.test.ts b/src/main/project-groups/nested-repo-discovery-queue.test.ts new file mode 100644 index 00000000000..16eb82fdd54 --- /dev/null +++ b/src/main/project-groups/nested-repo-discovery-queue.test.ts @@ -0,0 +1,122 @@ +import { posix, win32 } from 'node:path' +import { afterEach, expect, it, vi } from 'vitest' +import { scanNestedRepos } from './nested-repo-discovery' + +const branchNames = Array.from( + { length: 160 }, + (_, index) => `branch-${String(index).padStart(3, '0')}` +) +afterEach(() => vi.restoreAllMocks()) + +function fixture(paths: typeof posix, onRead: (count: number) => void = () => {}) { + const root = paths.resolve('/workspace') + const visits: string[] = [] + const branches = branchNames.map((name) => paths.join(root, name)) + const descendants = branches.map((path) => paths.join(path, 'deeper')) + const repositories = descendants.map((path) => paths.join(path, 'repository')) + return { + root, + visits, + branches, + descendants, + repositories, + filesystem: { + readDirectory: async (path: string) => { + visits.push(path) + onRead(visits.length) + const names = + path === root + ? branchNames.toReversed() + : paths.basename(path) === 'deeper' + ? ['repository'] + : ['ignored', 'deeper', '.gitignore'] + return names.map((name) => ({ name, isDirectory: name !== '.gitignore' })) + }, + readTextFile: async () => 'ignored/', + joinPath: paths.join, + basename: paths.basename, + hasGitMarker: (path: string) => paths.basename(path) === 'repository', + isSelectedPathGitRepo: () => false + } + } +} + +it.each([ + ['local Windows paths', win32], + ['SSH POSIX paths', posix] +] as const)('preserves broad BFS order and inherited ignores with %s', async (_label, paths) => { + const f = fixture(paths) + const result = await scanNestedRepos({ + path: f.root, + options: { maxRepos: 500 }, + filesystem: f.filesystem + }) + expect(f.visits).toEqual([f.root, ...f.branches, ...f.descendants]) + expect(result.repos.map(({ path }) => path)).toEqual(f.repositories) + expect(result.repos.every(({ depth }) => depth === 3)).toBe(true) + expect(result).toMatchObject({ + truncated: false, + stopped: false, + timedOut: false, + timeoutMs: null + }) +}) + +it('preserves max depth and result caps during broad traversal', async () => { + const depth = fixture(posix) + const boundedDepth = await scanNestedRepos({ + path: depth.root, + options: { maxDepth: 1 }, + filesystem: depth.filesystem + }) + expect(depth.visits).toEqual([depth.root, ...depth.branches]) + expect(boundedDepth.repos).toEqual([]) + const capped = fixture(posix) + const boundedResults = await scanNestedRepos({ + path: capped.root, + options: { maxRepos: 7 }, + filesystem: capped.filesystem + }) + expect(boundedResults.repos.map(({ path }) => path)).toEqual(capped.repositories.slice(0, 7)) + expect(boundedResults.truncated).toBe(true) +}) + +it('honors abort after a broad prefix has been consumed', async () => { + const controller = new AbortController() + const f = fixture(posix, (count) => { + if (count === 200) { + controller.abort() + } + }) + const result = await scanNestedRepos({ + path: f.root, + signal: controller.signal, + options: { maxRepos: 500 }, + filesystem: f.filesystem + }) + expect(f.visits).toHaveLength(200) + expect(result.repos.map(({ path }) => path)).toEqual(f.repositories.slice(0, 38)) + expect(result).toMatchObject({ stopped: true, timedOut: false }) +}) + +it.each([null, 500])( + 'preserves optional timeout=%s after consuming a broad prefix', + async (timeoutMs) => { + let now = 0 + vi.spyOn(Date, 'now').mockImplementation(() => now) + const f = fixture(posix, (count) => { + if (count === 200) { + now = 1_000 + } + }) + const result = await scanNestedRepos({ + path: f.root, + options: { maxRepos: 500, timeoutMs }, + filesystem: f.filesystem + }) + expect(result.repos.map(({ path }) => path)).toEqual( + timeoutMs === null ? f.repositories : f.repositories.slice(0, 38) + ) + expect(result).toMatchObject({ timedOut: timeoutMs !== null, timeoutMs, stopped: false }) + } +) diff --git a/src/main/project-groups/nested-repo-discovery.ts b/src/main/project-groups/nested-repo-discovery.ts index 84fddfd1167..1f25a85a26f 100644 --- a/src/main/project-groups/nested-repo-discovery.ts +++ b/src/main/project-groups/nested-repo-discovery.ts @@ -90,7 +90,7 @@ export async function scanNestedRepos(args: { return buildResult('non_git_folder') } - const foldersToTraverse: TraversalFolder[] = [ + const foldersToTraverse: (TraversalFolder | undefined)[] = [ { path: args.path, depth: 0, segments: [], ignoreRules: [] } ] let nextFolderIndex = 0 @@ -107,7 +107,13 @@ export async function scanNestedRepos(args: { if (noteAbort()) { break } - const currentFolder = foldersToTraverse[nextFolderIndex++] + const currentFolder = foldersToTraverse[nextFolderIndex++]! + // Release processed paths and inherited ignore rules before the next filesystem await. + foldersToTraverse[nextFolderIndex - 1] = undefined + if (nextFolderIndex >= 64 && nextFolderIndex * 2 >= foldersToTraverse.length) { + foldersToTraverse.splice(0, nextFolderIndex) + nextFolderIndex = 0 + } if (currentFolder.depth > options.maxDepth) { continue }