diff --git a/src/shared/cursor-sidecar-scan-directory.test.ts b/src/shared/cursor-sidecar-scan-directory.test.ts new file mode 100644 index 00000000000..02fad24c122 --- /dev/null +++ b/src/shared/cursor-sidecar-scan-directory.test.ts @@ -0,0 +1,289 @@ +import type { Dirent } from 'node:fs' +import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + CURSOR_DIR_MAX_ENTRIES_EXAMINED, + listLexicographicDirectoryNames, + retainLexicographic, + setStreamDirectoryIoForTests, + streamDirectoryNames +} from './cursor-sidecar-scan-directory' + +const tempRoots: string[] = [] + +afterEach(async () => { + setStreamDirectoryIoForTests() + await Promise.all(tempRoots.splice(0).map((root) => rm(root, { recursive: true, force: true }))) +}) + +async function tempDir(prefix: string): Promise { + const root = await mkdtemp(join(tmpdir(), prefix)) + tempRoots.push(root) + return root +} + +async function fillDirectory(dirPath: string, count: number, prefix = 'e'): Promise { + await mkdir(dirPath, { recursive: true }) + const names = Array.from( + { length: count }, + (_, index) => `${prefix}-${String(index).padStart(5, '0')}` + ) + await Promise.all(names.map((name) => writeFile(join(dirPath, name), ''))) + return names +} + +function unsupportedDirectoryStreamError(): NodeJS.ErrnoException { + return Object.assign(new Error('directory streams unavailable'), { code: 'ENOTSUP' }) +} + +describe('cursor sidecar directory examination bounds', () => { + it('reports examinationTruncated=false when the directory fits the exact budget', async () => { + const root = await tempDir('orca-cursor-dir-exact-') + const budget = 32 + await fillDirectory(root, budget) + + const visited: string[] = [] + const result = await streamDirectoryNames( + root, + (name) => { + visited.push(name) + }, + { maxEntriesExamined: budget } + ) + + expect(result.entriesExamined).toBe(budget) + expect(result.examinationTruncated).toBe(false) + expect(visited).toHaveLength(budget) + }) + + it('uses a bounded overflow probe so budget+1 truthfully reports truncation', async () => { + const root = await tempDir('orca-cursor-dir-overflow-') + const budget = 32 + await fillDirectory(root, budget + 1) + + const visited: string[] = [] + let onDirentCount = 0 + const result = await streamDirectoryNames( + root, + (name) => { + visited.push(name) + }, + { + maxEntriesExamined: budget, + onDirent: () => { + onDirentCount += 1 + } + } + ) + + expect(result.entriesExamined).toBe(budget) + expect(result.examinationTruncated).toBe(true) + expect(visited).toHaveLength(budget) + // Probe must not count as examination or fire onDirent. + expect(onDirentCount).toBe(budget) + }) + + it('hard-bounds the readdir fallback without retaining an unbounded listing', async () => { + const root = await tempDir('orca-cursor-dir-fallback-') + const budget = 16 + const names = await fillDirectory(root, budget + 8) + + let readdirCalls = 0 + let observedListingLength = 0 + setStreamDirectoryIoForTests({ + opendir: async () => { + throw unsupportedDirectoryStreamError() + }, + readdirWithFileTypes: async (dirPath) => { + readdirCalls += 1 + const entries = await readdir(dirPath, { withFileTypes: true }) + observedListingLength = entries.length + return entries + } + }) + + const visited: string[] = [] + const result = await streamDirectoryNames( + root, + (name) => { + visited.push(name) + }, + { maxEntriesExamined: budget } + ) + + expect(readdirCalls).toBe(1) + expect(result.entriesExamined).toBe(budget) + expect(result.examinationTruncated).toBe(true) + expect(visited).toHaveLength(budget) + expect(observedListingLength).toBe(names.length) + // After return, the eager listing must not remain as a retained examination set. + expect(visited.every((name) => names.includes(name))).toBe(true) + }) + + it('propagates cancellation from onDirent during examination', async () => { + const root = await tempDir('orca-cursor-dir-cancel-') + await fillDirectory(root, 64) + let seen = 0 + await expect( + streamDirectoryNames(root, () => undefined, { + maxEntriesExamined: 64, + onDirent: () => { + seen += 1 + if (seen >= 10) { + throw new Error('cursor_sidecar_scan_cancelled') + } + } + }) + ).rejects.toThrow('cursor_sidecar_scan_cancelled') + expect(seen).toBe(10) + }) + + it('propagates cancellation on the readdir fallback path', async () => { + const root = await tempDir('orca-cursor-dir-cancel-fallback-') + await fillDirectory(root, 40) + setStreamDirectoryIoForTests({ + opendir: async () => { + throw unsupportedDirectoryStreamError() + } + }) + let seen = 0 + await expect( + streamDirectoryNames(root, () => undefined, { + maxEntriesExamined: 40, + onDirent: () => { + seen += 1 + if (seen >= 5) { + throw new Error('cursor_sidecar_scan_cancelled') + } + } + }) + ).rejects.toThrow('cursor_sidecar_scan_cancelled') + expect(seen).toBe(5) + }) + + it('keeps listLexicographic examination-truncated when the walk hits the budget with more left', async () => { + const root = await tempDir('orca-cursor-dir-list-trunc-') + await fillDirectory(root, 20, 'z') + const listed = await listLexicographicDirectoryNames({ + dirPath: root, + limit: 5, + maxEntriesExamined: 10, + accept: () => true + }) + expect(listed.entriesExamined).toBe(10) + expect(listed.truncated).toBe(true) + expect(listed.names).toHaveLength(5) + }) + + it('does not mark truncated for an exact-budget complete directory with room in retention', async () => { + const root = await tempDir('orca-cursor-dir-list-exact-') + await fillDirectory(root, 10, 'a') + const listed = await listLexicographicDirectoryNames({ + dirPath: root, + limit: 20, + maxEntriesExamined: 10, + accept: () => true + }) + expect(listed.entriesExamined).toBe(10) + expect(listed.truncated).toBe(false) + expect(listed.names).toHaveLength(10) + }) + + it('bounds a 10k-entry directory walk for time, examination, and retained selection', async () => { + const root = await tempDir('orca-cursor-dir-10k-') + const total = 10_000 + await fillDirectory(root, total) + const budget = CURSOR_DIR_MAX_ENTRIES_EXAMINED + expect(total).toBeGreaterThan(budget) + + const visited: string[] = [] + const started = Date.now() + const result = await streamDirectoryNames( + root, + (name) => { + visited.push(name) + }, + { maxEntriesExamined: budget } + ) + const elapsedMs = Date.now() - started + + expect(result.entriesExamined).toBe(budget) + expect(result.examinationTruncated).toBe(true) + expect(visited).toHaveLength(budget) + // Adversarial cold walk must stay well under multi-second wall time on local SSDs. + expect(elapsedMs).toBeLessThan(15_000) + + const listed = await listLexicographicDirectoryNames({ + dirPath: root, + limit: 64, + maxEntriesExamined: budget, + accept: () => true + }) + expect(listed.entriesExamined).toBe(budget) + expect(listed.truncated).toBe(true) + expect(listed.names).toHaveLength(64) + }, 60_000) + + it('fallback path never visits more than the examination budget on a 10k store', async () => { + const root = await tempDir('orca-cursor-dir-10k-fallback-') + await fillDirectory(root, 10_000) + setStreamDirectoryIoForTests({ + opendir: async () => { + throw unsupportedDirectoryStreamError() + } + }) + const budget = 128 + let visits = 0 + const result = await streamDirectoryNames( + root, + () => { + visits += 1 + }, + { maxEntriesExamined: budget } + ) + expect(visits).toBe(budget) + expect(result.entriesExamined).toBe(budget) + expect(result.examinationTruncated).toBe(true) + }, 60_000) + + it('retainLexicographic keeps only the first `limit` names in order', () => { + const selected: string[] = [] + expect(retainLexicographic(selected, 'm', 3)).toBe(false) + expect(retainLexicographic(selected, 'a', 3)).toBe(false) + expect(retainLexicographic(selected, 'z', 3)).toBe(false) + expect(retainLexicographic(selected, 'b', 3)).toBe(true) + expect(selected).toEqual(['a', 'b', 'm']) + }) + + it('treats a zero examination budget as already truncated', async () => { + const root = await tempDir('orca-cursor-dir-zero-') + await fillDirectory(root, 3) + const visit = vi.fn() + const result = await streamDirectoryNames(root, visit, { maxEntriesExamined: 0 }) + expect(result).toEqual({ entriesExamined: 0, examinationTruncated: true }) + expect(visit).not.toHaveBeenCalled() + }) +}) + +describe('streamDirectoryNames Dirent typing for accept filters', () => { + it('passes real Dirent instances so isDirectory predicates work', async () => { + const root = await tempDir('orca-cursor-dir-types-') + await mkdir(join(root, 'bucket'), { recursive: true }) + await writeFile(join(root, 'file.txt'), '') + const kinds: string[] = [] + await streamDirectoryNames(root, (_name, entry: Dirent) => { + kinds.push( + entry.isDirectory() + ? 'dir' + : entry.isFile() + ? 'file' + : entry.isSymbolicLink() + ? 'link' + : 'other' + ) + }) + expect(kinds.sort()).toEqual(['dir', 'file']) + }) +}) diff --git a/src/shared/cursor-sidecar-scan-directory.ts b/src/shared/cursor-sidecar-scan-directory.ts index 3ed6320b7bb..de07463a783 100644 --- a/src/shared/cursor-sidecar-scan-directory.ts +++ b/src/shared/cursor-sidecar-scan-directory.ts @@ -1,4 +1,4 @@ -import type { Dirent } from 'node:fs' +import type { Dir, Dirent } from 'node:fs' import { opendir, readdir } from 'node:fs/promises' import { posix, win32 } from 'node:path' @@ -28,16 +28,40 @@ export function retainLexicographic(selected: string[], name: string, limit: num } export type StreamDirectoryNamesOptions = { - /** Invoked once per dirent observed (opendir read or readdir fallback entry). */ + /** Invoked once per examined dirent (not the overflow probe). */ onDirent?: () => void /** Hard stop after this many dirents; default CURSOR_DIR_MAX_ENTRIES_EXAMINED. */ maxEntriesExamined?: number } +type StreamDirectoryIo = { + opendir: (path: string) => ReturnType + readdirWithFileTypes: (path: string) => Promise +} + +const defaultStreamDirectoryIo: StreamDirectoryIo = { + opendir, + readdirWithFileTypes: (path) => readdir(path, { withFileTypes: true }) +} +let streamDirectoryIo: StreamDirectoryIo = defaultStreamDirectoryIo + +/** Test isolation for opendir vs readdir-fallback paths. */ +export function setStreamDirectoryIoForTests(next?: Partial): void { + streamDirectoryIo = next + ? { + opendir: next.opendir ?? defaultStreamDirectoryIo.opendir, + readdirWithFileTypes: + next.readdirWithFileTypes ?? defaultStreamDirectoryIo.readdirWithFileTypes + } + : defaultStreamDirectoryIo +} + /** - * Streams directory names without materializing the full listing. + * Streams directory names without materializing the full listing on opendir hosts. * Retention callers keep only their bounded selection; examination stops at - * maxEntriesExamined so cold adversarial directories cannot unbounded-walk. + * maxEntriesExamined. One extra dirent is read as an overflow probe so stopping + * exactly at the budget still reports examinationTruncated truthfully when more + * entries exist. */ export async function streamDirectoryNames( dirPath: string, @@ -45,46 +69,25 @@ export async function streamDirectoryNames( options: StreamDirectoryNamesOptions = {} ): Promise<{ entriesExamined: number; examinationTruncated: boolean }> { const maxEntries = options.maxEntriesExamined ?? CURSOR_DIR_MAX_ENTRIES_EXAMINED - let entriesExamined = 0 - let examinationTruncated = false - - const consider = (name: string, entry: Dirent): boolean => { - if (entriesExamined >= maxEntries) { - examinationTruncated = true - return false - } - entriesExamined += 1 - options.onDirent?.() - visit(name, entry) - return entriesExamined < maxEntries + if (maxEntries <= 0) { + return { entriesExamined: 0, examinationTruncated: true } } try { - const directory = await opendir(dirPath) + const directory = await streamDirectoryIo.opendir(dirPath) try { - for await (const entry of directory) { - if (!consider(entry.name, entry)) { - break - } - } + return await examineDirectoryStream(directory, visit, options, maxEntries) } finally { // Closing mid-iteration is best-effort; for-await also closes on break/throw. await directory.close().catch(() => undefined) } - return { entriesExamined, examinationTruncated } } catch (error) { // Fallback keeps remote-wire hosts that only expose readdir working. if (!isUnsupportedDirectoryStream(error)) { throw error } } - const entries = await readdir(dirPath, { withFileTypes: true }) - for (const entry of entries) { - if (!consider(entry.name, entry)) { - break - } - } - return { entriesExamined, examinationTruncated } + return examineReaddirFallback(dirPath, visit, options, maxEntries) } /** @@ -153,6 +156,55 @@ export function safeBasename(value: string): boolean { return Boolean(value && value !== '.' && value !== '..' && !/[\\/]/u.test(value)) } +async function examineDirectoryStream( + directory: Dir, + visit: (name: string, entry: Dirent) => void, + options: StreamDirectoryNamesOptions, + maxEntries: number +): Promise<{ entriesExamined: number; examinationTruncated: boolean }> { + let entriesExamined = 0 + let examinationTruncated = false + for await (const entry of directory) { + // Bounded overflow probe: reading one dirent past the budget proves that + // stopping exactly at maxEntries is truncation, without visiting it. + if (entriesExamined >= maxEntries) { + examinationTruncated = true + break + } + entriesExamined += 1 + options.onDirent?.() + visit(entry.name, entry) + } + return { entriesExamined, examinationTruncated } +} + +async function examineReaddirFallback( + dirPath: string, + visit: (name: string, entry: Dirent) => void, + options: StreamDirectoryNamesOptions, + maxEntries: number +): Promise<{ entriesExamined: number; examinationTruncated: boolean }> { + // Why: readdir eagerly allocates the full listing. Shrink immediately to the + // examination window so adversarial directories cannot keep an unbounded + // Dirent array alive past the budget (peak alloc is platform-limited). + const entries = await streamDirectoryIo.readdirWithFileTypes(dirPath) + let entriesExamined = 0 + try { + const examinationTruncated = entries.length > maxEntries + if (examinationTruncated) { + entries.length = maxEntries + } + for (const entry of entries) { + entriesExamined += 1 + options.onDirent?.() + visit(entry.name, entry) + } + return { entriesExamined, examinationTruncated } + } finally { + entries.length = 0 + } +} + function isUnsupportedDirectoryStream(error: unknown): boolean { const code = (error as NodeJS.ErrnoException | null)?.code return code === 'ERR_FEATURE_UNAVAILABLE_ON_PLATFORM' || code === 'ENOTSUP'