diff --git a/src/main/skills/skill-directory-read-ahead.ts b/src/main/skills/skill-directory-read-ahead.ts new file mode 100644 index 00000000000..e78979464d7 --- /dev/null +++ b/src/main/skills/skill-directory-read-ahead.ts @@ -0,0 +1,152 @@ +import type { Dirent } from 'node:fs' +import { readdir, realpath } from 'node:fs/promises' +import { join, sep } from 'node:path' + +// Why: keeps scans from flooding libuv's 4-thread pool. Up to 3 read-ahead reads, plus one demand read from `claim` (the walk claims serially): at most 4 reads, each up to two requests. Per walk: discovery walks roots concurrently. +const MAX_IN_FLIGHT_DIRECTORY_READS = 3 +// Finished listings the walk has not claimed yet; bounds memory on wide trees. +const MAX_UNCLAIMED_DIRECTORY_READS = 512 + +export type DirectoryRead = { resolvedPath: string | null; entries: Dirent[] | null } + +/** A directory's place in the walk: entry indexes from the root. Lexical order is walk order. */ +export type WalkOrder = readonly number[] + +type Ticket = { + dirPath: string + order: WalkOrder + /** Known for a plain child of a resolved directory: no symlink can change it. */ + resolvedPath: string | null + started: Promise | null + dropped: boolean +} + +function compareWalkOrder(left: WalkOrder, right: WalkOrder): number { + const length = Math.min(left.length, right.length) + for (let index = 0; index < length; index += 1) { + if (left[index] !== right[index]) { + return left[index] - right[index] + } + } + return left.length - right.length +} + +async function readDirectory(ticket: Ticket): Promise { + const [resolvedPath, entries] = await Promise.all([ + ticket.resolvedPath ?? realpath(ticket.dirPath).catch(() => null), + readdir(ticket.dirPath, { withFileTypes: true }).catch(() => null) + ]) + return { resolvedPath, entries } +} + +export type DirectoryReadAhead = { + /** The listing the walk needs now; a read started ahead is reused. */ + claim: (dirPath: string, order: WalkOrder) => Promise + /** The walk will not enter this directory's children; stop reading them ahead. */ + drop: (dirPath: string) => void +} + +/** + * Reads a directory tree ahead of a depth-first walk, in the order the walk will claim it. A busy event loop then costs one turn per batch of reads instead of one per directory. The walk still claims listings in its own order, so its results and cycle guards are unchanged. + */ +export function createDirectoryReadAhead(args: { + signal?: AbortSignal + /** Entry indexes of the plain child directories the walk will enter. */ + enteredChildIndexes: (dirPath: string, entries: readonly Dirent[]) => number[] +}): DirectoryReadAhead { + const { signal, enteredChildIndexes } = args + const tickets = new Map() + // Unstarted reads in reverse walk order, so the one the walk needs soonest pops off the end. + const queue: Ticket[] = [] + let inFlight = 0 + let unclaimed = 0 + + function start(ticket: Ticket): Promise { + inFlight += 1 + const started = readDirectory(ticket).then((read) => { + inFlight -= 1 + if (read.entries && !ticket.dropped && !signal?.aborted) { + queueChildren(ticket, read) + } + pump() + return read + }) + ticket.started = started + return started + } + + function queueChildren(parent: Ticket, read: DirectoryRead): void { + const entries = read.entries ?? [] + const children = enteredChildIndexes(parent.dirPath, entries).map((index): Ticket => { + const name = entries[index].name + const ticket: Ticket = { + dirPath: join(parent.dirPath, name), + order: [...parent.order, index], + resolvedPath: read.resolvedPath === null ? null : join(read.resolvedPath, name), + started: null, + dropped: false + } + tickets.set(ticket.dirPath, ticket) + return ticket + }) + // No queued read sits between a parent and its children, so they go in as one block after every later read. + let low = 0 + let high = queue.length + while (low < high) { + const mid = (low + high) >> 1 + if (compareWalkOrder(queue[mid].order, parent.order) > 0) { + low = mid + 1 + } else { + high = mid + } + } + // Loop instead of spread so a huge directory cannot exceed the argument limit. + for (const ticket of [...children.toReversed(), ...queue.splice(low)]) { + queue.push(ticket) + } + } + + function pump(): void { + while ( + !signal?.aborted && + inFlight < MAX_IN_FLIGHT_DIRECTORY_READS && + unclaimed < MAX_UNCLAIMED_DIRECTORY_READS && + queue.length > 0 + ) { + const ticket = queue.pop() + if (ticket && !ticket.started && !ticket.dropped) { + unclaimed += 1 + void start(ticket) + } + } + } + + return { + claim: (dirPath, order) => { + const ticket = tickets.get(dirPath) + tickets.delete(dirPath) + if (ticket?.started) { + unclaimed -= 1 + pump() + return ticket.started + } + // Not read yet: start it now rather than wait behind reads the walk needs later. + return start(ticket ?? { dirPath, order, resolvedPath: null, started: null, dropped: false }) + }, + drop: (dirPath) => { + // O(tickets): read-ahead tickets are capped by MAX_UNCLAIMED_DIRECTORY_READS, but queued ones are not. + const prefix = dirPath.endsWith(sep) ? dirPath : `${dirPath}${sep}` + for (const [path, ticket] of tickets) { + if (!path.startsWith(prefix)) { + continue + } + ticket.dropped = true + tickets.delete(path) + if (ticket.started) { + unclaimed -= 1 + } + } + pump() + } + } +} diff --git a/src/main/skills/skill-root-file-walk.ts b/src/main/skills/skill-root-file-walk.ts index b3cbaa39da5..95e91239bdd 100644 --- a/src/main/skills/skill-root-file-walk.ts +++ b/src/main/skills/skill-root-file-walk.ts @@ -1,7 +1,8 @@ import type { Dirent } from 'node:fs' -import { readdir, realpath, stat } from 'node:fs/promises' +import { stat } from 'node:fs/promises' import { isAbsolute, join, relative, sep } from 'node:path' import { isSkillStagingEntryName } from './skill-delete/staging-names' +import { createDirectoryReadAhead, type WalkOrder } from './skill-directory-read-ahead' export const SKILL_FILE_NAME = 'SKILL.md' @@ -17,12 +18,29 @@ function isWithinDepth(rootPath: string, childPath: string, maxDepth: number): b return rel.split(sep).length <= maxDepth } -async function readEntries(dirPath: string): Promise { - try { - return await readdir(dirPath, { withFileTypes: true }) - } catch { - return null +/** Entry indexes of the plain child directories a walk enters. */ +function enteredChildIndexes( + rootPath: string, + maxDepth: number, + dirPath: string, + entries: readonly Dirent[] +): number[] { + const indexes: number[] = [] + for (const [index, entry] of entries.entries()) { + if ( + !entry.isDirectory() || + entry.name === SKILL_FILE_NAME || + isSkillStagingEntryName(entry.name) + ) { + continue + } + // Directory entry names add one segment, so siblings share the depth verdict. + if (!isWithinDepth(rootPath, join(dirPath, entry.name), maxDepth)) { + return [] + } + indexes.push(index) } + return indexes } /** Abort prevents future filesystem work; dispatched readdir/realpath/stat calls cannot be cancelled and finish in libuv. */ @@ -33,27 +51,25 @@ export async function findSkillFiles( ): Promise { const out: string[] = [] const visitedDirectoryPaths = new Set() - async function visit(dirPath: string): Promise { + const readAhead = createDirectoryReadAhead({ + signal, + enteredChildIndexes: (dirPath, entries) => + enteredChildIndexes(rootPath, maxDepth, dirPath, entries) + }) + async function visit(dirPath: string, order: WalkOrder): Promise { // Reject on abort; a cached truncated listing would falsely report skills removed. signal?.throwIfAborted() - let resolvedDirPath: string - try { - resolvedDirPath = await realpath(dirPath) - } catch { - return - } - if (visitedDirectoryPaths.has(resolvedDirPath)) { + const { resolvedPath: resolvedDirPath, entries } = await readAhead.claim(dirPath, order) + if (resolvedDirPath === null || visitedDirectoryPaths.has(resolvedDirPath)) { + readAhead.drop(dirPath) return } visitedDirectoryPaths.add(resolvedDirPath) - - const entries = await readEntries(dirPath) if (!entries) { return } - // Directory entry names add one segment, so siblings share the depth verdict. - let childrenWithinDepth: boolean | undefined - for (const entry of entries) { + const childIndexes = new Set(enteredChildIndexes(rootPath, maxDepth, dirPath, entries)) + for (const [index, entry] of entries.entries()) { signal?.throwIfAborted() // Hide transaction siblings so in-flight skills cannot appear twice or be deleted again. if (isSkillStagingEntryName(entry.name)) { @@ -77,15 +93,12 @@ export async function findSkillFiles( continue } if (entry.isDirectory()) { - if ((childrenWithinDepth ??= isWithinDepth(rootPath, entryPath, maxDepth))) { - await visit(entryPath) + if (childIndexes.has(index)) { + await visit(entryPath, [...order, index]) } continue } - if ( - entry.isSymbolicLink() && - (childrenWithinDepth ??= isWithinDepth(rootPath, entryPath, maxDepth)) - ) { + if (entry.isSymbolicLink() && isWithinDepth(rootPath, entryPath, maxDepth)) { // Follow provider directory links but guard realpaths against cycles. let linksToDirectory = false try { @@ -95,11 +108,11 @@ export async function findSkillFiles( } // Keep recursive visit outside broken-link catch so nested aborts cannot become successful truncated scans. if (linksToDirectory) { - await visit(entryPath) + await visit(entryPath, [...order, index]) } } } } - await visit(rootPath) + await visit(rootPath, []) return out }