import { opendir } from 'node:fs/promises'
import type { Dirent, Dir } from 'node:fs'
import { join } from 'node:path'
import {
assertMarkdownDocumentPathWithinLimit,
createMarkdownDocumentListingBudget,
MarkdownDocumentListingCapacityError,
retainMarkdownRelativePath,
visitMarkdownDocumentListingEntry,
type MarkdownDocumentListingLimits
} from './markdown-document-listing-limits'
type MarkdownDirectoryReader = (path: string) => Promise
>
export type MarkdownDocumentDiscoveryOptions = {
shouldDescend: (relativePath: string, name: string) => boolean
ignoreNestedDirectoryErrors?: boolean
limits?: Partial
readDirectory?: MarkdownDirectoryReader
signal?: AbortSignal
}
export function isMarkdownDocumentPath(path: string): boolean {
const lowerPath = path.toLowerCase()
return lowerPath.endsWith('.md') || lowerPath.endsWith('.mdx') || lowerPath.endsWith('.markdown')
}
export async function discoverMarkdownRelativePaths(
rootPath: string,
options: MarkdownDocumentDiscoveryOptions
): Promise {
const budget = createMarkdownDocumentListingBudget(options.limits)
const documents: string[] = []
const readDirectory = options.readDirectory ?? opendir
assertMarkdownDocumentPathWithinLimit(rootPath, budget.limits.maxPathBytes)
const visitDirectory = async (
absoluteDirectoryPath: string,
relativeDirectoryPath: string,
depth: number
): Promise => {
throwIfAborted(options.signal)
let directory: Dir | AsyncIterable
try {
directory = await readDirectory(absoluteDirectoryPath)
} catch (error) {
if (depth > 0 && options.ignoreNestedDirectoryErrors) {
return
}
throw error
}
for await (const entry of directory) {
throwIfAborted(options.signal)
const relativePath = relativeDirectoryPath
? `${relativeDirectoryPath}/${entry.name}`
: entry.name
const nextDepth = depth + 1
const shouldDescend = entry.isDirectory() && options.shouldDescend(relativePath, entry.name)
visitMarkdownDocumentListingEntry(budget, relativePath, shouldDescend ? nextDepth : depth)
if (entry.isSymbolicLink()) {
continue
}
if (entry.isDirectory()) {
if (shouldDescend) {
await visitDirectory(join(absoluteDirectoryPath, entry.name), relativePath, nextDepth)
}
continue
}
if (entry.isFile() && isMarkdownDocumentPath(entry.name)) {
retainMarkdownRelativePath(budget, rootPath, relativePath)
documents.push(relativePath)
}
}
}
await visitDirectory(rootPath, '', 0)
return documents
}
function throwIfAborted(signal: AbortSignal | undefined): void {
if (!signal?.aborted) {
return
}
throw signal.reason instanceof Error ? signal.reason : new MarkdownDocumentListingCapacityError()
}