diff --git a/frontend/src/lib/ata/apis.ts b/frontend/src/lib/ata/apis.ts index 16ec445a5f..4d5d1163d1 100644 --- a/frontend/src/lib/ata/apis.ts +++ b/frontend/src/lib/ata/apis.ts @@ -1,21 +1,19 @@ -import type { ATABootstrapConfig } from './index' - // https://github.com/jsdelivr/data.jsdelivr.com -export const getNPMVersionsForModule = (config: ATABootstrapConfig, moduleName: string) => { +export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => { const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}` - return api<{ tags: Record; versions: string[] }>(config, url, { + return api<{ tags: Record; versions: string[] }>(url, resLimit, { cache: 'no-store' }) } export const getNPMVersionForModuleReference = ( - config: ATABootstrapConfig, moduleName: string, - reference: string + reference: string, + resLimit: ResLimit ) => { const url = `https://data.jsdelivr.com/v1/package/resolve/npm/${moduleName}@${reference}` - return api<{ version: string | null }>(config, url) + return api<{ version: string | null }>(url, resLimit) } export type NPMTreeMeta = { @@ -27,13 +25,13 @@ export type NPMTreeMeta = { } export const getFiletreeForModuleWithVersion = async ( - config: ATABootstrapConfig, moduleName: string, version: string, - raw: string + raw: string, + resLimit: ResLimit ) => { const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}@${version}/flat` - const res = await api(config, url) + const res = await api(url, resLimit) if (res instanceof Error) { return res } else { @@ -47,7 +45,6 @@ export const getFiletreeForModuleWithVersion = async ( } export const getDTSFileForModuleWithVersion = async ( - config: ATABootstrapConfig, moduleName: string, version: string, file: string @@ -62,10 +59,26 @@ export const getDTSFileForModuleWithVersion = async ( } } -function api(config: ATABootstrapConfig, url: string, init?: RequestInit): Promise { +export interface ResLimit { + usage: number +} + +function api(url: string, resLimit: ResLimit, init?: RequestInit): Promise { + if (resLimit.usage > 500000) { + console.warn( + `Exceeded limit of types downloaded for the needs of the assistant fetching: ${url}` + ) + return new Promise(() => new Error('Exceeded limit of 100MB of data downloaded.')) + } + return fetch(url, init).then((res) => { if (res.ok) { - return res.json().then((f) => f as T) + return res.text().then((text) => { + resLimit.usage += text.length + console.log('resLimit', url, resLimit.usage) + + return JSON.parse(text) as T + }) as Promise } else { return new Error('OK') } diff --git a/frontend/src/lib/ata/index.ts b/frontend/src/lib/ata/index.ts index 30d08dfc36..ff550bb7f6 100644 --- a/frontend/src/lib/ata/index.ts +++ b/frontend/src/lib/ata/index.ts @@ -3,7 +3,8 @@ import { getFiletreeForModuleWithVersion, getNPMVersionForModuleReference, getNPMVersionsForModule, - type NPMTreeMeta + type NPMTreeMeta, + type ResLimit } from './apis' import { isRelativePath, mapModuleNameToModule } from './edgeCases' @@ -54,7 +55,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { estimatedToDownload = 0 estimatedDownloaded = 0 - return resolveDeps(initialSourceFile, 0).then((t) => { + return resolveDeps(initialSourceFile, 0, { usage: 0 }).then((t) => { if (estimatedDownloaded > 0) { config.delegate.finished?.(fsMap) } @@ -73,11 +74,10 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { return 'latest' } - async function resolveDeps(initialSourceFile: string, depth: number) { + async function resolveDeps(initialSourceFile: string, depth: number, resLimit: ResLimit) { // if (depth > 2) { // return // } - let depsToGet = config .depsParser(initialSourceFile) .map((d: string) => { @@ -93,6 +93,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { if (depsToGet.length === 0) { return } + // Make it so it won't get re-downloaded depsToGet.forEach((dep) => moduleMap.set(dep.raw, { state: 'loading' })) @@ -113,24 +114,28 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { // Grab the module trees which gives us a list of files to download const trees = await Promise.all( - depsToGet.map((f) => getFileTreeForModuleWithTag(config, f.module, f.version, f.raw)) + depsToGet.map((f) => getFileTreeForModuleWithTag(f.module, f.version, f.raw, resLimit)) ) + console.log(trees, 'trees') const treesOnly = trees.filter((t) => !('error' in t)) as NPMTreeMeta[] // These are the modules which we can grab directly const hasDTS = treesOnly.filter((t) => t.files.find((f) => f.name.endsWith('.d.ts'))) const dtsFilesFromNPM = hasDTS.map((t) => treeToDTSFiles(t, `/node_modules/${t.raw}`)) - + // console.log(dtsFilesFromNPM, 'dtsFilesFromNPM') // These are ones we need to look on DT for (which may not be there, who knows) const mightBeOnDT = treesOnly.filter((t) => !hasDTS.includes(t)) + // console.log(mightBeOnDT, 'mightBeOnDT') + // return + const dtTrees = await Promise.all( // TODO: Switch from 'latest' to the version from the original tree which is user-controlled mightBeOnDT.map((f) => getFileTreeForModuleWithTag( - config, `@types/${getDTName(f.moduleName)}`, 'latest', - `@types/${getDTName(f.raw)}` + `@types/${getDTName(f.raw)}`, + resLimit ) ) ) @@ -150,7 +155,6 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { // Grab the package.jsons for each dependency for (const tree of treesOnly) { const pkgJSON = await getDTSFileForModuleWithVersion( - config, tree.moduleName, tree.version, '/package.json' @@ -172,7 +176,6 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { await Promise.all( allDTSFiles.map(async (dts) => { const dtsCode = await getDTSFileForModuleWithVersion( - config, dts.moduleName, dts.moduleVersion, dts.path @@ -192,7 +195,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { if (dts.moduleName != 'bun-types') { // Recurse through deps - await resolveDeps(dtsCode, depth + 1) + await resolveDeps(dtsCode, depth + 1, resLimit) } } }) @@ -225,10 +228,10 @@ function treeToDTSFiles(tree: NPMTreeMeta, vfsPrefix: string) { /** The bulk load of the work in getting the filetree based on how people think about npm names and versions */ export const getFileTreeForModuleWithTag = async ( - config: ATABootstrapConfig, moduleName: string, tag: string | undefined, - raw: string + raw: string, + resLimit: ResLimit ) => { let toDownload = tag || 'latest' @@ -237,7 +240,7 @@ export const getFileTreeForModuleWithTag = async ( if (toDownload.split('.').length < 2) { // The jsdelivr API needs a _version_ not a tag. So, we need to switch out // the tag to the version via an API request. - const response = await getNPMVersionForModuleReference(config, moduleName, toDownload) + const response = await getNPMVersionForModuleReference(moduleName, toDownload, resLimit) if (response instanceof Error) { return { error: response, @@ -247,7 +250,7 @@ export const getFileTreeForModuleWithTag = async ( const neededVersion = response.version if (!neededVersion) { - const versions = await getNPMVersionsForModule(config, moduleName) + const versions = await getNPMVersionsForModule(moduleName, resLimit) if (versions instanceof Error) { return { error: response, @@ -265,7 +268,7 @@ export const getFileTreeForModuleWithTag = async ( toDownload = neededVersion } - const res = await getFiletreeForModuleWithVersion(config, moduleName, toDownload, raw) + const res = await getFiletreeForModuleWithVersion(moduleName, toDownload, raw, resLimit) if (res instanceof Error) { return { error: res,