fix: add limits to bun auto-type fetching

frontend/src/lib/ata/apis.ts

@@ -1,21 +1,19 @@
-import type { ATABootstrapConfig } from './index'
-
 //  https://github.com/jsdelivr/data.jsdelivr.com
 
-export const getNPMVersionsForModule = (config: ATABootstrapConfig, moduleName: string) => {
+export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => {
 	const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}`
-	return api<{ tags: Record<string, string>; versions: string[] }>(config, url, {
+	return api<{ tags: Record<string, string>; versions: string[] }>(url, resLimit, {
 		cache: 'no-store'
 	})
 }
 
 export const getNPMVersionForModuleReference = (
-	config: ATABootstrapConfig,
 	moduleName: string,
-	reference: string
+	reference: string,
+	resLimit: ResLimit
 ) => {
 	const url = `https://data.jsdelivr.com/v1/package/resolve/npm/${moduleName}@${reference}`
-	return api<{ version: string | null }>(config, url)
+	return api<{ version: string | null }>(url, resLimit)
 }
 
 export type NPMTreeMeta = {
@@ -27,13 +25,13 @@ export type NPMTreeMeta = {
 }
 
 export const getFiletreeForModuleWithVersion = async (
-	config: ATABootstrapConfig,
 	moduleName: string,
 	version: string,
-	raw: string
+	raw: string,
+	resLimit: ResLimit
 ) => {
 	const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}@${version}/flat`
-	const res = await api<NPMTreeMeta>(config, url)
+	const res = await api<NPMTreeMeta>(url, resLimit)
 	if (res instanceof Error) {
 		return res
 	} else {
@@ -47,7 +45,6 @@ export const getFiletreeForModuleWithVersion = async (
 }
 
 export const getDTSFileForModuleWithVersion = async (
-	config: ATABootstrapConfig,
 	moduleName: string,
 	version: string,
 	file: string
@@ -62,10 +59,26 @@ export const getDTSFileForModuleWithVersion = async (
 	}
 }
 
-function api<T>(config: ATABootstrapConfig, url: string, init?: RequestInit): Promise<T | Error> {
+export interface ResLimit {
+	usage: number
+}
+
+function api<T>(url: string, resLimit: ResLimit, init?: RequestInit): Promise<T | Error> {
+	if (resLimit.usage > 500000) {
+		console.warn(
+			`Exceeded limit of types downloaded for the needs of the assistant fetching: ${url}`
+		)
+		return new Promise(() => new Error('Exceeded limit of 100MB of data downloaded.'))
+	}
+
 	return fetch(url, init).then((res) => {
 		if (res.ok) {
-			return res.json().then((f) => f as T)
+			return res.text().then((text) => {
+				resLimit.usage += text.length
+				console.log('resLimit', url, resLimit.usage)
+
+				return JSON.parse(text) as T
+			}) as Promise<T | Error>
 		} else {
 			return new Error('OK')
 		}

frontend/src/lib/ata/index.ts

@@ -3,7 +3,8 @@ import {
 	getFiletreeForModuleWithVersion,
 	getNPMVersionForModuleReference,
 	getNPMVersionsForModule,
-	type NPMTreeMeta
+	type NPMTreeMeta,
+	type ResLimit
 } from './apis'
 import { isRelativePath, mapModuleNameToModule } from './edgeCases'
 
@@ -54,7 +55,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 		estimatedToDownload = 0
 		estimatedDownloaded = 0
 
-		return resolveDeps(initialSourceFile, 0).then((t) => {
+		return resolveDeps(initialSourceFile, 0, { usage: 0 }).then((t) => {
 			if (estimatedDownloaded > 0) {
 				config.delegate.finished?.(fsMap)
 			}
@@ -73,11 +74,10 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 		return 'latest'
 	}
 
-	async function resolveDeps(initialSourceFile: string, depth: number) {
+	async function resolveDeps(initialSourceFile: string, depth: number, resLimit: ResLimit) {
 		// if (depth > 2) {
 		// 	return
 		// }
-
 		let depsToGet = config
 			.depsParser(initialSourceFile)
 			.map((d: string) => {
@@ -93,6 +93,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 		if (depsToGet.length === 0) {
 			return
 		}
+
 		// Make it so it won't get re-downloaded
 		depsToGet.forEach((dep) => moduleMap.set(dep.raw, { state: 'loading' }))
 
@@ -113,24 +114,28 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 
 		// Grab the module trees which gives us a list of files to download
 		const trees = await Promise.all(
-			depsToGet.map((f) => getFileTreeForModuleWithTag(config, f.module, f.version, f.raw))
+			depsToGet.map((f) => getFileTreeForModuleWithTag(f.module, f.version, f.raw, resLimit))
 		)
+		console.log(trees, 'trees')
 		const treesOnly = trees.filter((t) => !('error' in t)) as NPMTreeMeta[]
 
 		// These are the modules which we can grab directly
 		const hasDTS = treesOnly.filter((t) => t.files.find((f) => f.name.endsWith('.d.ts')))
 		const dtsFilesFromNPM = hasDTS.map((t) => treeToDTSFiles(t, `/node_modules/${t.raw}`))
-
+		// console.log(dtsFilesFromNPM, 'dtsFilesFromNPM')
 		// These are ones we need to look on DT for (which may not be there, who knows)
 		const mightBeOnDT = treesOnly.filter((t) => !hasDTS.includes(t))
+		// console.log(mightBeOnDT, 'mightBeOnDT')
+		// return
+
 		const dtTrees = await Promise.all(
 			// TODO: Switch from 'latest' to the version from the original tree which is user-controlled
 			mightBeOnDT.map((f) =>
 				getFileTreeForModuleWithTag(
-					config,
 					`@types/${getDTName(f.moduleName)}`,
 					'latest',
-					`@types/${getDTName(f.raw)}`
+					`@types/${getDTName(f.raw)}`,
+					resLimit
 				)
 			)
 		)
@@ -150,7 +155,6 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 		// Grab the package.jsons for each dependency
 		for (const tree of treesOnly) {
 			const pkgJSON = await getDTSFileForModuleWithVersion(
-				config,
 				tree.moduleName,
 				tree.version,
 				'/package.json'
@@ -172,7 +176,6 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 		await Promise.all(
 			allDTSFiles.map(async (dts) => {
 				const dtsCode = await getDTSFileForModuleWithVersion(
-					config,
 					dts.moduleName,
 					dts.moduleVersion,
 					dts.path
@@ -192,7 +195,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => {
 
 					if (dts.moduleName != 'bun-types') {
 						// Recurse through deps
-						await resolveDeps(dtsCode, depth + 1)
+						await resolveDeps(dtsCode, depth + 1, resLimit)
 					}
 				}
 			})
@@ -225,10 +228,10 @@ function treeToDTSFiles(tree: NPMTreeMeta, vfsPrefix: string) {
 
 /** The bulk load of the work in getting the filetree based on how people think about npm names and versions */
 export const getFileTreeForModuleWithTag = async (
-	config: ATABootstrapConfig,
 	moduleName: string,
 	tag: string | undefined,
-	raw: string
+	raw: string,
+	resLimit: ResLimit
 ) => {
 	let toDownload = tag || 'latest'
 
@@ -237,7 +240,7 @@ export const getFileTreeForModuleWithTag = async (
 	if (toDownload.split('.').length < 2) {
 		// The jsdelivr API needs a _version_ not a tag. So, we need to switch out
 		// the tag to the version via an API request.
-		const response = await getNPMVersionForModuleReference(config, moduleName, toDownload)
+		const response = await getNPMVersionForModuleReference(moduleName, toDownload, resLimit)
 		if (response instanceof Error) {
 			return {
 				error: response,
@@ -247,7 +250,7 @@ export const getFileTreeForModuleWithTag = async (
 
 		const neededVersion = response.version
 		if (!neededVersion) {
-			const versions = await getNPMVersionsForModule(config, moduleName)
+			const versions = await getNPMVersionsForModule(moduleName, resLimit)
 			if (versions instanceof Error) {
 				return {
 					error: response,
@@ -265,7 +268,7 @@ export const getFileTreeForModuleWithTag = async (
 		toDownload = neededVersion
 	}
 
-	const res = await getFiletreeForModuleWithVersion(config, moduleName, toDownload, raw)
+	const res = await getFiletreeForModuleWithVersion(moduleName, toDownload, raw, resLimit)
 	if (res instanceof Error) {
 		return {
 			error: res,