diff --git a/frontend/package-lock.json b/frontend/package-lock.json index cf7c437ab6..ca7e4c97c2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -40,6 +40,7 @@ "monaco-languageclient": "~7.0.1", "ol": "^7.4.0", "openai": "^4.47.1", + "p-limit": "^6.1.0", "pdfjs-dist": "^3.8.162", "quill": "^1.3.7", "rfc4648": "^1.5.3", @@ -7574,15 +7575,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz", + "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==", + "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.1.1" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -7603,6 +7604,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/pako": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/pako/-/pako-2.1.0.tgz", @@ -10927,12 +10957,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", + "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", + "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/frontend/package.json b/frontend/package.json index 737f6e340d..98ef453448 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -121,6 +121,7 @@ "monaco-languageclient": "~7.0.1", "ol": "^7.4.0", "openai": "^4.47.1", + "p-limit": "^6.1.0", "pdfjs-dist": "^3.8.162", "quill": "^1.3.7", "rfc4648": "^1.5.3", diff --git a/frontend/src/lib/ata/apis.ts b/frontend/src/lib/ata/apis.ts index 6bd8fad530..b2a18d3fec 100644 --- a/frontend/src/lib/ata/apis.ts +++ b/frontend/src/lib/ata/apis.ts @@ -1,5 +1,7 @@ // https://github.com/jsdelivr/data.jsdelivr.com +import pLimit from 'p-limit' + export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => { const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}` return api<{ tags: Record; versions: string[] }>(url, resLimit, { @@ -51,7 +53,7 @@ export const getDTSFileForModuleWithVersion = async ( ) => { // file comes with a prefix / const url = `https://cdn.jsdelivr.net/npm/${moduleName}@${version}${file}` - const res = await fetch(url) + const res = await limit(() => fetch(url)) if (res.ok) { return res.text() } else { @@ -67,6 +69,8 @@ export function isOverlimit(resLimit: ResLimit) { return resLimit.usage > 5000000 } +export const limit = pLimit(6) + function api(url: string, resLimit: ResLimit, init?: RequestInit): Promise { if (isOverlimit(resLimit)) { console.warn( @@ -75,16 +79,18 @@ function api(url: string, resLimit: ResLimit, init?: RequestInit): Promise new Error('Exceeded limit of 100MB of data downloaded.')) } - return fetch(url, init).then((res) => { - if (res.ok) { - return res.text().then((text) => { - resLimit.usage += text.length - console.log('resLimit', url, resLimit.usage) + return limit(() => + fetch(url, init).then((res) => { + if (res.ok) { + return res.text().then((text) => { + resLimit.usage += text.length + console.log('resLimit', url, resLimit.usage) - return JSON.parse(text) as T - }) as Promise - } else { - return new Error('OK') - } - }) + return JSON.parse(text) as T + }) as Promise + } else { + return new Error('OK') + } + }) + ) } diff --git a/frontend/src/lib/ata/index.ts b/frontend/src/lib/ata/index.ts index 4be56edb0f..1a2ade62f2 100644 --- a/frontend/src/lib/ata/index.ts +++ b/frontend/src/lib/ata/index.ts @@ -4,6 +4,7 @@ import { getNPMVersionForModuleReference, getNPMVersionsForModule, isOverlimit, + limit, type NPMTreeMeta, type ResLimit } from './apis' @@ -49,6 +50,7 @@ export const setupTypeAcquisition = (config: ATABootstrapConfig) => { const moduleMap = new Map() const fsMap = new Map() + limit.clearQueue() let estimatedToDownload = 0 let estimatedDownloaded = 0