mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-12 00:06:14 +00:00
feat: add private npm registry proxy support for ATA in webide (#7597)
This commit is contained in:
Generated
+2
@@ -15343,6 +15343,7 @@ dependencies = [
|
||||
"deno_core",
|
||||
"deno_error",
|
||||
"ed25519-dalek",
|
||||
"flate2",
|
||||
"futures",
|
||||
"git-version",
|
||||
"google-cloud-googleapis",
|
||||
@@ -15390,6 +15391,7 @@ dependencies = [
|
||||
"sha2 0.10.9",
|
||||
"sql-builder",
|
||||
"sqlx",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.17",
|
||||
"time",
|
||||
|
||||
@@ -405,6 +405,7 @@ async-once-cell = "0.5.4"
|
||||
aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
|
||||
crc = "^3"
|
||||
tar = "^0"
|
||||
flate2 = "^1"
|
||||
http = "^1"
|
||||
async-stream = "^0"
|
||||
|
||||
|
||||
@@ -157,6 +157,8 @@ google-cloud-googleapis = { workspace = true , optional = true }
|
||||
tonic = { workspace = true, optional = true }
|
||||
deno_error = { workspace = true, optional = true }
|
||||
deno_core = { workspace = true, optional = true }
|
||||
tar.workspace = true
|
||||
flate2.workspace = true
|
||||
backon = {workspace = true, optional = true}
|
||||
|
||||
[build-dependencies]
|
||||
|
||||
@@ -4977,6 +4977,141 @@ paths:
|
||||
items:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/npm_proxy/metadata/{package}:
|
||||
get:
|
||||
summary: get npm package metadata from private registry
|
||||
operationId: getNpmPackageMetadata
|
||||
tags:
|
||||
- npm_proxy
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: package
|
||||
description: npm package name
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: package metadata
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
tags:
|
||||
type: object
|
||||
additionalProperties:
|
||||
type: string
|
||||
versions:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/npm_proxy/resolve/{package}:
|
||||
get:
|
||||
summary: resolve npm package version from private registry
|
||||
operationId: resolveNpmPackageVersion
|
||||
tags:
|
||||
- npm_proxy
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: package
|
||||
description: npm package name
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: tag
|
||||
description: version tag or reference
|
||||
in: query
|
||||
required: false
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: resolved version
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
version:
|
||||
type: string
|
||||
nullable: true
|
||||
|
||||
/w/{workspace}/npm_proxy/filetree/{package}/{version}:
|
||||
get:
|
||||
summary: get npm package file tree from private registry
|
||||
operationId: getNpmPackageFiletree
|
||||
tags:
|
||||
- npm_proxy
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: package
|
||||
description: npm package name
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: version
|
||||
description: package version
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: package file tree
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
default:
|
||||
type: string
|
||||
files:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/npm_proxy/file/{package}/{version}/{filepath}:
|
||||
get:
|
||||
summary: get specific file from npm package in private registry
|
||||
operationId: getNpmPackageFile
|
||||
tags:
|
||||
- npm_proxy
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: package
|
||||
description: npm package name
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: version
|
||||
description: package version
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
- name: filepath
|
||||
description: file path within package
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
"200":
|
||||
description: file content
|
||||
content:
|
||||
text/plain:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/embeddings/query_resource_types:
|
||||
get:
|
||||
summary: query resource types by similarity
|
||||
|
||||
@@ -106,6 +106,7 @@ mod inkeep_oss;
|
||||
mod inputs;
|
||||
mod integration;
|
||||
mod live_migrations;
|
||||
mod npm_proxy;
|
||||
#[cfg(feature = "http_trigger")]
|
||||
mod openapi;
|
||||
#[cfg(all(feature = "private", feature = "parquet"))]
|
||||
@@ -496,6 +497,7 @@ pub async fn run_server(
|
||||
Router::new()
|
||||
})
|
||||
.nest("/ai", ai::workspaced_service())
|
||||
.nest("/npm_proxy", npm_proxy::workspaced_service())
|
||||
.nest("/raw_apps", raw_apps::workspaced_service())
|
||||
.nest("/resources", resources::workspaced_service())
|
||||
.nest("/schedules", schedule::workspaced_service())
|
||||
|
||||
@@ -0,0 +1,502 @@
|
||||
/*
|
||||
* This file provides a proxy endpoint for npm package requests
|
||||
* to support private registries in the frontend ATA (Automatic Type Acquisition)
|
||||
*/
|
||||
|
||||
use axum::{
|
||||
extract::{Path, Query},
|
||||
routing::get,
|
||||
Extension, Json, Router,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sqlx::types::JsonValue;
|
||||
use std::collections::HashMap;
|
||||
use tower_http::cors::{Any, CorsLayer};
|
||||
use windmill_common::{
|
||||
error::{Error, JsonResult, Result},
|
||||
global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
|
||||
utils::StripPath,
|
||||
};
|
||||
|
||||
use crate::{db::ApiAuthed, HTTP_CLIENT};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ProxyQuery {
|
||||
tag: Option<String>,
|
||||
}
|
||||
|
||||
/// Parse a scoped package path like "@scope/name" or "name" from a wildcard path
|
||||
fn parse_package_name(path: &str) -> String {
|
||||
// Remove leading slash if present
|
||||
path.trim_start_matches('/').to_string()
|
||||
}
|
||||
|
||||
/// Parse package and version from a path like "@scope/name/1.0.0" or "name/1.0.0"
|
||||
fn parse_package_and_version(path: &str) -> Result<(String, String)> {
|
||||
let path = path.trim_start_matches('/');
|
||||
|
||||
if path.starts_with('@') {
|
||||
// Scoped package: @scope/name/version
|
||||
let parts: Vec<&str> = path.splitn(3, '/').collect();
|
||||
if parts.len() < 3 {
|
||||
return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string()));
|
||||
}
|
||||
let package = format!("{}/{}", parts[0], parts[1]);
|
||||
let version = parts[2].to_string();
|
||||
Ok((package, version))
|
||||
} else {
|
||||
// Regular package: name/version
|
||||
let parts: Vec<&str> = path.splitn(2, '/').collect();
|
||||
if parts.len() < 2 {
|
||||
return Err(Error::BadRequest("Invalid package path, expected name/version".to_string()));
|
||||
}
|
||||
Ok((parts[0].to_string(), parts[1].to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse package, version, and filepath from a path like "@scope/name/1.0.0/index.d.ts"
|
||||
fn parse_package_version_and_file(path: &str) -> Result<(String, String, String)> {
|
||||
let path = path.trim_start_matches('/');
|
||||
|
||||
if path.starts_with('@') {
|
||||
// Scoped package: @scope/name/version/filepath
|
||||
let parts: Vec<&str> = path.splitn(4, '/').collect();
|
||||
if parts.len() < 4 {
|
||||
return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string()));
|
||||
}
|
||||
let package = format!("{}/{}", parts[0], parts[1]);
|
||||
let version = parts[2].to_string();
|
||||
let filepath = parts[3].to_string();
|
||||
Ok((package, version, filepath))
|
||||
} else {
|
||||
// Regular package: name/version/filepath
|
||||
let parts: Vec<&str> = path.splitn(3, '/').collect();
|
||||
if parts.len() < 3 {
|
||||
return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string()));
|
||||
}
|
||||
Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PackageVersions {
|
||||
tags: HashMap<String, String>,
|
||||
versions: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PackageVersion {
|
||||
version: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PackageFiletree {
|
||||
default: String,
|
||||
files: Vec<FileEntry>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct FileEntry {
|
||||
name: String,
|
||||
}
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
Router::new()
|
||||
// Use wildcards for package names to support scoped packages like @scope/package
|
||||
.route("/metadata/*package", get(get_package_metadata))
|
||||
.route("/resolve/*package", get(resolve_package_version))
|
||||
.route("/filetree/*package_version", get(get_package_filetree))
|
||||
.route("/file/*package_version_filepath", get(get_package_file))
|
||||
.layer(
|
||||
CorsLayer::new()
|
||||
.allow_origin(Any)
|
||||
.allow_methods(Any)
|
||||
.allow_headers(Any),
|
||||
)
|
||||
}
|
||||
|
||||
/// Get package metadata (versions and tags) from the private registry
|
||||
async fn get_package_metadata(
|
||||
_authed: ApiAuthed,
|
||||
Path((_w_id, package_path)): Path<(String, StripPath)>,
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageVersions> {
|
||||
let package = parse_package_name(package_path.to_path());
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package metadata from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Package {} not found in private registry",
|
||||
package
|
||||
)));
|
||||
}
|
||||
|
||||
let package_json: JsonValue = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Extract versions and dist-tags from the package metadata
|
||||
let mut versions = Vec::new();
|
||||
let mut tags = HashMap::new();
|
||||
|
||||
if let Some(versions_obj) = package_json.get("versions").and_then(|v| v.as_object()) {
|
||||
versions = versions_obj.keys().cloned().collect();
|
||||
}
|
||||
|
||||
if let Some(tags_obj) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
|
||||
for (tag, version) in tags_obj {
|
||||
if let Some(version_str) = version.as_str() {
|
||||
tags.insert(tag.clone(), version_str.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(PackageVersions { tags, versions }))
|
||||
}
|
||||
|
||||
/// Resolve a package tag/version reference to a specific version
|
||||
async fn resolve_package_version(
|
||||
_authed: ApiAuthed,
|
||||
Path((_w_id, package_path)): Path<(String, StripPath)>,
|
||||
Query(query): Query<ProxyQuery>,
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageVersion> {
|
||||
let package = parse_package_name(package_path.to_path());
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let reference = query.tag.unwrap_or_else(|| "latest".to_string());
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Resolving package version from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Package {} not found in private registry",
|
||||
package
|
||||
)));
|
||||
}
|
||||
|
||||
let package_json: JsonValue = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Try to resolve the reference as a tag first
|
||||
let version = if let Some(tags) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
|
||||
if let Some(version) = tags.get(&reference).and_then(|v| v.as_str()) {
|
||||
Some(version.to_string())
|
||||
} else {
|
||||
// If not a tag, check if it's a valid version
|
||||
if let Some(versions) = package_json.get("versions").and_then(|v| v.as_object()) {
|
||||
if versions.contains_key(&reference) {
|
||||
Some(reference)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
Ok(Json(PackageVersion { version }))
|
||||
}
|
||||
|
||||
/// Get the file tree for a specific package version
|
||||
async fn get_package_filetree(
|
||||
_authed: ApiAuthed,
|
||||
Path((_w_id, package_version_path)): Path<(String, StripPath)>,
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> JsonResult<PackageFiletree> {
|
||||
let (package, version) = parse_package_and_version(package_version_path.to_path())?;
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package filetree from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Package {} not found in private registry",
|
||||
package
|
||||
)));
|
||||
}
|
||||
|
||||
let package_json: JsonValue = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Get the tarball URL for this version
|
||||
let tarball_url = package_json
|
||||
.get("versions")
|
||||
.and_then(|v| v.get(&version))
|
||||
.and_then(|v| v.get("dist"))
|
||||
.and_then(|d| d.get("tarball"))
|
||||
.and_then(|t| t.as_str())
|
||||
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
|
||||
|
||||
// Download and extract tarball to get file list
|
||||
let tarball_response = HTTP_CLIENT
|
||||
.get(tarball_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
|
||||
|
||||
if !tarball_response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Failed to download tarball for {}@{}",
|
||||
package, version
|
||||
)));
|
||||
}
|
||||
|
||||
let tarball_bytes = tarball_response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
|
||||
|
||||
// Extract file list from tarball
|
||||
let files = extract_tarball_files(&tarball_bytes)?;
|
||||
|
||||
// Find the main entry point
|
||||
let main = package_json
|
||||
.get("versions")
|
||||
.and_then(|v| v.get(&version))
|
||||
.and_then(|v| v.get("main"))
|
||||
.and_then(|m| m.as_str())
|
||||
.unwrap_or("index.js")
|
||||
.to_string();
|
||||
|
||||
Ok(Json(PackageFiletree {
|
||||
default: main,
|
||||
files,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Get a specific file from a package version
|
||||
async fn get_package_file(
|
||||
_authed: ApiAuthed,
|
||||
Path((_w_id, full_path)): Path<(String, StripPath)>,
|
||||
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
|
||||
) -> Result<String> {
|
||||
let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
|
||||
let npm_registry = get_npm_registry(&db).await?;
|
||||
|
||||
if npm_registry.is_none() {
|
||||
return Err(Error::BadRequest(
|
||||
"No private npm registry configured".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let registry_url = npm_registry.unwrap();
|
||||
let package_url = format_registry_url(®istry_url, &package, None, None);
|
||||
|
||||
tracing::info!("Fetching package file from: {}", package_url);
|
||||
|
||||
let response = HTTP_CLIENT
|
||||
.get(&package_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
|
||||
|
||||
if !response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Package {} not found in private registry",
|
||||
package
|
||||
)));
|
||||
}
|
||||
|
||||
let package_json: JsonValue = response
|
||||
.json()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
|
||||
|
||||
// Get the tarball URL for this version
|
||||
let tarball_url = package_json
|
||||
.get("versions")
|
||||
.and_then(|v| v.get(&version))
|
||||
.and_then(|v| v.get("dist"))
|
||||
.and_then(|d| d.get("tarball"))
|
||||
.and_then(|t| t.as_str())
|
||||
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
|
||||
|
||||
// Download tarball
|
||||
let tarball_response = HTTP_CLIENT
|
||||
.get(tarball_url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
|
||||
|
||||
if !tarball_response.status().is_success() {
|
||||
return Err(Error::NotFound(format!(
|
||||
"Failed to download tarball for {}@{}",
|
||||
package, version
|
||||
)));
|
||||
}
|
||||
|
||||
let tarball_bytes = tarball_response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
|
||||
|
||||
// Extract the specific file from the tarball
|
||||
let file_content = extract_file_from_tarball(&tarball_bytes, &filepath)?;
|
||||
|
||||
Ok(file_content)
|
||||
}
|
||||
|
||||
/// Get the npm registry URL from global settings
|
||||
async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
|
||||
let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
|
||||
.await?
|
||||
.and_then(|v| v.as_str().map(|s| s.to_string()));
|
||||
|
||||
Ok(registry)
|
||||
}
|
||||
|
||||
/// Format a registry URL for a package
|
||||
fn format_registry_url(
|
||||
registry_base: &str,
|
||||
package: &str,
|
||||
version: Option<&str>,
|
||||
file: Option<&str>,
|
||||
) -> String {
|
||||
let registry_base = registry_base.trim_end_matches('/');
|
||||
|
||||
// Handle scoped packages (e.g., @types/node)
|
||||
let package_path = if package.starts_with('@') {
|
||||
// Scoped packages need to be URL encoded properly
|
||||
package.replace('/', "%2F")
|
||||
} else {
|
||||
package.to_string()
|
||||
};
|
||||
|
||||
match (version, file) {
|
||||
(Some(v), Some(f)) => {
|
||||
format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f)
|
||||
}
|
||||
(Some(v), None) => {
|
||||
format!("{}/{}/{}", registry_base, package_path, v)
|
||||
}
|
||||
_ => {
|
||||
format!("{}/{}", registry_base, package_path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract file list from a tarball
|
||||
fn extract_tarball_files(tarball_bytes: &[u8]) -> Result<Vec<FileEntry>> {
|
||||
use flate2::read::GzDecoder;
|
||||
use tar::Archive;
|
||||
|
||||
let gz = GzDecoder::new(tarball_bytes);
|
||||
let mut archive = Archive::new(gz);
|
||||
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
|
||||
{
|
||||
let entry = entry
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
|
||||
let path = entry
|
||||
.path()
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
|
||||
|
||||
// Remove the package/ prefix that npm tarballs have
|
||||
let path_str = path.to_string_lossy().to_string();
|
||||
if let Some(stripped) = path_str.strip_prefix("package/") {
|
||||
files.push(FileEntry {
|
||||
name: format!("/{}", stripped),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Extract a specific file from a tarball
|
||||
fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result<String> {
|
||||
use flate2::read::GzDecoder;
|
||||
use std::io::Read;
|
||||
use tar::Archive;
|
||||
|
||||
let gz = GzDecoder::new(tarball_bytes);
|
||||
let mut archive = Archive::new(gz);
|
||||
|
||||
// Normalize the target file path
|
||||
let target = target_file.trim_start_matches('/');
|
||||
|
||||
for entry in archive
|
||||
.entries()
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
|
||||
{
|
||||
let mut entry = entry
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
|
||||
let path = entry
|
||||
.path()
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
|
||||
|
||||
let path_str = path.to_string_lossy().to_string();
|
||||
|
||||
// Check if this is the file we're looking for
|
||||
if let Some(stripped) = path_str.strip_prefix("package/") {
|
||||
if stripped == target {
|
||||
let mut content = String::new();
|
||||
entry
|
||||
.read_to_string(&mut content)
|
||||
.map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?;
|
||||
return Ok(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(Error::NotFound(format!("File {} not found in tarball", target_file)))
|
||||
}
|
||||
@@ -1,21 +1,84 @@
|
||||
// https://github.com/jsdelivr/data.jsdelivr.com
|
||||
|
||||
import pLimit from 'p-limit'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
import { get } from 'svelte/store'
|
||||
|
||||
export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => {
|
||||
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}`
|
||||
return api<{ tags: Record<string, string>; versions: string[] }>(url, resLimit, {
|
||||
cache: 'no-store'
|
||||
})
|
||||
// Backend proxy fallback functions
|
||||
const getBackendProxyUrl = () => {
|
||||
const workspace = get(workspaceStore)
|
||||
if (!workspace) {
|
||||
throw new Error('No workspace available')
|
||||
}
|
||||
return `/api/w/${workspace}/npm_proxy`
|
||||
}
|
||||
|
||||
export const getNPMVersionForModuleReference = (
|
||||
const backendProxyApi = async <T>(endpoint: string, resLimit: ResLimit): Promise<T | Error> => {
|
||||
if (isOverlimit(resLimit)) {
|
||||
console.warn(
|
||||
`Exceeded limit of types downloaded for the needs of the assistant fetching: ${endpoint}, ${resLimit.usage}`
|
||||
)
|
||||
return new Error('Exceeded limit of 100MB of data downloaded.')
|
||||
}
|
||||
|
||||
try {
|
||||
const baseUrl = getBackendProxyUrl()
|
||||
const url = `${baseUrl}${endpoint}`
|
||||
|
||||
return limit(() =>
|
||||
fetch(url, { credentials: 'include' }).then((res) => {
|
||||
if (res.ok) {
|
||||
return res.text().then((text) => {
|
||||
resLimit.usage += text.length
|
||||
console.log('resLimit (backend proxy)', url, resLimit.usage)
|
||||
return JSON.parse(text) as T
|
||||
}) as Promise<T | Error>
|
||||
} else {
|
||||
return new Error('Backend proxy request failed')
|
||||
}
|
||||
})
|
||||
)
|
||||
} catch (e) {
|
||||
return new Error('Backend proxy not available')
|
||||
}
|
||||
}
|
||||
|
||||
export const getNPMVersionsForModule = async (moduleName: string, resLimit: ResLimit) => {
|
||||
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}`
|
||||
const result = await api<{ tags: Record<string, string>; versions: string[] }>(url, resLimit, {
|
||||
cache: 'no-store'
|
||||
})
|
||||
|
||||
// If jsdelivr fails, try backend proxy
|
||||
if (result instanceof Error) {
|
||||
console.log('jsdelivr failed for', moduleName, 'trying backend proxy')
|
||||
return backendProxyApi<{ tags: Record<string, string>; versions: string[] }>(
|
||||
`/metadata/${encodeURIComponent(moduleName)}`,
|
||||
resLimit
|
||||
)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export const getNPMVersionForModuleReference = async (
|
||||
moduleName: string,
|
||||
reference: string,
|
||||
resLimit: ResLimit
|
||||
) => {
|
||||
const url = `https://data.jsdelivr.com/v1/package/resolve/npm/${moduleName}@${reference}`
|
||||
return api<{ version: string | null }>(url, resLimit)
|
||||
const result = await api<{ version: string | null }>(url, resLimit)
|
||||
|
||||
// If jsdelivr fails, try backend proxy
|
||||
if (result instanceof Error) {
|
||||
console.log('jsdelivr failed for', moduleName, reference, 'trying backend proxy')
|
||||
return backendProxyApi<{ version: string | null }>(
|
||||
`/resolve/${encodeURIComponent(moduleName)}?tag=${encodeURIComponent(reference)}`,
|
||||
resLimit
|
||||
)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export type NPMTreeMeta = {
|
||||
@@ -35,7 +98,22 @@ export const getFiletreeForModuleWithVersion = async (
|
||||
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}@${version}/flat`
|
||||
const res = await api<NPMTreeMeta>(url, resLimit)
|
||||
if (res instanceof Error) {
|
||||
return res
|
||||
// Try backend proxy
|
||||
console.log('jsdelivr failed for', moduleName, version, 'trying backend proxy')
|
||||
const backendRes = await backendProxyApi<NPMTreeMeta>(
|
||||
`/filetree/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}`,
|
||||
resLimit
|
||||
)
|
||||
if (backendRes instanceof Error) {
|
||||
return res
|
||||
} else {
|
||||
return {
|
||||
...backendRes,
|
||||
moduleName,
|
||||
version,
|
||||
raw
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return {
|
||||
...res,
|
||||
@@ -57,6 +135,18 @@ export const getDTSFileForModuleWithVersion = async (
|
||||
if (res.ok) {
|
||||
return res.text()
|
||||
} else {
|
||||
// Try backend proxy
|
||||
console.log('jsdelivr failed for file', moduleName, version, file, 'trying backend proxy')
|
||||
try {
|
||||
const baseUrl = getBackendProxyUrl()
|
||||
const proxyUrl = `${baseUrl}/file/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}${file}`
|
||||
const proxyRes = await limit(() => fetch(proxyUrl, { credentials: 'include' }))
|
||||
if (proxyRes.ok) {
|
||||
return proxyRes.text()
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('Backend proxy failed for file', e)
|
||||
}
|
||||
return new Error('OK')
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user