feat: add private npm registry proxy support for ATA in webide (#7597)

This commit is contained in:
Ruben Fiszel
2026-01-17 10:44:30 +00:00
committed by GitHub
parent 8dd5e81a32
commit b3cb41efa4
7 changed files with 742 additions and 8 deletions
+2
View File
@@ -15343,6 +15343,7 @@ dependencies = [
"deno_core",
"deno_error",
"ed25519-dalek",
"flate2",
"futures",
"git-version",
"google-cloud-googleapis",
@@ -15390,6 +15391,7 @@ dependencies = [
"sha2 0.10.9",
"sql-builder",
"sqlx",
"tar",
"tempfile",
"thiserror 2.0.17",
"time",
+1
View File
@@ -405,6 +405,7 @@ async-once-cell = "0.5.4"
aws-smithy-types-convert = { version = "^0", features = ["convert-chrono"] }
crc = "^3"
tar = "^0"
flate2 = "^1"
http = "^1"
async-stream = "^0"
+2
View File
@@ -157,6 +157,8 @@ google-cloud-googleapis = { workspace = true , optional = true }
tonic = { workspace = true, optional = true }
deno_error = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
tar.workspace = true
flate2.workspace = true
backon = {workspace = true, optional = true}
[build-dependencies]
+135
View File
@@ -4977,6 +4977,141 @@ paths:
items:
type: string
/w/{workspace}/npm_proxy/metadata/{package}:
get:
summary: get npm package metadata from private registry
operationId: getNpmPackageMetadata
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
responses:
"200":
description: package metadata
content:
application/json:
schema:
type: object
properties:
tags:
type: object
additionalProperties:
type: string
versions:
type: array
items:
type: string
/w/{workspace}/npm_proxy/resolve/{package}:
get:
summary: resolve npm package version from private registry
operationId: resolveNpmPackageVersion
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: tag
description: version tag or reference
in: query
required: false
schema:
type: string
responses:
"200":
description: resolved version
content:
application/json:
schema:
type: object
properties:
version:
type: string
nullable: true
/w/{workspace}/npm_proxy/filetree/{package}/{version}:
get:
summary: get npm package file tree from private registry
operationId: getNpmPackageFiletree
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: version
description: package version
in: path
required: true
schema:
type: string
responses:
"200":
description: package file tree
content:
application/json:
schema:
type: object
properties:
default:
type: string
files:
type: array
items:
type: object
properties:
name:
type: string
/w/{workspace}/npm_proxy/file/{package}/{version}/{filepath}:
get:
summary: get specific file from npm package in private registry
operationId: getNpmPackageFile
tags:
- npm_proxy
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: package
description: npm package name
in: path
required: true
schema:
type: string
- name: version
description: package version
in: path
required: true
schema:
type: string
- name: filepath
description: file path within package
in: path
required: true
schema:
type: string
responses:
"200":
description: file content
content:
text/plain:
schema:
type: string
/w/{workspace}/embeddings/query_resource_types:
get:
summary: query resource types by similarity
+2
View File
@@ -106,6 +106,7 @@ mod inkeep_oss;
mod inputs;
mod integration;
mod live_migrations;
mod npm_proxy;
#[cfg(feature = "http_trigger")]
mod openapi;
#[cfg(all(feature = "private", feature = "parquet"))]
@@ -496,6 +497,7 @@ pub async fn run_server(
Router::new()
})
.nest("/ai", ai::workspaced_service())
.nest("/npm_proxy", npm_proxy::workspaced_service())
.nest("/raw_apps", raw_apps::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
+502
View File
@@ -0,0 +1,502 @@
/*
* This file provides a proxy endpoint for npm package requests
* to support private registries in the frontend ATA (Automatic Type Acquisition)
*/
use axum::{
extract::{Path, Query},
routing::get,
Extension, Json, Router,
};
use serde::{Deserialize, Serialize};
use sqlx::types::JsonValue;
use std::collections::HashMap;
use tower_http::cors::{Any, CorsLayer};
use windmill_common::{
error::{Error, JsonResult, Result},
global_settings::{load_value_from_global_settings, NPM_CONFIG_REGISTRY_SETTING},
utils::StripPath,
};
use crate::{db::ApiAuthed, HTTP_CLIENT};
#[derive(Deserialize)]
struct ProxyQuery {
tag: Option<String>,
}
/// Parse a scoped package path like "@scope/name" or "name" from a wildcard path
fn parse_package_name(path: &str) -> String {
// Remove leading slash if present
path.trim_start_matches('/').to_string()
}
/// Parse package and version from a path like "@scope/name/1.0.0" or "name/1.0.0"
fn parse_package_and_version(path: &str) -> Result<(String, String)> {
let path = path.trim_start_matches('/');
if path.starts_with('@') {
// Scoped package: @scope/name/version
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() < 3 {
return Err(Error::BadRequest("Invalid scoped package path, expected @scope/name/version".to_string()));
}
let package = format!("{}/{}", parts[0], parts[1]);
let version = parts[2].to_string();
Ok((package, version))
} else {
// Regular package: name/version
let parts: Vec<&str> = path.splitn(2, '/').collect();
if parts.len() < 2 {
return Err(Error::BadRequest("Invalid package path, expected name/version".to_string()));
}
Ok((parts[0].to_string(), parts[1].to_string()))
}
}
/// Parse package, version, and filepath from a path like "@scope/name/1.0.0/index.d.ts"
fn parse_package_version_and_file(path: &str) -> Result<(String, String, String)> {
let path = path.trim_start_matches('/');
if path.starts_with('@') {
// Scoped package: @scope/name/version/filepath
let parts: Vec<&str> = path.splitn(4, '/').collect();
if parts.len() < 4 {
return Err(Error::BadRequest("Invalid scoped package file path, expected @scope/name/version/filepath".to_string()));
}
let package = format!("{}/{}", parts[0], parts[1]);
let version = parts[2].to_string();
let filepath = parts[3].to_string();
Ok((package, version, filepath))
} else {
// Regular package: name/version/filepath
let parts: Vec<&str> = path.splitn(3, '/').collect();
if parts.len() < 3 {
return Err(Error::BadRequest("Invalid package file path, expected name/version/filepath".to_string()));
}
Ok((parts[0].to_string(), parts[1].to_string(), parts[2].to_string()))
}
}
#[derive(Serialize)]
struct PackageVersions {
tags: HashMap<String, String>,
versions: Vec<String>,
}
#[derive(Serialize)]
struct PackageVersion {
version: Option<String>,
}
#[derive(Serialize)]
struct PackageFiletree {
default: String,
files: Vec<FileEntry>,
}
#[derive(Serialize)]
struct FileEntry {
name: String,
}
pub fn workspaced_service() -> Router {
Router::new()
// Use wildcards for package names to support scoped packages like @scope/package
.route("/metadata/*package", get(get_package_metadata))
.route("/resolve/*package", get(resolve_package_version))
.route("/filetree/*package_version", get(get_package_filetree))
.route("/file/*package_version_filepath", get(get_package_file))
.layer(
CorsLayer::new()
.allow_origin(Any)
.allow_methods(Any)
.allow_headers(Any),
)
}
/// Get package metadata (versions and tags) from the private registry
async fn get_package_metadata(
_authed: ApiAuthed,
Path((_w_id, package_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersions> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package metadata from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Extract versions and dist-tags from the package metadata
let mut versions = Vec::new();
let mut tags = HashMap::new();
if let Some(versions_obj) = package_json.get("versions").and_then(|v| v.as_object()) {
versions = versions_obj.keys().cloned().collect();
}
if let Some(tags_obj) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
for (tag, version) in tags_obj {
if let Some(version_str) = version.as_str() {
tags.insert(tag.clone(), version_str.to_string());
}
}
}
Ok(Json(PackageVersions { tags, versions }))
}
/// Resolve a package tag/version reference to a specific version
async fn resolve_package_version(
_authed: ApiAuthed,
Path((_w_id, package_path)): Path<(String, StripPath)>,
Query(query): Query<ProxyQuery>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageVersion> {
let package = parse_package_name(package_path.to_path());
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let reference = query.tag.unwrap_or_else(|| "latest".to_string());
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Resolving package version from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Try to resolve the reference as a tag first
let version = if let Some(tags) = package_json.get("dist-tags").and_then(|v| v.as_object()) {
if let Some(version) = tags.get(&reference).and_then(|v| v.as_str()) {
Some(version.to_string())
} else {
// If not a tag, check if it's a valid version
if let Some(versions) = package_json.get("versions").and_then(|v| v.as_object()) {
if versions.contains_key(&reference) {
Some(reference)
} else {
None
}
} else {
None
}
}
} else {
None
};
Ok(Json(PackageVersion { version }))
}
/// Get the file tree for a specific package version
async fn get_package_filetree(
_authed: ApiAuthed,
Path((_w_id, package_version_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> JsonResult<PackageFiletree> {
let (package, version) = parse_package_and_version(package_version_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package filetree from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("dist"))
.and_then(|d| d.get("tarball"))
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download and extract tarball to get file list
let tarball_response = HTTP_CLIENT
.get(tarball_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
if !tarball_response.status().is_success() {
return Err(Error::NotFound(format!(
"Failed to download tarball for {}@{}",
package, version
)));
}
let tarball_bytes = tarball_response
.bytes()
.await
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
// Extract file list from tarball
let files = extract_tarball_files(&tarball_bytes)?;
// Find the main entry point
let main = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("main"))
.and_then(|m| m.as_str())
.unwrap_or("index.js")
.to_string();
Ok(Json(PackageFiletree {
default: main,
files,
}))
}
/// Get a specific file from a package version
async fn get_package_file(
_authed: ApiAuthed,
Path((_w_id, full_path)): Path<(String, StripPath)>,
Extension(db): Extension<sqlx::Pool<sqlx::Postgres>>,
) -> Result<String> {
let (package, version, filepath) = parse_package_version_and_file(full_path.to_path())?;
let npm_registry = get_npm_registry(&db).await?;
if npm_registry.is_none() {
return Err(Error::BadRequest(
"No private npm registry configured".to_string(),
));
}
let registry_url = npm_registry.unwrap();
let package_url = format_registry_url(&registry_url, &package, None, None);
tracing::info!("Fetching package file from: {}", package_url);
let response = HTTP_CLIENT
.get(&package_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to fetch package metadata: {}", e)))?;
if !response.status().is_success() {
return Err(Error::NotFound(format!(
"Package {} not found in private registry",
package
)));
}
let package_json: JsonValue = response
.json()
.await
.map_err(|e| Error::InternalErr(format!("Failed to parse package metadata: {}", e)))?;
// Get the tarball URL for this version
let tarball_url = package_json
.get("versions")
.and_then(|v| v.get(&version))
.and_then(|v| v.get("dist"))
.and_then(|d| d.get("tarball"))
.and_then(|t| t.as_str())
.ok_or_else(|| Error::NotFound(format!("Tarball not found for {}@{}", package, version)))?;
// Download tarball
let tarball_response = HTTP_CLIENT
.get(tarball_url)
.send()
.await
.map_err(|e| Error::InternalErr(format!("Failed to download tarball: {}", e)))?;
if !tarball_response.status().is_success() {
return Err(Error::NotFound(format!(
"Failed to download tarball for {}@{}",
package, version
)));
}
let tarball_bytes = tarball_response
.bytes()
.await
.map_err(|e| Error::InternalErr(format!("Failed to read tarball: {}", e)))?;
// Extract the specific file from the tarball
let file_content = extract_file_from_tarball(&tarball_bytes, &filepath)?;
Ok(file_content)
}
/// Get the npm registry URL from global settings
async fn get_npm_registry(db: &sqlx::Pool<sqlx::Postgres>) -> Result<Option<String>> {
let registry = load_value_from_global_settings(db, NPM_CONFIG_REGISTRY_SETTING)
.await?
.and_then(|v| v.as_str().map(|s| s.to_string()));
Ok(registry)
}
/// Format a registry URL for a package
fn format_registry_url(
registry_base: &str,
package: &str,
version: Option<&str>,
file: Option<&str>,
) -> String {
let registry_base = registry_base.trim_end_matches('/');
// Handle scoped packages (e.g., @types/node)
let package_path = if package.starts_with('@') {
// Scoped packages need to be URL encoded properly
package.replace('/', "%2F")
} else {
package.to_string()
};
match (version, file) {
(Some(v), Some(f)) => {
format!("{}/{}/{}/-/{}-{}/{}", registry_base, package_path, v, package, v, f)
}
(Some(v), None) => {
format!("{}/{}/{}", registry_base, package_path, v)
}
_ => {
format!("{}/{}", registry_base, package_path)
}
}
}
/// Extract file list from a tarball
fn extract_tarball_files(tarball_bytes: &[u8]) -> Result<Vec<FileEntry>> {
use flate2::read::GzDecoder;
use tar::Archive;
let gz = GzDecoder::new(tarball_bytes);
let mut archive = Archive::new(gz);
let mut files = Vec::new();
for entry in archive
.entries()
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
{
let entry = entry
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
let path = entry
.path()
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
// Remove the package/ prefix that npm tarballs have
let path_str = path.to_string_lossy().to_string();
if let Some(stripped) = path_str.strip_prefix("package/") {
files.push(FileEntry {
name: format!("/{}", stripped),
});
}
}
Ok(files)
}
/// Extract a specific file from a tarball
fn extract_file_from_tarball(tarball_bytes: &[u8], target_file: &str) -> Result<String> {
use flate2::read::GzDecoder;
use std::io::Read;
use tar::Archive;
let gz = GzDecoder::new(tarball_bytes);
let mut archive = Archive::new(gz);
// Normalize the target file path
let target = target_file.trim_start_matches('/');
for entry in archive
.entries()
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entries: {}", e)))?
{
let mut entry = entry
.map_err(|e| Error::InternalErr(format!("Failed to read tarball entry: {}", e)))?;
let path = entry
.path()
.map_err(|e| Error::InternalErr(format!("Failed to read entry path: {}", e)))?;
let path_str = path.to_string_lossy().to_string();
// Check if this is the file we're looking for
if let Some(stripped) = path_str.strip_prefix("package/") {
if stripped == target {
let mut content = String::new();
entry
.read_to_string(&mut content)
.map_err(|e| Error::InternalErr(format!("Failed to read file content: {}", e)))?;
return Ok(content);
}
}
}
Err(Error::NotFound(format!("File {} not found in tarball", target_file)))
}
+98 -8
View File
@@ -1,21 +1,84 @@
// https://github.com/jsdelivr/data.jsdelivr.com
import pLimit from 'p-limit'
import { workspaceStore } from '$lib/stores'
import { get } from 'svelte/store'
export const getNPMVersionsForModule = (moduleName: string, resLimit: ResLimit) => {
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}`
return api<{ tags: Record<string, string>; versions: string[] }>(url, resLimit, {
cache: 'no-store'
})
// Backend proxy fallback functions
const getBackendProxyUrl = () => {
const workspace = get(workspaceStore)
if (!workspace) {
throw new Error('No workspace available')
}
return `/api/w/${workspace}/npm_proxy`
}
export const getNPMVersionForModuleReference = (
const backendProxyApi = async <T>(endpoint: string, resLimit: ResLimit): Promise<T | Error> => {
if (isOverlimit(resLimit)) {
console.warn(
`Exceeded limit of types downloaded for the needs of the assistant fetching: ${endpoint}, ${resLimit.usage}`
)
return new Error('Exceeded limit of 100MB of data downloaded.')
}
try {
const baseUrl = getBackendProxyUrl()
const url = `${baseUrl}${endpoint}`
return limit(() =>
fetch(url, { credentials: 'include' }).then((res) => {
if (res.ok) {
return res.text().then((text) => {
resLimit.usage += text.length
console.log('resLimit (backend proxy)', url, resLimit.usage)
return JSON.parse(text) as T
}) as Promise<T | Error>
} else {
return new Error('Backend proxy request failed')
}
})
)
} catch (e) {
return new Error('Backend proxy not available')
}
}
export const getNPMVersionsForModule = async (moduleName: string, resLimit: ResLimit) => {
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}`
const result = await api<{ tags: Record<string, string>; versions: string[] }>(url, resLimit, {
cache: 'no-store'
})
// If jsdelivr fails, try backend proxy
if (result instanceof Error) {
console.log('jsdelivr failed for', moduleName, 'trying backend proxy')
return backendProxyApi<{ tags: Record<string, string>; versions: string[] }>(
`/metadata/${encodeURIComponent(moduleName)}`,
resLimit
)
}
return result
}
export const getNPMVersionForModuleReference = async (
moduleName: string,
reference: string,
resLimit: ResLimit
) => {
const url = `https://data.jsdelivr.com/v1/package/resolve/npm/${moduleName}@${reference}`
return api<{ version: string | null }>(url, resLimit)
const result = await api<{ version: string | null }>(url, resLimit)
// If jsdelivr fails, try backend proxy
if (result instanceof Error) {
console.log('jsdelivr failed for', moduleName, reference, 'trying backend proxy')
return backendProxyApi<{ version: string | null }>(
`/resolve/${encodeURIComponent(moduleName)}?tag=${encodeURIComponent(reference)}`,
resLimit
)
}
return result
}
export type NPMTreeMeta = {
@@ -35,7 +98,22 @@ export const getFiletreeForModuleWithVersion = async (
const url = `https://data.jsdelivr.com/v1/package/npm/${moduleName}@${version}/flat`
const res = await api<NPMTreeMeta>(url, resLimit)
if (res instanceof Error) {
return res
// Try backend proxy
console.log('jsdelivr failed for', moduleName, version, 'trying backend proxy')
const backendRes = await backendProxyApi<NPMTreeMeta>(
`/filetree/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}`,
resLimit
)
if (backendRes instanceof Error) {
return res
} else {
return {
...backendRes,
moduleName,
version,
raw
}
}
} else {
return {
...res,
@@ -57,6 +135,18 @@ export const getDTSFileForModuleWithVersion = async (
if (res.ok) {
return res.text()
} else {
// Try backend proxy
console.log('jsdelivr failed for file', moduleName, version, file, 'trying backend proxy')
try {
const baseUrl = getBackendProxyUrl()
const proxyUrl = `${baseUrl}/file/${encodeURIComponent(moduleName)}/${encodeURIComponent(version)}${file}`
const proxyRes = await limit(() => fetch(proxyUrl, { credentials: 'include' }))
if (proxyRes.ok) {
return proxyRes.text()
}
} catch (e) {
console.log('Backend proxy failed for file', e)
}
return new Error('OK')
}
}