feat(dependencies): complete npm package versions in package.json (#2596)

Completion inside a version string under dependencies, devDependencies,
peerDependencies or optionalDependencies lists the package's versions from the
npm registry (abbreviated metadata, shared version cache), newest first,
without deprecated versions and without pre-releases when the package has
releases. Aliases, paths, URLs and workspace protocols are left alone; the
scanner tolerates a document being edited.

Closes #2558

crates/ide-host/src/ops/dependencies.rs

@@ -1,7 +1,7 @@
 //! Repository version lookups for manifest inspections and completion.
 use crate::{Host, rpc::Dispatcher};
 use ide_core::{CancellationToken, DocId, IdeError, IdeResult, RequestId};
-use ide_workspace::dependencies::{DependencyVersionSource, DependencyVersions, cargo, go};
+use ide_workspace::dependencies::{DependencyVersionSource, DependencyVersions, cargo, go, npm};
 use std::collections::BTreeMap;
 use std::hash::{DefaultHasher, Hash, Hasher};
 use std::path::{Path, PathBuf};
@@ -25,6 +25,31 @@ impl DependencyVersionSource for CratesIo {
     }
 }
 
+struct NpmRegistry;
+impl DependencyVersionSource for NpmRegistry {
+    fn id(&self) -> &'static str {
+        "npm"
+    }
+
+    /// Scoped names (`@scope/name`) aren't file-name safe.
+    fn cache_key(&self, package: &str) -> Option<String> {
+        let mut hash = DefaultHasher::new();
+        package.hash(&mut hash);
+        Some(format!("{}-{:016x}", self.id(), hash.finish()))
+    }
+
+    fn fetch(&self, package: &str, cancel: &CancellationToken) -> IdeResult<Vec<String>> {
+        let path = npm::registry_path(package).ok_or_else(|| IdeError::internal("invalid package name"))?;
+        let search_path = std::env::var_os("PATH").unwrap_or_default();
+        // The abbreviated metadata: the full document of a large package runs to tens of megabytes.
+        let text = ide_tools::Http::new(&search_path)?
+            .https_only()
+            .accept("application/vnd.npm.install-v1+json")
+            .get_text_limited(&format!("https://registry.npmjs.org/{path}"), Some(32 * 1024 * 1024), cancel)?;
+        npm::registry_versions(&text).ok_or_else(|| IdeError::internal("invalid registry metadata"))
+    }
+}
+
 struct GoModules {
     root: PathBuf,
     env: BTreeMap<String, String>,
@@ -122,6 +147,13 @@ pub async fn dependency_versions(
                 let versions = cache.versions(&source, &dependency.package, &cancel)?;
                 return Ok(vec![DependencyVersions { dependency, versions, update: None }]);
             }
+            if snap.path.file_name().is_some_and(|name| name == "package.json") {
+                let Some(dependency) = offset.and_then(|offset| npm::dependency_at(&snap.text, offset)) else {
+                    return Ok(Vec::new());
+                };
+                let versions = cache.versions(&NpmRegistry, &dependency.package, &cancel)?;
+                return Ok(vec![DependencyVersions { dependency, versions, update: None }]);
+            }
             if snap.path.file_name().is_none_or(|name| name != "Cargo.toml") {
                 return Ok(Vec::new());
             }
@@ -208,4 +240,51 @@ mod tests {
         assert!(dependency_versions(&host, "main", info.id, None, RequestId(414)).await.unwrap().is_empty());
         host.close_project("main");
     }
+
+    struct CachedNpm;
+    impl DependencyVersionSource for CachedNpm {
+        fn id(&self) -> &'static str {
+            "npm"
+        }
+        fn cache_key(&self, package: &str) -> Option<String> {
+            NpmRegistry.cache_key(package)
+        }
+        fn fetch(&self, _: &str, _: &CancellationToken) -> IdeResult<Vec<String>> {
+            Ok(vec!["20.11.0".into(), "18.19.3".into()])
+        }
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn package_json_completes_the_version_at_the_cursor() {
+        let temp = tempfile::tempdir().unwrap();
+        let base = dunce::canonicalize(temp.path()).unwrap();
+        let root = base.join("project");
+        std::fs::create_dir(&root).unwrap();
+        let text = "{\n  \"devDependencies\": { \"@types/node\": \"^20\" }\n}\n";
+        let path = root.join("package.json");
+        std::fs::write(&path, text).unwrap();
+        let host = Host::new(HostConfig {
+            config_dir: base.join("config"),
+            data_dir: base.join("data"),
+            local_data_dir: base.join("local"),
+            languages: Arc::new(ide_language::LanguageRegistry::load(None).unwrap()),
+            syntax: Arc::default(),
+            sink_for: Box::new(|_| Arc::new(|_| {})),
+            owner: |w| w,
+            runtime: tokio::runtime::Handle::current(),
+            project_environment: false,
+        });
+        let project = host.open_project("main", &root).unwrap();
+        let info = project.documents.open(&path).unwrap();
+        host.dependency_versions.versions(&CachedNpm, "@types/node", &CancellationToken::new()).unwrap();
+        let offset = text.find("^20").unwrap() as u32 + 2;
+        let found = dependency_versions(&host, "main", info.id, Some(offset), RequestId(420)).await.unwrap();
+        assert_eq!(found.len(), 1);
+        assert_eq!(found[0].dependency.package, "@types/node");
+        assert_eq!(found[0].versions, ["20.11.0", "18.19.3"]);
+        // Inspections (no offset) and positions outside a version ask nothing.
+        assert!(dependency_versions(&host, "main", info.id, None, RequestId(421)).await.unwrap().is_empty());
+        assert!(dependency_versions(&host, "main", info.id, Some(2), RequestId(422)).await.unwrap().is_empty());
+        host.close_project("main");
+    }
 }

crates/ide-tools/src/process.rs

@@ -280,6 +280,7 @@ pub struct Http {
     explicit_github_token: Option<String>,
     https_only: bool,
     revoke_best_effort: bool,
+    accept: Option<&'static str>,
 }
 
 impl Http {
@@ -289,7 +290,7 @@ impl Http {
         let gh = which::which_in("gh", Some(search_path), ".").ok();
         let explicit_github_token = environment_github_token(|key| std::env::var(key).ok());
         let revoke_best_effort = cfg!(windows) && revoke_best_effort(&curl);
-        Ok(Self { curl, gh, explicit_github_token, https_only: false, revoke_best_effort })
+        Ok(Self { curl, gh, explicit_github_token, https_only: false, revoke_best_effort, accept: None })
     }
 
     /// Refuses every protocol but HTTPS, redirects included.
@@ -298,6 +299,12 @@ impl Http {
         self
     }
 
+    /// Sends `Accept: <mime>` (e.g. the npm registry's abbreviated metadata).
+    pub fn accept(mut self, mime: &'static str) -> Self {
+        self.accept = Some(mime);
+        self
+    }
+
     fn base(&self, url: &str, cancel: &CancellationToken) -> IdeResult<(Command, Option<String>)> {
         if cancel.is_cancelled() {
             return Err(IdeError::cancelled());
@@ -330,6 +337,9 @@ impl Http {
         if self.revoke_best_effort {
             cmd.arg("--ssl-revoke-best-effort");
         }
+        if let Some(mime) = self.accept {
+            cmd.arg("--header").arg(format!("Accept: {mime}"));
+        }
         cmd.stdin(Stdio::null());
         if token.is_some() {
             // Keep credentials out of process arguments, inherited environment,
@@ -700,6 +710,7 @@ mod tests {
             explicit_github_token: None,
             https_only: false,
             revoke_best_effort: false,
+            accept: None,
         };
         for (code, exit, expected) in [(404, 22, Some("[]")), (200, 0, Some("payload")), (403, 22, None), (0, 7, None)]
         {
@@ -852,6 +863,7 @@ mod github_auth_tests {
                         explicit_github_token: None,
                         https_only: false,
                         revoke_best_effort: false,
+                        accept: None,
                     },
                 };
                 fixture.script(

crates/ide-tools/src/process/download_tests.rs

@@ -94,7 +94,14 @@ fn http() -> Option<Http> {
         eprintln!("skipping download test: curl is not installed");
         return None;
     };
-    Some(Http { curl, gh: None, explicit_github_token: None, https_only: false, revoke_best_effort: false })
+    Some(Http {
+        curl,
+        gh: None,
+        explicit_github_token: None,
+        https_only: false,
+        revoke_best_effort: false,
+        accept: None,
+    })
 }
 
 #[test]

crates/ide-workspace/src/dependencies/mod.rs

@@ -2,6 +2,7 @@
 mod cache;
 pub mod cargo;
 pub mod go;
+pub mod npm;
 
 pub use cache::VersionCache;
 use ide_core::{CancellationToken, IdeResult};

crates/ide-workspace/src/dependencies/npm.rs

@@ -0,0 +1,227 @@
+//! `package.json` dependency versions and npm registry metadata.
+use super::ManifestDependency;
+
+/// Top-level objects whose string values are version ranges.
+const SECTIONS: [&str; 4] = ["dependencies", "devDependencies", "peerDependencies", "optionalDependencies"];
+
+/// npm package names: lowercase, optionally scoped (`@scope/name`).
+fn package_name(name: &str) -> bool {
+    let part = |p: &str| {
+        !p.is_empty()
+            && !p.starts_with(['.', '_'])
+            && p.bytes().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || b"-._~".contains(&c))
+    };
+    name.len() <= 214
+        && match name.strip_prefix('@') {
+            Some(scoped) => scoped.split_once('/').is_some_and(|(scope, rest)| part(scope) && part(rest)),
+            None => part(name),
+        }
+}
+
+enum Frame {
+    /// `key`: the last key read; `value`: a `:` followed it.
+    Object {
+        key: Option<String>,
+        value: bool,
+    },
+    Array,
+}
+
+/// The version string at a UTF-16 `offset` in a dependency section, also in a document
+/// being edited (an unclosed string ends at the line end). Aliases (`npm:`), paths, URLs
+/// and workspace protocols are not versions.
+pub fn dependency_at(text: &str, offset: u32) -> Option<ManifestDependency> {
+    // Each container with the key it is the value of.
+    let mut stack: Vec<(Frame, Option<String>)> = Vec::new();
+    let mut pos = 0u32;
+    let mut chars = text.chars().peekable();
+    while let Some(c) = chars.next() {
+        let here = pos;
+        pos += c.len_utf16() as u32;
+        match c {
+            '{' | '[' => {
+                let key = match stack.last_mut() {
+                    Some((Frame::Object { key, value }, _)) if *value => {
+                        *value = false;
+                        key.take()
+                    }
+                    _ => None,
+                };
+                let frame = if c == '{' { Frame::Object { key: None, value: false } } else { Frame::Array };
+                stack.push((frame, key));
+            }
+            '}' | ']' => {
+                stack.pop();
+            }
+            ':' => {
+                if let Some((Frame::Object { value, .. }, _)) = stack.last_mut() {
+                    *value = true;
+                }
+            }
+            ',' => {
+                if let Some((Frame::Object { key, value }, _)) = stack.last_mut() {
+                    (*key, *value) = (None, false);
+                }
+            }
+            '"' => {
+                let from = here + 1;
+                let mut content = String::new();
+                let mut to = from;
+                while let Some(&n) = chars.peek() {
+                    if n == '"' || n == '\n' || n == '\r' {
+                        break;
+                    }
+                    chars.next();
+                    pos += n.len_utf16() as u32;
+                    if n == '\\'
+                        && let Some(e) = chars.next()
+                    {
+                        pos += e.len_utf16() as u32;
+                        content.push(e);
+                    } else {
+                        content.push(n);
+                    }
+                    to = pos;
+                }
+                if chars.peek() == Some(&'"') {
+                    chars.next();
+                    pos += 1;
+                }
+                let in_section =
+                    stack.len() == 2 && matches!(&stack[1].1, Some(section) if SECTIONS.contains(&section.as_str()));
+                match stack.last_mut() {
+                    Some((Frame::Object { key, value }, _)) if *value => {
+                        if in_section
+                            && from <= offset
+                            && offset <= to
+                            && let Some(package) = key.as_deref().filter(|k| package_name(k))
+                            && !content.contains([':', '/'])
+                        {
+                            return Some(ManifestDependency {
+                                package: package.to_owned(),
+                                requirement: content,
+                                from,
+                                to,
+                            });
+                        }
+                        *value = false;
+                    }
+                    Some((Frame::Object { key, .. }, _)) => *key = Some(content),
+                    _ => {}
+                }
+            }
+            _ => {}
+        }
+    }
+    None
+}
+
+/// Registry URL path of a package (`@scope/name` → `@scope%2fname`).
+pub fn registry_path(package: &str) -> Option<String> {
+    package_name(package).then(|| package.replacen('/', "%2f", 1))
+}
+
+/// Versions in a registry document (full or abbreviated metadata), newest first: deprecated
+/// versions left out, and pre-releases only when the package has no release.
+pub fn registry_versions(text: &str) -> Option<Vec<String>> {
+    let doc: serde_json::Value = serde_json::from_str(text).ok()?;
+    let mut versions: Vec<semver::Version> = doc
+        .get("versions")?
+        .as_object()?
+        .iter()
+        .filter(|(_, meta)| meta.get("deprecated").is_none_or(|d| d.is_null() || d == false))
+        .filter_map(|(v, _)| semver::Version::parse(v).ok())
+        .collect();
+    if versions.iter().any(|v| v.pre.is_empty()) {
+        versions.retain(|v| v.pre.is_empty());
+    }
+    versions.sort_unstable_by(|a, b| b.cmp(a));
+    Some(versions.iter().map(ToString::to_string).collect())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn at(text: &str) -> Option<ManifestDependency> {
+        let offset = text.split_once('|').unwrap().0.encode_utf16().count() as u32;
+        dependency_at(&text.replace('|', ""), offset)
+    }
+
+    #[test]
+    fn finds_versions_in_dependency_sections_only() {
+        let doc = r#"{
+  "name": "app 中文 😀",
+  "version": "1.0.|0",
+  "dependencies": { "react": "^18.|2.0", "@types/node": "~2|0" },
+  "devDependencies": {
+    "vite": "|"
+  },
+  "scripts": { "dev": "vi|te" },
+  "overrides": { "foo": { "bar": "1.|0" } }
+}"#;
+        let found: Vec<_> = (0..doc.matches('|').count())
+            .map(|i| {
+                let mut text = String::new();
+                for (j, part) in doc.split('|').enumerate() {
+                    text.push_str(part);
+                    if j == i {
+                        text.push('|');
+                    }
+                }
+                at(&text).map(|d| (d.package, d.requirement))
+            })
+            .collect();
+        assert_eq!(
+            found,
+            [
+                None,
+                Some(("react".into(), "^18.2.0".into())),
+                Some(("@types/node".into(), "~20".into())),
+                Some(("vite".into(), "".into())),
+                None,
+                None,
+            ]
+        );
+    }
+
+    #[test]
+    fn utf16_range_covers_the_string_contents_also_while_typing() {
+        for (text, contents) in [
+            ("{\"x\": \"😀\", \"dependencies\": {\"lodash\": \"^4.|17\"}}", "^4.17"),
+            ("{\"dependencies\": {\n  \"lodash\": \"^4.|\n}}", "^4."),
+        ] {
+            let item = at(text).unwrap();
+            let units: Vec<u16> = text.replace('|', "").encode_utf16().collect();
+            assert_eq!(String::from_utf16(&units[item.from as usize..item.to as usize]).unwrap(), contents);
+        }
+    }
+
+    #[test]
+    fn skips_keys_names_and_protocols() {
+        for text in [
+            r#"{"dependencies": {"rea|ct": "^18"}}"#,
+            r#"{"dependencies": {"a": "npm:b@^|1"}}"#,
+            r#"{"dependencies": {"a": "file:../|a"}}"#,
+            r#"{"dependencies": {"a": "workspace:|*"}}"#,
+            r#"{"dependencies": {"a": "github:x/|y"}}"#,
+            r#"{"dependencies": {"Bad Name": "|1"}}"#,
+            r#"{"dependencies": ["|1"]}"#,
+        ] {
+            assert!(at(text).is_none(), "{text}");
+        }
+    }
+
+    #[test]
+    fn registry_versions_are_newest_first_without_deprecated_or_needless_prereleases() {
+        let doc = r#"{"name":"x","versions":{
+            "1.9.0":{}, "1.10.0":{}, "2.0.0-rc.1":{}, "0.1.0":{"deprecated":"old"}, "not-semver":{}
+        }}"#;
+        assert_eq!(registry_versions(doc).unwrap(), ["1.10.0", "1.9.0"]);
+        let only_pre = r#"{"versions":{"0.0.1-a":{},"0.0.1-b":{}}}"#;
+        assert_eq!(registry_versions(only_pre).unwrap(), ["0.0.1-b", "0.0.1-a"]);
+        assert!(registry_versions("{}").is_none());
+        assert_eq!(registry_path("@types/node").as_deref(), Some("@types%2fnode"));
+        assert_eq!(registry_path("../etc"), None);
+    }
+}

src/features/dependencies/index.ts

@@ -210,7 +210,7 @@ export function activate(): () => void {
     extensions: (doc) => {
       if (doc.virtual || doc.readOnly) return [];
       if (/(?:^|\/)Cargo\.toml$/.test(doc.path)) return extension(doc);
-      if (/(?:^|\/)go\.mod$/.test(doc.path)) return versionCompletion(doc);
+      if (/(?:^|\/)(?:go\.mod|package\.json)$/.test(doc.path)) return versionCompletion(doc);
       return [];
     },
   });