feat(libraries): show the Zig standard library and build.zig.zon packages in External Libraries (#3526)

A zig detector joins the External Libraries pipeline: the standard library from zig env (struct-literal and JSON output), and the fetched build.zig.zon dependencies from the global package cache, as p/<hash> folders (Zig <= 0.14) or p/<hash>.tar.gz blobs (Zig >= 0.15) browsed read-only through the archive VFS, which now reads gzip'd tars. Nothing is downloaded; path dependencies stay in the project tree. New setting languages.zig.zig (directory of zig, empty = PATH).

Co-authored-by: akbar1214 <43771058+akbar1214@users.noreply.github.com>
This commit is contained in:
Akbar Husain
2026-11-15 22:45:18 +08:00
committed by GitHub
parent a02a594010
commit acee0d5edb
14 changed files with 778 additions and 59 deletions
Generated
+4
View File
@@ -4799,6 +4799,7 @@ name = "ide-sdk"
version = "0.1.45"
dependencies = [
"dunce",
"flate2",
"ide-core",
"ide-language",
"ide-run",
@@ -4808,6 +4809,7 @@ dependencies = [
"serde",
"serde_json",
"specta",
"tar",
"tempfile",
"toml 0.9.12+spec-1.1.0",
"tracing",
@@ -4964,6 +4966,7 @@ dependencies = [
"chardetng",
"dunce",
"encoding_rs",
"flate2",
"gix",
"globset",
"ide-core",
@@ -4974,6 +4977,7 @@ dependencies = [
"serde",
"serde_json",
"specta",
"tar",
"tempfile",
"tracing",
"trash",
+2
View File
@@ -90,6 +90,8 @@ dunce = "1"
dirs = "6"
memmap2 = "0.9"
zip = { version = "9.2.3", default-features = false, features = ["deflate"] }
flate2 = "1"
tar = "0.4"
# search, index, language
grep-searcher = "0.1"
+9
View File
@@ -989,6 +989,7 @@ pub struct LanguagesSettings {
pub ruby: RubySettings,
pub markdown: MarkdownSettings,
pub arduino: ArduinoSettings,
pub zig: ZigSettings,
}
impl Default for LanguagesSettings {
@@ -1011,6 +1012,7 @@ impl Default for LanguagesSettings {
ruby: RubySettings::default(),
markdown: MarkdownSettings::default(),
arduino: ArduinoSettings::default(),
zig: ZigSettings::default(),
}
}
}
@@ -1222,6 +1224,13 @@ impl Default for RustSettings {
}
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Type)]
#[serde(default, rename_all = "camelCase")]
pub struct ZigSettings {
/// Directory holding the `zig` executable; empty = the one on `PATH`.
pub zig: String,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, Type)]
#[serde(default, rename_all = "camelCase")]
pub struct GoSettings {
+4
View File
@@ -21,3 +21,7 @@ toml.workspace = true
tracing.workspace = true
url = "2"
zip.workspace = true
[dev-dependencies]
flate2.workspace = true
tar.workspace = true
+6
View File
@@ -141,6 +141,12 @@ pub(crate) fn goroot(root: &Path, s: &Settings) -> Option<PathBuf> {
.or_else(|| pinned::pinned_install(root, Tool::Go).and_then(|(dir, _)| pinned::goroot_in_install(&dir)))
}
/// The directory of the configured Zig install (holding `zig`), else `None` for `PATH`.
pub(crate) fn zig_bin(root: &Path, s: &Settings) -> Option<PathBuf> {
setting_path(root, &s.languages.zig.zig)
.map(|p| if p.is_file() { p.parent().map(Path::to_path_buf).unwrap_or(p) } else { p })
}
/// Runtime choices for package-manager based IDE tool installs.
#[derive(Default)]
pub struct ToolInstallEnvironment {
+1
View File
@@ -26,6 +26,7 @@ pub mod pinned;
pub mod python;
pub mod ruby;
mod vue;
mod zig;
use serde::{Deserialize, Serialize};
use specta::Type;
+206 -3
View File
@@ -2,7 +2,8 @@
//! Project view): the folders holding the sources of the project's SDKs and
//! dependencies — the Python interpreter's standard library and site-packages,
//! the JDK, `node_modules` packages, Cargo crates and the Rust standard
//! library, GOROOT and the Go module cache.
//! library, GOROOT and the Go module cache, the Zig standard library and the
//! fetched packages of `build.zig.zon`.
//!
//! Ecosystems are detected in the project and its modules. SDK and graph probes
//! are cached until manifests or settings change; Node packages are read on refresh.
@@ -28,7 +29,7 @@ use std::time::{Duration, SystemTime};
pub struct LibraryGroup {
/// Display name, e.g. `Python 3.12 (.venv)`.
pub name: String,
/// `python` | `jdk` | `nodeModules` | `nodeCore` | `rust` | `cargo` | `go` | `goModules` | `maven` | `gradle` | `classpath`.
/// `python` | `jdk` | `nodeModules` | `nodeCore` | `rust` | `cargo` | `go` | `goModules` | `maven` | `gradle` | `classpath` | `zig` | `zigPackages`.
pub kind: String,
pub roots: Vec<LibraryRoot>,
}
@@ -56,6 +57,7 @@ const JAVA_MANIFESTS: &[&str] =
const CARGO_MANIFESTS: &[&str] = &["Cargo.toml", "Cargo.lock", "rust-toolchain", "rust-toolchain.toml"];
const GO_MANIFESTS: &[&str] = &["go.mod", "go.sum", "go.work"];
const RUBY_MANIFESTS: &[&str] = &["Gemfile", "Gemfile.lock", ".ruby-version", ".tool-versions", "mise.toml"];
const ZIG_MANIFESTS: &[&str] = &["build.zig.zon", "build.zig"];
/// A Maven / Gradle dependency of the project, from the build model.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
@@ -83,7 +85,7 @@ pub fn project_libraries_stream(
on_groups: &(dyn Fn(Vec<LibraryGroup>) + Sync),
) -> Vec<LibraryGroup> {
type Detect = fn(&Path, &Settings, &CancellationToken) -> Vec<LibraryGroup>;
let detectors: [Detect; 5] = [python_groups, jdk_groups, node_groups, go_groups, ruby_groups];
let detectors: [Detect; 6] = [python_groups, jdk_groups, node_groups, go_groups, ruby_groups, zig_groups];
let roots = library_project_roots(root, cancel);
let settings = module_settings(root, settings);
std::thread::scope(|s| {
@@ -193,6 +195,8 @@ fn library_project_roots(root: &Path, cancel: &CancellationToken) -> Vec<PathBuf
"setup.py",
"requirements.txt",
".python-version",
"build.zig",
"build.zig.zon",
".tool-versions",
"mise.toml",
]
@@ -252,6 +256,7 @@ fn module_settings(root: &Path, settings: &Settings) -> Settings {
&mut l.rust.stdlib,
&mut l.go.goroot,
&mut l.go.gopath,
&mut l.zig.zig,
&mut l.java.jdk,
] {
// `target:<id>` names a run target, not a path.
@@ -998,6 +1003,80 @@ pub(crate) fn parse_go_modules(text: &str) -> Vec<(String, PathBuf)> {
.collect()
}
// ------------------------------------------------------------------ zig
fn zig_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec<LibraryGroup> {
let Some(files) = manifests(root, ZIG_MANIFESTS) else { return Vec::new() };
let Some(zig) = tool(crate::apply::zig_bin(root, s), "zig") else { return Vec::new() };
cached_cancellable(root, "zig", zig.display().to_string(), &files, cancel, || {
let Some(env) = query_in_cancellable(&zig, &["env"], root, &[], QUERY_TIMEOUT, cancel)
.as_deref()
.and_then(crate::zig::parse_zig_env)
else {
return Vec::new();
};
let mut out = Vec::new();
let version =
env.version.clone().or_else(|| query_in_cancellable(&zig, &["version"], root, &[], QUERY_TIMEOUT, cancel));
let name = match version.as_deref().map(str::trim) {
Some(v) if !v.is_empty() => format!("Zig {v}"),
_ => "Zig".into(),
};
out.extend(env.std_sources().and_then(|std| group(name, "zig", library_roots(vec![std]))));
if let Some(cache) = env.global_cache_dir.as_deref() {
let zon = std::fs::read_to_string(root.join("build.zig.zon")).unwrap_or_default();
let deps = crate::zig::zon_dependencies(&zon);
out.extend(group("Zig packages", "zigPackages", zig_package_roots(cache, &deps)));
}
out
})
}
/// The fetched sources of a `build.zig.zon`'s dependencies: `p/<hash>` folders
/// (Zig ≤ 0.14 unpacks them) or the package root inside `p/<hash>.tar.gz` (Zig
/// ≥ 0.15 keeps packages as blobs, browsed through the archive VFS). Path
/// dependencies stay in the project tree; unfetched packages are simply absent —
/// listing libraries never downloads.
fn zig_package_roots(cache: &Path, deps: &[crate::zig::ZonDependency]) -> Vec<LibraryRoot> {
let mut seen = std::collections::HashSet::new();
deps.iter()
.filter_map(|d| {
let hash = d.hash.as_deref()?;
let dir = cache.join("p").join(hash);
if dir.is_dir() {
let zon = std::fs::read_to_string(dir.join("build.zig.zon")).unwrap_or_default();
let name =
match (crate::zig::zon_string_field(&zon, "name"), crate::zig::zon_string_field(&zon, "version")) {
(Some(name), Some(version)) => format!("{name} {version}"),
(Some(name), None) => name,
_ => hash.to_owned(),
};
return Some((name, dir));
}
let blob = cache.join("p").join(format!("{hash}.tar.gz"));
blob.is_file().then_some(hash).map(|hash| {
// The blob's only top-level folder is the package (`<hash>/`); fall back to
// the archive root should that layout ever change.
let entry = if ide_vfs::read_archive_dir(&blob, hash).is_ok() { hash } else { "" };
(package_label(d, hash), ide_vfs::archive_entry_path(&blob, entry))
})
})
.filter(|(_, path)| seen.insert(path.to_string_lossy().into_owned()))
.map(|(name, path)| LibraryRoot { name, path: path.into() })
.collect()
}
/// `probe_pkg 0.1.0` from a Zig ≥ 0.15 package hash (`name-version-<hash>`).
fn package_label(dep: &crate::zig::ZonDependency, hash: &str) -> String {
let Some((rest, _)) = hash.rsplit_once('-') else { return dep.name.clone() };
let mut parts: Vec<&str> = rest.split('-').collect();
let version = parts.pop().unwrap_or_default();
match parts.is_empty() || !version.starts_with(|c: char| c.is_ascii_digit()) || !version.contains('.') {
true => dep.name.clone(),
false => format!("{} {version}", parts.join("-")),
}
}
#[cfg(test)]
mod go_cache_tests {
use super::*;
@@ -1387,3 +1466,127 @@ mod idea_project_tests {
assert!(idea_java_project(root));
}
}
#[cfg(test)]
mod zig_tests {
use super::*;
use crate::zig::ZonDependency;
/// Writes a Zig package blob (`p/<hash>.tar.gz`) with a `<hash>/` root folder.
fn write_package_blob(cache: &Path, hash: &str, files: &[(&str, &str)]) {
use flate2::write::GzEncoder;
let blob = cache.join("p").join(format!("{hash}.tar.gz"));
std::fs::create_dir_all(blob.parent().unwrap()).unwrap();
let enc = GzEncoder::new(std::fs::File::create(blob).unwrap(), flate2::Compression::default());
let mut tar = tar::Builder::new(enc);
for (name, text) in files {
let mut header = tar::Header::new_gnu();
header.set_size(text.len() as u64);
header.set_mode(0o644);
header.set_cksum();
tar.append_data(&mut header, name, text.as_bytes()).unwrap();
}
tar.into_inner().unwrap().finish().unwrap();
}
#[test]
fn zig_packages_come_from_the_global_cache_without_fetching() {
let tmp = tempfile::tempdir().unwrap();
let cache = tmp.path();
let (hash, old) = ("zap-0.1.0-ISc0DC0BAAD0X6wRO4jcg4yLywsgmBYSlOsNhFDSnwV8", "1220b7c0");
write_package_blob(cache, hash, &[(&format!("{hash}/src/lib.zig"), "pub fn hello() {}\n")]);
let unpacked = cache.join("p").join(old);
std::fs::create_dir_all(&unpacked).unwrap();
std::fs::write(unpacked.join("build.zig.zon"), r#".{ .name = .old_pkg, .version = "0.2.0" }"#).unwrap();
let deps = [
ZonDependency { name: "zap".into(), hash: Some(hash.into()), path: None, lazy: false },
ZonDependency { name: "shared".into(), hash: None, path: Some("../shared".into()), lazy: false },
ZonDependency { name: "gone".into(), hash: Some("gone-1.0.0-BBBBBBBBBB".into()), path: None, lazy: true },
ZonDependency { name: "old".into(), hash: Some(old.into()), path: None, lazy: false },
];
let roots = zig_package_roots(cache, &deps);
let summary: Vec<(String, String)> =
roots.iter().map(|r| (r.name.clone(), r.path.to_string_lossy().into_owned())).collect();
let blob = cache.join("p").join(format!("{hash}.tar.gz"));
assert_eq!(
summary,
[
("zap 0.1.0".into(), format!("{}!/{hash}", blob.display())),
("old_pkg 0.2.0".into(), unpacked.to_string_lossy().into_owned())
]
);
}
#[test]
fn zig_package_labels_parse_from_the_hash() {
let dep = |name: &str| ZonDependency { name: name.into(), hash: None, path: None, lazy: false };
assert_eq!(package_label(&dep("zap"), "zap-0.1.0-ISc0DC0BAAD0X6wRO4jcg4yLywsgmBYSlOsNhFDSnwV8"), "zap 0.1.0");
assert_eq!(
package_label(&dep("diffz"), "diffz-0.0.1-G2tlISvOAQDORzPTSxDgiKwlHuADKeJMdJrw4kRfLufj"),
"diffz 0.0.1"
);
assert_eq!(
package_label(&dep("zig-anytype"), "zig-anytype-0.1.0-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"),
"zig-anytype 0.1.0"
);
assert_eq!(package_label(&dep("old"), "1220b7c0"), "old");
}
#[cfg(unix)]
#[test]
fn zig_libraries_list_the_stdlib_and_fetched_packages() {
use std::os::unix::fs::PermissionsExt;
let tmp = tempfile::tempdir().unwrap();
let root = tmp.path();
let stdlib = root.join("stdlib");
std::fs::create_dir_all(&stdlib).unwrap();
std::fs::write(root.join("build.zig"), "").unwrap();
let hash = "zap-0.1.0-ISc0DC0BAAD0X6wRO4jcg4yLywsgmBYSlOsNhFDSnwV8";
let cache = root.join("zig-cache");
write_package_blob(&cache, hash, &[(&format!("{hash}/src/lib.zig"), "pub fn hello() {}\n")]);
std::fs::write(
root.join("build.zig.zon"),
r#".{
.name = .app,
.dependencies = .{
// the package cache holds this one
.zap = .{ .url = "https://example.com/zap.tar.gz", .hash = "zap-0.1.0-ISc0DC0BAAD0X6wRO4jcg4yLywsgmBYSlOsNhFDSnwV8" },
.local = .{ .path = "../shared" },
.gone = .{ .url = "https://example.com/gone.tar.gz", .hash = "gone-1.0.0-BBBBBBBBBB" },
},
}
"#,
)
.unwrap();
let bin = root.join("tools");
std::fs::create_dir(&bin).unwrap();
let zig = bin.join("zig");
std::fs::write(
&zig,
format!(
"#!/bin/sh\ncase \"$1\" in\n env) printf '%s\\n' '.{{\n .zig_exe = \"{}\",\n .std_dir = \"{}\",\n .global_cache_dir = \"{}\",\n .version = \"0.16.0\",\n}}' ;;\n version) echo 0.16.0 ;;\nesac\n",
zig.display(),
stdlib.display(),
cache.display(),
),
)
.unwrap();
std::fs::set_permissions(&zig, std::fs::Permissions::from_mode(0o755)).unwrap();
let mut settings = Settings::default();
settings.languages.zig.zig = "tools".into();
let groups = zig_groups(root, &settings, &CancellationToken::new());
let summary: Vec<(&str, &str, Vec<&str>)> = groups
.iter()
.map(|g| (g.name.as_str(), g.kind.as_str(), g.roots.iter().map(|r| r.name.as_str()).collect()))
.collect();
assert_eq!(
summary,
[("Zig 0.16.0", "zig", vec!["stdlib"]), ("Zig packages", "zigPackages", vec!["zap 0.1.0"])]
);
let packages = groups.iter().find(|g| g.kind == "zigPackages").unwrap();
assert_eq!(
packages.roots[0].path.to_string_lossy(),
format!("{}!/{hash}", cache.join("p").join(format!("{hash}.tar.gz")).display())
);
}
}
+79
View File
@@ -1589,6 +1589,85 @@ fn go_modules_skip_the_main_module_and_missing_downloads() {
);
}
#[test]
fn build_zig_zon_dependencies_are_parsed() {
let zon = r#"// the manifest is a Zig struct literal
.{
.name = .app,
.version = "0.1.0",
.fingerprint = 0x348e9ddfb4b9e03c,
.dependencies = .{
.zap = .{ .url = "https://example.com/zap.tar.gz", .hash = "zap-0.1.0-AAA" },
.@"weird name" = .{
.url = "https://example.com/w.tar.gz",
.hash = "weird-2.0.0-BBB",
.lazy = true,
},
.shared = .{ .path = "../shared" },
},
.paths = .{ "build.zig", "src" },
}
"#;
assert_eq!(
crate::zig::zon_dependencies(zon),
[
crate::zig::ZonDependency {
name: "zap".into(),
hash: Some("zap-0.1.0-AAA".into()),
path: None,
lazy: false
},
crate::zig::ZonDependency {
name: "weird name".into(),
hash: Some("weird-2.0.0-BBB".into()),
path: None,
lazy: true
},
crate::zig::ZonDependency {
name: "shared".into(),
hash: None,
path: Some("../shared".into()),
lazy: false
},
]
);
assert!(crate::zig::zon_dependencies(".{ .name = .app, .dependencies = .{} }").is_empty());
assert!(crate::zig::zon_dependencies(".{ .name = .app }").is_empty());
assert!(crate::zig::zon_dependencies("{ broken").is_empty());
assert_eq!(
crate::zig::zon_string_field(".{ .name = .probe_pkg, .version = \"0.1.0\" }", "name").as_deref(),
Some("probe_pkg")
);
}
#[test]
fn zig_env_is_parsed_from_json_and_struct_literals() {
let env = crate::zig::parse_zig_env(
r#"{"zig_exe":"/opt/zig/zig","lib_dir":"/opt/zig/lib","std_dir":"/opt/zig/lib/std","global_cache_dir":"/home/u/.cache/zig","version":"0.14.1"}"#,
)
.unwrap();
assert_eq!(env.std_sources().as_deref(), Some(std::path::Path::new("/opt/zig/lib/std")));
assert_eq!(env.global_cache_dir.as_deref(), Some(std::path::Path::new("/home/u/.cache/zig")));
// Zig ≥ 0.15 prints a struct literal; a relative std_dir resolves against the install dir.
let env = crate::zig::parse_zig_env(
r#".{
.zig_exe = "/opt/zig/zig",
.lib_dir = "/opt/zig/lib",
.std_dir = "lib/std",
.global_cache_dir = "/home/u/.cache/zig",
.version = "0.16.0",
.target = "aarch64-macos",
.env = .{ .ZIG_GLOBAL_CACHE_DIR = null, .HOME = "/home/u" },
}"#,
)
.unwrap();
assert_eq!(env.std_sources().as_deref(), Some(std::path::Path::new("/opt/zig/lib/std")));
assert_eq!(env.version.as_deref(), Some("0.16.0"));
assert_eq!(env.global_cache_dir.as_deref(), Some(std::path::Path::new("/home/u/.cache/zig")));
assert!(crate::zig::parse_zig_env("error: nothing").is_none());
}
#[test]
fn jdk_library_is_the_jdk_home() {
let tmp = tempfile::tempdir().unwrap();
+261
View File
@@ -0,0 +1,261 @@
//! Reading Zig's own text formats for External Libraries, without running Zig:
//! the package manifest (`build.zig.zon`) and the `zig env` output. Both are
//! Zig struct literals; `zig env` printed JSON before Zig 0.15.
use serde::Deserialize;
use std::path::{Path, PathBuf};
/// One `.dependencies` entry of a `build.zig.zon`.
#[derive(Clone, Debug, PartialEq)]
pub(crate) struct ZonDependency {
pub name: String,
/// Package hash: its sources are fetched into the global cache as `p/<hash>`
/// (Zig ≤ 0.14 unpacks them, Zig ≥ 0.15 keeps `p/<hash>.tar.gz` blobs).
pub hash: Option<String>,
/// A path dependency, part of the project tree rather than the package cache.
pub path: Option<String>,
/// Fetched only when the build actually references it.
pub lazy: bool,
}
/// The `.dependencies` entries of a `build.zig.zon`, in order. Tolerant of
/// partially edited files: whatever does not parse is skipped.
pub(crate) fn zon_dependencies(zon: &str) -> Vec<ZonDependency> {
let src = code_only(zon);
let Some(top) = struct_body(&src) else { return Vec::new() };
let Some((_, value)) = fields(top).into_iter().find(|(name, _)| *name == "dependencies") else {
return Vec::new();
};
let Some(deps) = struct_body(value) else { return Vec::new() };
fields(deps)
.into_iter()
.map(|(name, value)| {
let entry = struct_body(value).map(|body| fields(body)).unwrap_or_default();
ZonDependency {
name: name.to_owned(),
hash: field(&entry, "hash"),
path: field(&entry, "path"),
lazy: entry.iter().any(|(n, v)| *n == "lazy" && v.trim() == "true"),
}
})
.collect()
}
/// A top-level string (or enum-literal, `.name`) field of a Zig struct literal.
pub(crate) fn zon_string_field(text: &str, name: &str) -> Option<String> {
let src = code_only(text);
let top = struct_body(&src)?;
field(&fields(top), name)
}
/// What `zig env` reports that External Libraries needs.
#[derive(Clone, Debug, Default, PartialEq)]
pub(crate) struct ZigEnv {
pub zig_exe: Option<PathBuf>,
pub lib_dir: Option<PathBuf>,
pub std_dir: Option<PathBuf>,
pub global_cache_dir: Option<PathBuf>,
pub version: Option<String>,
}
impl ZigEnv {
/// The standard library sources: `std_dir`, else `lib_dir/std`. A relative
/// path resolves against the zig install dir (stage builds print relative paths).
pub(crate) fn std_sources(&self) -> Option<PathBuf> {
let resolve = |p: &Path| match (p.is_relative(), self.zig_exe.as_ref().and_then(|e| e.parent())) {
(true, Some(install)) => install.join(p),
_ => p.to_path_buf(),
};
self.std_dir
.as_deref()
.map(Path::new)
.map(resolve)
.or_else(|| self.lib_dir.as_deref().map(Path::new).map(|dir| resolve(dir).join("std")))
}
}
/// The `zig env` output: JSON (older Zig) or a struct literal (Zig ≥ 0.15).
pub(crate) fn parse_zig_env(out: &str) -> Option<ZigEnv> {
#[derive(Deserialize)]
struct Json {
zig_exe: Option<PathBuf>,
lib_dir: Option<PathBuf>,
std_dir: Option<PathBuf>,
global_cache_dir: Option<PathBuf>,
version: Option<String>,
}
if let Ok(json) = serde_json::from_str::<Json>(out.trim()) {
return Some(ZigEnv {
zig_exe: json.zig_exe,
lib_dir: json.lib_dir,
std_dir: json.std_dir,
global_cache_dir: json.global_cache_dir,
version: json.version,
});
}
let src = code_only(out);
let top = struct_body(&src)?;
let f = fields(top);
Some(ZigEnv {
zig_exe: field(&f, "zig_exe").map(PathBuf::from),
lib_dir: field(&f, "lib_dir").map(PathBuf::from),
std_dir: field(&f, "std_dir").map(PathBuf::from),
global_cache_dir: field(&f, "global_cache_dir").map(PathBuf::from),
version: field(&f, "version"),
})
}
/// The string (or enum-literal, `.foo`) value of a field of a struct body.
fn field<'a>(entry: &[(&'a str, &'a str)], name: &str) -> Option<String> {
let value = entry.iter().find(|(n, _)| *n == name)?.1;
if let Some(string) = string_value(value) {
return Some(string);
}
value
.strip_prefix('.')
.filter(|v| v.chars().next().is_some_and(|c| c.is_ascii_alphanumeric() || c == '_'))
.map(str::to_owned)
}
/// The contents of the string literal starting at `value`, `\"` unescaped.
fn string_value(value: &str) -> Option<String> {
let rest = value.trim_start().strip_prefix('"')?;
let mut out = String::new();
let mut chars = rest.chars();
while let Some(c) = chars.next() {
match c {
'"' => return Some(out),
'\\' => out.push(chars.next()?),
c => out.push(c),
}
}
None
}
/// `src` with comments blanked out and string contents kept.
fn code_only(src: &str) -> String {
let mut out = String::with_capacity(src.len());
for line in src.lines() {
let bytes = line.as_bytes();
let mut in_string = false;
let mut escaped = false;
let mut end = line.len();
for (i, &b) in bytes.iter().enumerate() {
if in_string {
match b {
_ if escaped => escaped = false,
b'\\' => escaped = true,
b'"' => in_string = false,
_ => {}
}
} else if b == b'"' {
in_string = true;
} else if b == b'/' && bytes.get(i + 1) == Some(&b'/') {
end = i;
break;
}
}
out.push_str(&line[..end]);
out.push('\n');
}
out
}
/// The inside of the first `{ … }` of `text`, strings respected.
fn struct_body(text: &str) -> Option<&str> {
let bytes = text.as_bytes();
let mut start = 0;
while start < bytes.len() {
match bytes[start] {
b'"' => start = skip_string(text, start),
b'{' => break,
_ => start += 1,
}
}
let mut depth = 0;
let mut i = start;
while i < bytes.len() {
match bytes[i] {
b'"' => i = skip_string(text, i),
b'{' => {
depth += 1;
i += 1;
}
b'}' => {
if depth == 1 {
return Some(&text[start + 1..i]);
}
depth -= 1;
i += 1;
}
_ => i += 1,
}
}
None
}
/// The index just past the string literal starting at `i` (an opening `"`).
fn skip_string(text: &str, i: usize) -> usize {
let bytes = text.as_bytes();
let mut i = i + 1;
while i < bytes.len() {
match bytes[i] {
b'\\' => i += 2,
b'"' => return i + 1,
_ => i += 1,
}
}
i
}
/// The `.name = value` fields of a struct literal body, in order: `name` is the
/// identifier (or `@"…"` unquoted), `value` trimmed and may itself be `.{ … }`.
fn fields(body: &str) -> Vec<(&str, &str)> {
let bytes = body.as_bytes();
let mut out = Vec::new();
let mut depth = 0;
let mut field: Option<usize> = None;
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
b'"' => i = skip_string(body, i),
b'{' | b'[' | b'(' => {
depth += 1;
i += 1;
}
b'}' | b']' | b')' => {
if depth == 0 {
break;
}
depth -= 1;
i += 1;
}
b',' if depth == 0 => {
if let Some(start) = field.take() {
push_field(&mut out, body, start, i);
}
i += 1;
}
b'.' if depth == 0 && field.is_none() => {
if bytes.get(i + 1).is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || *b == b'@') {
field = Some(i + 1);
}
i += 1;
}
_ => i += 1,
}
}
if let Some(start) = field {
push_field(&mut out, body, start, bytes.len());
}
out
}
fn push_field<'a>(out: &mut Vec<(&'a str, &'a str)>, body: &'a str, start: usize, end: usize) {
let text = &body[start..end];
let Some(eq) = text.find('=') else { return };
let name = text[..eq].trim().trim_start_matches('@').trim_matches('"');
if !name.is_empty() {
out.push((name, text[eq + 1..].trim()));
}
}
+2
View File
@@ -20,6 +20,8 @@ trash.workspace = true
tracing.workspace = true
parking_lot.workspace = true
zip.workspace = true
flate2.workspace = true
tar.workspace = true
[dev-dependencies]
tempfile.workspace = true
+163 -44
View File
@@ -1,6 +1,7 @@
//! Read-only browsing of zip archives (jar, zip, war…), IDEA's `jar://` file
//! system: `<archive>!` is the archive's root and `<archive>!/<entry>` an entry
//! inside it. Listing and reading go through a small cache of parsed central
//! Read-only browsing of zip archives (jar, zip, war…) and gzip'd tars (Zig
//! keeps its fetched packages as `.tar.gz` blobs), IDEA's `jar://` file system:
//! `<archive>!` is the archive's root and `<archive>!/<entry>` an entry inside
//! it. Listing and reading go through a small cache of parsed central
//! directories, refreshed when the archive changes on disk.
use crate::{EntryKind, FileEntry, mtime_ms, natural_cmp};
@@ -8,7 +9,7 @@ use ide_core::{IdeError, IdeResult};
use parking_lot::Mutex;
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::io::{Read, Seek};
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock};
use std::time::SystemTime;
@@ -22,13 +23,21 @@ pub const ARCHIVE_SEPARATOR: char = '!';
/// Parsed archives kept open.
const CACHE_SIZE: usize = 8;
/// Whether `path` names a file browsed as an archive (by extension).
/// Whether `path` names a file browsed as an archive: a zip extension, or a
/// gzip'd tar (`x.tar.gz`).
pub fn is_archive_file(path: &Path) -> bool {
if is_tar_gz(path) {
return true;
}
path.extension()
.and_then(|e| e.to_str())
.is_some_and(|e| ARCHIVE_EXTENSIONS.iter().any(|a| a.eq_ignore_ascii_case(e)))
}
fn is_tar_gz(path: &Path) -> bool {
path.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.to_ascii_lowercase().ends_with(".tar.gz"))
}
/// `(archive, entry)` for `<archive>!` (entry `""`) or `<archive>!/<entry>`;
/// `None` for ordinary paths. The entry is `/`-separated without leading or trailing `/`.
pub fn split_archive_path(path: &Path) -> Option<(PathBuf, String)> {
@@ -84,7 +93,14 @@ struct Archive {
mtime_ms: f64,
/// Entry path (`""` = root) → node.
nodes: HashMap<String, Node>,
zip: Mutex<zip::ZipArchive<File>>,
contents: Contents,
}
/// Where entry contents come from: a random-access zip, or a gzip'd tar that
/// is re-scanned per read (package tarballs are small).
enum Contents {
Zip(Mutex<zip::ZipArchive<File>>),
Tar(Mutex<File>),
}
/// Most recently used first.
@@ -115,43 +131,83 @@ fn parent_of(name: &str) -> (&str, &str) {
impl Archive {
fn open(path: &Path, stamp: (Option<SystemTime>, u64), mtime_ms: f64) -> IdeResult<Self> {
let mut zip = zip::ZipArchive::new(File::open(path)?).map_err(|e| zip_error(path, e))?;
let mut nodes: HashMap<String, Node> = HashMap::new();
nodes.insert(String::new(), Node { dir: true, size: 0, index: None, children: Vec::new() });
for i in 0..zip.len() {
let Ok(file) = zip.by_index_raw(i) else { continue };
let Ok(name) = file.name() else { continue };
let Some(name) = clean_name(&name) else { continue };
let (dir, size) = (file.is_dir(), file.size());
drop(file);
let mut parent = parent_of(&name).0;
while !nodes.contains_key(parent) {
nodes.insert(parent.to_owned(), Node { dir: true, size: 0, index: None, children: Vec::new() });
parent = parent_of(parent).0;
}
let node = nodes.entry(name).or_insert(Node { dir, size: 0, index: None, children: Vec::new() });
node.dir |= dir;
node.index = Some(i);
node.size = if node.dir { 0 } else { size };
}
let links: Vec<(String, String)> = nodes
.keys()
.filter(|k| !k.is_empty())
.map(|k| {
let (parent, base) = parent_of(k);
(parent.to_owned(), base.to_owned())
})
.collect();
for (parent, base) in links {
if let Some(p) = nodes.get_mut(&parent) {
p.dir = true;
p.children.push(base);
}
}
Ok(Self { stamp, mtime_ms, nodes, zip: Mutex::new(zip) })
let file = File::open(path)?;
let (nodes, contents) = if is_tar_gz(path) {
(tar_nodes(&file)?, Contents::Tar(Mutex::new(file)))
} else {
let mut zip = zip::ZipArchive::new(file).map_err(|e| zip_error(path, e))?;
let nodes = zip_nodes(&mut zip)?;
(nodes, Contents::Zip(Mutex::new(zip)))
};
Ok(Self { stamp, mtime_ms, nodes, contents })
}
}
/// The directory tree of a zip, with each entry's central-directory index.
fn zip_nodes(zip: &mut zip::ZipArchive<File>) -> IdeResult<HashMap<String, Node>> {
let mut nodes: HashMap<String, Node> = HashMap::new();
nodes.insert(String::new(), Node { dir: true, size: 0, index: None, children: Vec::new() });
for i in 0..zip.len() {
let Ok(file) = zip.by_index_raw(i) else { continue };
let Ok(name) = file.name() else { continue };
let Some(name) = clean_name(&name) else { continue };
let (dir, size) = (file.is_dir(), file.size());
drop(file);
add_entry(&mut nodes, name, i, dir, size);
}
Ok(link_children(nodes))
}
/// The directory tree of a gzip'd tar, with each entry's ordinal position.
/// Entries are re-read by streaming the (decompressed) tar again.
fn tar_nodes(file: &File) -> IdeResult<HashMap<String, Node>> {
let mut nodes: HashMap<String, Node> = HashMap::new();
nodes.insert(String::new(), Node { dir: true, size: 0, index: None, children: Vec::new() });
let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(file));
for (i, entry) in tar.entries()?.enumerate() {
let Ok(entry) = entry else { break };
let Ok(name) = entry.path().map(|p| p.to_string_lossy().into_owned()) else { continue };
let Some(name) = clean_name(&name) else { continue };
let (dir, size) = match entry.header().entry_type().is_dir() {
true => (true, 0),
false => (false, entry.header().size().unwrap_or_default()),
};
add_entry(&mut nodes, name, i, dir, size);
}
Ok(link_children(nodes))
}
fn add_entry(nodes: &mut HashMap<String, Node>, name: String, index: usize, dir: bool, size: u64) {
let mut parent = parent_of(&name).0;
while !nodes.contains_key(parent) {
nodes.insert(parent.to_owned(), Node { dir: true, size: 0, index: None, children: Vec::new() });
parent = parent_of(parent).0;
}
let node = nodes.entry(name).or_insert(Node { dir, size: 0, index: None, children: Vec::new() });
node.dir |= dir;
node.index = Some(index);
node.size = if node.dir { 0 } else { size };
}
/// Makes every named entry a child of its parent folder node.
fn link_children(mut nodes: HashMap<String, Node>) -> HashMap<String, Node> {
let links: Vec<(String, String)> = nodes
.keys()
.filter(|k| !k.is_empty())
.map(|k| {
let (parent, base) = parent_of(k);
(parent.to_owned(), base.to_owned())
})
.collect();
for (parent, base) in links {
if let Some(p) = nodes.get_mut(&parent) {
p.dir = true;
p.children.push(base);
}
}
nodes
}
/// The parsed archive at `path`, from the cache unless it changed on disk.
fn archive(path: &Path) -> IdeResult<Arc<Archive>> {
let meta = std::fs::metadata(path)?;
@@ -235,10 +291,31 @@ pub fn read_archive_entry(archive_path: &Path, entry: &str) -> IdeResult<(Vec<u8
Some(_) => return Err(IdeError::invalid(format!("{entry} is a directory"))),
None => return Err(IdeError::not_found(format!("{entry} not found in {}", archive_path.display()))),
};
let mut zip = a.zip.lock();
let mut file = zip.by_index(index).map_err(|e| zip_error(archive_path, e))?;
let mut bytes = Vec::with_capacity(file.size().min(crate::MAX_OPEN_SIZE) as usize);
file.read_to_end(&mut bytes)?;
let bytes = match &a.contents {
Contents::Zip(zip) => {
let mut zip = zip.lock();
let mut file = zip.by_index(index).map_err(|e| zip_error(archive_path, e))?;
let mut bytes = Vec::with_capacity(file.size().min(crate::MAX_OPEN_SIZE) as usize);
file.read_to_end(&mut bytes)?;
bytes
}
Contents::Tar(file) => {
let mut file = file.lock();
file.seek(std::io::SeekFrom::Start(0))?;
let mut tar = tar::Archive::new(flate2::read::GzDecoder::new(&mut *file));
for (i, entry) in tar.entries()?.enumerate() {
if i != index {
continue;
}
let mut entry = entry.map_err(|e| IdeError::invalid(format!("{}: {e}", archive_path.display())))?;
let mut bytes =
Vec::with_capacity(entry.header().size().unwrap_or_default().min(crate::MAX_OPEN_SIZE) as usize);
entry.read_to_end(&mut bytes)?;
return Ok((bytes, a.mtime_ms));
}
return Err(IdeError::not_found(format!("{entry} not found in {}", archive_path.display())));
}
};
Ok((bytes, a.mtime_ms))
}
@@ -262,6 +339,20 @@ pub(crate) mod tests {
w.finish().unwrap();
}
/// Writes a gzip'd tar with `files`, the layout of the Zig package cache blobs.
pub(crate) fn write_tar_gz(path: &Path, files: &[(&str, &str)]) {
let enc = flate2::write::GzEncoder::new(File::create(path).unwrap(), flate2::Compression::default());
let mut b = tar::Builder::new(enc);
for (name, text) in files {
let mut header = tar::Header::new_gnu();
header.set_size(text.len() as u64);
header.set_mode(0o644);
header.set_cksum();
b.append_data(&mut header, name, text.as_bytes()).unwrap();
}
b.into_inner().unwrap().finish().unwrap();
}
#[test]
fn splits_archive_paths() {
assert_eq!(split_archive_path(Path::new("/a/b.txt")), None);
@@ -317,6 +408,34 @@ pub(crate) mod tests {
assert_eq!(names(""), vec![("only.txt".into(), EntryKind::File)]);
}
#[test]
fn lists_and_reads_gzipped_tars() {
let tmp = tempfile::tempdir().unwrap();
let hash = "probe_pkg-0.1.0-ISc0DC0BAAD0X6wRO4jcg4yLywsgmBYSlOsNhFDSnwV8";
let blob = tmp.path().join(format!("{hash}.tar.gz"));
let (zon, lib) = (format!("{hash}/build.zig.zon"), format!("{hash}/src/lib.zig"));
write_tar_gz(&blob, &[(zon.as_str(), ".{ .name = .probe_pkg }"), (lib.as_str(), "pub fn hello() {}\n")]);
let root = |entry: &str| PathBuf::from(format!("{}!/{entry}", blob.display()));
let names = |dir: &str| -> Vec<String> {
crate::read_dir(&root(&format!("{hash}/{dir}")), &[]).unwrap().into_iter().map(|e| e.name).collect()
};
assert_eq!(names(""), vec!["src", "build.zig.zon"]);
assert_eq!(names("src"), vec!["lib.zig"]);
let lib = crate::load_text(&root(&format!("{hash}/src/lib.zig"))).unwrap();
assert_eq!((lib.text.as_str(), lib.binary), ("pub fn hello() {}\n", false));
// A rewritten tar is re-read, and a plain `.gz` stays an ordinary file.
write_tar_gz(&blob, &[("only.zig", "1")]);
let f = File::options().write(true).open(&blob).unwrap();
f.set_modified(SystemTime::now() + std::time::Duration::from_secs(5)).unwrap();
assert_eq!(
crate::read_dir(&root(""), &[]).unwrap().iter().map(|e| e.name.as_str()).collect::<Vec<_>>(),
["only.zig"]
);
assert!(!is_archive_file(Path::new("/a/data.gz")));
assert!(is_archive_file(Path::new("/a/pack.TAR.GZ")));
}
#[test]
fn read_dir_and_load_text_go_through_archives() {
let tmp = tempfile::tempdir().unwrap();
+7 -1
View File
@@ -6993,6 +6993,7 @@ export type LanguagesSettings = {
ruby?: RubySettings,
markdown?: MarkdownSettings,
arduino?: ArduinoSettings,
zig?: ZigSettings,
};
/** What a freshly opened window should do once its frontend is up. */
@@ -7081,7 +7082,7 @@ export type Launch_Serialize = {
export type LibraryGroup = {
/** Display name, e.g. `Python 3.12 (.venv)`. */
name: string,
/** `python` | `jdk` | `nodeModules` | `nodeCore` | `rust` | `cargo` | `go` | `goModules` | `maven` | `gradle` | `classpath`. */
/** `python` | `jdk` | `nodeModules` | `nodeCore` | `rust` | `cargo` | `go` | `goModules` | `maven` | `gradle` | `classpath` | `zig` | `zigPackages`. */
kind: string,
roots: LibraryRoot[],
};
@@ -11011,6 +11012,11 @@ export type Worktree = {
prunable: boolean,
};
export type ZigSettings = {
/** Directory holding the `zig` executable; empty = the one on `PATH`. */
zig?: string,
};
/* Tauri Specta runtime */
async function typedError<T, E>(result: Promise<T>): Promise<{ status: "ok"; data: T } | { status: "error"; error: E }> {
try {
+9 -4
View File
@@ -2,9 +2,10 @@
* External Libraries: the project's SDK / library source folders (the Python
* interpreter's standard library and site-packages, the JDK, node_modules
* packages, Cargo crates and the Rust standard library, GOROOT and Go modules,
* Maven / Gradle dependency jars) as a Project view root. Their files open
* read-only, like IDEA's library sources. Archives (the JDK's `src.zip`, jars)
* are browsed through the backend's `<archive>!/<entry>` paths.
* the Zig standard library and its fetched build.zig.zon packages, Maven /
* Gradle dependency jars) as a Project view root. Their files open read-only,
* like IDEA's library sources. Archives (the JDK's `src.zip`, jars, Zig
* package tarballs) are browsed through the backend's `<archive>!/<entry>` paths.
*/
import { Channel } from "@tauri-apps/api/core";
import { commands, type LibraryGroup, type RequestId } from "@/bindings";
@@ -32,10 +33,12 @@ const GROUP_ICONS: Record<string, string> = {
classpath: "file-java",
ruby: "file-ruby",
gems: "file-ruby",
zig: "file-zig",
zigPackages: "file-zig",
};
/** SDK groups, which IDEA writes in angle brackets (`< 21 >`); library groups (`Maven: g:a:v`) go without. */
const SDK_KINDS = new Set(["jdk", "python", "ruby", "go", "rust"]);
const SDK_KINDS = new Set(["jdk", "python", "ruby", "go", "rust", "zig"]);
/** An archive's root (`/jdk/lib/src.zip!`): its entries live below `<archive>!/`. */
const isArchiveRoot = (path: string) => path.endsWith("!");
@@ -62,6 +65,7 @@ const MANIFESTS = new Set([
"go.mod",
"go.sum",
"go.work",
"build.zig.zon",
".classpath",
"pyproject.toml",
"setup.py",
@@ -129,6 +133,7 @@ const sdkKey = (s: ReturnType<typeof useProject.getState>) => {
l?.go?.goroot,
l?.go?.gopath,
l?.go?.modules,
l?.zig?.zig,
s.settings?.projectStructure?.libraries,
s.settings?.projectStructure?.sourceRoots,
]);
+25 -7
View File
@@ -29,13 +29,24 @@ describe("External Libraries", () => {
it("gives each ecosystem's groups their icon", () => {
const root = librariesRoot(
["jdk", "nodeCore", "nodeModules", "rust", "cargo", "go", "goModules", "maven", "gradle", "other"].map(
(kind) => ({
name: kind,
kind,
roots: [{ name: "r", path: `/libs/${kind}` }],
}),
),
[
"jdk",
"nodeCore",
"nodeModules",
"rust",
"cargo",
"go",
"goModules",
"maven",
"gradle",
"zig",
"zigPackages",
"other",
].map((kind) => ({
name: kind,
kind,
roots: [{ name: "r", path: `/libs/${kind}` }],
})),
)!;
expect(Object.fromEntries(root.children!.map((g) => [g.label, g.icon]))).toEqual({
"< jdk >": "file-java",
@@ -47,6 +58,8 @@ describe("External Libraries", () => {
goModules: "file-gomod",
maven: "maven-project",
gradle: "file-gradle",
"< zig >": "file-zig",
zigPackages: "file-zig",
other: "library",
});
});
@@ -63,18 +76,22 @@ describe("External Libraries", () => {
group("Go SDK 1.27.4", "go"),
group("Maven: com.acme:core:2.1.27", "maven"),
group("Node.js Core", "nodeCore"),
group("Zig 0.16.0", "zig"),
group("Zig packages", "zigPackages"),
];
const labels = (groups: typeof detected) => librariesRoot(groups)!.children!.map((g) => g.label);
expect(labels(detected)).toEqual([
"< Go SDK 1.27.4 >",
"< Python 3.14 >",
"< temurin-21 >",
"< Zig 0.16.0 >",
"Cargo dependencies",
"Gradle: com.acme:lib:1.0",
"Maven: com.acme:core:2.1.27",
"Maven: org.junit:junit:4.13",
"node_modules",
"Node.js Core",
"Zig packages",
]);
expect(labels([...detected].reverse())).toEqual(labels(detected));
});
@@ -104,6 +121,7 @@ describe("External Libraries", () => {
expect(touchesManifest(["C:\\p\\go.mod"])).toBe(true);
expect(touchesManifest(["/p/.venv/pyvenv.cfg"])).toBe(true);
expect(touchesManifest(["/p/pnpm-workspace.yaml", "/p/Gemfile.lock"])).toBe(true);
expect(touchesManifest(["/p/build.zig.zon", "/p/go.work"])).toBe(true);
expect(touchesManifest(["/p/node_modules/react/package.json"])).toBe(false);
expect(touchesManifest(["/p/src/package.json.bak", "/p/README.md"])).toBe(false);
});