diff --git a/crates/ide-tools/src/lib.rs b/crates/ide-tools/src/lib.rs index d0a8b65d..dbdddfa5 100644 --- a/crates/ide-tools/src/lib.rs +++ b/crates/ide-tools/src/lib.rs @@ -1221,10 +1221,26 @@ impl Tools { cancel: &CancellationToken, ) -> IdeResult { match source { - ReleaseSource::GitHub { repo } => { - let json = http.get_text(&sources::github_latest_url(repo), cancel)?; - sources::github_pick(&json, assets, self.platform) - } + ReleaseSource::GitHub { repo } => match http.get_text(&sources::github_latest_url(repo), cancel) { + Ok(json) => sources::github_pick(&json, assets, self.platform), + Err(e) if e.kind == ErrorKind::Cancelled => Err(e), + // Without a token the API allows 60 requests an hour per IP address, often used + // up behind a shared one (403/429); the release's web pages have no such limit. + Err(api) => self.resolve_github_web(http, repo, assets, cancel).map_err(|web| { + if web.kind == ErrorKind::Cancelled { + return web; + } + tracing::warn!("GitHub release lookup for {repo} failed: {api}; web pages: {web}"); + IdeError::new( + api.kind, + format!( + "{}\nDownload it from {} and set its Path in Settings › Languages & Frameworks › Language Servers.", + api.message, + sources::github_latest_page(repo) + ), + ) + }), + }, ReleaseSource::HashiCorp { product } => { let json = http.get_text(&sources::hashicorp_latest_url(product), cancel)?; sources::hashicorp_pick(&json, assets, self.platform) @@ -1251,6 +1267,27 @@ impl Tools { } } + /// [`resolve_release`](Self::resolve_release) for a GitHub release, from its web pages. + fn resolve_github_web( + &self, + http: &Http, + repo: &str, + assets: &[platform::AssetRule], + cancel: &CancellationToken, + ) -> IdeResult { + let page = http.effective_url(&sources::github_latest_page(repo), cancel)?; + let tag = sources::github_tag_from_url(&page) + .ok_or_else(|| IdeError::unavailable(format!("{repo} has no published release")))?; + let list = http.get_text(&sources::github_assets_page(repo, &tag), cancel)?; + match sources::github_pick_web(repo, &tag, &list, None, assets, self.platform) { + Err(e) if e.kind == ErrorKind::Unavailable => { + let notes = http.get_text(&sources::github_tag_page(repo, &tag), cancel)?; + sources::github_pick_web(repo, &tag, &list, Some(¬es), assets, self.platform) + } + result => result, + } + } + fn install_release( &self, spec: &ToolSpec, diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs index ac8a48b2..7d34423e 100644 --- a/crates/ide-tools/src/process.rs +++ b/crates/ide-tools/src/process.rs @@ -263,6 +263,15 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool { (major, minor) >= (7, 70) && first.split_whitespace().any(|w| w.eq_ignore_ascii_case("schannel")) } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Request { + Get, + /// Adoptium uses 404 as well as an empty array for absent platform builds. + GetOrEmptyOn404, + /// Headers, then the URL redirects ended at. + Head, +} + /// HTTP via the system `curl` (shipped with macOS, Windows 10+ and every /// mainstream Linux), so the IDE carries no TLS stack of its own. pub struct Http { @@ -336,26 +345,39 @@ impl Http { self.get_text_limited(url, None, cancel) } + /// Where `url` ends up after redirects (a HEAD request). + pub fn effective_url(&self, url: &str, cancel: &CancellationToken) -> IdeResult { + let out = self.get_text_response(url, None, Request::Head, cancel)?; + Ok(out.trim_end().rsplit('\n').next().unwrap_or_default().trim().to_owned()) + } + /// GETs `url` as text, failing when the response is larger than `max_bytes`. pub fn get_text_limited(&self, url: &str, max_bytes: Option, cancel: &CancellationToken) -> IdeResult { - self.get_text_response(url, max_bytes, false, cancel) + self.get_text_response(url, max_bytes, Request::Get, cancel) } /// Adoptium uses 404 as well as an empty array for absent platform builds. pub fn get_text_or_empty_on_404(&self, url: &str, cancel: &CancellationToken) -> IdeResult { - self.get_text_response(url, None, true, cancel) + self.get_text_response(url, None, Request::GetOrEmptyOn404, cancel) } fn get_text_response( &self, url: &str, max_bytes: Option, - empty_on_404: bool, + request: Request, cancel: &CancellationToken, ) -> IdeResult { let (mut cmd, token) = self.base(url, cancel)?; - if empty_on_404 { - cmd.args(["--write-out", "\n%{http_code}"]); + let empty_on_404 = request == Request::GetOrEmptyOn404; + match request { + Request::Get => {} + Request::GetOrEmptyOn404 => { + cmd.args(["--write-out", "\n%{http_code}"]); + } + Request::Head => { + cmd.args(["--head", "--write-out", "\n%{url_effective}"]); + } } if let Some(max) = max_bytes { cmd.arg("--max-filesize").arg(max.to_string()); diff --git a/crates/ide-tools/src/sources.rs b/crates/ide-tools/src/sources.rs index 2337bc75..0e5da748 100644 --- a/crates/ide-tools/src/sources.rs +++ b/crates/ide-tools/src/sources.rs @@ -47,38 +47,146 @@ pub fn github_latest_url(repo: &str) -> String { format!("https://api.github.com/repos/{repo}/releases/latest") } +/// The latest release's web page; it redirects to `…/releases/tag/`. Web pages don't +/// count against the API's rate limit for unauthenticated clients (60 requests an hour). +pub fn github_latest_page(repo: &str) -> String { + format!("https://github.com/{repo}/releases/latest") +} + +/// The tag `github_latest_page` redirected to. +pub fn github_tag_from_url(url: &str) -> Option { + let (_, tag) = url.split_once("/releases/tag/")?; + let tag = tag.split(['?', '#']).next().unwrap_or(tag).trim_end_matches('/'); + (!tag.is_empty()).then(|| percent_decode(tag)) +} + +pub fn github_tag_page(repo: &str, tag: &str) -> String { + format!("https://github.com/{repo}/releases/tag/{tag}") +} + +/// The asset list the release page loads (HTML). +pub fn github_assets_page(repo: &str, tag: &str) -> String { + format!("https://github.com/{repo}/releases/expanded_assets/{tag}") +} + +/// One file of a release, from the API or its web page. +struct ReleaseAsset { + name: String, + url: String, + size: Option, + sha256: Option, +} + /// Picks the platform asset from a GitHub `releases/latest` response. Links /// in the release notes count as assets too (some projects host binaries on /// a CDN and only link them). pub fn github_pick(json: &str, rules: &[AssetRule], platform: Platform) -> IdeResult { let rel: GhRelease = serde_json::from_str(json).map_err(|e| proto(format!("unexpected GitHub response: {e}")))?; - let version = rel.tag_name.trim_start_matches('v').to_string(); - let names: Vec<&str> = rel.assets.iter().map(|a| a.name.as_str()).collect(); + let assets: Vec = rel + .assets + .into_iter() + .map(|a| ReleaseAsset { + sha256: a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase), + name: a.name, + url: a.browser_download_url, + size: a.size, + }) + .collect(); + pick_release(&rel.tag_name, &assets, rel.body.as_deref().unwrap_or(""), rules, platform) +} + +/// [`github_pick`] from the release's web pages: `assets_html` from [`github_assets_page`] and, +/// for projects that only link their builds, `notes_html` from [`github_tag_page`]. +pub fn github_pick_web( + repo: &str, + tag: &str, + assets_html: &str, + notes_html: Option<&str>, + rules: &[AssetRule], + platform: Platform, +) -> IdeResult { + let prefix = format!("href=\"/{repo}/releases/download/"); + let mut assets: Vec = Vec::new(); + let mut rest = assets_html; + while let Some(i) = rest.find(&prefix) { + let tail = &rest[i + "href=\"".len()..]; + let Some(end) = tail.find('"') else { break }; + let path = html_unescape(&tail[..end]); + rest = &tail[end..]; + let name = percent_decode(file_name(&path)); + if name.is_empty() || assets.iter().any(|a| a.name == name) { + continue; + } + // `` + let sha256 = assets_html.find(&format!("digest for {}\"", html_escape(&name))).and_then(|at| { + let tag_end = assets_html[at..].find('>').map_or(assets_html.len(), |e| at + e); + let value = assets_html[at..tag_end].split_once("value=\"sha256:")?.1; + let hex: String = value.chars().take_while(char::is_ascii_hexdigit).collect(); + (hex.len() == 64).then(|| hex.to_ascii_lowercase()) + }); + assets.push(ReleaseAsset { url: format!("https://github.com{path}"), name, size: None, sha256 }); + } + pick_release(tag, &assets, &html_unescape(notes_html.unwrap_or("")), rules, platform) +} + +fn pick_release( + tag: &str, + assets: &[ReleaseAsset], + notes: &str, + rules: &[AssetRule], + platform: Platform, +) -> IdeResult { + let version = tag.trim_start_matches('v').to_string(); + let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect(); if let Some(name) = select_asset(&names, rules, platform) { - let a = rel.assets.iter().find(|a| a.name == name).expect("selected from list"); - let sha256 = a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase); - let checksum_url = rel - .assets + let a = assets.iter().find(|a| a.name == name).expect("selected from list"); + let checksum_url = assets .iter() .find(|c| c.name == format!("{name}.sha256") || c.name == format!("{name}.sha256sum")) - .map(|c| c.browser_download_url.clone()); + .map(|c| c.url.clone()); return Ok(Download { version, name: a.name.clone(), - url: a.browser_download_url.clone(), + url: a.url.clone(), size: a.size, - sha256, + sha256: a.sha256.clone(), checksum_url, }); } - let links = extract_links(rel.body.as_deref().unwrap_or("")); + let links = extract_links(notes); let link_names: Vec<&str> = links.iter().map(|u| file_name(u)).collect(); if let Some(name) = select_asset(&link_names, rules, platform) { let url = links.iter().find(|u| file_name(u) == name).expect("selected from list").clone(); let checksum_url = links.iter().find(|u| *u == &format!("{url}.sha256")).cloned(); return Ok(Download { version, name: name.to_string(), url, size: None, sha256: None, checksum_url }); } - Err(IdeError::unavailable(format!("release {} has no build for this platform", rel.tag_name))) + Err(IdeError::unavailable(format!("release {tag} has no build for this platform"))) +} + +fn html_unescape(s: &str) -> String { + s.replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">").replace("&", "&") +} + +fn html_escape(s: &str) -> String { + s.replace('&', "&").replace('"', """).replace('<', "<").replace('>', ">") +} + +fn percent_decode(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' + && let Some(b) = s.get(i + 1..i + 3).and_then(|h| u8::from_str_radix(h, 16).ok()) + { + out.push(b); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).unwrap_or_else(|_| s.to_owned()) } fn file_name(url: &str) -> &str { @@ -336,6 +444,71 @@ mod tests { assert_eq!(d.name, "kotlin-lsp-0.253.10629-linux-x64.zip"); } + #[test] + fn github_tag_from_redirect() { + let url = "https://github.com/Kotlin/kotlin-lsp/releases/tag/kotlin-lsp/v263.7084.0"; + assert_eq!(github_tag_from_url(url).as_deref(), Some("kotlin-lsp/v263.7084.0")); + let url = "https://github.com/o/r/releases/tag/v1.0%2Bbuild"; + assert_eq!(github_tag_from_url(url).as_deref(), Some("v1.0+build")); + // A repository without releases stays on the list. + assert_eq!(github_tag_from_url("https://github.com/o/r/releases"), None); + } + + /// Trimmed from `github.com//releases/expanded_assets/`. + const ASSETS_HTML: &str = r#"
  • + sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789 +
    • +
  • lua-language-server-3.13.5-linux-x64.tar.gz
    • +
  • Source code (zip)
    • "#; + + #[test] + fn github_web_assets_with_digest() { + let rules = find("lua-language-server").unwrap().asset_rules(); + let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, MAC_ARM).unwrap(); + assert_eq!(d.version, "3.13.5"); + assert_eq!(d.name, "lua-language-server-3.13.5-darwin-arm64.tar.gz"); + assert_eq!( + d.url, + "https://github.com/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz" + ); + assert_eq!(d.sha256.as_deref(), Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789")); + let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, LINUX).unwrap(); + assert_eq!(d.sha256, None); + let win = Platform { os: Os::Windows, arch: Arch::X64 }; + let err = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, win).unwrap_err(); + assert_eq!(err.kind, ErrorKind::Unavailable); + } + + #[test] + fn github_web_sibling_checksum_and_notes_links() { + let html = r#" + "#; + let d = github_pick_web( + "artempyanykh/marksman", + "2024-12-18", + html, + None, + find("marksman").unwrap().asset_rules(), + MAC_ARM, + ) + .unwrap(); + assert_eq!( + d.checksum_url.as_deref(), + Some("https://github.com/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256") + ); + + let notes = r#"
  • macOS arm
    • "#; + let rules = find("kotlin-lsp").unwrap().asset_rules(); + let d = + github_pick_web("Kotlin/kotlin-lsp", "kotlin-lsp/v0.253.10629", "", Some(notes), rules, MAC_ARM).unwrap(); + assert_eq!(d.name, "kotlin-lsp-0.253.10629-mac-aarch64.zip"); + assert_eq!( + d.url, + "https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2" + ); + } + #[test] fn hashicorp() { let json = r#"{"version":"0.36.4","url_shasums":"https://r/terraform-ls_0.36.4_SHA256SUMS","builds":[