starred/openidea
1
0
Fork 0
mirror of https://github.com/l0ng-ai/openidea.git synced 2026-11-06 08:07:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(tools): install GitHub releases when the API is rate-limited (#1715)

Browse Source 
Language servers and debuggers published as GitHub releases were looked up only through api.github.com, which allows 60 unauthenticated requests an hour per IP address; behind a shared address installs failed with 403. When the API request fails, the release is now resolved from its web pages (latest-release redirect for the tag, the asset list for files and SHA-256 digests, the release notes for linked builds). This covers every GitHub-sourced tool, not only Kotlin. If both fail, the error names the release page and the Language Servers Path setting for a manual install.

Fixes #1651
This commit is contained in:
l0ng-ai
2026-11-06 13:24:59 +08:00
committed by GitHub
parent b370bb0564
commit c1aa092d75
3 changed files with 252 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/ide-tools/src/lib.rs
+41 -4
View File
@@ -1221,10 +1221,26 @@ impl Tools {
cancel: &CancellationToken,
) -> IdeResult<sources::Download> {
match source {
ReleaseSource::GitHub { repo } => {
let json = http.get_text(&sources::github_latest_url(repo), cancel)?;
sources::github_pick(&json, assets, self.platform)
}
ReleaseSource::GitHub { repo } => match http.get_text(&sources::github_latest_url(repo), cancel) {
Ok(json) => sources::github_pick(&json, assets, self.platform),
Err(e) if e.kind == ErrorKind::Cancelled => Err(e),
// Without a token the API allows 60 requests an hour per IP address, often used
// up behind a shared one (403/429); the release's web pages have no such limit.
Err(api) => self.resolve_github_web(http, repo, assets, cancel).map_err(|web| {
if web.kind == ErrorKind::Cancelled {
return web;
}
tracing::warn!("GitHub release lookup for {repo} failed: {api}; web pages: {web}");
IdeError::new(
api.kind,
format!(
"{}\nDownload it from {} and set its Path in Settings › Languages & Frameworks › Language Servers.",
api.message,
sources::github_latest_page(repo)
),
)
}),
},
ReleaseSource::HashiCorp { product } => {
let json = http.get_text(&sources::hashicorp_latest_url(product), cancel)?;
sources::hashicorp_pick(&json, assets, self.platform)
@@ -1251,6 +1267,27 @@ impl Tools {
}
}
/// [`resolve_release`](Self::resolve_release) for a GitHub release, from its web pages.
fn resolve_github_web(
&self,
http: &Http,
repo: &str,
assets: &[platform::AssetRule],
cancel: &CancellationToken,
) -> IdeResult<sources::Download> {
let page = http.effective_url(&sources::github_latest_page(repo), cancel)?;
let tag = sources::github_tag_from_url(&page)
.ok_or_else(|| IdeError::unavailable(format!("{repo} has no published release")))?;
let list = http.get_text(&sources::github_assets_page(repo, &tag), cancel)?;
match sources::github_pick_web(repo, &tag, &list, None, assets, self.platform) {
Err(e) if e.kind == ErrorKind::Unavailable => {
let notes = http.get_text(&sources::github_tag_page(repo, &tag), cancel)?;
sources::github_pick_web(repo, &tag, &list, Some(&notes), assets, self.platform)
}
result => result,
}
}
fn install_release(
&self,
spec: &ToolSpec,
crates/ide-tools/src/process.rs
+27 -5
View File
@@ -263,6 +263,15 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool {
(major, minor) >= (7, 70) && first.split_whitespace().any(|w| w.eq_ignore_ascii_case("schannel"))
}
#[derive(Clone, Copy, PartialEq, Eq)]
enum Request {
Get,
/// Adoptium uses 404 as well as an empty array for absent platform builds.
GetOrEmptyOn404,
/// Headers, then the URL redirects ended at.
Head,
}
/// HTTP via the system `curl` (shipped with macOS, Windows 10+ and every
/// mainstream Linux), so the IDE carries no TLS stack of its own.
pub struct Http {
@@ -336,26 +345,39 @@ impl Http {
self.get_text_limited(url, None, cancel)
}
/// Where `url` ends up after redirects (a HEAD request).
pub fn effective_url(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {
let out = self.get_text_response(url, None, Request::Head, cancel)?;
Ok(out.trim_end().rsplit('\n').next().unwrap_or_default().trim().to_owned())
}
/// GETs `url` as text, failing when the response is larger than `max_bytes`.
pub fn get_text_limited(&self, url: &str, max_bytes: Option<u64>, cancel: &CancellationToken) -> IdeResult<String> {
self.get_text_response(url, max_bytes, false, cancel)
self.get_text_response(url, max_bytes, Request::Get, cancel)
}
/// Adoptium uses 404 as well as an empty array for absent platform builds.
pub fn get_text_or_empty_on_404(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {
self.get_text_response(url, None, true, cancel)
self.get_text_response(url, None, Request::GetOrEmptyOn404, cancel)
}
fn get_text_response(
&self,
url: &str,
max_bytes: Option<u64>,
empty_on_404: bool,
request: Request,
cancel: &CancellationToken,
) -> IdeResult<String> {
let (mut cmd, token) = self.base(url, cancel)?;
if empty_on_404 {
cmd.args(["--write-out", "\n%{http_code}"]);
let empty_on_404 = request == Request::GetOrEmptyOn404;
match request {
Request::Get => {}
Request::GetOrEmptyOn404 => {
cmd.args(["--write-out", "\n%{http_code}"]);
}
Request::Head => {
cmd.args(["--head", "--write-out", "\n%{url_effective}"]);
}
}
if let Some(max) = max_bytes {
cmd.arg("--max-filesize").arg(max.to_string());
crates/ide-tools/src/sources.rs
+184 -11
View File
@@ -47,38 +47,146 @@ pub fn github_latest_url(repo: &str) -> String {
format!("https://api.github.com/repos/{repo}/releases/latest")
}
/// The latest release's web page; it redirects to `…/releases/tag/<tag>`. Web pages don't
/// count against the API's rate limit for unauthenticated clients (60 requests an hour).
pub fn github_latest_page(repo: &str) -> String {
format!("https://github.com/{repo}/releases/latest")
}
/// The tag `github_latest_page` redirected to.
pub fn github_tag_from_url(url: &str) -> Option<String> {
let (_, tag) = url.split_once("/releases/tag/")?;
let tag = tag.split(['?', '#']).next().unwrap_or(tag).trim_end_matches('/');
(!tag.is_empty()).then(|| percent_decode(tag))
}
pub fn github_tag_page(repo: &str, tag: &str) -> String {
format!("https://github.com/{repo}/releases/tag/{tag}")
}
/// The asset list the release page loads (HTML).
pub fn github_assets_page(repo: &str, tag: &str) -> String {
format!("https://github.com/{repo}/releases/expanded_assets/{tag}")
}
/// One file of a release, from the API or its web page.
struct ReleaseAsset {
name: String,
url: String,
size: Option<u64>,
sha256: Option<String>,
}
/// Picks the platform asset from a GitHub `releases/latest` response. Links
/// in the release notes count as assets too (some projects host binaries on
/// a CDN and only link them).
pub fn github_pick(json: &str, rules: &[AssetRule], platform: Platform) -> IdeResult<Download> {
let rel: GhRelease = serde_json::from_str(json).map_err(|e| proto(format!("unexpected GitHub response: {e}")))?;
let version = rel.tag_name.trim_start_matches('v').to_string();
let names: Vec<&str> = rel.assets.iter().map(|a| a.name.as_str()).collect();
let assets: Vec<ReleaseAsset> = rel
.assets
.into_iter()
.map(|a| ReleaseAsset {
sha256: a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase),
name: a.name,
url: a.browser_download_url,
size: a.size,
})
.collect();
pick_release(&rel.tag_name, &assets, rel.body.as_deref().unwrap_or(""), rules, platform)
}
/// [`github_pick`] from the release's web pages: `assets_html` from [`github_assets_page`] and,
/// for projects that only link their builds, `notes_html` from [`github_tag_page`].
pub fn github_pick_web(
repo: &str,
tag: &str,
assets_html: &str,
notes_html: Option<&str>,
rules: &[AssetRule],
platform: Platform,
) -> IdeResult<Download> {
let prefix = format!("href=\"/{repo}/releases/download/");
let mut assets: Vec<ReleaseAsset> = Vec::new();
let mut rest = assets_html;
while let Some(i) = rest.find(&prefix) {
let tail = &rest[i + "href=\"".len()..];
let Some(end) = tail.find('"') else { break };
let path = html_unescape(&tail[..end]);
rest = &tail[end..];
let name = percent_decode(file_name(&path));
if name.is_empty() || assets.iter().any(|a| a.name == name) {
continue;
}
// `<clipboard-copy aria-label="Copy to clipboard digest for <name>" … value="sha256:<hex>">`
let sha256 = assets_html.find(&format!("digest for {}\"", html_escape(&name))).and_then(|at| {
let tag_end = assets_html[at..].find('>').map_or(assets_html.len(), |e| at + e);
let value = assets_html[at..tag_end].split_once("value=\"sha256:")?.1;
let hex: String = value.chars().take_while(char::is_ascii_hexdigit).collect();
(hex.len() == 64).then(|| hex.to_ascii_lowercase())
});
assets.push(ReleaseAsset { url: format!("https://github.com{path}"), name, size: None, sha256 });
}
pick_release(tag, &assets, &html_unescape(notes_html.unwrap_or("")), rules, platform)
}
fn pick_release(
tag: &str,
assets: &[ReleaseAsset],
notes: &str,
rules: &[AssetRule],
platform: Platform,
) -> IdeResult<Download> {
let version = tag.trim_start_matches('v').to_string();
let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect();
if let Some(name) = select_asset(&names, rules, platform) {
let a = rel.assets.iter().find(|a| a.name == name).expect("selected from list");
let sha256 = a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase);
let checksum_url = rel
.assets
let a = assets.iter().find(|a| a.name == name).expect("selected from list");
let checksum_url = assets
.iter()
.find(|c| c.name == format!("{name}.sha256") || c.name == format!("{name}.sha256sum"))
.map(|c| c.browser_download_url.clone());
.map(|c| c.url.clone());
return Ok(Download {
version,
name: a.name.clone(),
url: a.browser_download_url.clone(),
url: a.url.clone(),
size: a.size,
sha256,
sha256: a.sha256.clone(),
checksum_url,
});
}
let links = extract_links(rel.body.as_deref().unwrap_or(""));
let links = extract_links(notes);
let link_names: Vec<&str> = links.iter().map(|u| file_name(u)).collect();
if let Some(name) = select_asset(&link_names, rules, platform) {
let url = links.iter().find(|u| file_name(u) == name).expect("selected from list").clone();
let checksum_url = links.iter().find(|u| *u == &format!("{url}.sha256")).cloned();
return Ok(Download { version, name: name.to_string(), url, size: None, sha256: None, checksum_url });
}
Err(IdeError::unavailable(format!("release {} has no build for this platform", rel.tag_name)))
Err(IdeError::unavailable(format!("release {tag} has no build for this platform")))
}
fn html_unescape(s: &str) -> String {
s.replace("&quot;", "\"").replace("&#39;", "'").replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
}
fn html_escape(s: &str) -> String {
s.replace('&', "&amp;").replace('"', "&quot;").replace('<', "&lt;").replace('>', "&gt;")
}
fn percent_decode(s: &str) -> String {
let bytes = s.as_bytes();
let mut out = Vec::with_capacity(bytes.len());
let mut i = 0;
while i < bytes.len() {
if bytes[i] == b'%'
&& let Some(b) = s.get(i + 1..i + 3).and_then(|h| u8::from_str_radix(h, 16).ok())
{
out.push(b);
i += 3;
} else {
out.push(bytes[i]);
i += 1;
}
}
String::from_utf8(out).unwrap_or_else(|_| s.to_owned())
}
fn file_name(url: &str) -> &str {
@@ -336,6 +444,71 @@ mod tests {
assert_eq!(d.name, "kotlin-lsp-0.253.10629-linux-x64.zip");
}
#[test]
fn github_tag_from_redirect() {
let url = "https://github.com/Kotlin/kotlin-lsp/releases/tag/kotlin-lsp/v263.7084.0";
assert_eq!(github_tag_from_url(url).as_deref(), Some("kotlin-lsp/v263.7084.0"));
let url = "https://github.com/o/r/releases/tag/v1.0%2Bbuild";
assert_eq!(github_tag_from_url(url).as_deref(), Some("v1.0+build"));
// A repository without releases stays on the list.
assert_eq!(github_tag_from_url("https://github.com/o/r/releases"), None);
}
/// Trimmed from `github.com/<repo>/releases/expanded_assets/<tag>`.
const ASSETS_HTML: &str = r#"<li><div><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz" rel="nofollow" data-turbo="false" class="wb-break-all">
<span class="text-bold">lua-language-server-3.13.5-darwin-arm64.tar.gz</span></a></div>
<span class="Truncate-text">sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789</span>
<clipboard-copy id="clipboard-button-sha256:abc" aria-label="Copy to clipboard digest for lua-language-server-3.13.5-darwin-arm64.tar.gz" type="button" value="sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789" class="Button"></clipboard-copy></li>
<li><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-linux-x64.tar.gz" rel="nofollow">lua-language-server-3.13.5-linux-x64.tar.gz</a></li>
<li><a href="/LuaLS/lua-language-server/archive/refs/tags/3.13.5.zip" rel="nofollow">Source code (zip)</a></li>"#;
#[test]
fn github_web_assets_with_digest() {
let rules = find("lua-language-server").unwrap().asset_rules();
let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, MAC_ARM).unwrap();
assert_eq!(d.version, "3.13.5");
assert_eq!(d.name, "lua-language-server-3.13.5-darwin-arm64.tar.gz");
assert_eq!(
d.url,
"https://github.com/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz"
);
assert_eq!(d.sha256.as_deref(), Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"));
let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, LINUX).unwrap();
assert_eq!(d.sha256, None);
let win = Platform { os: Os::Windows, arch: Arch::X64 };
let err = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, win).unwrap_err();
assert_eq!(err.kind, ErrorKind::Unavailable);
}
#[test]
fn github_web_sibling_checksum_and_notes_links() {
let html = r#"<a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos" rel="nofollow">
<a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256" rel="nofollow">"#;
let d = github_pick_web(
"artempyanykh/marksman",
"2024-12-18",
html,
None,
find("marksman").unwrap().asset_rules(),
MAC_ARM,
)
.unwrap();
assert_eq!(
d.checksum_url.as_deref(),
Some("https://github.com/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256")
);
let notes = r#"<li><a href="https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&amp;b=2" rel="nofollow">macOS arm</a></li>"#;
let rules = find("kotlin-lsp").unwrap().asset_rules();
let d =
github_pick_web("Kotlin/kotlin-lsp", "kotlin-lsp/v0.253.10629", "", Some(notes), rules, MAC_ARM).unwrap();
assert_eq!(d.name, "kotlin-lsp-0.253.10629-mac-aarch64.zip");
assert_eq!(
d.url,
"https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2"
);
}
#[test]
fn hashicorp() {
let json = r#"{"version":"0.36.4","url_shasums":"https://r/terraform-ls_0.36.4_SHA256SUMS","builds":[