From d44e2414caed98f987ca0d5863c789d3af3c10fd Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Fri, 30 Oct 2026 08:18:53 +0800 Subject: [PATCH] fix(build): decode tool output in the Windows code page (#299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Chinese Windows the Maven tool window showed sync errors as mojibake: tool output was decoded as UTF-8, but while JVMs are told to write UTF-8, cmd.exe, batch launchers (mvn.cmd, gradlew.bat) and native tools write their diagnostics in the ANSI code page (GBK). ide_core::output_encoding::decode_tool_output keeps valid UTF-8 and decodes other lines in the system code page (GetACP). It is used wherever OpenIDEA shows output of tools it launches itself: Maven/Gradle sync and tasks, tool installers, SDK probes, Docker, coverage, environment setup, Cargo stderr for debugging, agent version probes and notebook kernel stderr. Run consoles and protocol streams keep their own encodings. 未在 Windows 实机验证。 Refs #262 --- Cargo.lock | 2 + crates/ide-agent/src/native/claude/mod.rs | 2 +- crates/ide-agent/src/native/mod.rs | 2 +- crates/ide-build/src/process.rs | 2 +- crates/ide-core/Cargo.toml | 4 ++ crates/ide-core/src/output_encoding.rs | 77 +++++++++++++++++++++++ crates/ide-coverage/src/lib.rs | 8 ++- crates/ide-dap/src/cargo.rs | 6 +- crates/ide-docker/src/process.rs | 4 +- crates/ide-host/src/environment.rs | 3 +- crates/ide-notebook/src/kernel/process.rs | 37 ++++++----- crates/ide-sdk/src/exec.rs | 21 ++++--- crates/ide-tools/src/process.rs | 27 ++++---- 13 files changed, 146 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54c5d2b7..6d49a14d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4093,6 +4093,7 @@ name = "ide-core" version = "0.1.25" dependencies = [ "dunce", + "encoding_rs", "serde", "serde_json", "specta", @@ -4100,6 +4101,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio-util", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/ide-agent/src/native/claude/mod.rs b/crates/ide-agent/src/native/claude/mod.rs index a15c8860..8fd87ad8 100644 --- a/crates/ide-agent/src/native/claude/mod.rs +++ b/crates/ide-agent/src/native/claude/mod.rs @@ -296,7 +296,7 @@ impl Shim { let mut cmd = std::process::Command::new(&self.config.program); ide_core::process::prepare(&mut cmd); let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?; - let text = String::from_utf8_lossy(&out.stdout); + let text = ide_core::output_encoding::decode_tool_output(&out.stdout); text.split_whitespace().next().filter(|v| out.status.success() && !v.is_empty()).map(str::to_owned) } diff --git a/crates/ide-agent/src/native/mod.rs b/crates/ide-agent/src/native/mod.rs index 46de082e..10f11309 100644 --- a/crates/ide-agent/src/native/mod.rs +++ b/crates/ide-agent/src/native/mod.rs @@ -64,7 +64,7 @@ fn version_of(program: &Path) -> Option> { let mut cmd = std::process::Command::new(program); ide_core::process::prepare(&mut cmd); let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?; - parse_version(&String::from_utf8_lossy(&out.stdout)) + parse_version(&ide_core::output_encoding::decode_tool_output(&out.stdout)) } fn parse_version(text: &str) -> Option> { diff --git a/crates/ide-build/src/process.rs b/crates/ide-build/src/process.rs index 0ac37580..fcec4b45 100644 --- a/crates/ide-build/src/process.rs +++ b/crates/ide-build/src/process.rs @@ -68,7 +68,7 @@ pub(crate) fn run( match reader.read_until(b'\n', &mut buf) { Ok(0) | Err(_) => break, Ok(_) => { - let line = String::from_utf8_lossy(&buf); + let line = ide_core::output_encoding::decode_tool_output(&buf); let line = line.trim_end_matches(['\n', '\r']).to_owned(); if tx.send((kind, line)).is_err() { break; diff --git a/crates/ide-core/Cargo.toml b/crates/ide-core/Cargo.toml index 9daa648b..af0d6b99 100644 --- a/crates/ide-core/Cargo.toml +++ b/crates/ide-core/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] dunce.workspace = true +encoding_rs.workspace = true serde.workspace = true serde_json.workspace = true specta.workspace = true @@ -16,3 +17,6 @@ tokio-util.workspace = true [dev-dependencies] tempfile.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Globalization"] } diff --git a/crates/ide-core/src/output_encoding.rs b/crates/ide-core/src/output_encoding.rs index 5d3a3688..f5e85f3d 100644 --- a/crates/ide-core/src/output_encoding.rs +++ b/crates/ide-core/src/output_encoding.rs @@ -18,6 +18,60 @@ use std::collections::BTreeMap; +/// Decodes complete lines or collected output from IDE-managed tools, preferring UTF-8. +/// Windows launcher/native diagnostics fall back to the system ANSI code page. +/// Callers must buffer incomplete lines so multibyte characters are never split. +/// Run consoles and protocol streams have their own encoding contracts. +pub fn decode_tool_output(bytes: &[u8]) -> String { + #[cfg(windows)] + // SAFETY: GetACP is an argument-less query with no preconditions. + let code_page = unsafe { windows_sys::Win32::Globalization::GetACP() }; + #[cfg(not(windows))] + let code_page = 65001; + decode_tool_output_with_code_page(bytes, code_page) +} + +fn decode_tool_output_with_code_page(bytes: &[u8], code_page: u32) -> String { + if let Ok(text) = std::str::from_utf8(bytes) { + return text.to_owned(); + } + let encoding = encoding_for_code_page(code_page).unwrap_or(encoding_rs::UTF_8); + // Collected output can mix UTF-8 JVM lines with native launcher diagnostics. + let mut text = String::new(); + for line in bytes.split_inclusive(|b| *b == b'\n') { + match std::str::from_utf8(line) { + Ok(line) => text.push_str(line), + Err(_) => text.push_str(&encoding.decode_without_bom_handling(line).0), + } + } + text +} + +fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> { + Some(match cp { + 65001 => encoding_rs::UTF_8, + 936 => encoding_rs::GBK, + 54936 => encoding_rs::GB18030, + 950 => encoding_rs::BIG5, + 932 => encoding_rs::SHIFT_JIS, + 949 => encoding_rs::EUC_KR, + 866 => encoding_rs::IBM866, + 874 => encoding_rs::WINDOWS_874, + 1250 => encoding_rs::WINDOWS_1250, + 1251 => encoding_rs::WINDOWS_1251, + 1252 => encoding_rs::WINDOWS_1252, + 1253 => encoding_rs::WINDOWS_1253, + 1254 => encoding_rs::WINDOWS_1254, + 1255 => encoding_rs::WINDOWS_1255, + 1256 => encoding_rs::WINDOWS_1256, + 1257 => encoding_rs::WINDOWS_1257, + 1258 => encoding_rs::WINDOWS_1258, + 20866 => encoding_rs::KOI8_R, + 21866 => encoding_rs::KOI8_U, + _ => return None, + }) +} + const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"]; const FILE_ENCODING: &str = "file.encoding"; const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS"; @@ -237,6 +291,29 @@ impl NoticeFilter { mod tests { use super::*; + #[test] + fn tool_output_prefers_utf8() { + for cp in [936, 1252, 65001, 0] { + for text in ["", "plain ASCII\r\n", "中文 café 🦀\n"] { + assert_eq!(decode_tool_output_with_code_page(text.as_bytes(), cp), text); + } + } + } + + #[test] + fn tool_output_decodes_native_diagnostics_and_mixed_lines() { + const GBK: &[u8] = b"\xcf\xb5\xcd\xb3\xd5\xd2\xb2\xbb\xb5\xbd\xd6\xb8\xb6\xa8\xb5\xc4\xc2\xb7\xbe\xb6\xa1\xa3"; + assert_eq!(decode_tool_output_with_code_page(GBK, 936), "系统找不到指定的路径。"); + let mut mixed = "JVM: 中文\r\n".as_bytes().to_vec(); + mixed.extend_from_slice(GBK); + mixed.extend_from_slice(b"\r\nlast"); + assert_eq!(decode_tool_output_with_code_page(&mixed, 936), "JVM: 中文\r\n系统找不到指定的路径。\r\nlast"); + assert_eq!(decode_tool_output_with_code_page(b"caf\xe9", 1252), "café"); + for cp in [65001, 0] { + assert_eq!(decode_tool_output_with_code_page(b"a\xffb", cp), "a�b"); + } + } + const STD_UTF8: [&str; 4] = [ "-Dstdout.encoding=UTF-8", "-Dstderr.encoding=UTF-8", diff --git a/crates/ide-coverage/src/lib.rs b/crates/ide-coverage/src/lib.rs index 1fc48db2..75781203 100644 --- a/crates/ide-coverage/src/lib.rs +++ b/crates/ide-coverage/src/lib.rs @@ -213,8 +213,12 @@ pub fn collect(spec: &CoverageReportSpec) -> IdeResult { .output() .map_err(|e| IdeError::unavailable(format!("cannot run {program}: {e}")))?; if !out.status.success() { - let text = String::from_utf8_lossy(&out.stderr).trim().to_owned(); - let text = if text.is_empty() { String::from_utf8_lossy(&out.stdout).trim().to_owned() } else { text }; + let text = ide_core::output_encoding::decode_tool_output(&out.stderr).trim().to_owned(); + let text = if text.is_empty() { + ide_core::output_encoding::decode_tool_output(&out.stdout).trim().to_owned() + } else { + text + }; return Err(IdeError::not_found(format!("exporting the coverage data failed: {text}"))); } let text = read_report(lcov)?; diff --git a/crates/ide-dap/src/cargo.rs b/crates/ide-dap/src/cargo.rs index 6de0813b..30fb88a7 100644 --- a/crates/ide-dap/src/cargo.rs +++ b/crates/ide-dap/src/cargo.rs @@ -70,8 +70,10 @@ pub async fn build( let on_output = std::sync::Arc::new(on_output); let err_out = on_output.clone(); let err_task = tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { + let mut lines = BufReader::new(stderr).split(b'\n'); + while let Ok(Some(line)) = lines.next_segment().await { + let line = ide_core::output_encoding::decode_tool_output(&line); + let line = line.trim_end_matches('\r'); err_out(format!("{line}\n")); } }); diff --git a/crates/ide-docker/src/process.rs b/crates/ide-docker/src/process.rs index e9dc25a3..6c08c005 100644 --- a/crates/ide-docker/src/process.rs +++ b/crates/ide-docker/src/process.rs @@ -54,7 +54,7 @@ fn read_all(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle, s let mut pending: Vec = Vec::new(); let flush = |pending: &mut Vec| { if !pending.is_empty() { - let line = String::from_utf8_lossy(pending).trim_end().to_string(); + let line = ide_core::output_encoding::decode_tool_output(pending).trim_end().to_string(); pending.clear(); let _ = tx.send(Event::Line(stream, line)); } diff --git a/crates/ide-host/src/environment.rs b/crates/ide-host/src/environment.rs index 4dabd350..0ef84ec5 100644 --- a/crates/ide-host/src/environment.rs +++ b/crates/ide-host/src/environment.rs @@ -281,7 +281,8 @@ fn run_one(argv: &[String], root: &Path, cancel: &CancellationToken, on_line: &d for pipe in pipes.into_iter().flatten() { let tx = tx.clone(); std::thread::spawn(move || { - for line in BufReader::new(pipe).lines().map_while(Result::ok) { + for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) { + let line = ide_core::output_encoding::decode_tool_output(&line).trim_end_matches('\r').to_owned(); if tx.send(line).is_err() { break; } diff --git a/crates/ide-notebook/src/kernel/process.rs b/crates/ide-notebook/src/kernel/process.rs index 791939e0..afcdda93 100644 --- a/crates/ide-notebook/src/kernel/process.rs +++ b/crates/ide-notebook/src/kernel/process.rs @@ -1,13 +1,12 @@ use ide_core::{IdeError, IdeResult}; use jupyter_protocol::ConnectionInfo; use parking_lot::Mutex; -use std::collections::VecDeque; use std::io::Write; use std::path::Path; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::task::JoinHandle; use uuid::Uuid; @@ -18,7 +17,7 @@ pub(super) struct Process { child: Child, #[cfg(unix)] pid: Option, - stderr: Arc>>, + stderr: Arc>, drains: Vec>, // Kept until the process and sockets have stopped; contains the HMAC secret. _directory: tempfile::TempDir, @@ -103,26 +102,31 @@ impl Process { child, #[cfg(unix)] pid, - stderr: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_BYTES))), + stderr: Arc::new(Mutex::new(String::new())), drains: Vec::new(), _directory: directory, #[cfg(windows)] windows, }; - if let Some(mut stderr) = process.child.stderr.take() { + if let Some(stderr) = process.child.stderr.take() { let tail = process.stderr.clone(); process.drains.push(tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - while let Ok(count) = stderr.read(&mut buffer).await { - if count == 0 { - break; - } - let mut tail = tail.lock(); - for byte in &buffer[..count] { - if tail.len() == STDERR_BYTES { - tail.pop_front(); + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + loop { + line.clear(); + match reader.read_until(b'\n', &mut line).await { + Ok(0) | Err(_) => break, + Ok(_) => { + let text = ide_core::output_encoding::decode_tool_output(&line); + let mut tail = tail.lock(); + tail.push_str(&text); + let mut start = tail.len().saturating_sub(STDERR_BYTES); + while !tail.is_char_boundary(start) { + start += 1; + } + tail.drain(..start); } - tail.push_back(*byte); } } })); @@ -143,8 +147,7 @@ impl Process { } pub fn stderr(&self) -> String { - let bytes = self.stderr.lock().iter().copied().collect::>(); - String::from_utf8_lossy(&bytes).into_owned() + self.stderr.lock().clone() } pub fn death(&mut self) -> IdeResult> { diff --git a/crates/ide-sdk/src/exec.rs b/crates/ide-sdk/src/exec.rs index aa2296c6..03333c50 100644 --- a/crates/ide-sdk/src/exec.rs +++ b/crates/ide-sdk/src/exec.rs @@ -1,6 +1,7 @@ //! Running toolchain commands: short queries with a timeout and long-running //! streaming commands that honour cancellation. +use ide_core::output_encoding::decode_tool_output; use ide_core::{CancellationToken, IdeError, IdeResult}; use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; @@ -77,14 +78,14 @@ pub(crate) fn query_in( let mut stdout = child.stdout.take()?; let mut stderr = child.stderr.take()?; let out = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stdout.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stdout.read_to_end(&mut s); + decode_tool_output(&s) }); let err = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let deadline = Instant::now() + timeout; let status = loop { @@ -131,16 +132,16 @@ pub(crate) fn stream( if let Some(out) = child.stdout.take() { let tx = tx.clone(); readers.push(std::thread::spawn(move || { - for line in BufReader::new(out).lines().map_while(Result::ok) { - let _ = tx.send(line); + for line in BufReader::new(out).split(b'\n').map_while(Result::ok) { + let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned()); } })); } if let Some(err) = child.stderr.take() { let tx = tx.clone(); readers.push(std::thread::spawn(move || { - for line in BufReader::new(err).lines().map_while(Result::ok) { - let _ = tx.send(line); + for line in BufReader::new(err).split(b'\n').map_while(Result::ok) { + let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned()); } })); } diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs index 159a7d60..81c44bf9 100644 --- a/crates/ide-tools/src/process.rs +++ b/crates/ide-tools/src/process.rs @@ -3,6 +3,7 @@ use crate::InstallEvent; use ide_core::error::ErrorKind; +use ide_core::output_encoding::decode_tool_output; use ide_core::{CancellationToken, IdeError, IdeResult}; use std::io::{BufRead, BufReader, Read}; use std::path::Path; @@ -86,8 +87,9 @@ pub fn run_collect(mut cmd: Command, cancel: &CancellationToken, emit: Emit) -> for pipe in pipes.into_iter().flatten() { let tx = tx.clone(); std::thread::spawn(move || { - for line in BufReader::new(pipe).lines() { + for line in BufReader::new(pipe).split(b'\n') { let Ok(line) = line else { break }; + let line = decode_tool_output(&line).trim_end_matches('\r').to_owned(); if tx.send(line).is_err() { break; } @@ -127,14 +129,14 @@ pub fn run_quiet(mut cmd: Command, timeout: Duration) -> Option<(bool, String)> let mut stdout = child.stdout.take()?; let mut stderr = child.stderr.take()?; let out_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stdout.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stdout.read_to_end(&mut s); + decode_tool_output(&s) }); let err_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let deadline = Instant::now() + timeout; let status = loop { @@ -251,8 +253,9 @@ impl Http { return Err(IdeError::new(ErrorKind::Io, format!("{url}: response is larger than {max} bytes"))); } if !status.success() { - let mut err = String::new(); - let _ = stderr.read_to_string(&mut err); + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + let err = decode_tool_output(&err); return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim()))); } String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8"))) @@ -272,9 +275,9 @@ impl Http { let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?; let mut stderr = child.stderr.take().expect("piped"); let err_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let mut last = Instant::now(); let mut last_bytes = 0;