fix(build): decode tool output in the Windows code page (#299)

On Chinese Windows the Maven tool window showed sync errors as mojibake: tool output was decoded as UTF-8, but while JVMs are told to write UTF-8, cmd.exe, batch launchers (mvn.cmd, gradlew.bat) and native tools write their diagnostics in the ANSI code page (GBK). ide_core::output_encoding::decode_tool_output keeps valid UTF-8 and decodes other lines in the system code page (GetACP). It is used wherever OpenIDEA shows output of tools it launches itself: Maven/Gradle sync and tasks, tool installers, SDK probes, Docker, coverage, environment setup, Cargo stderr for debugging, agent version probes and notebook kernel stderr. Run consoles and protocol streams keep their own encodings. 未在 Windows 实机验证。

Refs #262

Cargo.lock

@@ -4093,6 +4093,7 @@ name = "ide-core"
 version = "0.1.25"
 dependencies = [
  "dunce",
+ "encoding_rs",
  "serde",
  "serde_json",
  "specta",
@@ -4100,6 +4101,7 @@ dependencies = [
  "tempfile",
  "thiserror 2.0.21",
  "tokio-util",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]

crates/ide-agent/src/native/claude/mod.rs

@@ -296,7 +296,7 @@ impl Shim {
         let mut cmd = std::process::Command::new(&self.config.program);
         ide_core::process::prepare(&mut cmd);
         let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;
-        let text = String::from_utf8_lossy(&out.stdout);
+        let text = ide_core::output_encoding::decode_tool_output(&out.stdout);
         text.split_whitespace().next().filter(|v| out.status.success() && !v.is_empty()).map(str::to_owned)
     }
 

crates/ide-agent/src/native/mod.rs

@@ -64,7 +64,7 @@ fn version_of(program: &Path) -> Option<Vec<u64>> {
     let mut cmd = std::process::Command::new(program);
     ide_core::process::prepare(&mut cmd);
     let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?;
-    parse_version(&String::from_utf8_lossy(&out.stdout))
+    parse_version(&ide_core::output_encoding::decode_tool_output(&out.stdout))
 }
 
 fn parse_version(text: &str) -> Option<Vec<u64>> {

crates/ide-build/src/process.rs

@@ -68,7 +68,7 @@ pub(crate) fn run(
                 match reader.read_until(b'\n', &mut buf) {
                     Ok(0) | Err(_) => break,
                     Ok(_) => {
-                        let line = String::from_utf8_lossy(&buf);
+                        let line = ide_core::output_encoding::decode_tool_output(&buf);
                         let line = line.trim_end_matches(['\n', '\r']).to_owned();
                         if tx.send((kind, line)).is_err() {
                             break;

crates/ide-core/Cargo.toml

@@ -7,6 +7,7 @@ license.workspace = true
 
 [dependencies]
 dunce.workspace = true
+encoding_rs.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 specta.workspace = true
@@ -16,3 +17,6 @@ tokio-util.workspace = true
 
 [dev-dependencies]
 tempfile.workspace = true
+
+[target.'cfg(windows)'.dependencies]
+windows-sys = { version = "0.61", features = ["Win32_Globalization"] }

crates/ide-core/src/output_encoding.rs

@@ -18,6 +18,60 @@
 
 use std::collections::BTreeMap;
 
+/// Decodes complete lines or collected output from IDE-managed tools, preferring UTF-8.
+/// Windows launcher/native diagnostics fall back to the system ANSI code page.
+/// Callers must buffer incomplete lines so multibyte characters are never split.
+/// Run consoles and protocol streams have their own encoding contracts.
+pub fn decode_tool_output(bytes: &[u8]) -> String {
+    #[cfg(windows)]
+    // SAFETY: GetACP is an argument-less query with no preconditions.
+    let code_page = unsafe { windows_sys::Win32::Globalization::GetACP() };
+    #[cfg(not(windows))]
+    let code_page = 65001;
+    decode_tool_output_with_code_page(bytes, code_page)
+}
+
+fn decode_tool_output_with_code_page(bytes: &[u8], code_page: u32) -> String {
+    if let Ok(text) = std::str::from_utf8(bytes) {
+        return text.to_owned();
+    }
+    let encoding = encoding_for_code_page(code_page).unwrap_or(encoding_rs::UTF_8);
+    // Collected output can mix UTF-8 JVM lines with native launcher diagnostics.
+    let mut text = String::new();
+    for line in bytes.split_inclusive(|b| *b == b'\n') {
+        match std::str::from_utf8(line) {
+            Ok(line) => text.push_str(line),
+            Err(_) => text.push_str(&encoding.decode_without_bom_handling(line).0),
+        }
+    }
+    text
+}
+
+fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> {
+    Some(match cp {
+        65001 => encoding_rs::UTF_8,
+        936 => encoding_rs::GBK,
+        54936 => encoding_rs::GB18030,
+        950 => encoding_rs::BIG5,
+        932 => encoding_rs::SHIFT_JIS,
+        949 => encoding_rs::EUC_KR,
+        866 => encoding_rs::IBM866,
+        874 => encoding_rs::WINDOWS_874,
+        1250 => encoding_rs::WINDOWS_1250,
+        1251 => encoding_rs::WINDOWS_1251,
+        1252 => encoding_rs::WINDOWS_1252,
+        1253 => encoding_rs::WINDOWS_1253,
+        1254 => encoding_rs::WINDOWS_1254,
+        1255 => encoding_rs::WINDOWS_1255,
+        1256 => encoding_rs::WINDOWS_1256,
+        1257 => encoding_rs::WINDOWS_1257,
+        1258 => encoding_rs::WINDOWS_1258,
+        20866 => encoding_rs::KOI8_R,
+        21866 => encoding_rs::KOI8_U,
+        _ => return None,
+    })
+}
+
 const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"];
 const FILE_ENCODING: &str = "file.encoding";
 const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS";
@@ -237,6 +291,29 @@ impl NoticeFilter {
 mod tests {
     use super::*;
 
+    #[test]
+    fn tool_output_prefers_utf8() {
+        for cp in [936, 1252, 65001, 0] {
+            for text in ["", "plain ASCII\r\n", "中文 café 🦀\n"] {
+                assert_eq!(decode_tool_output_with_code_page(text.as_bytes(), cp), text);
+            }
+        }
+    }
+
+    #[test]
+    fn tool_output_decodes_native_diagnostics_and_mixed_lines() {
+        const GBK: &[u8] = b"\xcf\xb5\xcd\xb3\xd5\xd2\xb2\xbb\xb5\xbd\xd6\xb8\xb6\xa8\xb5\xc4\xc2\xb7\xbe\xb6\xa1\xa3";
+        assert_eq!(decode_tool_output_with_code_page(GBK, 936), "系统找不到指定的路径。");
+        let mut mixed = "JVM: 中文\r\n".as_bytes().to_vec();
+        mixed.extend_from_slice(GBK);
+        mixed.extend_from_slice(b"\r\nlast");
+        assert_eq!(decode_tool_output_with_code_page(&mixed, 936), "JVM: 中文\r\n系统找不到指定的路径。\r\nlast");
+        assert_eq!(decode_tool_output_with_code_page(b"caf\xe9", 1252), "café");
+        for cp in [65001, 0] {
+            assert_eq!(decode_tool_output_with_code_page(b"a\xffb", cp), "a�b");
+        }
+    }
+
     const STD_UTF8: [&str; 4] = [
         "-Dstdout.encoding=UTF-8",
         "-Dstderr.encoding=UTF-8",

crates/ide-coverage/src/lib.rs

@@ -213,8 +213,12 @@ pub fn collect(spec: &CoverageReportSpec) -> IdeResult<CoverageSuite> {
                 .output()
                 .map_err(|e| IdeError::unavailable(format!("cannot run {program}: {e}")))?;
             if !out.status.success() {
-                let text = String::from_utf8_lossy(&out.stderr).trim().to_owned();
-                let text = if text.is_empty() { String::from_utf8_lossy(&out.stdout).trim().to_owned() } else { text };
+                let text = ide_core::output_encoding::decode_tool_output(&out.stderr).trim().to_owned();
+                let text = if text.is_empty() {
+                    ide_core::output_encoding::decode_tool_output(&out.stdout).trim().to_owned()
+                } else {
+                    text
+                };
                 return Err(IdeError::not_found(format!("exporting the coverage data failed: {text}")));
             }
             let text = read_report(lcov)?;

crates/ide-dap/src/cargo.rs

@@ -70,8 +70,10 @@ pub async fn build(
     let on_output = std::sync::Arc::new(on_output);
     let err_out = on_output.clone();
     let err_task = tokio::spawn(async move {
-        let mut lines = BufReader::new(stderr).lines();
-        while let Ok(Some(line)) = lines.next_line().await {
+        let mut lines = BufReader::new(stderr).split(b'\n');
+        while let Ok(Some(line)) = lines.next_segment().await {
+            let line = ide_core::output_encoding::decode_tool_output(&line);
+            let line = line.trim_end_matches('\r');
             err_out(format!("{line}\n"));
         }
     });

crates/ide-docker/src/process.rs

@@ -54,7 +54,7 @@ fn read_all(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle<Str
     std::thread::spawn(move || {
         let mut buf = Vec::new();
         let _ = pipe.read_to_end(&mut buf);
-        String::from_utf8_lossy(&buf).into_owned()
+        ide_core::output_encoding::decode_tool_output(&buf)
     })
 }
 
@@ -115,7 +115,7 @@ fn spawn_reader(mut pipe: impl Read + Send + 'static, tx: mpsc::Sender<Event>, s
         let mut pending: Vec<u8> = Vec::new();
         let flush = |pending: &mut Vec<u8>| {
             if !pending.is_empty() {
-                let line = String::from_utf8_lossy(pending).trim_end().to_string();
+                let line = ide_core::output_encoding::decode_tool_output(pending).trim_end().to_string();
                 pending.clear();
                 let _ = tx.send(Event::Line(stream, line));
             }

crates/ide-host/src/environment.rs

@@ -281,7 +281,8 @@ fn run_one(argv: &[String], root: &Path, cancel: &CancellationToken, on_line: &d
     for pipe in pipes.into_iter().flatten() {
         let tx = tx.clone();
         std::thread::spawn(move || {
-            for line in BufReader::new(pipe).lines().map_while(Result::ok) {
+            for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) {
+                let line = ide_core::output_encoding::decode_tool_output(&line).trim_end_matches('\r').to_owned();
                 if tx.send(line).is_err() {
                     break;
                 }

crates/ide-notebook/src/kernel/process.rs

@@ -1,13 +1,12 @@
 use ide_core::{IdeError, IdeResult};
 use jupyter_protocol::ConnectionInfo;
 use parking_lot::Mutex;
-use std::collections::VecDeque;
 use std::io::Write;
 use std::path::Path;
 use std::process::Stdio;
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::io::AsyncReadExt;
+use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
 use tokio::process::{Child, Command};
 use tokio::task::JoinHandle;
 use uuid::Uuid;
@@ -18,7 +17,7 @@ pub(super) struct Process {
     child: Child,
     #[cfg(unix)]
     pid: Option<u32>,
-    stderr: Arc<Mutex<VecDeque<u8>>>,
+    stderr: Arc<Mutex<String>>,
     drains: Vec<JoinHandle<()>>,
     // Kept until the process and sockets have stopped; contains the HMAC secret.
     _directory: tempfile::TempDir,
@@ -103,26 +102,31 @@ impl Process {
             child,
             #[cfg(unix)]
             pid,
-            stderr: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_BYTES))),
+            stderr: Arc::new(Mutex::new(String::new())),
             drains: Vec::new(),
             _directory: directory,
             #[cfg(windows)]
             windows,
         };
-        if let Some(mut stderr) = process.child.stderr.take() {
+        if let Some(stderr) = process.child.stderr.take() {
             let tail = process.stderr.clone();
             process.drains.push(tokio::spawn(async move {
-                let mut buffer = [0_u8; 4096];
-                while let Ok(count) = stderr.read(&mut buffer).await {
-                    if count == 0 {
-                        break;
-                    }
-                    let mut tail = tail.lock();
-                    for byte in &buffer[..count] {
-                        if tail.len() == STDERR_BYTES {
-                            tail.pop_front();
+                let mut reader = BufReader::new(stderr);
+                let mut line = Vec::new();
+                loop {
+                    line.clear();
+                    match reader.read_until(b'\n', &mut line).await {
+                        Ok(0) | Err(_) => break,
+                        Ok(_) => {
+                            let text = ide_core::output_encoding::decode_tool_output(&line);
+                            let mut tail = tail.lock();
+                            tail.push_str(&text);
+                            let mut start = tail.len().saturating_sub(STDERR_BYTES);
+                            while !tail.is_char_boundary(start) {
+                                start += 1;
+                            }
+                            tail.drain(..start);
                         }
-                        tail.push_back(*byte);
                     }
                 }
             }));
@@ -143,8 +147,7 @@ impl Process {
     }
 
     pub fn stderr(&self) -> String {
-        let bytes = self.stderr.lock().iter().copied().collect::<Vec<_>>();
-        String::from_utf8_lossy(&bytes).into_owned()
+        self.stderr.lock().clone()
     }
 
     pub fn death(&mut self) -> IdeResult<Option<String>> {

crates/ide-sdk/src/exec.rs

@@ -1,6 +1,7 @@
 //! Running toolchain commands: short queries with a timeout and long-running
 //! streaming commands that honour cancellation.
 
+use ide_core::output_encoding::decode_tool_output;
 use ide_core::{CancellationToken, IdeError, IdeResult};
 use std::ffi::OsString;
 use std::io::{BufRead, BufReader, Read};
@@ -77,14 +78,14 @@ pub(crate) fn query_in(
     let mut stdout = child.stdout.take()?;
     let mut stderr = child.stderr.take()?;
     let out = std::thread::spawn(move || {
-        let mut s = String::new();
-        let _ = stdout.read_to_string(&mut s);
-        s
+        let mut s = Vec::new();
+        let _ = stdout.read_to_end(&mut s);
+        decode_tool_output(&s)
     });
     let err = std::thread::spawn(move || {
-        let mut s = String::new();
-        let _ = stderr.read_to_string(&mut s);
-        s
+        let mut s = Vec::new();
+        let _ = stderr.read_to_end(&mut s);
+        decode_tool_output(&s)
     });
     let deadline = Instant::now() + timeout;
     let status = loop {
@@ -131,16 +132,16 @@ pub(crate) fn stream(
     if let Some(out) = child.stdout.take() {
         let tx = tx.clone();
         readers.push(std::thread::spawn(move || {
-            for line in BufReader::new(out).lines().map_while(Result::ok) {
-                let _ = tx.send(line);
+            for line in BufReader::new(out).split(b'\n').map_while(Result::ok) {
+                let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());
             }
         }));
     }
     if let Some(err) = child.stderr.take() {
         let tx = tx.clone();
         readers.push(std::thread::spawn(move || {
-            for line in BufReader::new(err).lines().map_while(Result::ok) {
-                let _ = tx.send(line);
+            for line in BufReader::new(err).split(b'\n').map_while(Result::ok) {
+                let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned());
             }
         }));
     }

crates/ide-tools/src/process.rs

@@ -3,6 +3,7 @@
 
 use crate::InstallEvent;
 use ide_core::error::ErrorKind;
+use ide_core::output_encoding::decode_tool_output;
 use ide_core::{CancellationToken, IdeError, IdeResult};
 use std::io::{BufRead, BufReader, Read};
 use std::path::Path;
@@ -86,8 +87,9 @@ pub fn run_collect(mut cmd: Command, cancel: &CancellationToken, emit: Emit) ->
     for pipe in pipes.into_iter().flatten() {
         let tx = tx.clone();
         std::thread::spawn(move || {
-            for line in BufReader::new(pipe).lines() {
+            for line in BufReader::new(pipe).split(b'\n') {
                 let Ok(line) = line else { break };
+                let line = decode_tool_output(&line).trim_end_matches('\r').to_owned();
                 if tx.send(line).is_err() {
                     break;
                 }
@@ -127,14 +129,14 @@ pub fn run_quiet(mut cmd: Command, timeout: Duration) -> Option<(bool, String)>
     let mut stdout = child.stdout.take()?;
     let mut stderr = child.stderr.take()?;
     let out_t = std::thread::spawn(move || {
-        let mut s = String::new();
-        let _ = stdout.read_to_string(&mut s);
-        s
+        let mut s = Vec::new();
+        let _ = stdout.read_to_end(&mut s);
+        decode_tool_output(&s)
     });
     let err_t = std::thread::spawn(move || {
-        let mut s = String::new();
-        let _ = stderr.read_to_string(&mut s);
-        s
+        let mut s = Vec::new();
+        let _ = stderr.read_to_end(&mut s);
+        decode_tool_output(&s)
     });
     let deadline = Instant::now() + timeout;
     let status = loop {
@@ -251,8 +253,9 @@ impl Http {
             return Err(IdeError::new(ErrorKind::Io, format!("{url}: response is larger than {max} bytes")));
         }
         if !status.success() {
-            let mut err = String::new();
-            let _ = stderr.read_to_string(&mut err);
+            let mut err = Vec::new();
+            let _ = stderr.read_to_end(&mut err);
+            let err = decode_tool_output(&err);
             return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim())));
         }
         String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8")))
@@ -272,9 +275,9 @@ impl Http {
         let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?;
         let mut stderr = child.stderr.take().expect("piped");
         let err_t = std::thread::spawn(move || {
-            let mut s = String::new();
-            let _ = stderr.read_to_string(&mut s);
-            s
+            let mut s = Vec::new();
+            let _ = stderr.read_to_end(&mut s);
+            decode_tool_output(&s)
         });
         let mut last = Instant::now();
         let mut last_bytes = 0;