diff --git a/Cargo.lock b/Cargo.lock index 54c5d2b7..6d49a14d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4093,6 +4093,7 @@ name = "ide-core" version = "0.1.25" dependencies = [ "dunce", + "encoding_rs", "serde", "serde_json", "specta", @@ -4100,6 +4101,7 @@ dependencies = [ "tempfile", "thiserror 2.0.21", "tokio-util", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/ide-agent/src/native/claude/mod.rs b/crates/ide-agent/src/native/claude/mod.rs index a15c8860..8fd87ad8 100644 --- a/crates/ide-agent/src/native/claude/mod.rs +++ b/crates/ide-agent/src/native/claude/mod.rs @@ -296,7 +296,7 @@ impl Shim { let mut cmd = std::process::Command::new(&self.config.program); ide_core::process::prepare(&mut cmd); let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?; - let text = String::from_utf8_lossy(&out.stdout); + let text = ide_core::output_encoding::decode_tool_output(&out.stdout); text.split_whitespace().next().filter(|v| out.status.success() && !v.is_empty()).map(str::to_owned) } diff --git a/crates/ide-agent/src/native/mod.rs b/crates/ide-agent/src/native/mod.rs index 46de082e..10f11309 100644 --- a/crates/ide-agent/src/native/mod.rs +++ b/crates/ide-agent/src/native/mod.rs @@ -64,7 +64,7 @@ fn version_of(program: &Path) -> Option> { let mut cmd = std::process::Command::new(program); ide_core::process::prepare(&mut cmd); let out = cmd.arg("--version").envs(ide_core::process::shell_env()).output().ok()?; - parse_version(&String::from_utf8_lossy(&out.stdout)) + parse_version(&ide_core::output_encoding::decode_tool_output(&out.stdout)) } fn parse_version(text: &str) -> Option> { diff --git a/crates/ide-build/src/process.rs b/crates/ide-build/src/process.rs index 0ac37580..fcec4b45 100644 --- a/crates/ide-build/src/process.rs +++ b/crates/ide-build/src/process.rs @@ -68,7 +68,7 @@ pub(crate) fn run( match reader.read_until(b'\n', &mut buf) { Ok(0) | Err(_) => break, Ok(_) => { - let line = String::from_utf8_lossy(&buf); + let line = ide_core::output_encoding::decode_tool_output(&buf); let line = line.trim_end_matches(['\n', '\r']).to_owned(); if tx.send((kind, line)).is_err() { break; diff --git a/crates/ide-core/Cargo.toml b/crates/ide-core/Cargo.toml index 9daa648b..af0d6b99 100644 --- a/crates/ide-core/Cargo.toml +++ b/crates/ide-core/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] dunce.workspace = true +encoding_rs.workspace = true serde.workspace = true serde_json.workspace = true specta.workspace = true @@ -16,3 +17,6 @@ tokio-util.workspace = true [dev-dependencies] tempfile.workspace = true + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Globalization"] } diff --git a/crates/ide-core/src/output_encoding.rs b/crates/ide-core/src/output_encoding.rs index 5d3a3688..f5e85f3d 100644 --- a/crates/ide-core/src/output_encoding.rs +++ b/crates/ide-core/src/output_encoding.rs @@ -18,6 +18,60 @@ use std::collections::BTreeMap; +/// Decodes complete lines or collected output from IDE-managed tools, preferring UTF-8. +/// Windows launcher/native diagnostics fall back to the system ANSI code page. +/// Callers must buffer incomplete lines so multibyte characters are never split. +/// Run consoles and protocol streams have their own encoding contracts. +pub fn decode_tool_output(bytes: &[u8]) -> String { + #[cfg(windows)] + // SAFETY: GetACP is an argument-less query with no preconditions. + let code_page = unsafe { windows_sys::Win32::Globalization::GetACP() }; + #[cfg(not(windows))] + let code_page = 65001; + decode_tool_output_with_code_page(bytes, code_page) +} + +fn decode_tool_output_with_code_page(bytes: &[u8], code_page: u32) -> String { + if let Ok(text) = std::str::from_utf8(bytes) { + return text.to_owned(); + } + let encoding = encoding_for_code_page(code_page).unwrap_or(encoding_rs::UTF_8); + // Collected output can mix UTF-8 JVM lines with native launcher diagnostics. + let mut text = String::new(); + for line in bytes.split_inclusive(|b| *b == b'\n') { + match std::str::from_utf8(line) { + Ok(line) => text.push_str(line), + Err(_) => text.push_str(&encoding.decode_without_bom_handling(line).0), + } + } + text +} + +fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> { + Some(match cp { + 65001 => encoding_rs::UTF_8, + 936 => encoding_rs::GBK, + 54936 => encoding_rs::GB18030, + 950 => encoding_rs::BIG5, + 932 => encoding_rs::SHIFT_JIS, + 949 => encoding_rs::EUC_KR, + 866 => encoding_rs::IBM866, + 874 => encoding_rs::WINDOWS_874, + 1250 => encoding_rs::WINDOWS_1250, + 1251 => encoding_rs::WINDOWS_1251, + 1252 => encoding_rs::WINDOWS_1252, + 1253 => encoding_rs::WINDOWS_1253, + 1254 => encoding_rs::WINDOWS_1254, + 1255 => encoding_rs::WINDOWS_1255, + 1256 => encoding_rs::WINDOWS_1256, + 1257 => encoding_rs::WINDOWS_1257, + 1258 => encoding_rs::WINDOWS_1258, + 20866 => encoding_rs::KOI8_R, + 21866 => encoding_rs::KOI8_U, + _ => return None, + }) +} + const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"]; const FILE_ENCODING: &str = "file.encoding"; const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS"; @@ -237,6 +291,29 @@ impl NoticeFilter { mod tests { use super::*; + #[test] + fn tool_output_prefers_utf8() { + for cp in [936, 1252, 65001, 0] { + for text in ["", "plain ASCII\r\n", "中文 café 🦀\n"] { + assert_eq!(decode_tool_output_with_code_page(text.as_bytes(), cp), text); + } + } + } + + #[test] + fn tool_output_decodes_native_diagnostics_and_mixed_lines() { + const GBK: &[u8] = b"\xcf\xb5\xcd\xb3\xd5\xd2\xb2\xbb\xb5\xbd\xd6\xb8\xb6\xa8\xb5\xc4\xc2\xb7\xbe\xb6\xa1\xa3"; + assert_eq!(decode_tool_output_with_code_page(GBK, 936), "系统找不到指定的路径。"); + let mut mixed = "JVM: 中文\r\n".as_bytes().to_vec(); + mixed.extend_from_slice(GBK); + mixed.extend_from_slice(b"\r\nlast"); + assert_eq!(decode_tool_output_with_code_page(&mixed, 936), "JVM: 中文\r\n系统找不到指定的路径。\r\nlast"); + assert_eq!(decode_tool_output_with_code_page(b"caf\xe9", 1252), "café"); + for cp in [65001, 0] { + assert_eq!(decode_tool_output_with_code_page(b"a\xffb", cp), "a�b"); + } + } + const STD_UTF8: [&str; 4] = [ "-Dstdout.encoding=UTF-8", "-Dstderr.encoding=UTF-8", diff --git a/crates/ide-coverage/src/lib.rs b/crates/ide-coverage/src/lib.rs index 1fc48db2..75781203 100644 --- a/crates/ide-coverage/src/lib.rs +++ b/crates/ide-coverage/src/lib.rs @@ -213,8 +213,12 @@ pub fn collect(spec: &CoverageReportSpec) -> IdeResult { .output() .map_err(|e| IdeError::unavailable(format!("cannot run {program}: {e}")))?; if !out.status.success() { - let text = String::from_utf8_lossy(&out.stderr).trim().to_owned(); - let text = if text.is_empty() { String::from_utf8_lossy(&out.stdout).trim().to_owned() } else { text }; + let text = ide_core::output_encoding::decode_tool_output(&out.stderr).trim().to_owned(); + let text = if text.is_empty() { + ide_core::output_encoding::decode_tool_output(&out.stdout).trim().to_owned() + } else { + text + }; return Err(IdeError::not_found(format!("exporting the coverage data failed: {text}"))); } let text = read_report(lcov)?; diff --git a/crates/ide-dap/src/cargo.rs b/crates/ide-dap/src/cargo.rs index 6de0813b..30fb88a7 100644 --- a/crates/ide-dap/src/cargo.rs +++ b/crates/ide-dap/src/cargo.rs @@ -70,8 +70,10 @@ pub async fn build( let on_output = std::sync::Arc::new(on_output); let err_out = on_output.clone(); let err_task = tokio::spawn(async move { - let mut lines = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = lines.next_line().await { + let mut lines = BufReader::new(stderr).split(b'\n'); + while let Ok(Some(line)) = lines.next_segment().await { + let line = ide_core::output_encoding::decode_tool_output(&line); + let line = line.trim_end_matches('\r'); err_out(format!("{line}\n")); } }); diff --git a/crates/ide-docker/src/process.rs b/crates/ide-docker/src/process.rs index e9dc25a3..6c08c005 100644 --- a/crates/ide-docker/src/process.rs +++ b/crates/ide-docker/src/process.rs @@ -54,7 +54,7 @@ fn read_all(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle, s let mut pending: Vec = Vec::new(); let flush = |pending: &mut Vec| { if !pending.is_empty() { - let line = String::from_utf8_lossy(pending).trim_end().to_string(); + let line = ide_core::output_encoding::decode_tool_output(pending).trim_end().to_string(); pending.clear(); let _ = tx.send(Event::Line(stream, line)); } diff --git a/crates/ide-host/src/environment.rs b/crates/ide-host/src/environment.rs index 4dabd350..0ef84ec5 100644 --- a/crates/ide-host/src/environment.rs +++ b/crates/ide-host/src/environment.rs @@ -281,7 +281,8 @@ fn run_one(argv: &[String], root: &Path, cancel: &CancellationToken, on_line: &d for pipe in pipes.into_iter().flatten() { let tx = tx.clone(); std::thread::spawn(move || { - for line in BufReader::new(pipe).lines().map_while(Result::ok) { + for line in BufReader::new(pipe).split(b'\n').map_while(Result::ok) { + let line = ide_core::output_encoding::decode_tool_output(&line).trim_end_matches('\r').to_owned(); if tx.send(line).is_err() { break; } diff --git a/crates/ide-notebook/src/kernel/process.rs b/crates/ide-notebook/src/kernel/process.rs index 791939e0..afcdda93 100644 --- a/crates/ide-notebook/src/kernel/process.rs +++ b/crates/ide-notebook/src/kernel/process.rs @@ -1,13 +1,12 @@ use ide_core::{IdeError, IdeResult}; use jupyter_protocol::ConnectionInfo; use parking_lot::Mutex; -use std::collections::VecDeque; use std::io::Write; use std::path::Path; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; -use tokio::io::AsyncReadExt; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::{Child, Command}; use tokio::task::JoinHandle; use uuid::Uuid; @@ -18,7 +17,7 @@ pub(super) struct Process { child: Child, #[cfg(unix)] pid: Option, - stderr: Arc>>, + stderr: Arc>, drains: Vec>, // Kept until the process and sockets have stopped; contains the HMAC secret. _directory: tempfile::TempDir, @@ -103,26 +102,31 @@ impl Process { child, #[cfg(unix)] pid, - stderr: Arc::new(Mutex::new(VecDeque::with_capacity(STDERR_BYTES))), + stderr: Arc::new(Mutex::new(String::new())), drains: Vec::new(), _directory: directory, #[cfg(windows)] windows, }; - if let Some(mut stderr) = process.child.stderr.take() { + if let Some(stderr) = process.child.stderr.take() { let tail = process.stderr.clone(); process.drains.push(tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - while let Ok(count) = stderr.read(&mut buffer).await { - if count == 0 { - break; - } - let mut tail = tail.lock(); - for byte in &buffer[..count] { - if tail.len() == STDERR_BYTES { - tail.pop_front(); + let mut reader = BufReader::new(stderr); + let mut line = Vec::new(); + loop { + line.clear(); + match reader.read_until(b'\n', &mut line).await { + Ok(0) | Err(_) => break, + Ok(_) => { + let text = ide_core::output_encoding::decode_tool_output(&line); + let mut tail = tail.lock(); + tail.push_str(&text); + let mut start = tail.len().saturating_sub(STDERR_BYTES); + while !tail.is_char_boundary(start) { + start += 1; + } + tail.drain(..start); } - tail.push_back(*byte); } } })); @@ -143,8 +147,7 @@ impl Process { } pub fn stderr(&self) -> String { - let bytes = self.stderr.lock().iter().copied().collect::>(); - String::from_utf8_lossy(&bytes).into_owned() + self.stderr.lock().clone() } pub fn death(&mut self) -> IdeResult> { diff --git a/crates/ide-sdk/src/exec.rs b/crates/ide-sdk/src/exec.rs index aa2296c6..03333c50 100644 --- a/crates/ide-sdk/src/exec.rs +++ b/crates/ide-sdk/src/exec.rs @@ -1,6 +1,7 @@ //! Running toolchain commands: short queries with a timeout and long-running //! streaming commands that honour cancellation. +use ide_core::output_encoding::decode_tool_output; use ide_core::{CancellationToken, IdeError, IdeResult}; use std::ffi::OsString; use std::io::{BufRead, BufReader, Read}; @@ -77,14 +78,14 @@ pub(crate) fn query_in( let mut stdout = child.stdout.take()?; let mut stderr = child.stderr.take()?; let out = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stdout.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stdout.read_to_end(&mut s); + decode_tool_output(&s) }); let err = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let deadline = Instant::now() + timeout; let status = loop { @@ -131,16 +132,16 @@ pub(crate) fn stream( if let Some(out) = child.stdout.take() { let tx = tx.clone(); readers.push(std::thread::spawn(move || { - for line in BufReader::new(out).lines().map_while(Result::ok) { - let _ = tx.send(line); + for line in BufReader::new(out).split(b'\n').map_while(Result::ok) { + let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned()); } })); } if let Some(err) = child.stderr.take() { let tx = tx.clone(); readers.push(std::thread::spawn(move || { - for line in BufReader::new(err).lines().map_while(Result::ok) { - let _ = tx.send(line); + for line in BufReader::new(err).split(b'\n').map_while(Result::ok) { + let _ = tx.send(decode_tool_output(&line).trim_end_matches('\r').to_owned()); } })); } diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs index 159a7d60..81c44bf9 100644 --- a/crates/ide-tools/src/process.rs +++ b/crates/ide-tools/src/process.rs @@ -3,6 +3,7 @@ use crate::InstallEvent; use ide_core::error::ErrorKind; +use ide_core::output_encoding::decode_tool_output; use ide_core::{CancellationToken, IdeError, IdeResult}; use std::io::{BufRead, BufReader, Read}; use std::path::Path; @@ -86,8 +87,9 @@ pub fn run_collect(mut cmd: Command, cancel: &CancellationToken, emit: Emit) -> for pipe in pipes.into_iter().flatten() { let tx = tx.clone(); std::thread::spawn(move || { - for line in BufReader::new(pipe).lines() { + for line in BufReader::new(pipe).split(b'\n') { let Ok(line) = line else { break }; + let line = decode_tool_output(&line).trim_end_matches('\r').to_owned(); if tx.send(line).is_err() { break; } @@ -127,14 +129,14 @@ pub fn run_quiet(mut cmd: Command, timeout: Duration) -> Option<(bool, String)> let mut stdout = child.stdout.take()?; let mut stderr = child.stderr.take()?; let out_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stdout.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stdout.read_to_end(&mut s); + decode_tool_output(&s) }); let err_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let deadline = Instant::now() + timeout; let status = loop { @@ -251,8 +253,9 @@ impl Http { return Err(IdeError::new(ErrorKind::Io, format!("{url}: response is larger than {max} bytes"))); } if !status.success() { - let mut err = String::new(); - let _ = stderr.read_to_string(&mut err); + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + let err = decode_tool_output(&err); return Err(IdeError::new(ErrorKind::Io, format!("GET {url} failed: {}", err.trim()))); } String::from_utf8(body).map_err(|_| IdeError::new(ErrorKind::Protocol, format!("{url}: response is not UTF-8"))) @@ -272,9 +275,9 @@ impl Http { let mut child = cmd.spawn().map_err(|e| IdeError::unavailable(format!("cannot run curl: {e}")))?; let mut stderr = child.stderr.take().expect("piped"); let err_t = std::thread::spawn(move || { - let mut s = String::new(); - let _ = stderr.read_to_string(&mut s); - s + let mut s = Vec::new(); + let _ = stderr.read_to_end(&mut s); + decode_tool_output(&s) }); let mut last = Instant::now(); let mut last_bytes = 0;