fix(run): UTF-8 default charset for application JVMs forked by Maven and Gradle on Windows (#751)

* fix(run): give the spring-boot:run fork a UTF-8 default charset on Windows

On Windows, mvn spring-boot:run passes the stdout/stderr encoding properties to the
forked application JVM through JAVA_TOOL_OPTIONS, but not file.encoding (it would
change how Maven reads sources). On JDK 8-17, logback / log4j2 console appenders
encode with the default charset (GBK on Chinese Windows) and ignore stdout.encoding,
so Chinese log lines showed as U+FFFD in the UTF-8 Run console.

For a Maven launch with a spring-boot:run goal and a UTF-8 console, add
-Dfile.encoding=UTF-8 to -Dspring-boot.run.jvmArguments, merged into the user's
VM options. It reaches only the forked JVM; Maven's own JVM is unchanged. A
file.encoding the user set (in jvmArguments or JAVA_TOOL_OPTIONS) wins.

Fixes #697

* fix(run): give Gradle JavaExec forks a UTF-8 default charset on Windows

Applications started by Gradle (bootRun, run, custom JavaExec tasks) kept the
ANSI default charset (GBK on Chinese Windows, JDK 8-17) while the Run console
decodes UTF-8. Code that converts text with the default charset then breaks:
Nacos config with Chinese text fails to parse (MalformedInputException) and
logback/log4j2 console output is garbled.

For a UTF-8 console on Windows, a Gradle launch now gets an init script that
adds -Dfile.encoding=UTF-8 to each JavaExec task when it runs, unless the task
or JAVA_TOOL_OPTIONS already sets file.encoding. Gradle's own JVM and compiler
source encoding are unchanged. Maven and Gradle are told apart so the
spring-boot:run rewrite only applies to Maven.

* fix(run): set Gradle charset before build task configuration

---------

Co-authored-by: DPhngNam <pnam3072004@gmail.com>

crates/ide-core/src/output_encoding.rs

@@ -13,7 +13,10 @@
 //!   they fork get them too (`spring-boot:run`, Surefire, `bootRun`, test workers).
 //!   Each JVM then reports "Picked up JAVA_TOOL_OPTIONS: …" on stderr (which a build tool
 //!   may copy to its stdout); that line is hidden again ([`NoticeFilter`]). `file.encoding` is left alone there: it would change
-//!   how a build reads sources that declare no encoding.
+//!   how a build reads sources that declare no encoding. The application JVMs a build
+//!   forks get `file.encoding=UTF-8` for a UTF-8 console instead, through the build's own
+//!   options: `-Dspring-boot.run.jvmArguments` for Maven `spring-boot:run`, an init script
+//!   ([`GRADLE_FILE_ENCODING_SCRIPT`]) for Gradle `JavaExec` tasks (`bootRun`, `run`).
 //! - Python: `PYTHONIOENCODING`.
 
 use std::collections::BTreeMap;
@@ -75,16 +78,34 @@ fn encoding_for_code_page(cp: u32) -> Option<&'static encoding_rs::Encoding> {
 const STD_PROPERTIES: &[&str] = &["stdout.encoding", "stderr.encoding", "sun.stdout.encoding", "sun.stderr.encoding"];
 const FILE_ENCODING: &str = "file.encoding";
 const TOOL_OPTIONS: &str = "JAVA_TOOL_OPTIONS";
+const SPRING_BOOT_JVM_ARGS: &str = "-Dspring-boot.run.jvmArguments=";
 
 /// Build tools and launcher scripts that run on (and may fork) JVMs.
-const JVM_TOOLS: &[&str] =
-    &["mvn", "mvnw", "mvnd", "gradle", "gradlew", "sbt", "ant", "kotlin", "kotlinc", "groovy", "scala", "jbang"];
+const JVM_TOOLS: &[&str] = &["sbt", "ant", "kotlin", "kotlinc", "groovy", "scala", "jbang"];
+const MAVEN: &[&str] = &["mvn", "mvnw", "mvnd"];
+const GRADLE: &[&str] = &["gradle", "gradlew"];
+
+/// Gradle init script giving the `JavaExec` tasks (`bootRun`, `run`, custom ones) a UTF-8
+/// default charset before the build configures them, so the build can override it.
+/// Gradle's own JVM and the compiler's source encoding are untouched.
+pub const GRADLE_FILE_ENCODING_SCRIPT: &str = "\
+// Written by OpenIDEA: a UTF-8 default charset for applications run by Gradle, as the Run console decodes UTF-8.
+allprojects {
+    tasks.withType(JavaExec).configureEach {
+        defaultCharacterEncoding = 'UTF-8'
+    }
+}
+";
 
 /// What a launch needs: `-D` options to put first on a `java` command line,
 /// environment variables to set, and the notice line its JVMs print because of them.
 #[derive(Debug, Default, PartialEq, Eq)]
 pub struct EncodingFix {
     pub java_args: Vec<String>,
+    /// The build tool's arguments, rewritten to pass options to the JVM it forks.
+    pub args: Option<Vec<String>>,
+    /// Gradle: pass [`GRADLE_FILE_ENCODING_SCRIPT`] with `--init-script`.
+    pub gradle_init_script: bool,
     pub env: Vec<(String, String)>,
     /// "Picked up JAVA_TOOL_OPTIONS: …" for a variable set here alone, to drop from the output.
     pub notice: Option<String>,
@@ -116,6 +137,8 @@ fn options(properties: &[&str], encoding: &str) -> Vec<String> {
 #[derive(Clone, Copy, PartialEq, Eq)]
 enum Kind {
     Java,
+    Maven,
+    Gradle,
     JvmTool,
     Python,
     Other,
@@ -125,6 +148,10 @@ fn kind(program: &str) -> Kind {
     let name = program_name(program);
     if name == "java" || name == "javaw" {
         Kind::Java
+    } else if MAVEN.contains(&name.as_str()) {
+        Kind::Maven
+    } else if GRADLE.contains(&name.as_str()) {
+        Kind::Gradle
     } else if JVM_TOOLS.contains(&name.as_str()) {
         Kind::JvmTool
     } else if is_python(&name) {
@@ -193,7 +220,17 @@ fn fix_kind(
                 out.java_args.push(format!("-D{FILE_ENCODING}=UTF-8"));
             }
         }
-        (Kind::Java | Kind::JvmTool, _) => {
+        (Kind::Java | Kind::Maven | Kind::Gradle | Kind::JvmTool, _) => {
+            if let Some(args) = args
+                && is_utf8(encoding)
+                && !sets(&tool_options, &[FILE_ENCODING])
+            {
+                match kind {
+                    Kind::Maven => out.args = spring_boot_file_encoding(args),
+                    Kind::Gradle => out.gradle_init_script = true,
+                    _ => {}
+                }
+            }
             if !sets(&tool_options, STD_PROPERTIES) {
                 let added = options(STD_PROPERTIES, encoding).join(" ");
                 let value = match tool_options.trim() {
@@ -214,6 +251,26 @@ fn fix_kind(
     out
 }
 
+fn spring_boot_file_encoding(args: &[String]) -> Option<Vec<String>> {
+    if !args.iter().any(|a| a == "spring-boot:run" || (a.contains(":spring-boot-maven-plugin:") && a.ends_with(":run")))
+    {
+        return None;
+    }
+    let option = format!("-D{FILE_ENCODING}=UTF-8");
+    let mut out = args.to_vec();
+    match out.iter_mut().rfind(|a| a.starts_with(SPRING_BOOT_JVM_ARGS)) {
+        Some(own) if sets(own, &[FILE_ENCODING]) => return None,
+        Some(own) => {
+            if own.len() > SPRING_BOOT_JVM_ARGS.len() {
+                own.push(' ');
+            }
+            own.push_str(&option);
+        }
+        None => out.push(format!("{SPRING_BOOT_JVM_ARGS}{option}")),
+    }
+    Some(out)
+}
+
 /// `vmArgs` of a launch by java-debug, which decodes the JVM's output as `encoding`
 /// (its launch attribute, UTF-8 when unset). java-debug sets `file.encoding` only
 /// when that attribute is nonblank; use the same defaults and user overrides as [`fix`].
@@ -435,6 +492,64 @@ mod tests {
         assert_eq!(run("python3", &[], &[], "GBK", false).env, [("PYTHONIOENCODING".to_owned(), "GBK".to_owned())]);
     }
 
+    #[test]
+    fn spring_boot_run_forks_get_a_utf8_default_charset() {
+        let boot = |args: &[&str], env: &[(&str, &str)]| run("mvn.cmd", args, env, "UTF-8", true).args;
+        let utf8 = "-Dspring-boot.run.jvmArguments=-Dfile.encoding=UTF-8";
+        assert_eq!(boot(&["spring-boot:run"], &[]).unwrap(), ["spring-boot:run", utf8]);
+        let plugin = "org.springframework.boot:spring-boot-maven-plugin:3.2.0:run";
+        assert_eq!(boot(&["-pl", "app", plugin], &[]).unwrap(), ["-pl", "app", plugin, utf8]);
+        // The user's VM options are kept; their own `file.encoding` wins.
+        assert_eq!(
+            boot(&["spring-boot:run", "-Dspring-boot.run.jvmArguments=-Xmx1g \"-Da=b c\""], &[]).unwrap(),
+            ["spring-boot:run", "-Dspring-boot.run.jvmArguments=-Xmx1g \"-Da=b c\" -Dfile.encoding=UTF-8"]
+        );
+        assert_eq!(boot(&["spring-boot:run", "-Dspring-boot.run.jvmArguments="], &[]).unwrap()[1], utf8);
+        assert!(boot(&["spring-boot:run", "-Dspring-boot.run.jvmArguments=-Dfile.encoding=GBK"], &[]).is_none());
+        assert!(boot(&["spring-boot:run"], &[(TOOL_OPTIONS, "-Dfile.encoding=GBK")]).is_none());
+        // Other goals and tools, other consoles, and other platforms are left alone.
+        assert!(boot(&["package"], &[]).is_none());
+        assert!(run("mvn", &["spring-boot:run"], &[], "GBK", true).args.is_none());
+        assert!(run("mvn", &["spring-boot:run"], &[], "UTF-8", false).args.is_none());
+    }
+
+    #[test]
+    fn forked_application_jvms_get_a_utf8_default_charset() {
+        // Reported: a Spring Boot app run by the build on Chinese Windows (JDK 8–17, GBK default).
+        for (tool, args) in [
+            ("C:\\p\\gradlew.bat", &["bootRun"][..]),
+            ("gradle", &[":app:run"]),
+            ("mvnw.cmd", &["spring-boot:run"]),
+            ("mvnd", &["org.springframework.boot:spring-boot-maven-plugin:2.7.18:run"]),
+        ] {
+            let f = run(tool, args, &[], "UTF-8", true);
+            let gradle = tool.contains("gradle");
+            assert_eq!(f.gradle_init_script, gradle, "{tool}");
+            assert_eq!(f.args.is_some(), !gradle, "{tool}");
+            // The build tool's own JVM keeps its default charset.
+            assert!(!tool_env(&f).unwrap().contains("file.encoding"), "{tool}");
+            // The user's choice, a GBK console and other platforms are left alone.
+            let own = run(tool, args, &[(TOOL_OPTIONS, "-Dfile.encoding=GBK")], "UTF-8", true);
+            assert!(!own.gradle_init_script && own.args.is_none(), "{tool}");
+            let gbk = run(tool, args, &[], "GBK", true);
+            assert!(!gbk.gradle_init_script && gbk.args.is_none(), "{tool}");
+            assert_eq!(run(tool, args, &[], "UTF-8", false), EncodingFix::default(), "{tool}");
+        }
+        // Shell command lines can't take arguments; other JVM tools fork no application.
+        assert!(!fix_command_line("gradlew bootRun", &BTreeMap::new(), |_| None, "UTF-8", true).gradle_init_script);
+        let sbt = run("sbt", &["run"], &[], "UTF-8", true);
+        assert!(!sbt.gradle_init_script && sbt.args.is_none());
+    }
+
+    #[test]
+    fn gradle_script_respects_the_builds_own_file_encoding() {
+        let s = GRADLE_FILE_ENCODING_SCRIPT;
+        assert!(s.contains("tasks.withType(JavaExec).configureEach"), "{s}");
+        assert!(s.contains("defaultCharacterEncoding = 'UTF-8'"), "{s}");
+        assert!(!s.contains("doFirst"), "the build must be able to override the default: {s}");
+        assert!(!s.contains("withType(Test)") && !s.contains("JavaCompile"), "{s}");
+    }
+
     #[test]
     fn shell_command_lines_get_variables_from_their_first_word() {
         let line = |l: &str| fix_command_line(l, &BTreeMap::new(), |_| None, "UTF-8", true);

crates/ide-run/src/gradle_jvm.rs

@@ -41,17 +41,22 @@ pub fn default_dir() -> PathBuf {
 /// Writes the script for `jvm_args` under `dir` (named by its content, so launches share
 /// it and nothing needs deleting) and returns its path.
 pub fn write_init_script(dir: &Path, jvm_args: &[String]) -> std::io::Result<PathBuf> {
+    write_script(dir, "jvm-args", &init_script(jvm_args))
+}
+
+/// Writes the init script `text` under `dir` as `<prefix>-<hash>.init.gradle` (written
+/// once per content, so launches share it) and returns its path.
+pub fn write_script(dir: &Path, prefix: &str, text: &str) -> std::io::Result<PathBuf> {
     use std::hash::{Hash, Hasher};
-    let text = init_script(jvm_args);
     let mut h = std::collections::hash_map::DefaultHasher::new();
     text.hash(&mut h);
-    let file = dir.join(format!("jvm-args-{:016x}.init.gradle", h.finish()));
+    let file = dir.join(format!("{prefix}-{:016x}.init.gradle", h.finish()));
     if std::fs::read_to_string(&file).is_ok_and(|t| t == text) {
         return Ok(file);
     }
     std::fs::create_dir_all(dir)?;
     let tmp = file.with_extension(format!("{}.tmp", std::process::id()));
-    std::fs::write(&tmp, &text)?;
+    std::fs::write(&tmp, text)?;
     std::fs::rename(&tmp, &file).inspect_err(|_| {
         let _ = std::fs::remove_file(&tmp);
     })?;

crates/ide-run/src/process.rs

@@ -195,8 +195,21 @@ fn match_console_encoding(
         output_encoding::fix(&program, &inv.args, &env.vars, inherited, encoding, cfg!(windows))
     };
     env.vars.extend(fix.env);
-    if !fix.java_args.is_empty() && !inv.raw_args {
-        inv.args.splice(0..0, fix.java_args);
+    let mut prepend = fix.java_args;
+    if fix.gradle_init_script {
+        match crate::gradle_jvm::write_script(
+            &crate::gradle_jvm::default_dir(),
+            "file-encoding",
+            output_encoding::GRADLE_FILE_ENCODING_SCRIPT,
+        ) {
+            Ok(script) => prepend.extend(["--init-script".to_owned(), script.to_string_lossy().into_owned()]),
+            Err(e) => tracing::warn!("cannot write the Gradle file.encoding init script: {e}"),
+        }
+    }
+    let changed = fix.args.is_some() || !prepend.is_empty();
+    if changed && !inv.raw_args {
+        inv.args = fix.args.unwrap_or_else(|| std::mem::take(&mut inv.args));
+        inv.args.splice(0..0, prepend);
         let mut display = quote(config.command.trim());
         for a in &inv.args {
             display.push(' ');