From 670404ffe27fedc3858b46b0c6b3312fbe175e13 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 6 Sep 2026 11:44:06 +0000 Subject: [PATCH] fix: write and read python job files as utf-8, not the platform locale (#10994) * fix: write and read python job files as utf-8, not the platform locale Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE * refactor: trim the PYTHON_UTF8_ENVS comment to the 4-line limit Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE * chore: bump ee ref for the python runner-group utf8 companion Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE * chore: update ee-repo-ref to d33ea730c550cdbc7d050aeb6d40dcef3d134e07 This commit updates the EE repository reference after PR #782 was merged in windmill-ee-private. Previous ee-repo-ref: c8318661f8d91da9172a3c2dca050b70ba7afda2 New ee-repo-ref: d33ea730c550cdbc7d050aeb6d40dcef3d134e07 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/windmill-worker/loader.py | 6 +++- .../windmill-worker/src/python_executor.rs | 31 +++++++++++++------ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 048726821c..0ebbb1b5ea 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -313c572c9dcbcaafd8a1594df4054f9dd26f395c +d33ea730c550cdbc7d050aeb6d40dcef3d134e07 diff --git a/backend/windmill-worker/loader.py b/backend/windmill-worker/loader.py index ab10c36b86..bf67266d87 100644 --- a/backend/windmill-worker/loader.py +++ b/backend/windmill-worker/loader.py @@ -69,7 +69,11 @@ class WindmillFinder(MetaPathFinder): r = response.read().decode("utf-8") if r == "WINDMILL_IS_FOLDER": return ModuleSpec(name, WindmillLoader(name)) - with open(fullpath, "w+") as f: + # Python parses .py as UTF-8 regardless of locale, so the + # file has to be written as UTF-8. Without this the ANSI + # code page on a Windows worker re-encodes every + # non-ASCII literal and the import dies on a SyntaxError. + with open(fullpath, "w+", encoding="utf-8") as f: f.write(r) return spec_from_file_location(name, fullpath) except urllib.error.HTTPError as e: diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 38fb4bca32..18808672fc 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -117,6 +117,12 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto"); pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py"); +/// Every file exchanged with a job is UTF-8 by construction, so the interpreter +/// must agree. A job env carries no locale: Linux then picks UTF-8 on its own +/// (PEP 540), Windows picks the ANSI code page. Applied after the whitelisted +/// envs so the protocol is not the user's to opt out of. +pub const PYTHON_UTF8_ENVS: [(&str, &str); 1] = [("PYTHONUTF8", "1")]; + /// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a /// Python dict literal. Preview jobs pass a path -> temp-hash map so relative /// imports resolve from not-yet-deployed local content; deployed runs pass @@ -818,7 +824,7 @@ pub async fn handle_python_job( del pre_args[k] kwargs = inner_script.preprocessor(**pre_args) kwrags_json = res_to_json(kwargs, type(kwargs)) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(kwrags_json)"# ) } else { @@ -853,7 +859,7 @@ pub async fn handle_python_job( _pre_result = asyncio.run(_pre_result) kwargs = _pre_result if _pre_result is not None else {{}} _pre_json = json.dumps(kwargs, separators=(',', ':'), default=str) - with open("args.json", 'w') as f: + with open("args.json", 'w', encoding="utf-8") as f: f.write(_pre_json) sys.stdout.write("wm_res[preprocessed_args]:" + _pre_json + "\n") sys.stdout.flush()"# @@ -874,11 +880,11 @@ import sys from {module_dir_dot} import {last} as inner_script from wmill.client import _run_workflow -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) {transforms} -with open("checkpoint.json") as f: +with open("checkpoint.json", encoding="utf-8") as f: checkpoint = json.load(f, strict=False) result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") @@ -904,12 +910,12 @@ try: print("") print("--- WAC: complete ---") output_json = json.dumps(output, separators=(',', ':'), default=str) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(output_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -936,7 +942,7 @@ import sys from {module_dir_dot} import {last} as inner_script import re -with open("args.json") as f: +with open("args.json", encoding="utf-8") as f: kwargs = json.load(f, strict=False) args = {{}} {transforms} @@ -972,12 +978,12 @@ try: print("WM_STREAM: " + chunk.replace('\n', '\\n')) res = None res_json = res_to_json(res, typ) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: f.write(res_json) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.format_tb(exc_traceback) - with open(result_json, 'w') as f: + with open(result_json, 'w', encoding="utf-8") as f: err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }} extra = e.__dict__ if extra and len(extra) > 0: @@ -1117,6 +1123,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -1152,6 +1159,7 @@ mount {{ ) .await?, ) + .envs(PYTHON_UTF8_ENVS) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) @@ -3431,7 +3439,10 @@ pub async fn start_worker( ) .await; - let mut proc_envs = HashMap::new(); + let mut proc_envs: HashMap = PYTHON_UTF8_ENVS + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); let additional_python_paths_folders = additional_python_paths.iter().join(":"); proc_envs.insert("PYTHONPATH".to_string(), additional_python_paths_folders); proc_envs.insert("PATH".to_string(), PATH_ENV.to_string());