starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-07 00:01:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: write and read python job files as utf-8, not the platform locale (#10994)

Browse Source 
* fix: write and read python job files as utf-8, not the platform locale

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE

* refactor: trim the PYTHON_UTF8_ENVS comment to the 4-line limit

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE

* chore: bump ee ref for the python runner-group utf8 companion

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01ErBZtEpLpkFi1Y1W6eNZBE

* chore: update ee-repo-ref to d33ea730c550cdbc7d050aeb6d40dcef3d134e07

This commit updates the EE repository reference after PR #782 was merged in windmill-ee-private.

Previous ee-repo-ref: c8318661f8d91da9172a3c2dca050b70ba7afda2

New ee-repo-ref: d33ea730c550cdbc7d050aeb6d40dcef3d134e07

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
Ruben Fiszel
2026-09-06 11:44:06 +00:00
committed by GitHub
co-authored by Claude Opus 5 windmill-internal-app[bot]
parent c37f59e22a
commit 670404ffe2
3 changed files with 27 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
313c572c9dcbcaafd8a1594df4054f9dd26f395c
d33ea730c550cdbc7d050aeb6d40dcef3d134e07
backend/windmill-worker/loader.py
+5 -1
View File
@@ -69,7 +69,11 @@ class WindmillFinder(MetaPathFinder):
r = response.read().decode("utf-8")
if r == "WINDMILL_IS_FOLDER":
return ModuleSpec(name, WindmillLoader(name))
with open(fullpath, "w+") as f:
# Python parses .py as UTF-8 regardless of locale, so the
# file has to be written as UTF-8. Without this the ANSI
# code page on a Windows worker re-encodes every
# non-ASCII literal and the import dies on a SyntaxError.
with open(fullpath, "w+", encoding="utf-8") as f:
f.write(r)
return spec_from_file_location(name, fullpath)
except urllib.error.HTTPError as e:
backend/windmill-worker/src/python_executor.rs
+21 -10
View File
@@ -117,6 +117,12 @@ const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
pub const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
/// Every file exchanged with a job is UTF-8 by construction, so the interpreter
/// must agree. A job env carries no locale: Linux then picks UTF-8 on its own
/// (PEP 540), Windows picks the ANSI code page. Applied after the whitelisted
/// envs so the protocol is not the user's to opt out of.
pub const PYTHON_UTF8_ENVS: [(&str, &str); 1] = [("PYTHONUTF8", "1")];
/// Render loader.py with the TEMP_SCRIPT_REFS placeholder substituted by a
/// Python dict literal. Preview jobs pass a path -> temp-hash map so relative
/// imports resolve from not-yet-deployed local content; deployed runs pass
@@ -818,7 +824,7 @@ pub async fn handle_python_job(
del pre_args[k]
kwargs = inner_script.preprocessor(**pre_args)
kwrags_json = res_to_json(kwargs, type(kwargs))
with open("args.json", 'w') as f:
with open("args.json", 'w', encoding="utf-8") as f:
f.write(kwrags_json)"#
)
} else {
@@ -853,7 +859,7 @@ pub async fn handle_python_job(
_pre_result = asyncio.run(_pre_result)
kwargs = _pre_result if _pre_result is not None else {{}}
_pre_json = json.dumps(kwargs, separators=(',', ':'), default=str)
with open("args.json", 'w') as f:
with open("args.json", 'w', encoding="utf-8") as f:
f.write(_pre_json)
sys.stdout.write("wm_res[preprocessed_args]:" + _pre_json + "\n")
sys.stdout.flush()"#
@@ -874,11 +880,11 @@ import sys
from {module_dir_dot} import {last} as inner_script
from wmill.client import _run_workflow
with open("args.json") as f:
with open("args.json", encoding="utf-8") as f:
kwargs = json.load(f, strict=False)
{transforms}
with open("checkpoint.json") as f:
with open("checkpoint.json", encoding="utf-8") as f:
checkpoint = json.load(f, strict=False)
result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json")
@@ -904,12 +910,12 @@ try:
print("")
print("--- WAC: complete ---")
output_json = json.dumps(output, separators=(',', ':'), default=str)
with open(result_json, 'w') as f:
with open(result_json, 'w', encoding="utf-8") as f:
f.write(output_json)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.format_tb(exc_traceback)
with open(result_json, 'w') as f:
with open(result_json, 'w', encoding="utf-8") as f:
err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}
extra = e.__dict__
if extra and len(extra) > 0:
@@ -936,7 +942,7 @@ import sys
from {module_dir_dot} import {last} as inner_script
import re
with open("args.json") as f:
with open("args.json", encoding="utf-8") as f:
kwargs = json.load(f, strict=False)
args = {{}}
{transforms}
@@ -972,12 +978,12 @@ try:
print("WM_STREAM: " + chunk.replace('\n', '\\n'))
res = None
res_json = res_to_json(res, typ)
with open(result_json, 'w') as f:
with open(result_json, 'w', encoding="utf-8") as f:
f.write(res_json)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.format_tb(exc_traceback)
with open(result_json, 'w') as f:
with open(result_json, 'w', encoding="utf-8") as f:
err = {{ "message": str(e), "name": e.__class__.__name__, "stack": '\n'.join(tb[1:]) }}
extra = e.__dict__
if extra and len(extra) > 0:
@@ -1117,6 +1123,7 @@ mount {{
)
.await?,
)
.envs(PYTHON_UTF8_ENVS)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -1152,6 +1159,7 @@ mount {{
)
.await?,
)
.envs(PYTHON_UTF8_ENVS)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
@@ -3431,7 +3439,10 @@ pub async fn start_worker(
)
.await;
let mut proc_envs = HashMap::new();
let mut proc_envs: HashMap<String, String> = PYTHON_UTF8_ENVS
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
let additional_python_paths_folders = additional_python_paths.iter().join(":");
proc_envs.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
proc_envs.insert("PATH".to_string(), PATH_ENV.to_string());