From e732004180728a2dfa45083225d204d9fde89d06 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 24 Apr 2026 12:42:05 -0700 Subject: [PATCH] fix(nativets): forward OTEL-prefixed console logs to tracing events (#8937) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(nativets): forward OTEL-prefixed console logs to tracing events Nativets jobs run in-process and bypass the handle_child.rs stdout loop where `OTEL: ` lines are turned into `tracing::event!` calls when `OTEL_JOB_LOGS=true`. Apply the same prefix handling in the nativets log receiver so `console.log("OTEL: ...")` reaches the OTEL exporter like it does for other runtimes. Moves `OTEL_JOB_LOGS` and `OTEL_PREFIX` into windmill-common so both crates share the same definition. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(nativets): emit job_log tracing target so logs reach OTEL bridge For non-native runtimes, `lines_to_stream` → `process_streaming_log_lines` (EE) emits every stdout line as `tracing::info!(target: "windmill:job_log", ...)`, which is picked up by the EE `LogContextBridge` and exported to OTEL (the bridge's filter is `EnvFilter` only, not the targets filter that drops `windmill:job_log` from stdout/file sinks). Nativets delivers logs in-process via a channel, so it never goes through that path and console.log output only reached the Windmill UI. Emit the same `windmill:job_log` event per line from the nativets log receiver. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- backend/windmill-common/src/tracing_init.rs | 5 +++++ backend/windmill-runtime-nativets/src/lib.rs | 20 ++++++++++++++++++++ backend/windmill-worker/src/handle_child.rs | 4 +--- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 3b009a36eb..1464b8c4c6 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -37,12 +37,17 @@ fn compact_layer() -> Layer Targets { let targets = diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index e81925e7bb..dda57ae708 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -579,6 +579,7 @@ pub async fn eval_fetch_timeout( )); } + let w_id_for_tracing = w_id.to_string(); let result_f = tokio::task::spawn_blocking(move || { let CreatedRuntime { mut js_runtime, mut log_receiver, mut memory_limit_rx } = create_nativets_runtime(ann, spread)?; @@ -604,11 +605,30 @@ pub async fn eval_fetch_timeout( tracing::error!("failed to send extra logs: {e}"); } } + let w_id_for_tracing = w_id_for_tracing; let handle = tokio::spawn(async move { let mut result_stream = String::new(); let mut is_stream = false; while let Some(log) = log_receiver.recv().await { use windmill_common::result_stream::extract_stream_from_logs; + use windmill_common::tracing_init::{OTEL_JOB_LOGS, OTEL_PREFIX}; + + // Mirror `process_streaming_log_lines` (EE) + the OTEL_JOB_LOGS + // hook from handle_child.rs, neither of which runs for nativets + // since nativets delivers logs in-process via the log channel. + for line in log.lines() { + tracing::info!( + target: "windmill:job_log", + job_id = ?job_id, + workspace_id = ?w_id_for_tracing, + "{line}" + ); + if *OTEL_JOB_LOGS { + if let Some(otel_suffix) = line.strip_prefix(OTEL_PREFIX) { + tracing::event!(tracing::Level::INFO, otel_suffix); + } + } + } if let Some(stream) = extract_stream_from_logs(&log.trim_end_matches("\n")) { if !is_stream { diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 2edaa4f3a1..4eebb904d1 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -57,11 +57,10 @@ use crate::job_logger_oss::process_streaming_log_lines; use crate::worker_utils::{ping_job_status, update_worker_ping_from_job}; use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM}; -use windmill_common::tracing_init::{QUIET_MODE, VERBOSE_TARGET}; +use windmill_common::tracing_init::{OTEL_JOB_LOGS, OTEL_PREFIX, QUIET_MODE, VERBOSE_TARGET}; lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); - pub static ref OTEL_JOB_LOGS: bool = std::env::var("OTEL_JOB_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); } // - kill windows process along with all child processes @@ -357,7 +356,6 @@ pub async fn handle_child( } } -pub const OTEL_PREFIX: &str = "OTEL: "; pub const WAC_STEP_PREFIX: &str = "WM_WAC_STEP: "; pub async fn write_lines(