From f50a866430da8f5f43cb3163ec116fe254407ef9 Mon Sep 17 00:00:00 2001 From: Pyra <92104930+pyranota@users.noreply.github.com> Date: Wed, 28 Jan 2026 19:19:19 +0100 Subject: [PATCH] fix: nativets http tracing (#7716) * fix: nativets http tracing Signed-off-by: pyranota * ee repo Signed-off-by: pyranota * nit Signed-off-by: pyranota * ee ref Signed-off-by: pyranota * ee repo Signed-off-by: pyranota * ee repo Signed-off-by: pyranota * fix Signed-off-by: pyranota * fix Signed-off-by: pyranota * fix v2 Signed-off-by: pyranota * ee repo Signed-off-by: pyranota * chore: update ee-repo-ref to 5d841b358dd32130c9f34b54f59b96b5c322f213 This commit updates the EE repository reference after PR #396 was merged in windmill-ee-private. Previous ee-repo-ref: 250723c698fceccbc66ae9a6c6c7c09e33465819 New ee-repo-ref: 5d841b358dd32130c9f34b54f59b96b5c322f213 Automated by sync-ee-ref workflow. --------- Signed-off-by: pyranota Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 40 +++++++-------------- backend/windmill-worker/src/bun_executor.rs | 16 --------- backend/windmill-worker/src/js_eval.rs | 30 ++++++++++++++-- backend/windmill-worker/src/lib.rs | 2 +- backend/windmill-worker/src/runtime.js | 7 ++-- backend/windmill-worker/src/worker.rs | 22 +++++------- 7 files changed, 54 insertions(+), 65 deletions(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a45bb85c51..14c33389f7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -371efb2d7307f588c5ce00d63fd036751cc068f2 \ No newline at end of file +5d841b358dd32130c9f34b54f59b96b5c322f213 diff --git a/backend/src/main.rs b/backend/src/main.rs index 739eb90993..70ea4fb15d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -668,10 +668,11 @@ async fn windmill_main() -> anyhow::Result<()> { // Load OTEL tracing proxy settings and initialize deno_telemetry if nativets tracing is enabled // This must happen before any Deno runtime is created - #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] + #[cfg(all(feature = "private", feature = "enterprise"))] { reload_otel_tracing_proxy_setting(&Connection::Sql(db.clone())).await; + #[cfg(feature = "deno_core")] if windmill_worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets).await { match windmill_worker::load_internal_otel_exporter().await { Ok(()) => { @@ -1572,34 +1573,17 @@ Windmill Community Edition {GIT_VERSION} let otel_tracing_proxy_f = async { #[cfg(all(feature = "private", feature = "enterprise"))] - { - // Start OTEL tracing proxy for HTTP request interception - // Only enabled when: setting is on, worker mode (not server), and single worker (to avoid race conditions) - if worker_mode - && num_workers == 1 - && windmill_worker::OTEL_TRACING_PROXY_SETTINGS - .read() - .await - .enabled - { - if let Some(db) = conn.as_sql() { - tracing::info!( - "Starting jobs OTEL tracing (ports will be dynamically assigned)" - ); - if let Err(e) = - windmill_worker::start_jobs_otel_tracing(db.clone(), otel_killpill_rx) - .await - { - tracing::error!("Jobs OTEL tracing error: {}", e); - } - } - } else if windmill_worker::OTEL_TRACING_PROXY_SETTINGS - .read() + if worker_mode { + if let Some(db) = conn.as_sql() { + if let Err(e) = windmill_worker::start_jobs_otel_tracing( + db.clone(), + otel_killpill_rx, + num_workers, + ) .await - .enabled - && num_workers > 1 - { - tracing::warn!("OTEL tracing proxy is enabled but num_workers > 1. Disabling to avoid race conditions. Set NUM_WORKERS=1 to enable."); + { + tracing::error!("Jobs OTEL tracing error: {}", e); + } } } diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 40c2b94295..cf77467895 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1328,22 +1328,6 @@ try {{ let stream_notifier = StreamNotifier::new(conn, job); - // Set job context for OTEL tracing (EE only) - #[cfg(all(feature = "private", feature = "enterprise"))] - { - let tracing_enabled = - crate::worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets) - .await; - tracing::debug!( - "nativets job {}: OTEL tracing enabled={}", - job.id, - tracing_enabled - ); - if tracing_enabled { - crate::otel_tracing_proxy_ee::set_current_job_context(job.id).await; - } - } - let result = crate::js_eval::eval_fetch_timeout( env_code, inner_content.clone(), diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index b13f01539e..543b8d20c1 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -1135,7 +1135,9 @@ pub async fn eval_fetch_timeout( // We call the function exposed by runtime.js since we can't dynamically import ext: modules. #[cfg(all(feature = "private", feature = "enterprise"))] if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { - if let Err(e) = js_runtime.execute_script("", "globalThis.__bootstrapOtel()") { + if let Err(e) = + js_runtime.execute_script("", "globalThis.__bootstrapOtel()") + { tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); } } @@ -1331,6 +1333,26 @@ async fn eval_fetch( .context("failed to load module")?; let main_override = script_entrypoint_override.unwrap_or("main".to_string()); + + // Inject parent trace context using enterSpan with a duck-typed span object. + // Uses job_id as trace_id so all spans are linked to the job. + // span_id is a placeholder - it gets overwritten by the OTLP handler with the real parent span_id. + #[cfg(all(feature = "private", feature = "enterprise"))] + let otel_context_inject = if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { + let trace_id = job_id.as_simple().to_string(); + format!( +r#"globalThis.__enterSpan?.({{ + isRecording: () => true, + spanContext: () => ({{ traceId: "{trace_id}", spanId: "ffffffffffffffff", traceFlags: 1 }}) +}});"# + ) + } else { + String::new() + }; + + #[cfg(not(all(feature = "private", feature = "enterprise")))] + let otel_context_inject = ""; + let script = js_runtime .execute_script( "", @@ -1343,7 +1365,7 @@ function isAsyncIterable(obj) {{ function processStreamIterative(res) {{ const iterator = res[Symbol.asyncIterator](); - + function processLoop() {{ return new Promise(function(resolve) {{ function step() {{ @@ -1363,10 +1385,12 @@ function processStreamIterative(res) {{ step(); }}); }} - + return processLoop(); }} +{otel_context_inject} + let args = Deno.core.ops.op_get_static_args().map(JSON.parse) import("file:///eval.ts").then((module) => module.{main_override}(...args)) .then(res => {{ diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 060ab7ece8..406a4174dc 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -86,7 +86,7 @@ pub use otel_tracing_proxy_ee::{ set_current_job_context, start_jobs_otel_tracing, TRACING_PROXY_PORT, }; #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] -pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED}; +pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED, OTLP_COLLECTOR_PORT}; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/runtime.js b/backend/windmill-worker/src/runtime.js index f4267a9629..0feafb59ec 100644 --- a/backend/windmill-worker/src/runtime.js +++ b/backend/windmill-worker/src/runtime.js @@ -52,9 +52,12 @@ Object.assign(globalThis, { // Expose bootstrapOtel globally so it can be called from Rust after runtime creation. // We use dynamic import so deno_telemetry isn't loaded during snapshot creation. // Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] +// consoleConfig: 0=ignore, 1=capture, 2=replace globalThis.__bootstrapOtel = () => { - import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap }) => { - bootstrap([1, 0, 0, 0]); + import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { + bootstrap([1, 0, 1, 0]); + // Expose enterSpan for setting parent trace context + globalThis.__enterSpan = enterSpan; }); }; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 4a232981b2..e9a4ebc145 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -675,7 +675,6 @@ async fn get_otel_tracing_proxy_envs() -> anyhow::Result None, }); + // Set job context for OTEL tracing before entering handle_code_execution_job's span + #[cfg(all(feature = "private", feature = "enterprise"))] + if matches!(job.script_lang, Some(ScriptLang::Nativets) | Some(ScriptLang::Bunnative)) + && is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets).await + { + crate::otel_tracing_proxy_ee::set_current_job_context(job.id).await; + } + // Box::pin to move large future to heap let r = Box::pin(handle_code_execution_job( job.as_ref(),