diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 04243ca738..6b640de580 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16095,6 +16095,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-bedrockruntime", "aws-smithy-types", + "axum 0.7.9", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -16149,12 +16150,14 @@ dependencies = [ "postgres-native-tls 0.5.1", "process-wrap", "prometheus", + "prost", "rand 0.9.0", "rcgen", "regex", "reqwest 0.13.1", "reqwest-middleware", "rust_decimal", + "rustls-pemfile 2.2.0", "serde", "serde_json", "sha2 0.10.9", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index a412be58d2..d99e149c1d 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -296,6 +296,7 @@ deno_permissions = "0.49.0" deno_runtime = { version = "0.198.0", features = ["transpile"] } deno_telemetry = "0.12.0" deno_error = "=0.5.5" +rustls-pemfile = "2.2.0" # only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed libffi-sys = { version = "2.3.0", features = ["system"]} @@ -423,6 +424,7 @@ opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] } opentelemetry-appender-tracing = "0.27.0" opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] } opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] } +prost = "0.13" bollard = "0.18.1" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index a6d272f583..f8a88c7599 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -13825341581c1b9eb7a415f0ae25dbc9ba84059f \ No newline at end of file +e9f7a4485b2d21746c6a4a4c38fa1cbac2b942af diff --git a/backend/src/main.rs b/backend/src/main.rs index 6dafd54d74..739eb90993 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -665,6 +665,25 @@ async fn windmill_main() -> anyhow::Result<()> { .flatten() .unwrap_or_else(|| "UNKNOWN".to_string()) ); + + // Load OTEL tracing proxy settings and initialize deno_telemetry if nativets tracing is enabled + // This must happen before any Deno runtime is created + #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] + { + reload_otel_tracing_proxy_setting(&Connection::Sql(db.clone())).await; + + if windmill_worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets).await { + match windmill_worker::load_internal_otel_exporter().await { + Ok(()) => { + tracing::info!("Internal OTEL exporter initialized for nativets tracing"); + } + Err(e) => { + tracing::error!("Failed to initialize internal OTEL exporter: {}", e); + } + } + } + } + load_otel(&db).await; println!("Database connected"); @@ -1565,13 +1584,13 @@ Windmill Community Edition {GIT_VERSION} { if let Some(db) = conn.as_sql() { tracing::info!( - "Starting OTEL tracing proxy (port will be dynamically assigned)" + "Starting jobs OTEL tracing (ports will be dynamically assigned)" ); if let Err(e) = - windmill_worker::start_otel_tracing_proxy(db.clone(), otel_killpill_rx) + windmill_worker::start_jobs_otel_tracing(db.clone(), otel_killpill_rx) .await { - tracing::error!("OTEL tracing proxy error: {}", e); + tracing::error!("Jobs OTEL tracing error: {}", e); } } } else if windmill_worker::OTEL_TRACING_PROXY_SETTINGS diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 9039ea7393..4271e758ca 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -324,7 +324,6 @@ pub async fn initial_load( reload_maven_repos_setting(&conn).await; reload_no_default_maven_setting(&conn).await; reload_ruby_repos_setting(&conn).await; - reload_otel_tracing_proxy_setting(&conn).await; } } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 5790d16b05..4b7d30caf9 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -22,7 +22,7 @@ flow_testing = [] cloud = [] sqlx = [] deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", - "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi"] + "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile"] libffi_mac = ["dep:libffi-sys"] otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"] dind = ["dep:bollard"] @@ -105,6 +105,7 @@ deno_tls = { workspace = true, optional = true } deno_permissions = { workspace = true, optional = true } deno_io = { workspace = true, optional = true } deno_error = { workspace = true, optional = true } +rustls-pemfile = { workspace = true, optional = true } async-stream.workspace = true postgres-native-tls.workspace = true @@ -140,6 +141,8 @@ libloading = { workspace = true, optional = true } opentelemetry-proto.workspace = true opentelemetry = { workspace = true, optional = true } tracing-opentelemetry = { workspace = true, optional = true } +prost.workspace = true +axum.workspace = true bollard = { workspace = true, optional = true } oracle = { workspace = true, optional = true } hudsucker.workspace = true diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index cf77467895..40c2b94295 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1328,6 +1328,22 @@ try {{ let stream_notifier = StreamNotifier::new(conn, job); + // Set job context for OTEL tracing (EE only) + #[cfg(all(feature = "private", feature = "enterprise"))] + { + let tracing_enabled = + crate::worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets) + .await; + tracing::debug!( + "nativets job {}: OTEL tracing enabled={}", + job.id, + tracing_enabled + ); + if tracing_enabled { + crate::otel_tracing_proxy_ee::set_current_job_context(job.id).await; + } + } + let result = crate::js_eval::eval_fetch_timeout( env_code, inner_content.clone(), diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index cce5d8ef5e..b13f01539e 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -1131,6 +1131,15 @@ pub async fn eval_fetch_timeout( let mut js_runtime: JsRuntime = JsRuntime::new(options); // tracing::info!("ttc: {:?}", instant.elapsed()); + // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized. + // We call the function exposed by runtime.js since we can't dynamically import ext: modules. + #[cfg(all(feature = "private", feature = "enterprise"))] + if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) { + if let Err(e) = js_runtime.execute_script("", "globalThis.__bootstrapOtel()") { + tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e); + } + } + js_runtime.add_near_heap_limit_callback(move |x,y| { tracing::error!("heap limit reached: {x} {y}"); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 90aa00bc08..060ab7ece8 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -81,9 +81,12 @@ pub use worker::*; pub use worker_lockfiles::{ process_relative_imports, trigger_dependents_to_recompute_dependencies, }; -pub use otel_tracing_proxy_oss::start_otel_tracing_proxy; #[cfg(all(feature = "private", feature = "enterprise"))] -pub use otel_tracing_proxy_oss::{set_current_job_context, TRACING_PROXY_PORT}; +pub use otel_tracing_proxy_ee::{ + set_current_job_context, start_jobs_otel_tracing, TRACING_PROXY_PORT, +}; +#[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))] +pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED}; pub use result_processor::handle_job_error; diff --git a/backend/windmill-worker/src/otel_tracing_proxy_oss.rs b/backend/windmill-worker/src/otel_tracing_proxy_oss.rs index 8a0e6ba03f..de6e6f187d 100644 --- a/backend/windmill-worker/src/otel_tracing_proxy_oss.rs +++ b/backend/windmill-worker/src/otel_tracing_proxy_oss.rs @@ -1,13 +1 @@ //! OSS stubs for OTEL tracing proxy (EE feature) - -#[cfg(all(feature = "private", feature = "enterprise"))] -pub use crate::otel_tracing_proxy_ee::*; - -/// Start the OTEL tracing proxy (no-op in OSS) -#[cfg(not(all(feature = "private", feature = "enterprise")))] -pub async fn start_otel_tracing_proxy( - _db: windmill_common::DB, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> Result<(), Box> { - Ok(()) -} diff --git a/backend/windmill-worker/src/runtime.js b/backend/windmill-worker/src/runtime.js index 8e8ba6bf26..f4267a9629 100644 --- a/backend/windmill-worker/src/runtime.js +++ b/backend/windmill-worker/src/runtime.js @@ -49,6 +49,15 @@ Object.assign(globalThis, { setTimeout: timers.setTimeout, }); +// Expose bootstrapOtel globally so it can be called from Rust after runtime creation. +// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. +// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] +globalThis.__bootstrapOtel = () => { + import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap }) => { + bootstrap([1, 0, 0, 0]); + }); +}; + // Object.assign(globalThis, { // console: nonEnumerable( // new console.Console((msg, level) => core.print(msg, level > 1)) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 117c051266..4a232981b2 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -675,6 +675,7 @@ async fn get_otel_tracing_proxy_envs() -> anyhow::Result