feat: add otlp/http internal collector (#7690)

* feat: add otlp/http internal collector [merge-ee-first]

Signed-off-by: pyranota <pyra@duck.com>

* remove unused import

Signed-off-by: pyranota <pyra@duck.com>

* chore: update ee-repo-ref to b2e51eceaf00719b7b6794b516d42f20a1d96ff7

This commit updates the EE repository reference after PR #390 was merged in windmill-ee-private.

Previous ee-repo-ref: b576da804a0dd3fadf1ad2fbcc0a0df12684d795

New ee-repo-ref: b2e51eceaf00719b7b6794b516d42f20a1d96ff7

Automated by sync-ee-ref workflow.

* fix panic

Signed-off-by: pyranota <pyra@duck.com>

* finish

Signed-off-by: pyranota <pyra@duck.com>

* chore: update ee-repo-ref to c23a9fb439a71bf574139422b8a43770167ed233

This commit updates the EE repository reference after PR #392 was merged in windmill-ee-private.

Previous ee-repo-ref: 657a61696699d7d9751dc8e30935372bb5af2db7

New ee-repo-ref: c23a9fb439a71bf574139422b8a43770167ed233

Automated by sync-ee-ref workflow.

* defensive programming

Signed-off-by: pyranota <pyra@duck.com>

* ee repo ref

Signed-off-by: pyranota <pyra@duck.com>

* Apply suggestions from code review

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>

* fix silly claude

Signed-off-by: pyranota <pyra@duck.com>

* chore: update ee-repo-ref to e9f7a4485b2d21746c6a4a4c38fa1cbac2b942af

This commit updates the EE repository reference after PR #393 was merged in windmill-ee-private.

Previous ee-repo-ref: bc6c149c01d7063171488b7930f40ffce73a65d3

New ee-repo-ref: e9f7a4485b2d21746c6a4a4c38fa1cbac2b942af

Automated by sync-ee-ref workflow.

---------

Signed-off-by: pyranota <pyra@duck.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>

backend/Cargo.lock

@@ -16095,6 +16095,7 @@ dependencies = [
  "aws-credential-types",
  "aws-sdk-bedrockruntime",
  "aws-smithy-types",
+ "axum 0.7.9",
  "backon",
  "base64 0.22.1",
  "bit-vec 0.6.3",
@@ -16149,12 +16150,14 @@ dependencies = [
  "postgres-native-tls 0.5.1",
  "process-wrap",
  "prometheus",
+ "prost",
  "rand 0.9.0",
  "rcgen",
  "regex",
  "reqwest 0.13.1",
  "reqwest-middleware",
  "rust_decimal",
+ "rustls-pemfile 2.2.0",
  "serde",
  "serde_json",
  "sha2 0.10.9",

backend/Cargo.toml

@@ -296,6 +296,7 @@ deno_permissions = "0.49.0"
 deno_runtime = { version = "0.198.0", features = ["transpile"] }
 deno_telemetry = "0.12.0"
 deno_error = "=0.5.5"
+rustls-pemfile = "2.2.0"
 
 # only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed
 libffi-sys = { version = "2.3.0", features = ["system"]}
@@ -423,6 +424,7 @@ opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
 opentelemetry-appender-tracing = "0.27.0"
 opentelemetry-semantic-conventions = { version = "0.27.0", features = ["semconv_experimental"] }
 opentelemetry-proto = { version = "0.29.0", features = ["with-serde", "gen-tonic"] }
+prost = "0.13"
 
 bollard = "0.18.1"
 

backend/ee-repo-ref.txt

@@ -1 +1 @@
-13825341581c1b9eb7a415f0ae25dbc9ba84059f
+e9f7a4485b2d21746c6a4a4c38fa1cbac2b942af

backend/src/main.rs

@@ -665,6 +665,25 @@ async fn windmill_main() -> anyhow::Result<()> {
                 .flatten()
                 .unwrap_or_else(|| "UNKNOWN".to_string())
         );
+
+        // Load OTEL tracing proxy settings and initialize deno_telemetry if nativets tracing is enabled
+        // This must happen before any Deno runtime is created
+        #[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))]
+        {
+            reload_otel_tracing_proxy_setting(&Connection::Sql(db.clone())).await;
+
+            if windmill_worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets).await {
+                match windmill_worker::load_internal_otel_exporter().await {
+                    Ok(()) => {
+                        tracing::info!("Internal OTEL exporter initialized for nativets tracing");
+                    }
+                    Err(e) => {
+                        tracing::error!("Failed to initialize internal OTEL exporter: {}", e);
+                    }
+                }
+            }
+        }
+
         load_otel(&db).await;
 
         println!("Database connected");
@@ -1565,13 +1584,13 @@ Windmill Community Edition {GIT_VERSION}
                 {
                     if let Some(db) = conn.as_sql() {
                         tracing::info!(
-                            "Starting OTEL tracing proxy (port will be dynamically assigned)"
+                            "Starting jobs OTEL tracing (ports will be dynamically assigned)"
                         );
                         if let Err(e) =
-                            windmill_worker::start_otel_tracing_proxy(db.clone(), otel_killpill_rx)
+                            windmill_worker::start_jobs_otel_tracing(db.clone(), otel_killpill_rx)
                                 .await
                         {
-                            tracing::error!("OTEL tracing proxy error: {}", e);
+                            tracing::error!("Jobs OTEL tracing error: {}", e);
                         }
                     }
                 } else if windmill_worker::OTEL_TRACING_PROXY_SETTINGS

backend/src/monitor.rs

@@ -324,7 +324,6 @@ pub async fn initial_load(
         reload_maven_repos_setting(&conn).await;
         reload_no_default_maven_setting(&conn).await;
         reload_ruby_repos_setting(&conn).await;
-        reload_otel_tracing_proxy_setting(&conn).await;
     }
 }
 

backend/windmill-worker/Cargo.toml

@@ -22,7 +22,7 @@ flow_testing = []
 cloud = []
 sqlx = []
 deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
-    "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi"]
+    "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions", "dep:deno_io", "dep:deno_runtime", "dep:deno_telemetry", "dep:deno_error", "dep:winapi", "dep:rustls-pemfile"]
 libffi_mac = ["dep:libffi-sys"]
 otel = ["windmill-common/otel", "dep:opentelemetry", "dep:tracing-opentelemetry"]
 dind = ["dep:bollard"]
@@ -105,6 +105,7 @@ deno_tls = { workspace = true, optional = true }
 deno_permissions = { workspace = true, optional = true }
 deno_io = { workspace = true, optional = true }
 deno_error = { workspace = true, optional = true }
+rustls-pemfile = { workspace = true, optional = true }
 async-stream.workspace = true
 
 postgres-native-tls.workspace = true
@@ -140,6 +141,8 @@ libloading = { workspace = true, optional = true }
 opentelemetry-proto.workspace = true
 opentelemetry = { workspace = true, optional = true }
 tracing-opentelemetry = { workspace = true, optional = true }
+prost.workspace = true
+axum.workspace = true
 bollard = { workspace = true, optional = true }
 oracle = { workspace = true, optional = true }
 hudsucker.workspace = true

backend/windmill-worker/src/bun_executor.rs

@@ -1328,6 +1328,22 @@ try {{
 
             let stream_notifier = StreamNotifier::new(conn, job);
 
+            // Set job context for OTEL tracing (EE only)
+            #[cfg(all(feature = "private", feature = "enterprise"))]
+            {
+                let tracing_enabled =
+                    crate::worker::is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets)
+                        .await;
+                tracing::debug!(
+                    "nativets job {}: OTEL tracing enabled={}",
+                    job.id,
+                    tracing_enabled
+                );
+                if tracing_enabled {
+                    crate::otel_tracing_proxy_ee::set_current_job_context(job.id).await;
+                }
+            }
+
             let result = crate::js_eval::eval_fetch_timeout(
                 env_code,
                 inner_content.clone(),

backend/windmill-worker/src/js_eval.rs

@@ -1131,6 +1131,15 @@ pub async fn eval_fetch_timeout(
         let mut js_runtime: JsRuntime = JsRuntime::new(options);
         // tracing::info!("ttc: {:?}", instant.elapsed());
 
+        // Bootstrap OpenTelemetry for fetch auto-instrumentation if OTEL was initialized.
+        // We call the function exposed by runtime.js since we can't dynamically import ext: modules.
+        #[cfg(all(feature = "private", feature = "enterprise"))]
+        if crate::DENO_OTEL_INITIALIZED.load(std::sync::atomic::Ordering::SeqCst) {
+            if let Err(e) = js_runtime.execute_script("<otel_bootstrap>", "globalThis.__bootstrapOtel()") {
+                tracing::warn!("Failed to bootstrap OTEL telemetry: {}", e);
+            }
+        }
+
         js_runtime.add_near_heap_limit_callback(move |x,y| {
             tracing::error!("heap limit reached: {x} {y}");
 

backend/windmill-worker/src/lib.rs

@@ -81,9 +81,12 @@ pub use worker::*;
 pub use worker_lockfiles::{
     process_relative_imports, trigger_dependents_to_recompute_dependencies,
 };
-pub use otel_tracing_proxy_oss::start_otel_tracing_proxy;
 #[cfg(all(feature = "private", feature = "enterprise"))]
-pub use otel_tracing_proxy_oss::{set_current_job_context, TRACING_PROXY_PORT};
+pub use otel_tracing_proxy_ee::{
+    set_current_job_context, start_jobs_otel_tracing, TRACING_PROXY_PORT,
+};
+#[cfg(all(feature = "private", feature = "enterprise", feature = "deno_core"))]
+pub use otel_tracing_proxy_ee::{load_internal_otel_exporter, DENO_OTEL_INITIALIZED};
 
 pub use result_processor::handle_job_error;
 

backend/windmill-worker/src/otel_tracing_proxy_oss.rs

@@ -1,13 +1 @@
 //! OSS stubs for OTEL tracing proxy (EE feature)
-
-#[cfg(all(feature = "private", feature = "enterprise"))]
-pub use crate::otel_tracing_proxy_ee::*;
-
-/// Start the OTEL tracing proxy (no-op in OSS)
-#[cfg(not(all(feature = "private", feature = "enterprise")))]
-pub async fn start_otel_tracing_proxy(
-    _db: windmill_common::DB,
-    _killpill_rx: tokio::sync::broadcast::Receiver<()>,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
-    Ok(())
-}

backend/windmill-worker/src/runtime.js

@@ -49,6 +49,15 @@ Object.assign(globalThis, {
   setTimeout: timers.setTimeout,
 });
 
+// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
+// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
+// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
+globalThis.__bootstrapOtel = () => {
+  import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap }) => {
+    bootstrap([1, 0, 0, 0]);
+  });
+};
+
 // Object.assign(globalThis, {
 //   console: nonEnumerable(
 //     new console.Console((msg, level) => core.print(msg, level > 1))

backend/windmill-worker/src/worker.rs

@@ -675,6 +675,7 @@ async fn get_otel_tracing_proxy_envs() -> anyhow::Result<Vec<(&'static str, Stri
     ])
 }
 
+
 #[cfg(windows)]
 lazy_static::lazy_static! {
     pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string());
@@ -2703,6 +2704,19 @@ async fn do_nativets(
 
     let stream_notifier = StreamNotifier::new(conn, job);
 
+    // Set job context for OTEL tracing (EE only)
+    #[cfg(all(feature = "private", feature = "enterprise"))]
+    {
+        let tracing_enabled = is_otel_tracing_proxy_enabled_for_lang(&ScriptLang::Nativets).await;
+        tracing::debug!(
+            "nativets job {}: OTEL tracing enabled={}",
+            job.id, tracing_enabled
+        );
+        if tracing_enabled {
+            crate::otel_tracing_proxy_ee::set_current_job_context(job.id).await;
+        }
+    }
+
     Ok(eval_fetch_timeout(
         env_code,
         code.clone(),

frontend/src/lib/components/instanceSettings.ts

@@ -2,6 +2,7 @@ import type { ButtonType } from './common/button/model'
 
 // Languages that support HTTP request tracing via OTEL proxy
 export const OTEL_TRACING_PROXY_LANGUAGES = [
+	'nativets',
 	'python3',
 	'deno',
 	'bun',