starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-12 08:02:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[ee] chore: upgrade native runtime to current deno crates and v8 150 (#11657)

Browse Source 
* chore: upgrade the native runtime to the current deno crates and v8 150

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep data and blob fetches working under no_network

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: pin the simdutf v8 archive and keep telemetry on http exporters

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: force the http otlp exporter for native telemetry and pin v8

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-10-09 19:54:57 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 0a19d26d5f
commit 93e81ec189
12 changed files with 1485 additions and 895 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+1143 -507
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+24 -28
View File
@@ -451,7 +451,7 @@ headers = "^0"
hyper = { version = "^1", features = ["full"] }
hyper-tls = "^0.6"
hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
tokio = { version = "=1.47.1", features = ["full", "tracing", "time"] }
tokio-stream = { version = "0.1.17" }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
@@ -519,33 +519,29 @@ aws-sdk-rds = "^1"
async-trait = "0.1.88"
v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix
# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0.
# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4,
# the first swc_common patch that dropped `pub use serde::__private as serde;`
# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other
# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`,
# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls
# 0.198+ tightened that to exact `=0.23.28`, which would have made any
# meaningful deno bump resolver-impossible against aws-sdk.
deno_fetch = "0.233.0"
deno_tls = "0.196.0"
deno_console = "0.209.0"
deno_url = "0.209.0"
deno_webidl = "0.209.0"
deno_web = "0.240.0"
# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack
# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer
# deno_crypto would force bumping the whole deno stack.
deno_crypto = "0.223.0"
deno_io = "0.119.0"
deno_fs = "0.119.0"
deno_net = "0.201.0"
deno_core = "0.352.0"
# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core
# 0.412).
# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that
# version: update its version and hash there with any bump. No default features,
# so the pin leaves the feature set deno_core picks untouched.
v8 = { version = "=150.4.0", default-features = false }
# These release as a matched set: each pins its siblings, deno_error exactly,
# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below
# must equal it). A bump moves all of them at once.
# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls
# swc_common 17, which would force bumping every swc_* pin the parsers use.
deno_fetch = "0.283.0"
deno_tls = "0.246.0"
deno_webidl = "0.259.0"
deno_web = "0.290.0"
deno_crypto = "0.273.0"
deno_net = "0.251.0"
deno_core = "0.412.0"
deno_ast = { version = "=0.51.0", features = ["transpiling"] }
deno_permissions = "0.68.0"
deno_telemetry = "0.31.0"
deno_error = "=0.6.1"
deno_permissions = "0.118.0"
deno_telemetry = "0.81.0"
deno_error = "=0.7.1"
sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] }
rustls-pemfile = "2.2.0"
# only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed
@@ -682,7 +678,7 @@ aws-sdk-sqs = "^1.89.0"
aws-sdk-sts = "^1.91.0"
aws-sdk-sso = "=1.77.0"
aws-sdk-ssooidc = "=1.78.0"
rustls = "=0.23.35"
rustls = "=0.23.40"
async-once-cell = "0.5.4"
size = "0.5.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
5d7726c4979070ee8db4fb9e8014152872a641ba
599eae0034b34c5e0d3d42df37cf18bbe01f9b49
backend/windmill-runtime-nativets/Cargo.toml
+3 -10
View File
@@ -22,16 +22,15 @@ deno_webidl.workspace = true
deno_web.workspace = true
deno_crypto.workspace = true
deno_net.workspace = true
deno_console.workspace = true
deno_url.workspace = true
deno_core.workspace = true
deno_ast.workspace = true
deno_tls.workspace = true
deno_permissions.workspace = true
deno_io.workspace = true
deno_fs.workspace = true
deno_telemetry.workspace = true
deno_error.workspace = true
sys_traits.workspace = true
# Not used directly: depending on it is what makes the workspace pin bind.
v8.workspace = true
winapi.workspace = true
itertools.workspace = true
@@ -58,14 +57,8 @@ deno_webidl.workspace = true
deno_web.workspace = true
deno_crypto.workspace = true
deno_net.workspace = true
deno_console.workspace = true
deno_url.workspace = true
deno_core.workspace = true
deno_ast.workspace = true
deno_tls.workspace = true
deno_permissions.workspace = true
deno_io.workspace = true
deno_fs.workspace = true
deno_telemetry.workspace = true
deno_error.workspace = true
winapi.workspace = true
backend/windmill-runtime-nativets/build.rs
+85 -137
View File
@@ -1,107 +1,11 @@
use deno_ast::{MediaType, ParseParams};
use deno_core::{ModuleCodeString, ModuleName, SourceMapData};
use deno_core::{Extension, ModuleCodeString, ModuleName, SourceMapData};
use deno_error::JsErrorBox;
use deno_fetch::FetchPermissions;
use deno_net::NetPermissions;
use deno_web::{BlobStore, TimersPermission};
use std::borrow::Cow;
use std::collections::HashSet;
use std::env;
use std::fmt::Write as _;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::Arc;
pub struct PermissionsContainer;
impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_net_url(
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
#[inline(always)]
fn check_read<'a>(
&mut self,
_path: Cow<'a, Path>,
_api_name: &str,
_get_path: &'a dyn deno_fs::GetPath,
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
unreachable!("snapshotting")
}
#[inline(always)]
fn check_write<'a>(
&mut self,
_path: Cow<'a, Path>,
_api_name: &str,
_get_path: &'a dyn deno_fs::GetPath,
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
unreachable!("snapshotting")
}
#[inline(always)]
fn check_net_vsock(
&mut self,
_cid: u32,
_port: u32,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
}
impl TimersPermission for PermissionsContainer {
#[inline(always)]
fn allow_hrtime(&mut self) -> bool {
true
}
}
impl NetPermissions for PermissionsContainer {
fn check_read(
&mut self,
_p: &str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write(
&mut self,
_p: &str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_net<T: AsRef<str>>(
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_write_path<'a>(
&mut self,
_p: Cow<'a, Path>,
_api_name: &str,
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
fn check_vsock(
&mut self,
_cid: u32,
_port: u32,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
}
deno_core::extension!(
fetch,
@@ -109,38 +13,29 @@ deno_core::extension!(
esm = ["src/runtime.js"],
);
// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`.
// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`,
// also applied to the residual lazy sources below.
//
// Specialized to our snapshot's inputs. Of the eight deno_* extensions
// we register via `init()`, seven ship pre-built `.js` files
// in their `esm` lists (webidl/url/console/web/crypto/fetch/net) — only
// `deno_telemetry`'s `extension!` macro lists `.ts` files
// (`telemetry.ts`, `util.ts`), so the TypeScript branch is needed
// solely for that crate. Our local `fetch` extension contributes
// `src/runtime.js` (pure JS). No `node:` imports happen at snapshot
// build time, no `.mjs`, no user-supplied modules. So:
// Specialized to our inputs: the deno_* extensions ship `.js`, except
// `deno_telemetry` whose lazy scripts are `.ts`. Our local `fetch` extension
// contributes `src/runtime.js`. No `node:` imports, no `.mjs`, no
// user-supplied modules. So:
// - `.js` → pass through.
// - `.ts` → transpile via deno_ast (deno_telemetry only).
// - anything else → build bug (deno shipping an unexpected file type
// or us mislabelling one), panic loudly rather than emit a broken
// snapshot.
//
// No source maps: the snapshot is a binary blob the runtime loads — source
// maps would never be consumed.
// No source maps: nothing consumes them.
//
// The signature still returns `Result<_, JsErrorBox>` because that's what
// `extension_transpiler` expects, but we never construct one — parse and
// transpile failures are build-time bugs in deno's own .ts internals (or
// in our runtime.js, if we ever change its extension), so they panic.
// The signature returns `Result<_, JsErrorBox>` because that's what
// `extension_transpiler` expects, but we never construct one: parse and
// transpile failures are build-time bugs, so they panic.
//
// This replaces a call to `deno_runtime::transpile::maybe_transpile_source`
// from `deno_runtime 0.198.0`. The original is more general (handles
// `node:` modules, `.mjs`, emits source maps in debug builds, plumbs
// errors via `JsErrorBox`); none of that surface is reachable in our
// build. Dropping the `deno_runtime` dep eliminates a
// `deno_cache → rusqlite → libsqlite3-sys 0.35` transitive chain that
// collides with sqlx-sqlite's `libsqlite3-sys 0.30` (cargo's
// `links = "sqlite3"` rule).
// `deno_runtime::transpile::maybe_transpile_source` is the general version;
// depending on `deno_runtime` would pull a
// `deno_cache → rusqlite → libsqlite3-sys` chain that collides with
// sqlx-sqlite's `libsqlite3-sys` (cargo's `links = "sqlite3"` rule).
fn maybe_transpile_source(
name: ModuleName,
source: ModuleCodeString,
@@ -154,7 +49,7 @@ fn maybe_transpile_source(
let parsed = deno_ast::parse_module(ParseParams {
specifier: deno_core::url::Url::parse(&name).unwrap(),
text: source.into(),
text: source.as_str().into(),
media_type,
capture_tokens: false,
scope_analysis: false,
@@ -169,7 +64,13 @@ fn maybe_transpile_source(
..Default::default()
},
&deno_ast::TranspileModuleOptions::default(),
&deno_ast::EmitOptions::default(),
// The default appends an inline source map comment, which is a
// syntax error where deno_core evaluates a lazy script as an
// expression.
&deno_ast::EmitOptions {
source_map: deno_ast::SourceMapOption::None,
..Default::default()
},
)
.unwrap_or_else(|e| panic!("snapshot transpile: emit failed for {name}: {e}"))
.into_source();
@@ -177,27 +78,66 @@ fn maybe_transpile_source(
Ok((transpiled.text.into(), None))
}
/// One `(specifier, source)` table of the extension scripts the snapshot did
/// not evaluate. The snapshot only carries scripts that ran while it was built,
/// so a script first loaded in a live isolate has to come from here.
///
/// deno_core binary-searches the table and reads each source as ASCII without
/// checking, so it must be sorted by specifier, free of duplicates, and ASCII.
fn residual_table(
name: &str,
files: impl Iterator<Item = deno_core::ExtensionFileSource>,
consumed: &HashSet<String>,
) -> String {
let mut files: Vec<_> = files.filter(|f| !consumed.contains(f.specifier)).collect();
files.sort_by_key(|f| f.specifier);
files.dedup_by_key(|f| f.specifier);
let mut out = format!("pub static {name}: &[(&str, &str)] = &[\n");
for file in files {
let source = file
.load()
.unwrap_or_else(|e| panic!("could not load {}: {e}", file.specifier));
let (code, _) =
maybe_transpile_source(ModuleName::from_static(file.specifier), source).unwrap();
assert!(
code.as_str().is_ascii(),
"{} is not ASCII after transpiling",
file.specifier
);
writeln!(out, " ({:?}, {:?}),", file.specifier, code.as_str()).unwrap();
}
out.push_str("];\n");
out
}
fn main() {
println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
deno_telemetry::deno_telemetry::init(),
deno_webidl::deno_webidl::init(),
deno_url::deno_url::init(),
deno_console::deno_console::init(),
deno_web::deno_web::init::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
deno_crypto::deno_crypto::init(None),
deno_fetch::deno_fetch::init::<PermissionsContainer>(Default::default()),
deno_net::deno_net::init::<PermissionsContainer>(None, None),
// Same list and order as `create_nativets_runtime`, which supplies their
// state through `lazy_init_extensions`.
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::lazy_init(),
deno_webidl::deno_webidl::lazy_init(),
deno_web::deno_web::lazy_init(),
deno_crypto::deno_crypto::lazy_init(),
deno_fetch::deno_fetch::lazy_init(),
deno_net::deno_net::lazy_init(),
fetch::init(),
];
// Build the file path to the snapshot.
let lazy_js: Vec<_> = exts
.iter()
.flat_map(|e| e.lazy_loaded_js_files.iter().cloned())
.collect();
let lazy_esm: Vec<_> = exts
.iter()
.flat_map(|e| e.lazy_loaded_esm_files.iter().cloned())
.collect();
let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
let snapshot_path = o.join("FETCH_SNAPSHOT.bin");
// Create the snapshot.
let output = deno_core::snapshot::create_snapshot(
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
@@ -213,9 +153,17 @@ fn main() {
)
.unwrap();
let mut file = std::fs::File::create(snapshot_path).unwrap();
let mut file = std::fs::File::create(o.join("FETCH_SNAPSHOT.bin")).unwrap();
file.write_all(&output.output).unwrap();
let consumed: HashSet<String> = output.consumed_lazy_specifiers.into_iter().collect();
let residual = format!(
"{}{}",
residual_table("RESIDUAL_LAZY_JS", lazy_js.into_iter(), &consumed),
residual_table("RESIDUAL_LAZY_ESM", lazy_esm.into_iter(), &consumed),
);
std::fs::write(o.join("residual_lazy_sources.rs"), residual).unwrap();
for path in output.files_loaded_during_snapshot {
println!("cargo:rerun-if-changed={}", path.display());
}
backend/windmill-runtime-nativets/src/lib.rs
+110 -159
View File
@@ -22,9 +22,7 @@ mod smoke_tests;
mod cert_tests;
use std::{
borrow::Cow,
cell::RefCell,
path::PathBuf,
rc::Rc,
sync::{Arc, LazyLock, Mutex},
};
@@ -32,17 +30,55 @@ use std::{
// Re-export deno_telemetry for use by windmill-worker's otel proxy
pub use deno_telemetry;
/// The real environment and filesystem, except that the OTLP protocol always
/// reads as HTTP.
///
/// The caller points `OTEL_EXPORTER_OTLP_ENDPOINT` at its own HTTP collector
/// only while [`init_telemetry`] runs, and the instance's own exporter shares
/// `OTEL_EXPORTER_OTLP_PROTOCOL`. Honouring `grpc` there would build an exporter
/// that reads the endpoint again on every export, and `console` one that never
/// reads it, so either would send job telemetry past that collector.
struct HttpOtlpSys;
impl sys_traits::BaseEnvVar for HttpOtlpSys {
fn base_env_var_os(&self, key: &std::ffi::OsStr) -> Option<std::ffi::OsString> {
if key == "OTEL_EXPORTER_OTLP_PROTOCOL" {
return Some("http/protobuf".into());
}
sys_traits::impls::RealSys.base_env_var_os(key)
}
}
impl sys_traits::BaseFsRead for HttpOtlpSys {
fn base_fs_read(
&self,
path: &std::path::Path,
) -> std::io::Result<std::borrow::Cow<'static, [u8]>> {
sys_traits::impls::RealSys.base_fs_read(path)
}
}
/// `deno_telemetry::init` for the isolates of this process.
pub fn init_telemetry(
rt_config: deno_telemetry::OtelRuntimeConfig,
config: deno_telemetry::OtelConfig,
) -> anyhow::Result<()> {
deno_telemetry::init(&HttpOtlpSys, rt_config, config)
}
use deno_ast::ParseParams;
use deno_core::{
error::CoreErrorKind,
op2, serde_v8, url,
v8::{self, IsolateHandle},
Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
};
use deno_error::JsErrorBox;
use deno_fetch::FetchPermissions;
use deno_net::NetPermissions;
use deno_permissions::{
Permissions, PermissionsContainer, PermissionsOptions, RuntimePermissionDescriptorParser,
};
use deno_tls::{rustls::pki_types::CertificateDer, rustls::RootCertStore, RootCertStoreProvider};
use deno_web::{BlobStore, TimersPermission};
use deno_web::{BlobStore, InMemoryBroadcastChannel};
use itertools::Itertools;
use lazy_static::lazy_static;
use regex::Regex;
@@ -57,156 +93,55 @@ use windmill_common::worker::{write_file, Connection, WINDMILL_DIR};
// ── Snapshot-matched extensions ──────────────────────────────────────
//
// `deno_core` 0.352 validates that the snapshot's extension list is a
// *prefix* of the runtime's extension list (snapshot does not need an
// exact match — runtime is allowed to add extensions at the tail, but
// must not reorder or omit any that the snapshot baked in).
// The snapshot (build.rs) is built from the deno_* extensions in their
// `lazy_init()` form followed by this local `fetch` ext. deno_core requires the
// runtime's extension list to start with that same list in that same order; it
// may append more, which is how the windmill `ext` carrying our own ops gets in.
//
// Our snapshot (in build.rs) is the same eight deno_* extensions ending
// with this local `fetch` ext. The runtime adds one extra entry at the
// end — the windmill `ext` carrying our own ops — which is fine because
// it's after the snapshot prefix.
//
// This local `fetch` extension declaration must be present in both
// build.rs and lib.rs so the type passes through the `init()` macro.
// The ESM is already in the snapshot, so this `init()` call at runtime
// is a no-op for esm — the registration just records the ext.
// This declaration must exist in both build.rs and lib.rs. The ESM is already
// in the snapshot, so `init()` at runtime only records the extension.
deno_core::extension!(
fetch,
esm_entry_point = "ext:fetch/src/runtime.js",
esm = ["src/runtime.js"],
);
// ── Permission container ─────────────────────────────────────────────
include!(concat!(env!("OUT_DIR"), "/residual_lazy_sources.rs"));
/// `no_network` is what `//no_network` promises: no outbound traffic of any kind,
/// the Windmill API included (the client module reaches it through `fetch`).
/// Enforced here and by disabling the deno_net ops, never in JS: user code can
/// call `Deno.core.ops.*` directly and skip any JS-level guard.
pub struct PermissionsContainer {
pub no_network: bool,
}
const NO_NETWORK_ANNOTATION: &str = "no_network";
// ── Permissions ──────────────────────────────────────────────────────
/// Every op deno_net registers, see `create_nativets_runtime`. Read from the
/// extension itself so an upgrade that adds an op cannot leave it enabled.
static NO_NETWORK_DISABLED_OPS: LazyLock<std::collections::HashSet<&'static str>> =
LazyLock::new(|| {
deno_net::deno_net::init::<PermissionsContainer>(None, None)
deno_net::deno_net::lazy_init()
.ops
.iter()
.map(|op| op.name)
.collect()
});
impl PermissionsContainer {
fn deny_net(&self, target: &str) -> Result<(), deno_permissions::PermissionCheckError> {
if self.no_network {
Err(deno_permissions::PermissionDeniedError::Fatal {
access: format!(
"net access to {target} (the script is annotated //{NO_NETWORK_ANNOTATION})"
),
}
.into())
} else {
Ok(())
}
}
}
impl FetchPermissions for PermissionsContainer {
#[inline(always)]
fn check_net_url(
&mut self,
url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
self.deny_net(&format!("\"{}\"", url.host_str().unwrap_or_default()))
}
#[inline(always)]
fn check_read<'a>(
&mut self,
path: Cow<'a, std::path::Path>,
_api_name: &str,
_get_path: &'a dyn deno_fs::GetPath,
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
Ok(deno_fs::CheckedPath::Unresolved(path))
}
#[inline(always)]
fn check_write<'a>(
&mut self,
path: Cow<'a, std::path::Path>,
_api_name: &str,
_get_path: &'a dyn deno_fs::GetPath,
) -> Result<deno_fs::CheckedPath<'a>, deno_io::fs::FsError> {
Ok(deno_fs::CheckedPath::Unresolved(path))
}
#[inline(always)]
fn check_net_vsock(
&mut self,
cid: u32,
port: u32,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
self.deny_net(&format!("vsock {cid}:{port}"))
}
}
impl TimersPermission for PermissionsContainer {
#[inline(always)]
fn allow_hrtime(&mut self) -> bool {
true
}
}
impl NetPermissions for PermissionsContainer {
fn check_read(
&mut self,
p: &str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
self.deny_net(&format!("unix socket \"{p}\""))?;
Ok(PathBuf::from(p))
}
fn check_write(
&mut self,
p: &str,
_api_name: &str,
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
self.deny_net(&format!("unix socket \"{p}\""))?;
Ok(PathBuf::from(p))
}
fn check_net<T: AsRef<str>>(
&mut self,
host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
self.deny_net(&format!("\"{}\"", host.0.as_ref()))
}
fn check_write_path<'a>(
&mut self,
p: Cow<'a, std::path::Path>,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
self.deny_net(&format!("unix socket \"{}\"", p.display()))?;
Ok(p)
}
fn check_vsock(
&mut self,
cid: u32,
port: u32,
_api_name: &str,
) -> Result<(), deno_permissions::PermissionCheckError> {
self.deny_net(&format!("vsock {cid}:{port}"))
}
/// What an isolate may do: reach the network, unless `no_network`, and nothing
/// else (no filesystem, environment, subprocess or FFI). Nothing prompts.
///
/// `no_network` is what `//no_network` promises: no outbound traffic of any
/// kind, the Windmill API included (the client module reaches it through
/// `fetch`). Enforced here and by disabling the deno_net ops, never in JS: user
/// code can call `Deno.core.ops.*` directly and skip any JS-level guard.
fn isolate_permissions(no_network: bool) -> anyhow::Result<PermissionsContainer> {
let parser = Arc::new(RuntimePermissionDescriptorParser::new(
sys_traits::impls::RealSys,
));
// An empty list means "all hosts", for a grant and for a denial alike.
let options = PermissionsOptions {
allow_net: (!no_network).then(Vec::new),
deny_net: no_network.then(Vec::new),
prompt: false,
..Default::default()
};
let permissions = Permissions::from_options(parser.as_ref(), &options)
.map_err(|e| anyhow::anyhow!("invalid isolate permissions: {e}"))?;
Ok(PermissionsContainer::new(parser, permissions))
}
// ── Types ────────────────────────────────────────────────────────────
@@ -228,7 +163,7 @@ pub struct NativeAnnotation {
/// script; `None` leaves the default in force.
pub fetch_response_timeout_secs: Option<u64>,
/// `//no_network`: the isolate may open no connection at all, see
/// [`PermissionsContainer`].
/// [`isolate_permissions`].
pub no_network: bool,
}
@@ -413,13 +348,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
// deno_fetch requires a TLS provider; install ring as default (idempotent).
let _ = rustls::crypto::ring::default_provider().install_default();
let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
"--stack-size=1024".to_string(),
"--no-harmony-import-assertions".to_string(),
])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
let unrecognized_v8_flags = deno_core::v8_set_flags(vec!["--stack-size=1024".to_string()])
.into_iter()
.skip(1)
.collect::<Vec<_>>();
if !unrecognized_v8_flags.is_empty() {
init_err = Some(format!(
@@ -435,7 +367,7 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> {
// SIGSEGV in WasmCodePointerTable::AllocateUninitializedEntry() on x86_64 Linux.
// See: https://github.com/denoland/deno_core/issues/952
let platform = deno_core::v8::new_unprotected_default_platform(0, false).make_shared();
deno_core::JsRuntime::init_platform(Some(platform), false);
deno_core::JsRuntime::init_platform(Some(platform));
});
if let Some(msg) = init_err {
@@ -666,17 +598,15 @@ pub(crate) fn create_nativets_runtime(
..Default::default()
};
// Same list and order as the snapshot (build.rs); their state follows in
// `lazy_init_extensions` below, which must match it too.
let exts: Vec<Extension> = vec![
deno_telemetry::deno_telemetry::init(),
deno_webidl::deno_webidl::init(),
deno_url::deno_url::init(),
deno_console::deno_console::init(),
deno_web::deno_web::init::<PermissionsContainer>(Arc::new(BlobStore::default()), None),
// Registered after deno_web to keep the snapshot (build.rs) a prefix of
// the runtime extension list; deno_crypto declares deps = [deno_webidl, deno_web].
deno_crypto::deno_crypto::init(None),
deno_fetch::deno_fetch::init::<PermissionsContainer>(fetch_options),
deno_net::deno_net::init::<PermissionsContainer>(None, None),
deno_telemetry::deno_telemetry::lazy_init(),
deno_webidl::deno_webidl::lazy_init(),
deno_web::deno_web::lazy_init(),
deno_crypto::deno_crypto::lazy_init(),
deno_fetch::deno_fetch::lazy_init(),
deno_net::deno_net::lazy_init(),
fetch::init(),
ext,
];
@@ -688,6 +618,8 @@ pub(crate) fn create_nativets_runtime(
deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128),
),
startup_snapshot: Some(RUNTIME_SNAPSHOT),
residual_lazy_js_sources: RESIDUAL_LAZY_JS,
residual_lazy_esm_sources: RESIDUAL_LAZY_ESM,
module_loader: Some(Rc::new(deno_core::FsModuleLoader)),
extension_transpiler: None,
..Default::default()
@@ -714,12 +646,28 @@ pub(crate) fn create_nativets_runtime(
y * 2
});
js_runtime
.lazy_init_extensions(vec![
deno_telemetry::deno_telemetry::args(),
deno_webidl::deno_webidl::args(),
deno_web::deno_web::args(
Arc::new(BlobStore::default()),
None,
false,
InMemoryBroadcastChannel::default(),
),
deno_crypto::deno_crypto::args(None),
deno_fetch::deno_fetch::args(fetch_options),
deno_net::deno_net::args(None, None),
])
.map_err(windmill_common::error::to_anyhow)?;
let (log_sender, log_receiver) = mpsc::unbounded_channel::<String>();
{
let op_state = js_runtime.op_state();
let mut op_state = op_state.borrow_mut();
op_state.put(PermissionsContainer { no_network });
op_state.put(isolate_permissions(no_network)?);
op_state.put(MainArgs { args: initial_args });
op_state.put(LogString { s: log_sender });
}
@@ -731,7 +679,7 @@ pub(crate) fn create_nativets_runtime(
.execute_script(
"<wm_init>",
format!(
"globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})"
"globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms}, noNetwork: {no_network} }})"
),
)
.map_err(windmill_common::error::to_anyhow)?;
@@ -777,7 +725,7 @@ pub(crate) fn extract_global_string(
js_runtime: &mut JsRuntime,
global: v8::Global<v8::Value>,
) -> Result<Box<RawValue>, String> {
let scope = &mut js_runtime.handle_scope();
deno_core::scope!(scope, js_runtime);
let local = v8::Local::new(scope, global);
match serde_v8::from_v8::<Option<String>>(scope, local) {
Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))),
@@ -1169,7 +1117,10 @@ import("file:///eval.ts").then((module) => module.{main_fn}(...args))
Ok(global) => {
extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e))
}
Err(deno_core::error::CoreError::Js(e)) => {
Err(e) if matches!(e.as_kind(), CoreErrorKind::Js(_)) => {
let CoreErrorKind::Js(e) = e.into_kind() else {
unreachable!("matched as a JS error above")
};
let source = e.frames.first().and_then(|f| {
f.file_name
.as_ref()
backend/windmill-runtime-nativets/src/runtime.js
+82 -39
View File
@@ -1,39 +1,42 @@
import * as abortSignal from "ext:deno_web/03_abort_signal.js";
import * as domException from "ext:deno_web/01_dom_exception.js";
import * as base64 from "ext:deno_web/05_base64.js";
import * as console from "ext:deno_console/01_console.js";
import * as encoding from "ext:deno_web/08_text_encoding.js";
import * as event from "ext:deno_web/02_event.js";
import * as fetch from "ext:deno_fetch/26_fetch.js";
import * as file from "ext:deno_web/09_file.js";
import * as fileReader from "ext:deno_web/10_filereader.js";
import * as formData from "ext:deno_fetch/21_formdata.js";
import * as headers from "ext:deno_fetch/20_headers.js";
import * as streams from "ext:deno_web/06_streams.js";
import * as timers from "ext:deno_web/02_timers.js";
import * as url from "ext:deno_url/00_url.js";
import * as net from "ext:deno_net/01_net.js";
import * as tls from "ext:deno_net/02_tls.js";
import * as urlPattern from "ext:deno_url/01_urlpattern.js";
import * as webidl from "ext:deno_webidl/00_webidl.js";
import * as crypto from "ext:deno_crypto/00_crypto.js";
import * as response from "ext:deno_fetch/23_response.js";
import * as request from "ext:deno_fetch/23_request.js";
import "ext:deno_web/02_structured_clone.js";
import * as globalInterfaces from "ext:deno_web/04_global_interfaces.js";
// Namespace imports (not side-effect-only) so their constructors are reachable
// for the globalThis wiring below. The module bodies still execute on
// evaluation, so their side effects apply.
import * as messagePort from "ext:deno_web/13_message_port.js";
import * as compression from "ext:deno_web/14_compression.js";
import * as performance from "ext:deno_web/15_performance.js";
import "ext:deno_web/16_image_data.js";
import "ext:deno_fetch/27_eventsource.js";
import { core, internals } from "ext:core/mod.js";
// deno_crypto brands CryptoKeys with this symbol and reads it from `internals`
// when it evaluates; in Deno a deno_node module sets it, which is not loaded here.
internals.kKeyObject ??= Symbol("kKeyObject");
// Every script loaded here runs at snapshot-build time and is baked into the
// snapshot. One that is first loaded at run time comes from the residual table
// build.rs emits instead.
const abortSignal = core.loadExtScript("ext:deno_web/03_abort_signal.js");
const domException = core.loadExtScript("ext:deno_web/01_dom_exception.js");
const base64 = core.loadExtScript("ext:deno_web/05_base64.js");
const console = core.loadExtScript("ext:deno_web/01_console.js");
const encoding = core.loadExtScript("ext:deno_web/08_text_encoding.js");
const event = core.loadExtScript("ext:deno_web/02_event.js");
const fetch = core.loadExtScript("ext:deno_fetch/26_fetch.js");
const file = core.loadExtScript("ext:deno_web/09_file.js");
const fileReader = core.loadExtScript("ext:deno_web/10_filereader.js");
const formData = core.loadExtScript("ext:deno_fetch/21_formdata.js");
const headers = core.loadExtScript("ext:deno_fetch/20_headers.js");
const streams = core.loadExtScript("ext:deno_web/06_streams.js");
const timers = core.loadExtScript("ext:deno_web/02_timers.js");
const url = core.loadExtScript("ext:deno_web/00_url.js");
const urlPattern = core.loadExtScript("ext:deno_web/01_urlpattern.js");
const webidl = core.loadExtScript("ext:deno_webidl/00_webidl.js");
const crypto = core.loadExtScript("ext:deno_crypto/00_crypto.js");
const response = core.loadExtScript("ext:deno_fetch/23_response.js");
const request = core.loadExtScript("ext:deno_fetch/23_request.js");
const globalInterfaces = core.loadExtScript(
"ext:deno_web/04_global_interfaces.js",
);
const messagePort = core.loadExtScript("ext:deno_web/13_message_port.js");
const compression = core.loadExtScript("ext:deno_web/14_compression.js");
const performance = core.loadExtScript("ext:deno_web/15_performance.js");
// deno_fetch applies no deadline, so a peer that accepts a request and then
// never answers leaves `await fetch(...)` pending until the job timeout, which
// self-hosted defaults to 7 days.
const ORIGINAL_FETCH = fetch.fetch;
const DENO_FETCH = fetch.fetch;
// deno_web's timers reject any `this` other than undefined/globalThis, so
// `timers.setTimeout(...)` passes the module namespace and throws "Illegal
@@ -50,6 +53,14 @@ const abortControllerAbort = Function.prototype.call.bind(
);
const ReflectApply = Reflect.apply;
// Forwards the original argument count, like every caller below relies on. The
// extra promise hop under `no_network` costs the same-tick settling of an
// aborted fetch, which only matters where a request could have been made.
function ORIGINAL_FETCH() {
const p = ReflectApply(DENO_FETCH, undefined, arguments);
return noNetwork ? promiseThen(p, undefined, rewordNoNetworkDenial) : p;
}
// Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop
// for the impossible case of fetch running before that init.
let fetchResponseTimeoutMs = 900_000;
@@ -74,6 +85,21 @@ function fetchResponseTimeoutError(requestUrl, timeoutMs) {
);
}
// Set per isolate for a `//no_network` script. The denial itself is the
// isolate's permissions (Rust); this only replaces deno's "run again with the
// --allow-net flag" hint, which names a flag a script author cannot pass.
let noNetwork = false;
function rewordNoNetworkDenial(e) {
if (e?.name === "NotCapable" && typeof e.message === "string") {
e.message = e.message.replace(
", run again with the --allow-net flag",
": the script is annotated //no_network",
);
}
throw e;
}
globalThis.atob = base64.atob;
globalThis.btoa = base64.btoa;
// Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT
@@ -152,7 +178,22 @@ globalThis.console = new console.Console((msg, level) =>
);
globalThis.AbortController = abortSignal.AbortController;
globalThis.AbortSignal = abortSignal.AbortSignal;
globalThis.crypto = crypto.crypto;
// A getter, not a value: `crypto.crypto` mints a cppgc object, and the cppgc
// heap is not attached while this module runs at snapshot-build time.
Object.defineProperty(globalThis, "crypto", {
configurable: true,
enumerable: true,
get: () => crypto.crypto,
// Assignment has to keep working, as it does on a plain property.
set(value) {
Object.defineProperty(globalThis, "crypto", {
configurable: true,
enumerable: true,
writable: true,
value,
});
},
});
globalThis.Crypto = crypto.Crypto;
globalThis.CryptoKey = crypto.CryptoKey;
globalThis.SubtleCrypto = crypto.SubtleCrypto;
@@ -233,6 +274,7 @@ globalThis.__wmInitPerIsolate = (config) => {
if (config != null && typeof config.fetchResponseTimeoutMs === "number") {
fetchResponseTimeoutMs = config.fetchResponseTimeoutMs;
}
noNetwork = config?.noNetwork === true;
// setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock;
// without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN.
@@ -256,13 +298,14 @@ globalThis.__wmInitPerIsolate = (config) => {
};
// Expose bootstrapOtel globally so it can be called from Rust after runtime creation.
// We use dynamic import so deno_telemetry isn't loaded during snapshot creation.
// Loaded on first call so deno_telemetry isn't evaluated during snapshot creation.
// Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic]
// consoleConfig: 0=ignore, 1=capture, 2=replace
globalThis.__bootstrapOtel = () => {
import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => {
bootstrap([1, 0, 1, 0]);
// Expose enterSpan for setting parent trace context
globalThis.__enterSpan = enterSpan;
});
const { bootstrap, enterSpan } = core.loadExtScript(
"ext:deno_telemetry/telemetry.ts",
);
bootstrap([1, 0, 1, 0]);
// Expose enterSpan for setting parent trace context
globalThis.__enterSpan = enterSpan;
};
backend/windmill-runtime-nativets/src/smoke_tests.rs
+4
View File
@@ -451,6 +451,10 @@ export async function main(): Promise<{ uuid: string; nonzero: boolean; sha256:
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
// `crypto` is installed as an accessor; polyfill-style assignment must
// still work rather than throw for want of a setter.
(globalThis as any).crypto = globalThis.crypto;
return { uuid, nonzero, sha256 };
}
"#;
backend/windmill-runtime-nativets/tests/no_network.rs
+12
View File
@@ -55,6 +55,9 @@ async function attempt(f) {{
export async function main() {{
return {{
fetch: await attempt(() => fetch("http://127.0.0.1:{port}/").then((r) => r.text())),
data_url: await attempt(() => fetch("data:text/plain,ok").then((r) => r.text())),
blob_url: await attempt(() => fetch(URL.createObjectURL(new Blob(["ok"]))).then((r) => r.text())),
raw_fetch: await attempt(() => Deno.core.ops.op_fetch("GET", "http://127.0.0.1:{port}/", [], null, false, null, null)),
tcp: await attempt(() => Deno.core.ops.op_net_connect_tcp({{ hostname: "127.0.0.1", port: {port} }}, null, null)),
unix: await attempt(() => Deno.core.ops.op_net_connect_unix("/var/run/docker.sock")),
quic: await attempt(() => Deno.core.ops.op_quic_endpoint_create({{ hostname: "localhost", port: 0 }}, true)),
@@ -76,6 +79,15 @@ async fn no_network_denies_fetch_and_raw_socket_ops() {
fetch.contains("no_network"),
"fetch was not denied: {fetch}"
);
// Neither opens a connection, so neither is network access.
assert_eq!(out["data_url"], "allowed", "{out}");
assert_eq!(out["blob_url"], "allowed", "{out}");
// The global `fetch` only words the refusal; the op under it is what refuses.
let raw_fetch = out["raw_fetch"].as_str().unwrap();
assert!(
raw_fetch.contains("Requires net access"),
"the fetch op was not denied: {raw_fetch}"
);
for key in ["tcp", "unix", "quic"] {
let msg = out[key].as_str().unwrap();
assert!(
backend/windmill-runtime-nativets/tests/otel_e2e.rs
+7 -8
View File
@@ -71,7 +71,7 @@ async fn spawn_capturing_http(captured: Arc<Mutex<Vec<Vec<u8>>>>) -> u16 {
/// drifts away from `tracing_enabled + Capture`, this test breaks
/// before the customer-facing panic does.
fn init_with_ee_otel_config() {
deno_telemetry::init(
windmill_runtime_nativets::init_telemetry(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets".into(),
runtime_version: "0".into(),
@@ -157,13 +157,12 @@ export async function main(): Promise<number> {{
"fetch target should have been hit exactly once"
);
// 4. Force the BatchSpanProcessor to flush so the exporter posts
// to our mock collector synchronously (default flush interval
// is ~5s; tests can't wait that long).
deno_telemetry::flush();
// Exporter is async over the OTel runtime; give it a beat to
// actually send the HTTP request.
tokio::time::sleep(Duration::from_millis(500)).await;
// 4. The BatchSpanProcessor exports on its own schedule (about 5s by
// default) and deno_telemetry exposes no flush, so wait for it.
let deadline = tokio::time::Instant::now() + Duration::from_secs(20);
while otlp_hits.lock().await.is_empty() && tokio::time::Instant::now() < deadline {
tokio::time::sleep(Duration::from_millis(100)).await;
}
let otlp_captured = otlp_hits.lock().await;
assert!(
backend/windmill-runtime-nativets/tests/otel_init.rs
+9 -2
View File
@@ -46,7 +46,7 @@ async fn ee_otel_init_config_populates_otel_globals() {
// Step 1: reproduce the footgun. `OtelConfig::default()` has
// tracing/metrics off and `console = Ignore`, which trips the
// 0.31 early-return — Ok(()) but OnceCell stays empty.
deno_telemetry::init(
windmill_runtime_nativets::init_telemetry(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets-test".into(),
runtime_version: "0".into(),
@@ -63,10 +63,17 @@ async fn ee_otel_init_config_populates_otel_globals() {
tracing_enabled/console fields."
);
// The instance's own exporter shares this variable and may be set to gRPC,
// or to anything else. Native telemetry must not read it: a value
// deno_telemetry rejects outright shows it is never consulted.
// SAFETY: the only test in this binary, so nothing reads the environment
// concurrently.
unsafe { std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "not-a-protocol") };
// Step 2: re-call init with the exact config shape that the EE
// `load_internal_otel_exporter` ships — this is what production
// hits, so the test exercises the actual prod call path.
deno_telemetry::init(
windmill_runtime_nativets::init_telemetry(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets".into(),
runtime_version: "0".into(),
flake.nix
+5 -4
View File
@@ -78,15 +78,16 @@
];
# ---------------------------------------------------------------
# Prebuilt V8 binary (must match version in Cargo.toml)
# Prebuilt V8 binary (must match the `v8` pin in backend/Cargo.toml).
# The simdutf variant: deno_core enables that feature of the v8 crate.
# ---------------------------------------------------------------
rustyV8Archive = let
version = "137.1.0";
version = "150.4.0";
target = stdenv.hostPlatform.rust.rustcTarget;
sha256 = {
x86_64-linux =
"sha256-Tiscfy2bzYGR3s0T+SC1IB3xWvTVpVcSEdjq3MCRoRw=";
"sha256-9IdiyhDR8fxgWkQcWuQw7Izh6egPFNePvELLh4wwtHY=";
aarch64-linux = lib.fakeHash;
x86_64-darwin = lib.fakeHash;
aarch64-darwin = lib.fakeHash;
@@ -94,7 +95,7 @@
in pkgs.fetchurl {
name = "librusty_v8-${version}";
url =
"https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz";
"https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_simdutf_release_${target}.a.gz";
inherit sha256;
};