mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-11 08:07:15 +00:00
feat: add memory limits to the go build subprocess (#10666)
* feat: bound go compilation memory with GOMEMLIMIT Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: bound the whole go build tree, not each toolchain process Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: keep the go build memlimit and parallelism atomic Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: log the go limits actually installed and stop serializing small workers Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: make go build parallelism authoritative over persisted GOFLAGS Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: canonicalize the go build -p value and floor the module-step budget Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: parse GOMAXPROCS for -p the way the go runtime does Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: read GOMAXPROCS with go's own grammar and report limits neutrally Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: derive go build parallelism from the cgroup quota over its own period Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: keep go's minimum build parallelism under sub-CPU quotas Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: keep the windows 1CU cap out of go's two-compiler floor Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * docs: record that a worker runs one job at a time Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * docs: scope the one-job-at-a-time rule away from native workers Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
co-authored by
Claude Opus 5
Claude Opus 5
parent
dad4c10c8b
commit
4cb51cf7bc
@@ -120,6 +120,18 @@ A crash like this takes down every job on that worker, not just yours, so check
|
||||
backend log after the run rather than only the job's own status. If you cannot run one,
|
||||
say which path went unexercised instead of implying it was verified.
|
||||
|
||||
### How many jobs a worker runs at once
|
||||
|
||||
`NUM_WORKERS > 1` falls back to 1 outside native mode (`backend/src/main.rs`, unless
|
||||
`I_ACK_NUM_WORKERS_IS_UNSAFE`), so a worker serving script tags — `go`, `python3`,
|
||||
`dependency`, `flow`, … — runs one job at a time: a per-job resource budget (memory, CPU,
|
||||
temp space) shares the worker with the worker process alone.
|
||||
|
||||
Native mode is the exception, and budgets for its tags must divide by its concurrency:
|
||||
it forces 8 workers, and `NATIVE_TAGS` includes executors that already claim a per-job
|
||||
share of the worker's memory (`postgresql` and `mysql` through `MAX_SQL_RESULT_SIZE`), so
|
||||
up to 8 of those run against the same limit at once.
|
||||
|
||||
## Banned Patterns
|
||||
|
||||
### `$bindable(default_value)` on optional props
|
||||
|
||||
@@ -1472,6 +1472,73 @@ pub fn get_vcpus() -> Option<i64> {
|
||||
(sys.cpus().len() * 100000).try_into().ok()
|
||||
}
|
||||
|
||||
/// The window `get_vcpus`'s quota is spent over, in the same microseconds. Only
|
||||
/// their ratio is a number of CPUs, and the window is configurable — 100ms is
|
||||
/// merely its usual value.
|
||||
#[cfg(not(windows))]
|
||||
pub fn get_cpu_period() -> Option<i64> {
|
||||
if Path::new("/sys/fs/cgroup/cpu/cpu.cfs_period_us").exists() {
|
||||
// cgroup v1
|
||||
parse_file("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
|
||||
} else {
|
||||
// cgroup v2: `cpu.max` is "<quota|max> <period>"
|
||||
let cgroup_path = get_cgroupv2_path()?;
|
||||
parse_file::<String>(&format!("{cgroup_path}/cpu.max"))?
|
||||
.split_whitespace()
|
||||
.nth(1)?
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
.filter(|period| *period > 0)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub fn get_cpu_period() -> Option<i64> {
|
||||
Some(100000)
|
||||
}
|
||||
|
||||
/// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count
|
||||
/// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds
|
||||
/// the quota in, so a fraction of a CPU makes it report a single-core machine.
|
||||
#[cfg(not(windows))]
|
||||
pub fn get_affinity_cpus() -> Option<usize> {
|
||||
// "Cpus_allowed_list:\t0-7,16-23"
|
||||
let status = parse_file::<String>("/proc/self/status")?;
|
||||
let list = status
|
||||
.split("Cpus_allowed_list:")
|
||||
.nth(1)?
|
||||
.lines()
|
||||
.next()?
|
||||
.trim();
|
||||
|
||||
let cpus = list
|
||||
.split(',')
|
||||
.map(|range| {
|
||||
let (first, last) = range.split_once('-').unwrap_or((range, range));
|
||||
let (first, last) = (first.trim().parse::<usize>(), last.trim().parse::<usize>());
|
||||
match (first, last) {
|
||||
(Ok(first), Ok(last)) if last >= first => Some(last - first + 1),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
.sum::<Option<usize>>()?;
|
||||
|
||||
(cpus > 0).then_some(cpus)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub fn get_affinity_cpus() -> Option<usize> {
|
||||
// The 1CU cap is a policy rather than a bandwidth quota, so it is the whole
|
||||
// answer here as it is for `get_vcpus` and `get_memory` — a consumer that reads
|
||||
// this as the hardware count would raise the worker back above the cap.
|
||||
if *LIMIT_WINDOWS_TO_1CU {
|
||||
return Some(1);
|
||||
}
|
||||
let mut sys = System::new();
|
||||
sys.refresh_cpu_all();
|
||||
Some(sys.cpus().len()).filter(|cpus| *cpus > 0)
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
fn get_memory_from_meminfo() -> Option<i64> {
|
||||
let memory_info = parse_file::<String>("/proc/meminfo")?;
|
||||
|
||||
@@ -26,8 +26,9 @@ use crate::{
|
||||
OccupancyMetrics, DEV_CONF_NSJAIL,
|
||||
},
|
||||
handle_child::handle_child,
|
||||
is_sandboxing_enabled, read_ee_registry, DISABLE_NUSER, GOPRIVATE, GOPROXY, GO_BIN_CACHE_DIR,
|
||||
GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
|
||||
is_sandboxing_enabled, read_ee_registry, GoBuildLimits, DISABLE_NUSER, GOPRIVATE, GOPROXY,
|
||||
GO_BIN_CACHE_DIR, GO_BUILD_LIMITS, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
|
||||
TRACING_PROXY_CA_CERT_PATH, TZ_ENV,
|
||||
};
|
||||
use windmill_common::client::AuthedClient;
|
||||
|
||||
@@ -84,6 +85,149 @@ lazy_static::lazy_static! {
|
||||
|
||||
pub const GO_OBJECT_STORE_PREFIX: &str =
|
||||
const_format::concatcp!(crate::global_cache::TARGET, "_gobin/");
|
||||
|
||||
/// Worker group env vars forwarded to the Go toolchain (`go build`, `go mod …`).
|
||||
///
|
||||
/// Restricted to the GC and scheduler knobs, which bound what a compilation costs
|
||||
/// without changing what it produces: the built binary is cached under a hash of
|
||||
/// the source and lockfile alone, so a var that alters codegen (`GOFLAGS`, build
|
||||
/// tags) would let two worker groups disagree about the contents of one cache
|
||||
/// entry — including the one shared through the object store.
|
||||
const GO_TOOLCHAIN_TUNING_ENVS: [&str; 3] = ["GOMEMLIMIT", "GOGC", "GOMAXPROCS"];
|
||||
|
||||
/// The two shapes a Go toolchain invocation takes, which spend the budget
|
||||
/// differently: a build fans out into a driver plus compilers, while the module
|
||||
/// steps are one process with nothing else running.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum GoToolchainStep {
|
||||
Build,
|
||||
Mod,
|
||||
}
|
||||
|
||||
impl GoToolchainStep {
|
||||
fn label(&self) -> &'static str {
|
||||
match self {
|
||||
GoToolchainStep::Build => "Go compilation",
|
||||
GoToolchainStep::Mod => "Go dependency resolution",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Memory and parallelism settings for the Go toolchain subprocesses, which
|
||||
/// otherwise inherit nothing (they are spawned with a cleared environment).
|
||||
///
|
||||
/// Must be applied before the explicit `.env` calls of each command so the
|
||||
/// worker's own `PATH`/`GOPATH`/`GOCACHE`/`HOME` win over a worker group setting
|
||||
/// the same names, as they do on the run step.
|
||||
fn go_toolchain_envs(step: GoToolchainStep) -> Vec<(String, String)> {
|
||||
let worker_config = windmill_common::worker::WORKER_CONFIG.load();
|
||||
let envs = merge_go_toolchain_envs(&worker_config.env_vars, *GO_BUILD_LIMITS, step);
|
||||
log_go_toolchain_limits(step, &envs);
|
||||
envs
|
||||
}
|
||||
|
||||
/// A slow compilation is the symptom of a limit set too low, and nothing else would
|
||||
/// tell an operator that one is in force or what it resolved to. Reports what the
|
||||
/// toolchain is actually given, since a worker group can pin values of its own, and
|
||||
/// once per distinct setting rather than per job, since it can move them at runtime.
|
||||
fn log_go_toolchain_limits(step: GoToolchainStep, envs: &[(String, String)]) {
|
||||
lazy_static::lazy_static! {
|
||||
static ref LOGGED: std::sync::Mutex<std::collections::HashSet<String>> =
|
||||
Default::default();
|
||||
}
|
||||
|
||||
let limits = envs
|
||||
.iter()
|
||||
.map(|(k, v)| format!("{k}={v}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
// Neutral wording: the settings are reported rather than characterized, since
|
||||
// a worker group can pin ones that lift the limit (`GOMEMLIMIT=off`) as easily
|
||||
// as ones that impose it.
|
||||
let line = if limits.is_empty() {
|
||||
format!("{} runs with no limits", step.label())
|
||||
} else {
|
||||
format!("{} runs with {limits}", step.label())
|
||||
};
|
||||
|
||||
let Ok(mut logged) = LOGGED.lock() else {
|
||||
return;
|
||||
};
|
||||
if logged.insert(line.clone()) {
|
||||
tracing::info!("{line}");
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_go_toolchain_envs(
|
||||
worker_envs: &HashMap<String, String>,
|
||||
derived: Option<GoBuildLimits>,
|
||||
step: GoToolchainStep,
|
||||
) -> Vec<(String, String)> {
|
||||
// Values reach the toolchain as the worker group wrote them, the same way they
|
||||
// reach the run step: a `GOMEMLIMIT` the Go runtime rejects then fails both
|
||||
// steps alike instead of compiling under a rewritten value and dying on the
|
||||
// binary it produced. An allowlisted name the worker never set resolves to an
|
||||
// empty string, which would shadow the derived pair with nothing.
|
||||
let configured = |k: &str| worker_envs.get(k).filter(|v| !v.trim().is_empty());
|
||||
|
||||
let mut envs: Vec<(String, String)> = GO_TOOLCHAIN_TUNING_ENVS
|
||||
.iter()
|
||||
.filter_map(|k| configured(k).map(|v| (k.to_string(), v.clone())))
|
||||
.collect();
|
||||
|
||||
// Half of the derived pair is not a weaker limit but no limit (see
|
||||
// `resolve_go_build_limits`), so a worker group that sets either one owns both.
|
||||
let pinned_by_worker_group = envs
|
||||
.iter()
|
||||
.any(|(k, _)| k == "GOMEMLIMIT" || k == "GOMAXPROCS");
|
||||
if let (false, Some(limits)) = (pinned_by_worker_group, derived) {
|
||||
match step {
|
||||
GoToolchainStep::Build => {
|
||||
envs.push(("GOMEMLIMIT".to_string(), limits.memlimit.to_string()));
|
||||
envs.push(("GOMAXPROCS".to_string(), limits.parallelism.to_string()));
|
||||
}
|
||||
// Nothing shares the budget with a single process, and its work queue
|
||||
// is sized from `GOMAXPROCS` — throttling it would only slow fetches
|
||||
// down without bounding anything.
|
||||
GoToolchainStep::Mod => {
|
||||
envs.push(("GOMEMLIMIT".to_string(), limits.budget.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
envs
|
||||
}
|
||||
|
||||
/// `go build`'s `-p`, so the parallelism the aggregate assumes is the one it gets.
|
||||
///
|
||||
/// `GOMAXPROCS` only supplies the *default* for `-p`, which a `GOFLAGS=-p=…`
|
||||
/// persisted in the toolchain's own env file outranks — and that file is read,
|
||||
/// since the command keeps `HOME`. A flag on the command line is applied last.
|
||||
fn go_build_parallelism_args(envs: &[(String, String)]) -> Vec<String> {
|
||||
envs.iter()
|
||||
.find(|(k, _)| k == "GOMAXPROCS")
|
||||
.and_then(|(_, v)| go_runtime_int32(v))
|
||||
.filter(|parallelism| *parallelism > 0)
|
||||
.map(|parallelism| vec!["-p".to_string(), parallelism.to_string()])
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `GOMAXPROCS` as the Go runtime reads it: a signed 32-bit decimal, no padding of
|
||||
/// any kind, and `None` for everything else — which the runtime silently ignores.
|
||||
///
|
||||
/// A worker group's value is forwarded verbatim, so it is not necessarily one the
|
||||
/// `-p` flag would take, and the two parsers disagree in both directions: `-p`
|
||||
/// infers the base and rejects the `08` the runtime reads as 8, while it accepts
|
||||
/// the `2147483648` and the `"8 "` the runtime discards — and it would then start
|
||||
/// that many build workers. Reading the value the runtime's way keeps `-p` in step
|
||||
/// with it, so a spelling the runtime ignores leaves the flag off and the build
|
||||
/// keeps the default the runtime itself would have used.
|
||||
fn go_runtime_int32(v: &str) -> Option<i32> {
|
||||
let digits = v.strip_prefix(['+', '-']).unwrap_or(v);
|
||||
(!digits.is_empty() && digits.bytes().all(|b| b.is_ascii_digit()))
|
||||
.then(|| v.parse::<i32>().ok())
|
||||
.flatten()
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
pub async fn handle_go_job(
|
||||
mem_peak: &mut i32,
|
||||
@@ -229,10 +373,19 @@ func Run(req Req) (interface{{}}, error){{
|
||||
}
|
||||
}
|
||||
|
||||
let toolchain_envs = go_toolchain_envs(GoToolchainStep::Build);
|
||||
let build_args = [
|
||||
vec!["build".to_string()],
|
||||
go_build_parallelism_args(&toolchain_envs),
|
||||
vec!["main.go".to_string()],
|
||||
]
|
||||
.concat();
|
||||
|
||||
let mut build_go_cmd = Command::new(GO_PATH.as_str());
|
||||
build_go_cmd
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(toolchain_envs)
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.env("BASE_INTERNAL_URL", base_internal_url)
|
||||
.env("GOPATH", {
|
||||
@@ -248,7 +401,7 @@ func Run(req Req) (interface{{}}, error){{
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.env("GOCACHE", GO_CACHE_DIR.as_str())
|
||||
.envs(PROXY_ENVS.clone())
|
||||
.args(vec!["build", "main.go"])
|
||||
.args(build_args)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
|
||||
@@ -526,6 +679,7 @@ pub async fn install_go_dependencies(
|
||||
child_cmd
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(go_toolchain_envs(GoToolchainStep::Mod))
|
||||
.args(vec!["mod", "init", "mymod"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
@@ -610,6 +764,7 @@ pub async fn install_go_dependencies(
|
||||
child_cmd
|
||||
.current_dir(job_dir)
|
||||
.env_clear()
|
||||
.envs(go_toolchain_envs(GoToolchainStep::Mod))
|
||||
.env("HOME", HOME_ENV.as_str())
|
||||
.env("PATH", PATH_ENV.as_str())
|
||||
.envs(PROXY_ENVS.clone())
|
||||
@@ -719,3 +874,113 @@ async fn gen_go_mymod(code: &str, job_dir: &str) -> error::Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod go_toolchain_envs_tests {
|
||||
use super::{
|
||||
go_build_parallelism_args, merge_go_toolchain_envs, GoBuildLimits, GoToolchainStep, HashMap,
|
||||
};
|
||||
|
||||
const DERIVED: Option<GoBuildLimits> =
|
||||
Some(GoBuildLimits { budget: 2048, memlimit: 512, parallelism: 3 });
|
||||
|
||||
fn worker_envs(pairs: &[(&str, &str)]) -> HashMap<String, String> {
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn merged(pairs: &[(&str, &str)], derived: Option<GoBuildLimits>) -> Vec<(String, String)> {
|
||||
let mut envs =
|
||||
merge_go_toolchain_envs(&worker_envs(pairs), derived, GoToolchainStep::Build);
|
||||
envs.sort();
|
||||
envs
|
||||
}
|
||||
|
||||
fn expect(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
|
||||
let mut envs: Vec<(String, String)> = pairs
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect();
|
||||
envs.sort();
|
||||
envs
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn worker_group_settings_replace_the_derived_pair_whole() {
|
||||
assert_eq!(
|
||||
merged(&[], DERIVED),
|
||||
expect(&[("GOMEMLIMIT", "512"), ("GOMAXPROCS", "3")])
|
||||
);
|
||||
// Either half configured hands the whole policy over, since a cap without a
|
||||
// process count (or the reverse) bounds nothing.
|
||||
assert_eq!(
|
||||
merged(&[("GOMEMLIMIT", "2GiB")], DERIVED),
|
||||
expect(&[("GOMEMLIMIT", "2GiB")])
|
||||
);
|
||||
assert_eq!(
|
||||
merged(&[("GOMAXPROCS", "32")], DERIVED),
|
||||
expect(&[("GOMAXPROCS", "32")])
|
||||
);
|
||||
// GOGC is orthogonal to the pair, so it rides along with it.
|
||||
assert_eq!(
|
||||
merged(&[("GOGC", "50")], DERIVED),
|
||||
expect(&[("GOGC", "50"), ("GOMEMLIMIT", "512"), ("GOMAXPROCS", "3")])
|
||||
);
|
||||
// Forwarded untouched: a trimmed value would compile under a spelling the
|
||||
// run step then rejects.
|
||||
assert_eq!(
|
||||
merged(&[("GOMEMLIMIT", " 2GiB ")], DERIVED),
|
||||
expect(&[("GOMEMLIMIT", " 2GiB ")])
|
||||
);
|
||||
// An allowlisted name the worker never set must not shadow the pair.
|
||||
assert_eq!(
|
||||
merged(&[("GOMEMLIMIT", " ")], DERIVED),
|
||||
expect(&[("GOMEMLIMIT", "512"), ("GOMAXPROCS", "3")])
|
||||
);
|
||||
assert_eq!(merged(&[], None), expect(&[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_module_steps_get_the_whole_budget() {
|
||||
// One process, nothing sharing with it, and no compilers to hold back.
|
||||
assert_eq!(
|
||||
merge_go_toolchain_envs(&worker_envs(&[]), DERIVED, GoToolchainStep::Mod),
|
||||
expect(&[("GOMEMLIMIT", "2048")])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parallelism_is_passed_on_the_command_line_when_it_is_a_number() {
|
||||
assert_eq!(
|
||||
go_build_parallelism_args(&merged(&[], DERIVED)),
|
||||
vec!["-p".to_string(), "3".to_string()]
|
||||
);
|
||||
// A worker group value is forwarded verbatim, so `-p` is only asserted over
|
||||
// `GOFLAGS` when it is one the toolchain would accept.
|
||||
assert!(go_build_parallelism_args(&merged(&[("GOMAXPROCS", "many")], DERIVED)).is_empty());
|
||||
// `-p` infers the base and rejects `08`, which the runtime reads as 8, so
|
||||
// the number is emitted rather than the spelling it arrived in.
|
||||
assert_eq!(
|
||||
go_build_parallelism_args(&merged(&[("GOMAXPROCS", "08")], DERIVED)),
|
||||
vec!["-p".to_string(), "8".to_string()]
|
||||
);
|
||||
// Past the signed 32-bit range, and padded with whitespace, the runtime
|
||||
// ignores the value, so `-p` has to as well: the flag would take either and
|
||||
// start that many workers.
|
||||
assert!(
|
||||
go_build_parallelism_args(&merged(&[("GOMAXPROCS", "2147483648")], DERIVED)).is_empty()
|
||||
);
|
||||
assert!(
|
||||
go_build_parallelism_args(&merged(&[("GOMAXPROCS", "2147483647 ")], DERIVED))
|
||||
.is_empty()
|
||||
);
|
||||
// A leading `+` is one the runtime does take, so the flag keeps it too.
|
||||
assert_eq!(
|
||||
go_build_parallelism_args(&merged(&[("GOMAXPROCS", "+8")], DERIVED)),
|
||||
vec!["-p".to_string(), "8".to_string()]
|
||||
);
|
||||
assert!(go_build_parallelism_args(&merged(&[], None)).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1275,6 +1275,28 @@ const SQL_RESULT_SIZE_FRACTION: f64 = 0.15;
|
||||
// only reject work that would have succeeded.
|
||||
const MIN_MAX_SQL_RESULT_SIZE: usize = 8 * 1024 * 1024;
|
||||
|
||||
// Share of the worker's memory budget a Go compilation may hold. Set high enough
|
||||
// that an ordinary build never approaches it, so the only builds whose behavior
|
||||
// changes are those that were about to take the worker down.
|
||||
//
|
||||
// What the rest covers is not the worker process, which is tens of MB and would
|
||||
// argue for a constant: it is everything `GOMEMLIMIT` does not count and that grows
|
||||
// with the build — the toolchain's mmapped inputs and outputs, its non-Go
|
||||
// allocations, and the page cache its writes charge to the cgroup.
|
||||
const GO_BUILD_MEMLIMIT_FRACTION: f64 = 0.75;
|
||||
// Heap a Go compiler is comfortable in: cores are only put to work while the budget
|
||||
// still affords each of them this much.
|
||||
const GO_BUILD_TARGET_MEMLIMIT: usize = 384 * 1024 * 1024;
|
||||
// Driver plus the compilers below which a build stops overlapping and starts
|
||||
// waiting. Measured on a dependency-heavy build: at a budget too small to give this
|
||||
// many the target share, splitting it further still compiles faster than handing
|
||||
// fewer processes more — one compiler alone costs ~3x what five of them do on the
|
||||
// same budget — so the process count holds and the share absorbs the difference.
|
||||
const MIN_GO_BUILD_PROCESSES: usize = 6;
|
||||
// Floor under the share, past which dividing the budget again buys nothing: the
|
||||
// processes only trade compiling time for collecting time.
|
||||
const MIN_GO_BUILD_MEMLIMIT: usize = 128 * 1024 * 1024;
|
||||
|
||||
/// `"512"`, `"512MB"`, `"2GiB"`, `"1.5GB"` -> bytes. Suffixes are case-insensitive
|
||||
/// and binary, so `MB` and `MiB` both mean 1024².
|
||||
///
|
||||
@@ -1351,6 +1373,193 @@ lazy_static::lazy_static! {
|
||||
.unwrap_or(usize::MAX),
|
||||
}
|
||||
};
|
||||
|
||||
/// What a Go compilation is allowed to cost, derived from the worker's memory
|
||||
/// budget. Nothing else bounds it: a compilation grows until the cgroup OOM
|
||||
/// killer takes the worker process down, every job colocated on it with it.
|
||||
/// `GOMEMLIMIT` is soft — the GC works harder as the heap nears it instead of
|
||||
/// failing the allocation — so a pathological build turns into a slow one.
|
||||
///
|
||||
/// `GO_BUILD_MEMLIMIT` overrides the budget (`512MB`, `2GiB`, …), and `0`/`off`
|
||||
/// disables the whole thing, as does a worker with no cgroup memory reading to
|
||||
/// scale from.
|
||||
pub(crate) static ref GO_BUILD_LIMITS: Option<GoBuildLimits> = resolve_go_build_limits(
|
||||
std::env::var("GO_BUILD_MEMLIMIT").ok().as_deref(),
|
||||
windmill_common::worker::get_memory(),
|
||||
worker_vcpus(),
|
||||
);
|
||||
}
|
||||
|
||||
/// How much memory the Go toolchain may hold while compiling a script, expressed
|
||||
/// the only way the toolchain understands it.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub(crate) struct GoBuildLimits {
|
||||
/// What the whole build may hold, and so what a step that is one process gets.
|
||||
pub budget: usize,
|
||||
/// `GOMEMLIMIT` for one process of a build that fans out.
|
||||
pub memlimit: usize,
|
||||
/// `GOMAXPROCS`, which is also `go build`'s default `-p`: how many compilers it
|
||||
/// runs at once.
|
||||
pub parallelism: usize,
|
||||
}
|
||||
|
||||
fn worker_vcpus() -> usize {
|
||||
effective_vcpus(
|
||||
windmill_common::worker::get_vcpus(),
|
||||
windmill_common::worker::get_cpu_period(),
|
||||
windmill_common::worker::get_affinity_cpus()
|
||||
.or_else(|| std::thread::available_parallelism().ok().map(|n| n.get()))
|
||||
.unwrap_or(1),
|
||||
)
|
||||
}
|
||||
|
||||
/// CPUs worth of work the worker can actually run at once.
|
||||
///
|
||||
/// The cgroup states its allowance as a quota over a period, and only their ratio
|
||||
/// is a number of CPUs — `1500m` is `150000/100000`. A fraction still runs work, so
|
||||
/// it rounds up the way the Go runtime's own container-aware `GOMAXPROCS` does:
|
||||
/// flooring would call that worker single-core and serialize its builds.
|
||||
///
|
||||
/// `host_cpus` counts the CPUs the worker may run on, quota aside, and is both the
|
||||
/// answer when there is no quota and the floor under one: Go's own container-aware
|
||||
/// default never drops below two while the machine has two to give, since even a
|
||||
/// fraction of a CPU compiles two packages faster than it compiles them in series.
|
||||
fn effective_vcpus(quota_us: Option<i64>, period_us: Option<i64>, host_cpus: usize) -> usize {
|
||||
quota_us
|
||||
.zip(period_us)
|
||||
.filter(|(quota, period)| *quota > 0 && *period > 0)
|
||||
.map(|(quota, period)| ((quota + period - 1) / period) as usize)
|
||||
.unwrap_or(host_cpus)
|
||||
.max(host_cpus.min(2))
|
||||
.max(1)
|
||||
}
|
||||
|
||||
/// Split a build budget into the per-process cap and the parallelism it assumes.
|
||||
///
|
||||
/// `GOMEMLIMIT` bounds one process, and a build is a driver plus up to `-p`
|
||||
/// compilers that each inherit the same value, so the budget only holds if the
|
||||
/// number of processes sharing it is pinned alongside it.
|
||||
fn resolve_go_build_limits(
|
||||
env_override: Option<&str>,
|
||||
worker_memory: Option<i64>,
|
||||
vcpus: usize,
|
||||
) -> Option<GoBuildLimits> {
|
||||
let derived = || {
|
||||
worker_memory
|
||||
.filter(|bytes| *bytes > 0)
|
||||
.map(|bytes| (bytes as f64 * GO_BUILD_MEMLIMIT_FRACTION) as usize)
|
||||
};
|
||||
|
||||
let budget = match env_override.map(str::trim).filter(|v| !v.is_empty()) {
|
||||
None => derived()?,
|
||||
Some(v) if v.eq_ignore_ascii_case("off") => return None,
|
||||
Some(v) => match parse_byte_size(v) {
|
||||
// A zero budget would have the GC hold the heap at nothing, so it reads
|
||||
// as "no limit" instead.
|
||||
Some(0) => return None,
|
||||
Some(bytes) => bytes,
|
||||
None => {
|
||||
// Falling back silently would leave the operator believing the limit
|
||||
// they wrote is in force.
|
||||
tracing::warn!(
|
||||
"Go build memory budget {v:?} is not a byte size (e.g. 512MB, \
|
||||
2GiB); falling back to the worker's memory-derived budget"
|
||||
);
|
||||
derived()?
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
// One compiler per core while the budget affords each the target share, never
|
||||
// so few that the build stops overlapping, and never more than the cores can
|
||||
// run. The floor is the last word: on a worker too small to honor both, the
|
||||
// budget is the one that gives, since processes squeezed under it make no
|
||||
// progress to bound.
|
||||
let cap = vcpus.max(1) + 1;
|
||||
let processes = (budget / GO_BUILD_TARGET_MEMLIMIT).clamp(MIN_GO_BUILD_PROCESSES.min(cap), cap);
|
||||
Some(GoBuildLimits {
|
||||
// Floored like the share it is an alternative to: a step that holds the
|
||||
// whole budget must never end up with less than one of six compilers.
|
||||
budget: budget.max(MIN_GO_BUILD_MEMLIMIT),
|
||||
memlimit: (budget / processes).max(MIN_GO_BUILD_MEMLIMIT),
|
||||
parallelism: processes - 1,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod go_build_limits_tests {
|
||||
use super::{
|
||||
effective_vcpus, resolve_go_build_limits, GoBuildLimits, GO_BUILD_TARGET_MEMLIMIT,
|
||||
MIN_GO_BUILD_MEMLIMIT,
|
||||
};
|
||||
|
||||
const GIB: i64 = 1024 * 1024 * 1024;
|
||||
|
||||
fn limits(budget: usize, memlimit: usize, parallelism: usize) -> Option<GoBuildLimits> {
|
||||
Some(GoBuildLimits { budget, memlimit, parallelism })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_the_cgroup_allowance_as_cpus() {
|
||||
// 1500m: a fraction of a CPU still runs work, so it is two compilers' worth
|
||||
// of concurrency rather than one.
|
||||
assert_eq!(effective_vcpus(Some(150_000), Some(100_000), 24), 2);
|
||||
assert_eq!(effective_vcpus(Some(400_000), Some(100_000), 24), 4);
|
||||
// The period is configurable, so only the ratio means anything.
|
||||
assert_eq!(effective_vcpus(Some(400_000), Some(50_000), 24), 8);
|
||||
// The quota is the answer whenever there is one to read, since the count
|
||||
// it would otherwise be clamped to has already floored it.
|
||||
assert_eq!(effective_vcpus(Some(4_000_000), Some(100_000), 4), 40);
|
||||
assert_eq!(effective_vcpus(None, None, 8), 8);
|
||||
// Under a whole CPU the floor is two, as the Go runtime's own default is —
|
||||
// but only where there are two to give.
|
||||
assert_eq!(effective_vcpus(Some(50_000), Some(100_000), 24), 2);
|
||||
assert_eq!(effective_vcpus(Some(50_000), Some(100_000), 1), 1);
|
||||
// A one-CPU allowance stays one when that is all the worker may use, which
|
||||
// is how the Windows 1CU cap reports itself.
|
||||
assert_eq!(effective_vcpus(Some(100_000), Some(100_000), 1), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn splits_the_budget_across_the_build_tree() {
|
||||
// Fewer cores than the budget could feed: every core gets a compiler, and
|
||||
// they share 3GiB with the driver.
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(None, Some(4 * GIB), 4),
|
||||
limits(3 * GIB as usize, 3 * GIB as usize / 5, 4)
|
||||
);
|
||||
// More cores than it can feed at the target share: the extra cores idle
|
||||
// rather than shrink every compiler.
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(None, Some(4 * GIB), 64),
|
||||
limits(3 * GIB as usize, GO_BUILD_TARGET_MEMLIMIT, 7)
|
||||
);
|
||||
// Too small to give even the minimum process count the target share: the
|
||||
// build keeps overlapping and the share absorbs it.
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(None, Some(GIB), 64),
|
||||
limits(3 * GIB as usize / 4, 3 * GIB as usize / 4 / 6, 5)
|
||||
);
|
||||
// Nothing to scale from leaves the toolchain unlimited, as it was before.
|
||||
assert_eq!(resolve_go_build_limits(None, None, 4), None);
|
||||
// Under the floor the budget gives instead of the share.
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(None, Some(GIB / 8), 4),
|
||||
limits(MIN_GO_BUILD_MEMLIMIT, MIN_GO_BUILD_MEMLIMIT, 4)
|
||||
);
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(Some("2GiB"), Some(4 * GIB), 4),
|
||||
limits(2 * GIB as usize, 2 * GIB as usize / 5, 4)
|
||||
);
|
||||
assert_eq!(resolve_go_build_limits(Some("off"), Some(4 * GIB), 4), None);
|
||||
assert_eq!(resolve_go_build_limits(Some("0MB"), Some(4 * GIB), 4), None);
|
||||
// An unparseable override falls back to the derived budget rather than
|
||||
// lifting the limit.
|
||||
assert_eq!(
|
||||
resolve_go_build_limits(Some("lots"), Some(4 * GIB), 4),
|
||||
limits(3 * GIB as usize, 3 * GIB as usize / 5, 4)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The limit postgres collection is bounded by: the cloud product cap where one
|
||||
|
||||
Reference in New Issue
Block a user