From 6bbd34a216accdea3c6a3bd30df8ab28386afdde Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Thu, 11 Jul 2024 10:20:14 +0300 Subject: [PATCH] Enable core dumps for postgres (#8272) Set core rmilit to ulimited in compute_ctl, so that all child processes inherit it. We could also set rlimit in relevant startup script, but that way we would depend on external setup and might inadvertently disable it again (core dumping worked in pods, but not in VMs with inittab-based startup). --- Cargo.lock | 10 ++++++++++ compute_tools/Cargo.toml | 1 + compute_tools/src/bin/compute_ctl.rs | 4 ++++ compute_tools/src/compute.rs | 2 +- 4 files changed, 16 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 776d95c3c7..9fb3f5385d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,7 @@ dependencies = [ "regex", "remote_storage", "reqwest 0.12.4", + "rlimit", "rust-ini", "serde", "serde_json", @@ -4901,6 +4902,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rlimit" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +dependencies = [ + "libc", +] + [[package]] name = "routerify" version = "3.0.0" diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 8f96530a9d..8ceb8f2ad2 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -44,3 +44,4 @@ vm_monitor = { version = "0.1", path = "../libs/vm_monitor/" } zstd = "0.13" bytes = "1.0" rust-ini = "0.20.0" +rlimit = "0.10.1" diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 7bf5db5a57..f4c396a85d 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -64,6 +64,7 @@ use compute_tools::monitor::launch_monitor; use compute_tools::params::*; use compute_tools::spec::*; use compute_tools::swap::resize_swap; +use rlimit::{setrlimit, Resource}; // this is an arbitrary build tag. Fine as a default / for testing purposes // in-case of not-set environment var @@ -72,6 +73,9 @@ const BUILD_TAG_DEFAULT: &str = "latest"; fn main() -> Result<()> { let (build_tag, clap_args) = init()?; + // enable core dumping for all child processes + setrlimit(Resource::CORE, rlimit::INFINITY, rlimit::INFINITY)?; + let (pg_handle, start_pg_result) = { // Enter startup tracing context let _startup_context_guard = startup_context_from_env(); diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index eced6fc0b2..1112795d30 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1116,7 +1116,7 @@ impl ComputeNode { // EKS worker nodes have following core dump settings: // /proc/sys/kernel/core_pattern -> core // /proc/sys/kernel/core_uses_pid -> 1 - // ulimint -c -> unlimited + // ulimit -c -> unlimited // which results in core dumps being written to postgres data directory as core.. // // Use that as a default location and pattern, except macos where core dumps are written