From 3dee29eb0069b2d31a5fb3a810d7ec4c8966f6af Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Thu, 6 Mar 2025 19:14:19 +0000 Subject: [PATCH] Spawn rsyslog from neonvm (#11111) then configure it from compute_ctl. to make it more robust in case of restarts and rsyslogd crashes. --- compute/compute-node.Dockerfile | 10 ++-- compute/vm-image-spec-bookworm.yaml | 16 ++++++ compute/vm-image-spec-bullseye.yaml | 15 ++++++ compute_tools/src/compute.rs | 4 +- ...nf => compute_audit_rsyslog_template.conf} | 0 compute_tools/src/rsyslog.rs | 49 +++++++++---------- 6 files changed, 60 insertions(+), 34 deletions(-) rename compute_tools/src/config_template/{compute_rsyslog_template.conf => compute_audit_rsyslog_template.conf} (100%) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 61d9d59f79..6e46185e36 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1980,12 +1980,10 @@ COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neo RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig # rsyslog config permissions -RUN chown postgres:postgres /etc/rsyslog.conf && \ - touch /etc/compute_rsyslog.conf && \ - chown -R postgres:postgres /etc/compute_rsyslog.conf && \ - # directory for rsyslogd pid file - mkdir /var/run/rsyslogd && \ - chown -R postgres:postgres /var/run/rsyslogd +# directory for rsyslogd pid file +RUN mkdir /var/run/rsyslogd && \ + chown -R postgres:postgres /var/run/rsyslogd && \ + chown -R postgres:postgres /etc/rsyslog.d/ ENV LANG=en_US.utf8 diff --git a/compute/vm-image-spec-bookworm.yaml b/compute/vm-image-spec-bookworm.yaml index 74ff3a8b6d..e6707381ac 100644 --- a/compute/vm-image-spec-bookworm.yaml +++ b/compute/vm-image-spec-bookworm.yaml @@ -39,6 +39,10 @@ commands: user: nobody sysvInitAction: respawn shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499' + - name: rsyslogd + user: postgres + sysvInitAction: respawn + shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf' shutdownHook: | su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' files: @@ -69,6 +73,12 @@ files: } memory {} } +# Create dummy rsyslog config, because it refuses to start without at least one action configured. +# compute_ctl will rewrite this file with the actual configuration, if needed. + - filename: compute_rsyslog.conf + content: | + *.* /dev/null + $IncludeConfig /etc/rsyslog.d/*.conf build: | # Build cgroup-tools # @@ -132,6 +142,12 @@ merge: | RUN set -e \ && chmod 0644 /etc/cgconfig.conf + + COPY compute_rsyslog.conf /etc/compute_rsyslog.conf + RUN chmod 0666 /etc/compute_rsyslog.conf + RUN chmod 0666 /var/log/ + + COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ diff --git a/compute/vm-image-spec-bullseye.yaml b/compute/vm-image-spec-bullseye.yaml index c1787ab018..c89ee112dc 100644 --- a/compute/vm-image-spec-bullseye.yaml +++ b/compute/vm-image-spec-bullseye.yaml @@ -39,6 +39,10 @@ commands: user: nobody sysvInitAction: respawn shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499' + - name: rsyslogd + user: postgres + sysvInitAction: respawn + shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf' shutdownHook: | su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' files: @@ -69,6 +73,12 @@ files: } memory {} } +# Create dummy rsyslog config, because it refuses to start without at least one action configured. +# compute_ctl will rewrite this file with the actual configuration, if needed. + - filename: compute_rsyslog.conf + content: | + *.* /dev/null + $IncludeConfig /etc/rsyslog.d/*.conf build: | # Build cgroup-tools # @@ -128,6 +138,11 @@ merge: | RUN set -e \ && chmod 0644 /etc/cgconfig.conf + COPY compute_rsyslog.conf /etc/compute_rsyslog.conf + RUN chmod 0666 /etc/compute_rsyslog.conf + RUN chmod 0666 /var/log/ + + COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index ddcfe12330..fed97ee2b2 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -37,7 +37,7 @@ use crate::logger::startup_context_from_env; use crate::lsn_lease::launch_lsn_lease_bg_task_for_static; use crate::monitor::launch_monitor; use crate::pg_helpers::*; -use crate::rsyslog::configure_and_start_rsyslog; +use crate::rsyslog::configure_audit_rsyslog; use crate::spec::*; use crate::swap::resize_swap; use crate::sync_sk::{check_if_synced, ping_safekeeper}; @@ -689,7 +689,7 @@ impl ComputeNode { let log_directory_path = Path::new(&self.params.pgdata).join("log"); // TODO: make this more robust // now rsyslog starts once and there is no monitoring or restart if it fails - configure_and_start_rsyslog( + configure_audit_rsyslog( log_directory_path.to_str().unwrap(), "hipaa", &remote_endpoint, diff --git a/compute_tools/src/config_template/compute_rsyslog_template.conf b/compute_tools/src/config_template/compute_audit_rsyslog_template.conf similarity index 100% rename from compute_tools/src/config_template/compute_rsyslog_template.conf rename to compute_tools/src/config_template/compute_audit_rsyslog_template.conf diff --git a/compute_tools/src/rsyslog.rs b/compute_tools/src/rsyslog.rs index 776ff14644..c8fba4fdcd 100644 --- a/compute_tools/src/rsyslog.rs +++ b/compute_tools/src/rsyslog.rs @@ -21,40 +21,34 @@ fn get_rsyslog_pid() -> Option { } } -// Start rsyslogd with the specified configuration file -// If it is already running, do nothing. -fn start_rsyslog(rsyslog_conf_path: &str) -> Result<()> { - let pid = get_rsyslog_pid(); - if let Some(pid) = pid { - info!("rsyslogd is already running with pid: {}", pid); - return Ok(()); - } +// Restart rsyslogd to apply the new configuration. +// This is necessary, because there is no other way to reload the rsyslog configuration. +// +// Rsyslogd shouldn't lose any messages, because of the restart, +// because it tracks the last read position in the log files +// and will continue reading from that position. +// TODO: test it properly +// +fn restart_rsyslog() -> Result<()> { + let old_pid = get_rsyslog_pid().context("rsyslogd is not running")?; + info!("rsyslogd is running with pid: {}, restart it", old_pid); - let _ = Command::new("/usr/sbin/rsyslogd") - .arg("-f") - .arg(rsyslog_conf_path) - .arg("-i") - .arg("/var/run/rsyslogd/rsyslogd.pid") + // kill it to restart + let _ = Command::new("pkill") + .arg("rsyslogd") .output() - .context("Failed to start rsyslogd")?; - - // Check that rsyslogd is running - if let Some(pid) = get_rsyslog_pid() { - info!("rsyslogd started successfully with pid: {}", pid); - } else { - return Err(anyhow::anyhow!("Failed to start rsyslogd")); - } + .context("Failed to stop rsyslogd")?; Ok(()) } -pub fn configure_and_start_rsyslog( +pub fn configure_audit_rsyslog( log_directory: &str, tag: &str, remote_endpoint: &str, ) -> Result<()> { let config_content: String = format!( - include_str!("config_template/compute_rsyslog_template.conf"), + include_str!("config_template/compute_audit_rsyslog_template.conf"), log_directory = log_directory, tag = tag, remote_endpoint = remote_endpoint @@ -62,7 +56,7 @@ pub fn configure_and_start_rsyslog( info!("rsyslog config_content: {}", config_content); - let rsyslog_conf_path = "/etc/compute_rsyslog.conf"; + let rsyslog_conf_path = "/etc/rsyslog.d/compute_audit_rsyslog.conf"; let mut file = OpenOptions::new() .create(true) .write(true) @@ -71,10 +65,13 @@ pub fn configure_and_start_rsyslog( file.write_all(config_content.as_bytes())?; - info!("rsyslog configuration added successfully. Starting rsyslogd"); + info!( + "rsyslog configuration file {} added successfully. Starting rsyslogd", + rsyslog_conf_path + ); // start the service, using the configuration - start_rsyslog(rsyslog_conf_path)?; + restart_rsyslog()?; Ok(()) }