Spawn rsyslog from neonvm (#11111)

then configure it from compute_ctl.
to make it more robust in case of restarts and rsyslogd crashes.

compute/compute-node.Dockerfile

@@ -1980,12 +1980,10 @@ COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neo
 RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
 
 # rsyslog config permissions
-RUN chown postgres:postgres /etc/rsyslog.conf && \
-    touch /etc/compute_rsyslog.conf && \
-    chown -R postgres:postgres /etc/compute_rsyslog.conf && \
-    # directory for rsyslogd pid file
-    mkdir /var/run/rsyslogd && \
-    chown -R postgres:postgres /var/run/rsyslogd
+# directory for rsyslogd pid file
+RUN mkdir /var/run/rsyslogd && \
+    chown -R postgres:postgres /var/run/rsyslogd && \
+    chown -R postgres:postgres /etc/rsyslog.d/
 
 
 ENV LANG=en_US.utf8

compute/vm-image-spec-bookworm.yaml

@@ -39,6 +39,10 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
+  - name: rsyslogd
+    user: postgres
+    sysvInitAction: respawn
+    shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf'
 shutdownHook: |
   su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10'
 files:
@@ -69,6 +73,12 @@ files:
           }
           memory {}
       }
+# Create dummy rsyslog config, because it refuses to start without at least one action configured.
+# compute_ctl will rewrite this file with the actual configuration, if needed.
+  - filename: compute_rsyslog.conf
+    content: |
+      *.*    /dev/null
+      $IncludeConfig /etc/rsyslog.d/*.conf
 build: |
   # Build cgroup-tools
   #
@@ -132,6 +142,12 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+
+  COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
+  RUN chmod 0666 /etc/compute_rsyslog.conf
+  RUN chmod 0666 /var/log/
+
+
   COPY --from=libcgroup-builder /libcgroup-install/bin/*  /usr/bin/
   COPY --from=libcgroup-builder /libcgroup-install/lib/*  /usr/lib/
   COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/

compute/vm-image-spec-bullseye.yaml

@@ -39,6 +39,10 @@ commands:
     user: nobody
     sysvInitAction: respawn
     shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
+  - name: rsyslogd
+    user: postgres
+    sysvInitAction: respawn
+    shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf'
 shutdownHook: |
   su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10'
 files:
@@ -69,6 +73,12 @@ files:
           }
           memory {}
       }
+# Create dummy rsyslog config, because it refuses to start without at least one action configured.
+# compute_ctl will rewrite this file with the actual configuration, if needed.
+  - filename: compute_rsyslog.conf
+    content: |
+      *.*    /dev/null
+      $IncludeConfig /etc/rsyslog.d/*.conf
 build: |
   # Build cgroup-tools
   #
@@ -128,6 +138,11 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+  COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
+  RUN chmod 0666 /etc/compute_rsyslog.conf
+  RUN chmod 0666 /var/log/
+
+
   COPY --from=libcgroup-builder /libcgroup-install/bin/*  /usr/bin/
   COPY --from=libcgroup-builder /libcgroup-install/lib/*  /usr/lib/
   COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/

compute_tools/src/compute.rs

@@ -37,7 +37,7 @@ use crate::logger::startup_context_from_env;
 use crate::lsn_lease::launch_lsn_lease_bg_task_for_static;
 use crate::monitor::launch_monitor;
 use crate::pg_helpers::*;
-use crate::rsyslog::configure_and_start_rsyslog;
+use crate::rsyslog::configure_audit_rsyslog;
 use crate::spec::*;
 use crate::swap::resize_swap;
 use crate::sync_sk::{check_if_synced, ping_safekeeper};
@@ -689,7 +689,7 @@ impl ComputeNode {
             let log_directory_path = Path::new(&self.params.pgdata).join("log");
             // TODO: make this more robust
             // now rsyslog starts once and there is no monitoring or restart if it fails
-            configure_and_start_rsyslog(
+            configure_audit_rsyslog(
                 log_directory_path.to_str().unwrap(),
                 "hipaa",
                 &remote_endpoint,

compute_tools/src/rsyslog.rs

@@ -21,40 +21,34 @@ fn get_rsyslog_pid() -> Option<String> {
     }
 }
 
-// Start rsyslogd with the specified configuration file
-// If it is already running, do nothing.
-fn start_rsyslog(rsyslog_conf_path: &str) -> Result<()> {
-    let pid = get_rsyslog_pid();
-    if let Some(pid) = pid {
-        info!("rsyslogd is already running with pid: {}", pid);
-        return Ok(());
-    }
+// Restart rsyslogd to apply the new configuration.
+// This is necessary, because there is no other way to reload the rsyslog configuration.
+//
+// Rsyslogd shouldn't lose any messages, because of the restart,
+// because it tracks the last read position in the log files
+// and will continue reading from that position.
+// TODO: test it properly
+//
+fn restart_rsyslog() -> Result<()> {
+    let old_pid = get_rsyslog_pid().context("rsyslogd is not running")?;
+    info!("rsyslogd is running with pid: {}, restart it", old_pid);
 
-    let _ = Command::new("/usr/sbin/rsyslogd")
-        .arg("-f")
-        .arg(rsyslog_conf_path)
-        .arg("-i")
-        .arg("/var/run/rsyslogd/rsyslogd.pid")
+    // kill it to restart
+    let _ = Command::new("pkill")
+        .arg("rsyslogd")
         .output()
-        .context("Failed to start rsyslogd")?;
-
-    // Check that rsyslogd is running
-    if let Some(pid) = get_rsyslog_pid() {
-        info!("rsyslogd started successfully with pid: {}", pid);
-    } else {
-        return Err(anyhow::anyhow!("Failed to start rsyslogd"));
-    }
+        .context("Failed to stop rsyslogd")?;
 
     Ok(())
 }
 
-pub fn configure_and_start_rsyslog(
+pub fn configure_audit_rsyslog(
     log_directory: &str,
     tag: &str,
     remote_endpoint: &str,
 ) -> Result<()> {
     let config_content: String = format!(
-        include_str!("config_template/compute_rsyslog_template.conf"),
+        include_str!("config_template/compute_audit_rsyslog_template.conf"),
         log_directory = log_directory,
         tag = tag,
         remote_endpoint = remote_endpoint
@@ -62,7 +56,7 @@ pub fn configure_and_start_rsyslog(
 
     info!("rsyslog config_content: {}", config_content);
 
-    let rsyslog_conf_path = "/etc/compute_rsyslog.conf";
+    let rsyslog_conf_path = "/etc/rsyslog.d/compute_audit_rsyslog.conf";
     let mut file = OpenOptions::new()
         .create(true)
         .write(true)
@@ -71,10 +65,13 @@ pub fn configure_and_start_rsyslog(
 
     file.write_all(config_content.as_bytes())?;
 
-    info!("rsyslog configuration added successfully. Starting rsyslogd");
+    info!(
+        "rsyslog configuration file {} added successfully. Starting rsyslogd",
+        rsyslog_conf_path
+    );
 
     // start the service, using the configuration
-    start_rsyslog(rsyslog_conf_path)?;
+    restart_rsyslog()?;
 
     Ok(())
 }