Add node id concept

Cargo.lock

@@ -2275,6 +2275,8 @@ dependencies = [
  "kumo-server-lifecycle",
  "kumo-server-memory",
  "kumo-server-runtime",
+ "libc",
+ "mac_address",
  "metrics",
  "metrics-prometheus",
  "metrics-tracing-context",
@@ -2299,6 +2301,7 @@ dependencies = [
  "tracing",
  "tracing-appender",
  "tracing-subscriber",
+ "uuid",
 ]
 
 [[package]]

crates/kumo-server-common/Cargo.toml

@@ -24,6 +24,8 @@ kumo-api-types = {path="../kumo-api-types"}
 kumo-server-lifecycle = {path="../kumo-server-lifecycle"}
 kumo-server-memory = {path="../kumo-server-memory"}
 kumo-server-runtime = {path="../kumo-server-runtime"}
+libc = "0.2.139"
+mac_address = "1.1"
 metrics = "0.20"
 metrics-prometheus = "0.3"
 metrics-tracing-context = "0.13"
@@ -48,3 +50,4 @@ toml = "0.7"
 tracing = "0.1"
 tracing-appender = "0.2"
 tracing-subscriber = {version="0.3", features=["env-filter", "std", "fmt", "json"]}
+uuid = {version="1.3", features=["v4", "fast-rng"]}

crates/kumo-server-common/src/lib.rs

@@ -5,6 +5,7 @@ use mod_redis::RedisConnKey;
 
 pub mod diagnostic_logging;
 pub mod http_server;
+pub mod nodeid;
 pub mod panic;
 pub mod start;
 pub mod tls_helpers;

crates/kumo-server-common/src/nodeid.rs

@@ -0,0 +1,136 @@
+use once_cell::sync::Lazy;
+use std::path::PathBuf;
+use uuid::Uuid;
+
+static NODEID: Lazy<NodeId> = Lazy::new(|| NodeId::new());
+const DEFAULT_NODE_ID_PATH: &str = "/opt/kumomta/etc/.nodeid";
+static MAC: Lazy<[u8; 6]> = Lazy::new(get_mac_address);
+
+/// Obtain the mac address of the first non-loopback interface on the system.
+/// If there are no candidate interfaces, fall back to the `gethostid()` function,
+/// which will attempt to load a host id from a file on the filesystem, or if that
+/// fails, resolve the hostname of the node to its IPv4 address using a reverse DNS
+/// lookup, and then derive some 32-bit number from that address through unspecified
+/// means.
+fn get_mac_address() -> [u8; 6] {
+    match mac_address::get_mac_address() {
+        Ok(Some(addr)) => addr.bytes(),
+        _ => {
+            // Fall back to gethostid, which is not great, but
+            // likely better than just random numbers
+            let host_id = unsafe { libc::gethostid() }.to_le_bytes();
+            let mac: [u8; 6] = [
+                host_id[0], host_id[1], host_id[2], host_id[3], host_id[4], host_id[5],
+            ];
+            mac
+        }
+    }
+}
+
+/// The NodeId is intended to identify a specific instance of KumoMTA
+/// within your own local cluster.
+/// It is a uuid that will be generated and persisted when the node
+/// starts up.
+///
+/// If persisting the id isn't possible, we fall back to generating
+/// a "stable" v1 uuid from the mac address or deriving a fake mac address
+/// from the hostid of the system. Those aren't great when running in
+/// some virtualization environments, so it is recommended to resolve
+/// any issues with persisting the id there. There are some environment
+/// variables that can be used to influence that if the default filesystem
+/// path is not suitable for whatever reason.
+///
+/// The intended use of the nodeid is disambiguation during reporting,
+/// and also for future configuration management/provisioning related
+/// functionality.
+#[derive(Debug, Clone)]
+pub struct NodeId {
+    /// Unique node id in the cluster
+    pub uuid: Uuid,
+
+    /// Captures any write error we may have experienced while generating
+    /// the uuid. This is surfaced by the `check` method.
+    write_error: Option<String>,
+}
+
+impl std::fmt::Display for NodeId {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        self.uuid.fmt(fmt)
+    }
+}
+
+impl NodeId {
+    pub fn get() -> Self {
+        (*NODEID).clone()
+    }
+
+    /// Raises an error if we don't have a persistent unique node id
+    pub fn check() -> anyhow::Result<()> {
+        let nodeid = Self::get();
+        if let Some(err) = &nodeid.write_error {
+            anyhow::bail!(
+                "Unable to determine the KUMO_NODE_ID. \
+                 Refusing to operate as part of a cluster. {err}"
+            );
+        }
+        Ok(())
+    }
+
+    pub fn new() -> Self {
+        let mut write_error = None;
+
+        let uuid = match std::env::var_os("KUMO_NODE_ID") {
+            Some(id_os) => match id_os.to_str() {
+                Some(id) => match Uuid::parse_str(id) {
+                    Ok(uuid) => uuid,
+                    Err(err) => {
+                        panic!("Env var KUMO_NODE_ID (`{id}`) is not a valid UUID: {err:#}")
+                    }
+                },
+                None => panic!("Env var KUMO_NODE_ID (`{id_os:?}`) is not valid UTF-8"),
+            },
+            None => {
+                let uuid_path: PathBuf = match std::env::var_os("KUMO_NODE_ID_PATH") {
+                    Some(node_path) => node_path.into(),
+                    None => DEFAULT_NODE_ID_PATH.into(),
+                };
+
+                match std::fs::read_to_string(&uuid_path) {
+                    Ok(id) => match Uuid::parse_str(&id) {
+                        Ok(uuid) => uuid,
+                        Err(err) => {
+                            panic!("File {uuid_path:?} content `{id}` is not a valid UUID: {err:#}")
+                        }
+                    },
+                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                        let uuid = Uuid::new_v4();
+
+                        match std::fs::write(&uuid_path, format!("{uuid}")) {
+                            Ok(_) => uuid,
+                            Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
+                                let err =
+                                    format!("Failed to write node id to {uuid_path:?}: {err:#}");
+                                tracing::debug!(
+                                    "{err}. Proceeding on the assumption that \
+                                     we're not in a cluster and switching to a \
+                                     stable v1 uuid based on the mac address"
+                                );
+                                write_error.replace(err);
+
+                                // Switch to a mac address based v1 uuid, with a fixed
+                                // timestamp. It looks like:
+                                // 00000000-0000-1000-8000-XXXXXXXXXXXX
+                                // where the X's are the hex digits from the mac address
+                                Uuid::new_v1(uuid::Timestamp::from_rfc4122(0, 0), &*MAC)
+                            }
+                            Err(err) => panic!("Failed to write node id to {uuid_path:?}: {err:#}"),
+                        }
+                    }
+                    Err(err) => panic!("File {uuid_path:?} could not be read: {err:#}"),
+                }
+            }
+        };
+
+        Self { uuid, write_error }
+    }
+}

crates/kumod/src/main.rs

@@ -132,6 +132,8 @@ fn main() -> anyhow::Result<()> {
 
 fn perform_init() -> Pin<Box<dyn Future<Output = anyhow::Result<()>>>> {
     Box::pin(async move {
+        let nodeid = kumo_server_common::nodeid::NodeId::get();
+        tracing::info!("NodeId is {nodeid}");
         let mut config = config::load_config().await?;
         config.async_call_callback("init", ()).await?;
 

crates/spool/src/spool_id.rs

@@ -11,8 +11,12 @@ fn get_mac_address() -> [u8; 6] {
     match mac_address::get_mac_address() {
         Ok(Some(addr)) => addr.bytes(),
         _ => {
-            let mut mac = [0u8; 6];
-            getrandom::getrandom(&mut mac).ok();
+            // Fall back to gethostid, which is not great, but
+            // likely better than just random numbers
+            let host_id = unsafe { libc::gethostid() }.to_le_bytes();
+            let mac: [u8; 6] = [
+                host_id[0], host_id[1], host_id[2], host_id[3], host_id[4], host_id[5],
+            ];
             mac
         }
     }