From 0a528efa5bfc2c74512f775f5f092dd0e9e1ddcc Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Thu, 27 Jul 2023 07:20:44 -0700 Subject: [PATCH] Add node id concept --- Cargo.lock | 3 + crates/kumo-server-common/Cargo.toml | 3 + crates/kumo-server-common/src/lib.rs | 1 + crates/kumo-server-common/src/nodeid.rs | 136 ++++++++++++++++++++++++ crates/kumod/src/main.rs | 2 + crates/kumod/src/nodeid.rs | 0 crates/spool/src/spool_id.rs | 8 +- 7 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 crates/kumo-server-common/src/nodeid.rs create mode 100644 crates/kumod/src/nodeid.rs diff --git a/Cargo.lock b/Cargo.lock index 09cd5fae..836cbd0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2275,6 +2275,8 @@ dependencies = [ "kumo-server-lifecycle", "kumo-server-memory", "kumo-server-runtime", + "libc", + "mac_address", "metrics", "metrics-prometheus", "metrics-tracing-context", @@ -2299,6 +2301,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "uuid", ] [[package]] diff --git a/crates/kumo-server-common/Cargo.toml b/crates/kumo-server-common/Cargo.toml index 821f25af..9e2e6304 100644 --- a/crates/kumo-server-common/Cargo.toml +++ b/crates/kumo-server-common/Cargo.toml @@ -24,6 +24,8 @@ kumo-api-types = {path="../kumo-api-types"} kumo-server-lifecycle = {path="../kumo-server-lifecycle"} kumo-server-memory = {path="../kumo-server-memory"} kumo-server-runtime = {path="../kumo-server-runtime"} +libc = "0.2.139" +mac_address = "1.1" metrics = "0.20" metrics-prometheus = "0.3" metrics-tracing-context = "0.13" @@ -48,3 +50,4 @@ toml = "0.7" tracing = "0.1" tracing-appender = "0.2" tracing-subscriber = {version="0.3", features=["env-filter", "std", "fmt", "json"]} +uuid = {version="1.3", features=["v4", "fast-rng"]} diff --git a/crates/kumo-server-common/src/lib.rs b/crates/kumo-server-common/src/lib.rs index 0a64d7b1..fa81bcd9 100644 --- a/crates/kumo-server-common/src/lib.rs +++ b/crates/kumo-server-common/src/lib.rs @@ -5,6 +5,7 @@ use mod_redis::RedisConnKey; pub mod diagnostic_logging; pub mod http_server; +pub mod nodeid; pub mod panic; pub mod start; pub mod tls_helpers; diff --git a/crates/kumo-server-common/src/nodeid.rs b/crates/kumo-server-common/src/nodeid.rs new file mode 100644 index 00000000..01e49d8f --- /dev/null +++ b/crates/kumo-server-common/src/nodeid.rs @@ -0,0 +1,136 @@ +use once_cell::sync::Lazy; +use std::path::PathBuf; +use uuid::Uuid; + +static NODEID: Lazy = Lazy::new(|| NodeId::new()); +const DEFAULT_NODE_ID_PATH: &str = "/opt/kumomta/etc/.nodeid"; +static MAC: Lazy<[u8; 6]> = Lazy::new(get_mac_address); + +/// Obtain the mac address of the first non-loopback interface on the system. +/// If there are no candidate interfaces, fall back to the `gethostid()` function, +/// which will attempt to load a host id from a file on the filesystem, or if that +/// fails, resolve the hostname of the node to its IPv4 address using a reverse DNS +/// lookup, and then derive some 32-bit number from that address through unspecified +/// means. +fn get_mac_address() -> [u8; 6] { + match mac_address::get_mac_address() { + Ok(Some(addr)) => addr.bytes(), + _ => { + // Fall back to gethostid, which is not great, but + // likely better than just random numbers + let host_id = unsafe { libc::gethostid() }.to_le_bytes(); + let mac: [u8; 6] = [ + host_id[0], host_id[1], host_id[2], host_id[3], host_id[4], host_id[5], + ]; + mac + } + } +} + +/// The NodeId is intended to identify a specific instance of KumoMTA +/// within your own local cluster. +/// It is a uuid that will be generated and persisted when the node +/// starts up. +/// +/// If persisting the id isn't possible, we fall back to generating +/// a "stable" v1 uuid from the mac address or deriving a fake mac address +/// from the hostid of the system. Those aren't great when running in +/// some virtualization environments, so it is recommended to resolve +/// any issues with persisting the id there. There are some environment +/// variables that can be used to influence that if the default filesystem +/// path is not suitable for whatever reason. +/// +/// The intended use of the nodeid is disambiguation during reporting, +/// and also for future configuration management/provisioning related +/// functionality. +#[derive(Debug, Clone)] +pub struct NodeId { + /// Unique node id in the cluster + pub uuid: Uuid, + + /// Captures any write error we may have experienced while generating + /// the uuid. This is surfaced by the `check` method. + write_error: Option, +} + +impl std::fmt::Display for NodeId { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.uuid.fmt(fmt) + } +} + +impl NodeId { + pub fn get() -> Self { + (*NODEID).clone() + } + + /// Raises an error if we don't have a persistent unique node id + pub fn check() -> anyhow::Result<()> { + let nodeid = Self::get(); + if let Some(err) = &nodeid.write_error { + anyhow::bail!( + "Unable to determine the KUMO_NODE_ID. \ + Refusing to operate as part of a cluster. {err}" + ); + } + Ok(()) + } + + pub fn new() -> Self { + let mut write_error = None; + + let uuid = match std::env::var_os("KUMO_NODE_ID") { + Some(id_os) => match id_os.to_str() { + Some(id) => match Uuid::parse_str(id) { + Ok(uuid) => uuid, + Err(err) => { + panic!("Env var KUMO_NODE_ID (`{id}`) is not a valid UUID: {err:#}") + } + }, + None => panic!("Env var KUMO_NODE_ID (`{id_os:?}`) is not valid UTF-8"), + }, + None => { + let uuid_path: PathBuf = match std::env::var_os("KUMO_NODE_ID_PATH") { + Some(node_path) => node_path.into(), + None => DEFAULT_NODE_ID_PATH.into(), + }; + + match std::fs::read_to_string(&uuid_path) { + Ok(id) => match Uuid::parse_str(&id) { + Ok(uuid) => uuid, + Err(err) => { + panic!("File {uuid_path:?} content `{id}` is not a valid UUID: {err:#}") + } + }, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let uuid = Uuid::new_v4(); + + match std::fs::write(&uuid_path, format!("{uuid}")) { + Ok(_) => uuid, + Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => { + let err = + format!("Failed to write node id to {uuid_path:?}: {err:#}"); + tracing::debug!( + "{err}. Proceeding on the assumption that \ + we're not in a cluster and switching to a \ + stable v1 uuid based on the mac address" + ); + write_error.replace(err); + + // Switch to a mac address based v1 uuid, with a fixed + // timestamp. It looks like: + // 00000000-0000-1000-8000-XXXXXXXXXXXX + // where the X's are the hex digits from the mac address + Uuid::new_v1(uuid::Timestamp::from_rfc4122(0, 0), &*MAC) + } + Err(err) => panic!("Failed to write node id to {uuid_path:?}: {err:#}"), + } + } + Err(err) => panic!("File {uuid_path:?} could not be read: {err:#}"), + } + } + }; + + Self { uuid, write_error } + } +} diff --git a/crates/kumod/src/main.rs b/crates/kumod/src/main.rs index cc4e1995..7c19db3a 100644 --- a/crates/kumod/src/main.rs +++ b/crates/kumod/src/main.rs @@ -132,6 +132,8 @@ fn main() -> anyhow::Result<()> { fn perform_init() -> Pin>>> { Box::pin(async move { + let nodeid = kumo_server_common::nodeid::NodeId::get(); + tracing::info!("NodeId is {nodeid}"); let mut config = config::load_config().await?; config.async_call_callback("init", ()).await?; diff --git a/crates/kumod/src/nodeid.rs b/crates/kumod/src/nodeid.rs new file mode 100644 index 00000000..e69de29b diff --git a/crates/spool/src/spool_id.rs b/crates/spool/src/spool_id.rs index 9a005214..3c9d80c5 100644 --- a/crates/spool/src/spool_id.rs +++ b/crates/spool/src/spool_id.rs @@ -11,8 +11,12 @@ fn get_mac_address() -> [u8; 6] { match mac_address::get_mac_address() { Ok(Some(addr)) => addr.bytes(), _ => { - let mut mac = [0u8; 6]; - getrandom::getrandom(&mut mac).ok(); + // Fall back to gethostid, which is not great, but + // likely better than just random numbers + let host_id = unsafe { libc::gethostid() }.to_le_bytes(); + let mac: [u8; 6] = [ + host_id[0], host_id[1], host_id[2], host_id[3], host_id[4], host_id[5], + ]; mac } }