From b824e2c988ee4eef021b97fc65ebcfa36a166d7f Mon Sep 17 00:00:00 2001 From: Amaury <1293565+amaury1093@users.noreply.github.com> Date: Sun, 16 Feb 2025 15:43:56 +0100 Subject: [PATCH] fix: Make new config backwards-compatible (#1567) * fix: Make new config backwards-compatible * fix test --- backend/backend_config.toml | 146 ++++++++++-------------- backend/src/config.rs | 144 +++++++++++++++-------- backend/src/http/v0/check_email/post.rs | 2 +- backend/src/main.rs | 1 + 4 files changed, 159 insertions(+), 134 deletions(-) diff --git a/backend/backend_config.toml b/backend/backend_config.toml index 72c420d9..86cc5409 100644 --- a/backend/backend_config.toml +++ b/backend/backend_config.toml @@ -15,13 +15,30 @@ http_host = "127.0.0.1" # Env variable: RCH__HTTP_PORT http_port = 8080 +# Name to use during the EHLO/HELO command in the SMTP conversation. +# Ideally, this should match the reverse DNS of the server's IP address. +# +# Env variable: RCH__HELLO_NAME +hello_name = "localhost" + +# Email to use during the MAIL FROM command in the SMTP conversation. +# Ideally, the domain of this email should match the "hello_name" above. +# +# Env variable: RCH__FROM_EMAIL +from_email = "hello@localhost" + +# Timeout for each SMTP connection, in seconds. Leaving it commented out will +# not set a timeout, i.e. the connection will wait indefinitely. +# +# Env variable: RCH__SMTP_TIMEOUT +# smtp_timeout = 45 + # Shared secret between a trusted client and the backend, required in the # `x-reacher-secret` header of all incoming requests. # # Env variable: RCH__HEADER_SECRET # header_secret = "my-secret" - # Optional Sentry DSN. If set, all errors will be sent to Sentry. # # Env variable: RCH__SENTRY_DSN @@ -32,6 +49,26 @@ http_port = 8080 # Env variable: RCH__WEBDRIVER_ADDR webdriver_addr = "http://localhost:9515" +# Uncomment the lines below to route all SMTP verification requests +# through a specified proxy. Note that the proxy must be a SOCKS5 proxy to work +# with the SMTP protocol. This proxy will not be used for headless +# verifications. +# +# The username and password are optional and only needed if the proxy requires +# authentication. +# +# Env variables: +# - RCH__PROXY__HOST +# - RCH__PROXY__PORT +# - RCH__PROXY__USERNAME +# - RCH__PROXY__PASSWORD +# +# [proxy] +# host = "my.proxy.com" +# port = 1080 +# username = "my-username" +# password = "my-password" + [webdriver] # Path to the Chrome binary. If not set, the default system Chrome will be used. # @@ -60,13 +97,14 @@ webdriver_addr = "http://localhost:9515" # username = "my-username" # password = "my-password" -# Verification method to use for each email provider. Each email provider can +# Override verification method to use for each email provider. Each email provider can # be verified using one of the following methods: # - Gmail: smtp # - Hotmail B2B: smtp # - Hotmail B2C: headless or smtp # - Yahoo: headless or smtp -# - Everything else: smtp +# - Mimecast: smtp +# - Proofpoint: smtp # # For the email providers you choose to verify using the "smtp" method, you # may add additional configuration, such as hello_name, from_email, and @@ -75,112 +113,50 @@ webdriver_addr = "http://localhost:9515" # If using proxies, the list of proxies must be defined in the "proxies" # section below, with a unique name for each proxy such as "proxy1", "proxy2", # etc. Then, in the email provider's SMTP configuration, set the value to the -# name of the proxy to use. -[verif_method] +# name of the proxy to use. To use the proxy defined the the top-level "proxy" +# section, set the value to "default". +[overrides] # Use the "proxies" configuration below to route SMTP verification requests # through a specified proxy. # # Reacher allows you to configure multiple proxies, each with a unique name. # We recommend simply using "proxy1", "proxy2", etc. as the proxy names. # -# In the `verif_method` section below, you can specify which proxy to use for +# In the `overrides` section below, you can specify which proxy to use for # each email provider. For example, to use "proxy1" for Gmail and "proxy2" for # Yahoo, set the `gmail` field to "proxy1" and the `yahoo` field to "proxy2". -[verif_method.proxies] +[overrides.proxies] # Uncomment the lines below to configure a proxy. The username and password are # optional and only needed if the proxy requires authentication. # # Env variables: -# - RCH__VERIF_METHOD__PROXIES__PROXY1__HOST -# - RCH__VERIF_METHOD__PROXIES__PROXY1__PORT -# - RCH__VERIF_METHOD__PROXIES__PROXY1__USERNAME -# - RCH__VERIF_METHOD__PROXIES__PROXY1__PASSWORD +# - RCH__OVERRIDES__PROXIES__PROXY1__HOST +# - RCH__OVERRIDES__PROXIES__PROXY1__PORT +# - RCH__OVERRIDES__PROXIES__PROXY1__USERNAME +# - RCH__OVERRIDES__PROXIES__PROXY1__PASSWORD # proxy1 = { host = "my.proxy1.com", port = 1080, username = "my-username1", password = "my-password1" } # proxy2 = { host = "my.proxy2.com", port = 1081 } -[verif_method.gmail] -# Verification method to use for Gmail. Gmail only supports the "smtp" method. -# You can add additional configuration such as hello_name, from_email, and -# whether to use a proxy or not, by uncommenting the lines below. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__TYPE -type = "smtp" -# The proxy to use to verify Gmail. If set, it must match one of the names -# defined in the "proxies" section above. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__PROXY +# Set overrides for Gmail. If uncommented, make sure to uncomment all fields. +# [overrides.gmail] +# type = "smtp" # proxy = "proxy1" - -# Name to use during the EHLO/HELO command in the SMTP conversation. -# Ideally, this should match the reverse DNS of the server's IP address. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__HELLO_NAME # hello_name = "my-domain.com" - -# Email to use during the MAIL FROM command in the SMTP conversation. -# Ideally, the domain of this email should match the "hello_name" above. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__FROM_EMAIL # from_email = "hello@my-domain.com" -# Timeout for each SMTP connection, in seconds. Leaving it commented out will -# not set a timeout, i.e. the connection will wait indefinitely. Both fields -# secs and nanos must be set. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__SMTP_TIMEOUT -# smtp_timeout = { secs = 25, nanos = 0 } +# For each email provider, you can override the verification method and set +# additional configuration. The available fields are the same as for the +# "gmail" section above. -# Number of retries for each email verification. Leaving it commented out will -# default to 1 retry. -# -# Env variable: RCH__VERIF_METHOD__GMAIL__RETRIES -# retries = 2 +# [overrides.hotmailb2b] -[verif_method.hotmailb2b] -# Verification method to use for Hotmail B2B. Hotmail B2B only supports the -# "smtp" method. You may copy the configuration from the Gmail section above to -# apply them here. -# -# Env variable: RCH__VERIF_METHOD__HOTMAILB2B__TYPE -type = "smtp" -# proxy = "proxy1" +# [overrides.hotmailb2c] -[verif_method.hotmailb2c] -# Verification method to use for Hotmail B2C. Hotmail B2C supports both the -# "headless" and "smtp" methods. The "headless" method is recommended. -# -# Env variable: RCH__VERIF_METHOD__HOTMAILB2C__TYPE -type = "headless" +# [overrides.mimecast] -[verif_method.mimecast] -# Verification method to use for emails behind Mimecast. Mimecast only supports -# the "smtp" method. You may copy the configuration from the Gmail section above -# to apply them here. -# -# Env variable: RCH__VERIF_METHOD__MIMECAST__TYPE -type = "smtp" +# [overrides.proofpoint] -[verif_method.proofpoint] -# Verification method to use for emails behind Proofpoint. Proofpoint only supports -# the "smtp" method. You may copy the configuration from the Gmail section above -# to apply them here. -# -# Env variable: RCH__VERIF_METHOD__PROOFPOINT__TYPE -type = "smtp" - -[verif_method.yahoo] -# Verification method to use for Yahoo. Yahoo supports both the "headless" and -# "smtp" methods. The "headless" method is recommended. -# -# Env variable: RCH__VERIF_METHOD__YAHOO__TYPE -type = "headless" - -[verif_method.everything_else] -# Verification method to use for all other email providers. By default, all -# other emails will be verified using the "smtp" method. You may copy the -# configuration from the Gmail section above to apply them here. -type = "smtp" -# proxy = "proxy1" +# [overrides.yahoo] # Throttle the maximum number of requests per second, per minute, per hour, and # per day for this worker. diff --git a/backend/src/config.rs b/backend/src/config.rs index 08f00089..92de5810 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -19,7 +19,11 @@ use crate::throttle::ThrottleManager; use crate::worker::do_work::TaskWebhook; use crate::worker::setup_rabbit_mq; use anyhow::{bail, Context}; -use check_if_email_exists::smtp::verif_method::VerifMethod; +use check_if_email_exists::smtp::verif_method::{ + EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, HotmailB2CVerifMethod, + MimecastVerifMethod, ProofpointVerifMethod, VerifMethod, VerifMethodSmtpConfig, + YahooVerifMethod, +}; use check_if_email_exists::{CheckEmailInputProxy, WebdriverConfig, LOG_TARGET}; use config::Config; use lapin::Channel; @@ -35,14 +39,19 @@ pub struct BackendConfig { /// Name of the backend. pub backend_name: String, + // Fields from VerifMethodSmtpConfig + pub from_email: String, + pub hello_name: String, + /// Timeout for each SMTP connection, in seconds. Leaving it commented out + /// will not set a timeout, i.e. the connection will wait indefinitely. + pub smtp_timeout: Option, // This field is deprecated, but kept for backwards compatibility. If set, // it will be moved to the "default" proxy in the `verif_method.proxies` // field. - #[deprecated] pub proxy: Option, - /// Verification methods per email provider. - pub verif_method: VerifMethod, + /// Overrides over the default verification method provided above. + pub overrides: OverridesConfig, /// Webdriver configuration. pub webdriver_addr: String, @@ -88,9 +97,11 @@ impl BackendConfig { backend_name: "".to_string(), webdriver_addr: "".to_string(), webdriver: WebdriverConfig::default(), - #[allow(deprecated)] + from_email: "".to_string(), + hello_name: "".to_string(), + smtp_timeout: None, proxy: None, - verif_method: VerifMethod::default(), + overrides: OverridesConfig::default(), http_host: "127.0.0.1".to_string(), http_port: 8080, header_secret: None, @@ -107,6 +118,56 @@ impl BackendConfig { } } + pub fn get_verif_method(&self) -> VerifMethod { + let mut proxies = self.overrides.proxies.clone(); + if let Some(proxy) = self.proxy.as_ref() { + proxies.insert("default".to_string(), proxy.clone()); + } + + let default_smtp_config = VerifMethodSmtpConfig { + from_email: self.from_email.clone(), + hello_name: self.hello_name.clone(), + proxy: self.proxy.as_ref().map(|_| "default".to_string()), + smtp_timeout: self.smtp_timeout.map(Duration::from_secs), + ..Default::default() + }; + + VerifMethod { + proxies, + gmail: self + .overrides + .gmail + .clone() + .unwrap_or_else(|| GmailVerifMethod::Smtp(default_smtp_config.clone())), + hotmailb2b: self + .overrides + .hotmailb2b + .clone() + .unwrap_or_else(|| HotmailB2BVerifMethod::Smtp(default_smtp_config.clone())), + hotmailb2c: self + .overrides + .hotmailb2c + .clone() + .unwrap_or_else(|| HotmailB2CVerifMethod::Headless), + mimecast: self + .overrides + .mimecast + .clone() + .unwrap_or_else(|| MimecastVerifMethod::Smtp(default_smtp_config.clone())), + proofpoint: self + .overrides + .proofpoint + .clone() + .unwrap_or_else(|| ProofpointVerifMethod::Smtp(default_smtp_config.clone())), + yahoo: self + .overrides + .yahoo + .clone() + .unwrap_or_else(|| YahooVerifMethod::Headless), + everything_else: EverythingElseVerifMethod::Smtp(default_smtp_config), + } + } + /// Get the worker configuration. /// /// # Panics @@ -177,22 +238,14 @@ impl BackendConfig { } #[derive(Debug, Default, Deserialize, Clone, Serialize)] -pub struct VerifMethodConfig { +pub struct OverridesConfig { pub proxies: HashMap, - pub gmail: check_if_email_exists::smtp::verif_method::GmailVerifMethod, - pub hotmailb2b: check_if_email_exists::smtp::verif_method::HotmailB2BVerifMethod, - pub hotmailb2c: check_if_email_exists::smtp::verif_method::HotmailB2CVerifMethod, - pub yahoo: check_if_email_exists::smtp::verif_method::YahooVerifMethod, - pub everything_else: check_if_email_exists::smtp::verif_method::EverythingElseVerifMethod, -} - -pub struct VerifMethodSmtpConfig { - pub from_email: Option, - pub hello_name: Option, - pub smtp_port: Option, - pub retries: Option, - pub proxy: Option, - pub smtp_timeout: Option, + pub gmail: Option, + pub hotmailb2b: Option, + pub hotmailb2c: Option, + pub mimecast: Option, + pub proofpoint: Option, + pub yahoo: Option, } #[derive(Debug, Default, Deserialize, Clone, Serialize)] @@ -268,7 +321,7 @@ pub async fn load_config() -> Result { .add_source(config::File::with_name("backend_config")) .add_source(config::Environment::with_prefix("RCH").separator("__")); - let mut cfg = cfg.build()?.try_deserialize::()?; + let cfg = cfg.build()?.try_deserialize::()?; // Perform additional checks @@ -282,21 +335,10 @@ pub async fn load_config() -> Result { } } - // 2. If the `proxy` field is set, move it to the `proxies` field with the - // "default" key. - #[allow(deprecated)] - if let Some(proxy) = cfg.proxy.take() { - cfg.verif_method - .proxies - .insert("default".to_string(), proxy); - - cfg.proxy = None; - } - - // 3. Validate the verif_method proxies, meaning that for each email + // 2. Validate the verif_method proxies, meaning that for each email // provider's verification method, the proxy (if set) must exist in the // `proxies` field. - cfg.verif_method.validate_proxies()?; + cfg.get_verif_method().validate_proxies()?; Ok(cfg) } @@ -304,28 +346,31 @@ pub async fn load_config() -> Result { #[cfg(test)] mod tests { use super::*; - use check_if_email_exists::smtp::verif_method::{ - EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, VerifMethodSmtpConfig, - }; use serial_test::serial; use std::{env, time::Duration}; + use { + EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, VerifMethodSmtpConfig, + }; #[tokio::test] #[serial] async fn test_proxies() { - env::set_var("RCH__VERIF_METHOD__PROXIES__PROXY3__HOST", "test-proxy"); - env::set_var("RCH__VERIF_METHOD__PROXIES__PROXY3__PORT", "1234"); + env::set_var("RCH__OVERRIDES__PROXIES__PROXY3__HOST", "test-proxy"); + env::set_var("RCH__OVERRIDES__PROXIES__PROXY3__PORT", "1234"); let cfg = load_config().await.unwrap(); // Proxies - assert_eq!(cfg.verif_method.proxies.len(), 1); + assert_eq!(cfg.get_verif_method().proxies.len(), 1); assert_eq!( - cfg.verif_method.proxies.get("proxy3").unwrap().host, + cfg.get_verif_method().proxies.get("proxy3").unwrap().host, "test-proxy" ); - assert_eq!(cfg.verif_method.proxies.get("proxy3").unwrap().port, 1234); + assert_eq!( + cfg.get_verif_method().proxies.get("proxy3").unwrap().port, + 1234 + ); - env::remove_var("RCH__VERIF_METHOD__PROXIES__PROXY3__HOST"); - env::remove_var("RCH__VERIF_METHOD__PROXIES__PROXY3__PORT"); + env::remove_var("RCH__OVERRIDES__PROXIES__PROXY3__HOST"); + env::remove_var("RCH__OVERRIDES__PROXIES__PROXY3__PORT"); } #[tokio::test] @@ -335,12 +380,15 @@ mod tests { env::set_var("RCH__PROXY__PORT", "5678"); let cfg = load_config().await.unwrap(); // Proxies - assert_eq!(cfg.verif_method.proxies.len(), 1); + assert_eq!(cfg.get_verif_method().proxies.len(), 1); assert_eq!( - cfg.verif_method.proxies.get("default").unwrap().host, + cfg.get_verif_method().proxies.get("default").unwrap().host, "test-default-proxy" ); - assert_eq!(cfg.verif_method.proxies.get("default").unwrap().port, 5678); + assert_eq!( + cfg.get_verif_method().proxies.get("default").unwrap().port, + 5678 + ); env::remove_var("RCH__PROXY__HOST"); env::remove_var("RCH__PROXY__PORT"); diff --git a/backend/src/http/v0/check_email/post.rs b/backend/src/http/v0/check_email/post.rs index af399f55..3a76d9c3 100644 --- a/backend/src/http/v0/check_email/post.rs +++ b/backend/src/http/v0/check_email/post.rs @@ -39,7 +39,7 @@ impl CheckEmailRequest { verif_method: self .verif_method .clone() - .unwrap_or(config.verif_method.clone()), + .unwrap_or(config.get_verif_method().clone()), sentry_dsn: config.sentry_dsn.clone(), backend_name: config.backend_name.clone(), webdriver_config: config.webdriver.clone(), diff --git a/backend/src/main.rs b/backend/src/main.rs index 6ca3eefd..4230b63e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -36,6 +36,7 @@ async fn main() -> Result<(), anyhow::Error> { let mut config = load_config().await?; config.connect().await?; debug!(target: LOG_TARGET, "{:#?}", config); + debug!(target: LOG_TARGET, "{:#?}", config.get_verif_method()); // Setup sentry bug tracking. let _guard: sentry::ClientInitGuard;