fix: Make new config backwards-compatible (#1567)

* fix: Make new config backwards-compatible

* fix test

backend/backend_config.toml

@@ -15,13 +15,30 @@ http_host = "127.0.0.1"
 # Env variable: RCH__HTTP_PORT
 http_port = 8080
 
+# Name to use during the EHLO/HELO command in the SMTP conversation.
+# Ideally, this should match the reverse DNS of the server's IP address.
+#
+# Env variable: RCH__HELLO_NAME
+hello_name = "localhost"
+
+# Email to use during the MAIL FROM command in the SMTP conversation.
+# Ideally, the domain of this email should match the "hello_name" above.
+#
+# Env variable: RCH__FROM_EMAIL
+from_email = "hello@localhost"
+
+# Timeout for each SMTP connection, in seconds. Leaving it commented out will
+# not set a timeout, i.e. the connection will wait indefinitely.
+#
+# Env variable: RCH__SMTP_TIMEOUT
+# smtp_timeout = 45
+
 # Shared secret between a trusted client and the backend, required in the
 # `x-reacher-secret` header of all incoming requests.
 #
 # Env variable: RCH__HEADER_SECRET
 # header_secret = "my-secret"
 
-
 # Optional Sentry DSN. If set, all errors will be sent to Sentry.
 #
 # Env variable: RCH__SENTRY_DSN
@@ -32,6 +49,26 @@ http_port = 8080
 # Env variable: RCH__WEBDRIVER_ADDR
 webdriver_addr = "http://localhost:9515"
 
+# Uncomment the lines below to route all SMTP verification requests
+# through a specified proxy. Note that the proxy must be a SOCKS5 proxy to work
+# with the SMTP protocol. This proxy will not be used for headless
+# verifications.
+#
+# The username and password are optional and only needed if the proxy requires
+# authentication.
+#
+# Env variables:
+# - RCH__PROXY__HOST
+# - RCH__PROXY__PORT
+# - RCH__PROXY__USERNAME
+# - RCH__PROXY__PASSWORD
+#
+# [proxy]
+# host = "my.proxy.com"
+# port = 1080
+# username = "my-username"
+# password = "my-password"
+
 [webdriver]
 # Path to the Chrome binary. If not set, the default system Chrome will be used.
 #
@@ -60,13 +97,14 @@ webdriver_addr = "http://localhost:9515"
 # username = "my-username"
 # password = "my-password"
 
-# Verification method to use for each email provider. Each email provider can
+# Override verification method to use for each email provider. Each email provider can
 # be verified using one of the following methods:
 # - Gmail: smtp
 # - Hotmail B2B: smtp
 # - Hotmail B2C: headless or smtp
 # - Yahoo: headless or smtp
-# - Everything else: smtp
+# - Mimecast: smtp
+# - Proofpoint: smtp
 #
 # For the email providers you choose to verify using the "smtp" method, you
 # may add additional configuration, such as hello_name, from_email, and
@@ -75,112 +113,50 @@ webdriver_addr = "http://localhost:9515"
 # If using proxies, the list of proxies must be defined in the "proxies"
 # section below, with a unique name for each proxy such as "proxy1", "proxy2",
 # etc. Then, in the email provider's SMTP configuration, set the value to the
-# name of the proxy to use.
-[verif_method]
+# name of the proxy to use. To use the proxy defined the the top-level "proxy"
+# section, set the value to "default".
+[overrides]
 # Use the "proxies" configuration below to route SMTP verification requests
 # through a specified proxy.
 #
 # Reacher allows you to configure multiple proxies, each with a unique name.
 # We recommend simply using "proxy1", "proxy2", etc. as the proxy names.
 #
-# In the `verif_method` section below, you can specify which proxy to use for
+# In the `overrides` section below, you can specify which proxy to use for
 # each email provider. For example, to use "proxy1" for Gmail and "proxy2" for
 # Yahoo, set the `gmail` field to "proxy1" and the `yahoo` field to "proxy2".
-[verif_method.proxies]
+[overrides.proxies]
 # Uncomment the lines below to configure a proxy. The username and password are
 # optional and only needed if the proxy requires authentication.
 #
 # Env variables:
-# - RCH__VERIF_METHOD__PROXIES__PROXY1__HOST
-# - RCH__VERIF_METHOD__PROXIES__PROXY1__PORT
-# - RCH__VERIF_METHOD__PROXIES__PROXY1__USERNAME
-# - RCH__VERIF_METHOD__PROXIES__PROXY1__PASSWORD
+# - RCH__OVERRIDES__PROXIES__PROXY1__HOST
+# - RCH__OVERRIDES__PROXIES__PROXY1__PORT
+# - RCH__OVERRIDES__PROXIES__PROXY1__USERNAME
+# - RCH__OVERRIDES__PROXIES__PROXY1__PASSWORD
 # proxy1 = { host = "my.proxy1.com", port = 1080, username = "my-username1", password = "my-password1" }
 # proxy2 = { host = "my.proxy2.com", port = 1081 }
 
-[verif_method.gmail]
-# Verification method to use for Gmail. Gmail only supports the "smtp" method.
-# You can add additional configuration such as hello_name, from_email, and
-# whether to use a proxy or not, by uncommenting the lines below.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__TYPE
-type = "smtp"
-# The proxy to use to verify Gmail. If set, it must match one of the names
-# defined in the "proxies" section above.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__PROXY
+# Set overrides for Gmail. If uncommented, make sure to uncomment all fields.
+# [overrides.gmail]
+# type = "smtp"
 # proxy = "proxy1"
-
-# Name to use during the EHLO/HELO command in the SMTP conversation.
-# Ideally, this should match the reverse DNS of the server's IP address.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__HELLO_NAME
 # hello_name = "my-domain.com"
-
-# Email to use during the MAIL FROM command in the SMTP conversation.
-# Ideally, the domain of this email should match the "hello_name" above.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__FROM_EMAIL
 # from_email = "hello@my-domain.com"
 
-# Timeout for each SMTP connection, in seconds. Leaving it commented out will
-# not set a timeout, i.e. the connection will wait indefinitely. Both fields
-# secs and nanos must be set.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__SMTP_TIMEOUT
-# smtp_timeout = { secs = 25, nanos = 0 }
+# For each email provider, you can override the verification method and set
+# additional configuration. The available fields are the same as for the
+# "gmail" section above.
 
-# Number of retries for each email verification. Leaving it commented out will
-# default to 1 retry.
-#
-# Env variable: RCH__VERIF_METHOD__GMAIL__RETRIES
-# retries = 2
+# [overrides.hotmailb2b]
 
-[verif_method.hotmailb2b]
-# Verification method to use for Hotmail B2B. Hotmail B2B only supports the
-# "smtp" method. You may copy the configuration from the Gmail section above to
-# apply them here.
-#
-# Env variable: RCH__VERIF_METHOD__HOTMAILB2B__TYPE
-type = "smtp"
-# proxy = "proxy1"
+# [overrides.hotmailb2c]
 
-[verif_method.hotmailb2c]
-# Verification method to use for Hotmail B2C. Hotmail B2C supports both the
-# "headless" and "smtp" methods. The "headless" method is recommended.
-#
-# Env variable: RCH__VERIF_METHOD__HOTMAILB2C__TYPE
-type = "headless"
+# [overrides.mimecast]
 
-[verif_method.mimecast]
-# Verification method to use for emails behind Mimecast. Mimecast only supports
-# the "smtp" method. You may copy the configuration from the Gmail section above
-# to apply them here.
-#
-# Env variable: RCH__VERIF_METHOD__MIMECAST__TYPE
-type = "smtp"
+# [overrides.proofpoint]
 
-[verif_method.proofpoint]
-# Verification method to use for emails behind Proofpoint. Proofpoint only supports
-# the "smtp" method. You may copy the configuration from the Gmail section above
-# to apply them here.
-#
-# Env variable: RCH__VERIF_METHOD__PROOFPOINT__TYPE
-type = "smtp"
-
-[verif_method.yahoo]
-# Verification method to use for Yahoo. Yahoo supports both the "headless" and
-# "smtp" methods. The "headless" method is recommended.
-#
-# Env variable: RCH__VERIF_METHOD__YAHOO__TYPE
-type = "headless"
-
-[verif_method.everything_else]
-# Verification method to use for all other email providers. By default, all
-# other emails will be verified using the "smtp" method. You may copy the
-# configuration from the Gmail section above to apply them here.
-type = "smtp"
-# proxy = "proxy1"
+# [overrides.yahoo]
 
 # Throttle the maximum number of requests per second, per minute, per hour, and
 # per day for this worker.

backend/src/config.rs

@@ -19,7 +19,11 @@ use crate::throttle::ThrottleManager;
 use crate::worker::do_work::TaskWebhook;
 use crate::worker::setup_rabbit_mq;
 use anyhow::{bail, Context};
-use check_if_email_exists::smtp::verif_method::VerifMethod;
+use check_if_email_exists::smtp::verif_method::{
+	EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, HotmailB2CVerifMethod,
+	MimecastVerifMethod, ProofpointVerifMethod, VerifMethod, VerifMethodSmtpConfig,
+	YahooVerifMethod,
+};
 use check_if_email_exists::{CheckEmailInputProxy, WebdriverConfig, LOG_TARGET};
 use config::Config;
 use lapin::Channel;
@@ -35,14 +39,19 @@ pub struct BackendConfig {
 	/// Name of the backend.
 	pub backend_name: String,
 
+	// Fields from VerifMethodSmtpConfig
+	pub from_email: String,
+	pub hello_name: String,
+	/// Timeout for each SMTP connection, in seconds. Leaving it commented out
+	/// will not set a timeout, i.e. the connection will wait indefinitely.
+	pub smtp_timeout: Option<u64>,
 	// This field is deprecated, but kept for backwards compatibility. If set,
 	// it will be moved to the "default" proxy in the `verif_method.proxies`
 	// field.
-	#[deprecated]
 	pub proxy: Option<CheckEmailInputProxy>,
 
-	/// Verification methods per email provider.
-	pub verif_method: VerifMethod,
+	/// Overrides over the default verification method provided above.
+	pub overrides: OverridesConfig,
 
 	/// Webdriver configuration.
 	pub webdriver_addr: String,
@@ -88,9 +97,11 @@ impl BackendConfig {
 			backend_name: "".to_string(),
 			webdriver_addr: "".to_string(),
 			webdriver: WebdriverConfig::default(),
-			#[allow(deprecated)]
+			from_email: "".to_string(),
+			hello_name: "".to_string(),
+			smtp_timeout: None,
 			proxy: None,
-			verif_method: VerifMethod::default(),
+			overrides: OverridesConfig::default(),
 			http_host: "127.0.0.1".to_string(),
 			http_port: 8080,
 			header_secret: None,
@@ -107,6 +118,56 @@ impl BackendConfig {
 		}
 	}
 
+	pub fn get_verif_method(&self) -> VerifMethod {
+		let mut proxies = self.overrides.proxies.clone();
+		if let Some(proxy) = self.proxy.as_ref() {
+			proxies.insert("default".to_string(), proxy.clone());
+		}
+
+		let default_smtp_config = VerifMethodSmtpConfig {
+			from_email: self.from_email.clone(),
+			hello_name: self.hello_name.clone(),
+			proxy: self.proxy.as_ref().map(|_| "default".to_string()),
+			smtp_timeout: self.smtp_timeout.map(Duration::from_secs),
+			..Default::default()
+		};
+
+		VerifMethod {
+			proxies,
+			gmail: self
+				.overrides
+				.gmail
+				.clone()
+				.unwrap_or_else(|| GmailVerifMethod::Smtp(default_smtp_config.clone())),
+			hotmailb2b: self
+				.overrides
+				.hotmailb2b
+				.clone()
+				.unwrap_or_else(|| HotmailB2BVerifMethod::Smtp(default_smtp_config.clone())),
+			hotmailb2c: self
+				.overrides
+				.hotmailb2c
+				.clone()
+				.unwrap_or_else(|| HotmailB2CVerifMethod::Headless),
+			mimecast: self
+				.overrides
+				.mimecast
+				.clone()
+				.unwrap_or_else(|| MimecastVerifMethod::Smtp(default_smtp_config.clone())),
+			proofpoint: self
+				.overrides
+				.proofpoint
+				.clone()
+				.unwrap_or_else(|| ProofpointVerifMethod::Smtp(default_smtp_config.clone())),
+			yahoo: self
+				.overrides
+				.yahoo
+				.clone()
+				.unwrap_or_else(|| YahooVerifMethod::Headless),
+			everything_else: EverythingElseVerifMethod::Smtp(default_smtp_config),
+		}
+	}
+
 	/// Get the worker configuration.
 	///
 	/// # Panics
@@ -177,22 +238,14 @@ impl BackendConfig {
 }
 
 #[derive(Debug, Default, Deserialize, Clone, Serialize)]
-pub struct VerifMethodConfig {
+pub struct OverridesConfig {
 	pub proxies: HashMap<String, CheckEmailInputProxy>,
-	pub gmail: check_if_email_exists::smtp::verif_method::GmailVerifMethod,
-	pub hotmailb2b: check_if_email_exists::smtp::verif_method::HotmailB2BVerifMethod,
-	pub hotmailb2c: check_if_email_exists::smtp::verif_method::HotmailB2CVerifMethod,
-	pub yahoo: check_if_email_exists::smtp::verif_method::YahooVerifMethod,
-	pub everything_else: check_if_email_exists::smtp::verif_method::EverythingElseVerifMethod,
-}
-
-pub struct VerifMethodSmtpConfig {
-	pub from_email: Option<String>,
-	pub hello_name: Option<String>,
-	pub smtp_port: Option<u16>,
-	pub retries: Option<u8>,
-	pub proxy: Option<String>,
-	pub smtp_timeout: Option<Duration>,
+	pub gmail: Option<GmailVerifMethod>,
+	pub hotmailb2b: Option<HotmailB2BVerifMethod>,
+	pub hotmailb2c: Option<HotmailB2CVerifMethod>,
+	pub mimecast: Option<MimecastVerifMethod>,
+	pub proofpoint: Option<ProofpointVerifMethod>,
+	pub yahoo: Option<YahooVerifMethod>,
 }
 
 #[derive(Debug, Default, Deserialize, Clone, Serialize)]
@@ -268,7 +321,7 @@ pub async fn load_config() -> Result<BackendConfig, anyhow::Error> {
 		.add_source(config::File::with_name("backend_config"))
 		.add_source(config::Environment::with_prefix("RCH").separator("__"));
 
-	let mut cfg = cfg.build()?.try_deserialize::<BackendConfig>()?;
+	let cfg = cfg.build()?.try_deserialize::<BackendConfig>()?;
 
 	// Perform additional checks
 
@@ -282,21 +335,10 @@ pub async fn load_config() -> Result<BackendConfig, anyhow::Error> {
 		}
 	}
 
-	// 2. If the `proxy` field is set, move it to the `proxies` field with the
-	// "default" key.
-	#[allow(deprecated)]
-	if let Some(proxy) = cfg.proxy.take() {
-		cfg.verif_method
-			.proxies
-			.insert("default".to_string(), proxy);
-
-		cfg.proxy = None;
-	}
-
-	// 3. Validate the verif_method proxies, meaning that for each email
+	// 2. Validate the verif_method proxies, meaning that for each email
 	// provider's verification method, the proxy (if set) must exist in the
 	// `proxies` field.
-	cfg.verif_method.validate_proxies()?;
+	cfg.get_verif_method().validate_proxies()?;
 
 	Ok(cfg)
 }
@@ -304,28 +346,31 @@ pub async fn load_config() -> Result<BackendConfig, anyhow::Error> {
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use check_if_email_exists::smtp::verif_method::{
-		EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, VerifMethodSmtpConfig,
-	};
 	use serial_test::serial;
 	use std::{env, time::Duration};
+	use {
+		EverythingElseVerifMethod, GmailVerifMethod, HotmailB2BVerifMethod, VerifMethodSmtpConfig,
+	};
 
 	#[tokio::test]
 	#[serial]
 	async fn test_proxies() {
-		env::set_var("RCH__VERIF_METHOD__PROXIES__PROXY3__HOST", "test-proxy");
-		env::set_var("RCH__VERIF_METHOD__PROXIES__PROXY3__PORT", "1234");
+		env::set_var("RCH__OVERRIDES__PROXIES__PROXY3__HOST", "test-proxy");
+		env::set_var("RCH__OVERRIDES__PROXIES__PROXY3__PORT", "1234");
 		let cfg = load_config().await.unwrap();
 		// Proxies
-		assert_eq!(cfg.verif_method.proxies.len(), 1);
+		assert_eq!(cfg.get_verif_method().proxies.len(), 1);
 		assert_eq!(
-			cfg.verif_method.proxies.get("proxy3").unwrap().host,
+			cfg.get_verif_method().proxies.get("proxy3").unwrap().host,
 			"test-proxy"
 		);
-		assert_eq!(cfg.verif_method.proxies.get("proxy3").unwrap().port, 1234);
+		assert_eq!(
+			cfg.get_verif_method().proxies.get("proxy3").unwrap().port,
+			1234
+		);
 
-		env::remove_var("RCH__VERIF_METHOD__PROXIES__PROXY3__HOST");
-		env::remove_var("RCH__VERIF_METHOD__PROXIES__PROXY3__PORT");
+		env::remove_var("RCH__OVERRIDES__PROXIES__PROXY3__HOST");
+		env::remove_var("RCH__OVERRIDES__PROXIES__PROXY3__PORT");
 	}
 
 	#[tokio::test]
@@ -335,12 +380,15 @@ mod tests {
 		env::set_var("RCH__PROXY__PORT", "5678");
 		let cfg = load_config().await.unwrap();
 		// Proxies
-		assert_eq!(cfg.verif_method.proxies.len(), 1);
+		assert_eq!(cfg.get_verif_method().proxies.len(), 1);
 		assert_eq!(
-			cfg.verif_method.proxies.get("default").unwrap().host,
+			cfg.get_verif_method().proxies.get("default").unwrap().host,
 			"test-default-proxy"
 		);
-		assert_eq!(cfg.verif_method.proxies.get("default").unwrap().port, 5678);
+		assert_eq!(
+			cfg.get_verif_method().proxies.get("default").unwrap().port,
+			5678
+		);
 
 		env::remove_var("RCH__PROXY__HOST");
 		env::remove_var("RCH__PROXY__PORT");

backend/src/http/v0/check_email/post.rs

@@ -39,7 +39,7 @@ impl CheckEmailRequest {
 			verif_method: self
 				.verif_method
 				.clone()
-				.unwrap_or(config.verif_method.clone()),
+				.unwrap_or(config.get_verif_method().clone()),
 			sentry_dsn: config.sentry_dsn.clone(),
 			backend_name: config.backend_name.clone(),
 			webdriver_config: config.webdriver.clone(),

backend/src/main.rs

@@ -36,6 +36,7 @@ async fn main() -> Result<(), anyhow::Error> {
 	let mut config = load_config().await?;
 	config.connect().await?;
 	debug!(target: LOG_TARGET, "{:#?}", config);
+	debug!(target: LOG_TARGET, "{:#?}", config.get_verif_method());
 
 	// Setup sentry bug tracking.
 	let _guard: sentry::ClientInitGuard;