From 9fa2fd0f58d3f9faac3ba7d1c74d00549298e00e Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Sat, 25 May 2024 01:16:54 -0700 Subject: [PATCH] implement hetzner server launch --- bin/core/src/api/execute/build.rs | 2 +- bin/core/src/api/execute/server_template.rs | 57 ++- bin/core/src/cloud/aws.rs | 7 +- bin/core/src/cloud/hetzner/client.rs | 148 ++++++++ bin/core/src/cloud/hetzner/common.rs | 193 +++++++++- bin/core/src/cloud/hetzner/create_server.rs | 31 +- bin/core/src/cloud/hetzner/create_volume.rs | 36 ++ bin/core/src/cloud/hetzner/mod.rs | 336 +++++++++++++++--- bin/core/src/helpers/alert.rs | 24 +- bin/core/src/resource/server_template.rs | 4 + client/core/rs/src/entities/alert.rs | 11 + client/core/rs/src/entities/config/core.rs | 18 +- .../rs/src/entities/server_template/aws.rs | 178 ++++++++++ .../src/entities/server_template/hetzner.rs | 193 ++++++++++ .../mod.rs} | 257 +++++--------- config_example/core.config.example.toml | 5 +- 16 files changed, 1258 insertions(+), 242 deletions(-) create mode 100644 bin/core/src/cloud/hetzner/client.rs create mode 100644 bin/core/src/cloud/hetzner/create_volume.rs create mode 100644 client/core/rs/src/entities/server_template/aws.rs create mode 100644 client/core/rs/src/entities/server_template/hetzner.rs rename client/core/rs/src/entities/{server_template.rs => server_template/mod.rs} (51%) diff --git a/bin/core/src/api/execute/build.rs b/bin/core/src/api/execute/build.rs index 15fc14d30..7677f56ff 100644 --- a/bin/core/src/api/execute/build.rs +++ b/bin/core/src/api/execute/build.rs @@ -14,7 +14,7 @@ use monitor_client::{ monitor_timestamp, permission::PermissionLevel, server::Server, - server_template::AwsServerTemplateConfig, + server_template::aws::AwsServerTemplateConfig, update::{Log, Update}, user::{auto_redeploy_user, User}, Operation, diff --git a/bin/core/src/api/execute/server_template.rs b/bin/core/src/api/execute/server_template.rs index c77ea9e3e..0e89acd93 100644 --- a/bin/core/src/api/execute/server_template.rs +++ b/bin/core/src/api/execute/server_template.rs @@ -15,7 +15,10 @@ use resolver_api::Resolve; use serror::serialize_error_pretty; use crate::{ - cloud::aws::launch_ec2_instance, helpers::update::{add_update, make_update, update_update}, resource, state::{db_client, State} + cloud::{aws::launch_ec2_instance, hetzner::launch_hetzner_server}, + helpers::update::{add_update, make_update, update_update}, + resource, + state::{db_client, State}, }; impl Resolve for State { @@ -64,17 +67,19 @@ impl Resolve for State { let config = match template.config { ServerTemplateConfig::Aws(config) => { let region = config.region.clone(); - let instance = launch_ec2_instance(&name, config).await; - if let Err(e) = &instance { - update.push_error_log( - "launch server", - format!("failed to launch aws instance\n\n{e:#?}"), - ); - update.finalize(); - update_update(update.clone()).await?; - return Ok(update); - } - let instance = instance.unwrap(); + let instance = match launch_ec2_instance(&name, config).await + { + Ok(instance) => instance, + Err(e) => { + update.push_error_log( + "launch server", + format!("failed to launch aws instance\n\n{e:#?}"), + ); + update.finalize(); + update_update(update.clone()).await?; + return Ok(update); + } + }; update.push_simple_log( "launch server", format!( @@ -88,6 +93,34 @@ impl Resolve for State { ..Default::default() } } + ServerTemplateConfig::Hetzner(config) => { + let datacenter = config.datacenter; + let server = match launch_hetzner_server(&name, config).await + { + Ok(server) => server, + Err(e) => { + update.push_error_log( + "launch server", + format!("failed to launch hetzner server\n\n{e:#?}"), + ); + update.finalize(); + update_update(update.clone()).await?; + return Ok(update); + } + }; + update.push_simple_log( + "launch server", + format!( + "successfully launched server {name} on ip {}", + server.ip + ), + ); + PartialServerConfig { + address: format!("http://{}:8120", server.ip).into(), + region: datacenter.as_ref().to_string().into(), + ..Default::default() + } + } }; match self.resolve(CreateServer { name, config }, user).await { diff --git a/bin/core/src/cloud/aws.rs b/bin/core/src/cloud/aws.rs index 8a77aedd5..765b14cce 100644 --- a/bin/core/src/cloud/aws.rs +++ b/bin/core/src/cloud/aws.rs @@ -16,7 +16,7 @@ use monitor_client::entities::{ alert::{Alert, AlertData, AlertDataVariant}, monitor_timestamp, server::stats::SeverityLevel, - server_template::AwsServerTemplateConfig, + server_template::aws::AwsServerTemplateConfig, update::ResourceTarget, }; @@ -165,7 +165,7 @@ pub async fn terminate_ec2_instance_with_retry( } Err(e) => { if i == MAX_TERMINATION_TRIES - 1 { - error!("failed to terminate instance {instance_id}."); + error!("failed to terminate aws instance {instance_id}."); let alert = Alert { id: Default::default(), ts: monitor_timestamp(), @@ -175,6 +175,7 @@ pub async fn terminate_ec2_instance_with_retry( variant: AlertDataVariant::AwsBuilderTerminationFailed, data: AlertData::AwsBuilderTerminationFailed { instance_id: instance_id.to_string(), + message: format!("{e:#}"), }, resolved_ts: None, }; @@ -191,7 +192,7 @@ pub async fn terminate_ec2_instance_with_retry( unreachable!() } -#[instrument] +#[instrument(skip(client))] async fn terminate_ec2_instance_inner( client: &Client, instance_id: &str, diff --git a/bin/core/src/cloud/hetzner/client.rs b/bin/core/src/cloud/hetzner/client.rs new file mode 100644 index 000000000..eb84cdeea --- /dev/null +++ b/bin/core/src/cloud/hetzner/client.rs @@ -0,0 +1,148 @@ +use anyhow::{anyhow, Context}; +use axum::http::{HeaderName, HeaderValue}; +use reqwest::{RequestBuilder, StatusCode}; +use serde::{de::DeserializeOwned, Serialize}; + +use super::{ + common::{ + HetznerActionResponse, HetznerDatacenterResponse, + HetznerServerResponse, HetznerVolumeResponse, + }, + create_server::{CreateServerBody, CreateServerResponse}, + create_volume::{CreateVolumeBody, CreateVolumeResponse}, +}; + +const BASE_URL: &str = "https://api.hetzner.cloud/v1"; + +pub struct HetznerClient(reqwest::Client); + +impl HetznerClient { + pub fn new(token: &str) -> HetznerClient { + HetznerClient( + reqwest::ClientBuilder::new() + .default_headers( + [( + HeaderName::from_static("authorization"), + HeaderValue::from_str(&format!("Bearer {token}")) + .unwrap(), + )] + .into_iter() + .collect(), + ) + .build() + .context("failed to build Hetzner request client") + .unwrap(), + ) + } + + pub async fn get_server( + &self, + id: i64, + ) -> anyhow::Result { + self.get(&format!("/servers/{id}")).await + } + + pub async fn create_server( + &self, + body: &CreateServerBody, + ) -> anyhow::Result { + self.post("/servers", body).await + } + + pub async fn delete_server( + &self, + id: i64, + ) -> anyhow::Result { + self.delete(&format!("/servers/{id}")).await + } + + pub async fn get_volume( + &self, + id: i64, + ) -> anyhow::Result { + self.get(&format!("/volumes/{id}")).await + } + + pub async fn create_volume( + &self, + body: &CreateVolumeBody, + ) -> anyhow::Result { + self.post("/volumes", body).await + } + + pub async fn delete_volume(&self, id: i64) -> anyhow::Result<()> { + let res = self + .0 + .delete(format!("{BASE_URL}/volumes/{id}")) + .send() + .await + .context("failed at request to delete volume")?; + + let status = res.status(); + + if status == StatusCode::NO_CONTENT { + Ok(()) + } else { + let text = res + .text() + .await + .context("failed to get response body as text")?; + Err(anyhow!("{status} | {text}")) + } + } + + pub async fn list_datacenters( + &self, + ) -> anyhow::Result { + self.get("/datacenters").await + } + + async fn get( + &self, + path: &str, + ) -> anyhow::Result { + let req = self.0.get(format!("{BASE_URL}{path}")); + handle_req(req).await.with_context(|| { + format!("failed at GET request to Hetzner | path: {path}") + }) + } + + async fn post( + &self, + path: &str, + body: &Body, + ) -> anyhow::Result { + let req = self.0.post(format!("{BASE_URL}{path}")).json(&body); + handle_req(req).await.with_context(|| { + format!("failed at POST request to Hetzner | path: {path}") + }) + } + + async fn delete( + &self, + path: &str, + ) -> anyhow::Result { + let req = self.0.delete(format!("{BASE_URL}{path}")); + handle_req(req).await.with_context(|| { + format!("failed at DELETE request to Hetzner | path: {path}") + }) + } +} + +async fn handle_req( + req: RequestBuilder, +) -> anyhow::Result { + let res = req.send().await?; + + let status = res.status(); + + if status == StatusCode::OK { + res.json().await.context("failed to parse response to json") + } else { + let text = res + .text() + .await + .context("failed to get response body as text")?; + Err(anyhow!("{status} | {text}")) + } +} diff --git a/bin/core/src/cloud/hetzner/common.rs b/bin/core/src/cloud/hetzner/common.rs index a49dedab0..93627059c 100644 --- a/bin/core/src/cloud/hetzner/common.rs +++ b/bin/core/src/cloud/hetzner/common.rs @@ -1,4 +1,11 @@ -use serde::Deserialize; +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerServerResponse { + pub server: HetznerServer, +} #[derive(Debug, Clone, Deserialize)] pub struct HetznerServer { @@ -8,13 +15,13 @@ pub struct HetznerServer { pub primary_disk_size: f64, pub private_net: Vec, pub public_net: HetznerPublicNet, - pub server_type: HetznerServerType, - pub status: String, + pub server_type: HetznerServerTypeDetails, + pub status: HetznerServerStatus, pub volumes: Vec, } #[derive(Debug, Clone, Deserialize)] -pub struct HetznerServerType { +pub struct HetznerServerTypeDetails { pub architecture: String, pub cores: i64, pub cpu_type: String, @@ -36,7 +43,8 @@ pub struct HetznerPrivateNet { #[derive(Debug, Clone, Deserialize)] pub struct HetznerPublicNet { pub firewalls: Vec, - pub ipv4: HetznerIpv4, + pub floating_ips: Vec, + pub ipv4: Option, } #[derive(Debug, Clone, Deserialize)] @@ -47,8 +55,9 @@ pub struct HetznerFirewall { #[derive(Debug, Clone, Deserialize)] pub struct HetznerIpv4 { + pub id: i64, pub blocked: bool, - pub dns_prt: String, + pub dns_ptr: String, pub ip: String, } @@ -61,10 +70,15 @@ pub struct HetznerImage { pub rapid_deploy: bool, } +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerActionResponse { + pub action: HetznerAction, +} + #[derive(Debug, Clone, Deserialize)] pub struct HetznerAction { pub command: String, - pub error: HetznerError, + pub error: Option, pub finished: Option, pub id: i64, pub progress: i32, @@ -85,3 +99,168 @@ pub struct HetznerResource { #[serde(rename = "type")] pub ty: String, } + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerVolumeResponse { + pub volume: HetznerVolume, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerVolume { + /// Name of the Resource. Must be unique per Project. + pub name: String, + /// Point in time when the Resource was created (in ISO-8601 format). + pub created: String, + /// Filesystem of the Volume if formatted on creation, null if not formatted on creation + pub format: Option, + /// ID of the Volume. + pub id: i64, + /// User-defined labels ( key/value pairs) for the Resource + pub labels: HashMap, + /// Device path on the file system for the Volume + pub linux_device: String, + /// Protection configuration for the Resource. + pub protection: HetznerProtection, + /// ID of the Server the Volume is attached to, null if it is not attached at all + pub server: Option, + /// Size in GB of the Volume + pub size: i64, + /// Current status of the Volume. Allowed: `creating`, `available` + pub status: HetznerVolumeStatus, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerProtection { + /// Prevent the Resource from being deleted. + pub delete: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerDatacenterResponse { + pub datacenters: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct HetznerDatacenterDetails { + pub id: i64, + pub name: String, + pub location: serde_json::Map, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum HetznerLocation { + #[serde(rename = "nbg1")] + Nuremberg1, + #[serde(rename = "hel1")] + Helsinki1, + #[serde(rename = "fsn1")] + Falkenstein1, + #[serde(rename = "ash")] + Ashburn, + #[serde(rename = "hil")] + Hillsboro, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub enum HetznerDatacenter { + #[serde(rename = "nbg1-dc3")] + Nuremberg1Dc3, + #[serde(rename = "hel1-dc2")] + Helsinki1Dc2, + #[serde(rename = "fsn1-dc14")] + Falkenstein1Dc14, + #[serde(rename = "ash-dc1")] + AshburnDc1, + #[serde(rename = "hil-dc1")] + HillsboroDc1, +} + +impl From for HetznerLocation { + fn from(value: HetznerDatacenter) -> Self { + match value { + HetznerDatacenter::Nuremberg1Dc3 => HetznerLocation::Nuremberg1, + HetznerDatacenter::Helsinki1Dc2 => HetznerLocation::Helsinki1, + HetznerDatacenter::Falkenstein1Dc14 => { + HetznerLocation::Falkenstein1 + } + HetznerDatacenter::AshburnDc1 => HetznerLocation::Ashburn, + HetznerDatacenter::HillsboroDc1 => HetznerLocation::Hillsboro, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HetznerVolumeFormat { + Xfs, + Ext4, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HetznerVolumeStatus { + Creating, + Available, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum HetznerServerStatus { + Running, + Initializing, + Starting, + Stopping, + Off, + Deleting, + Migrating, + Rebuilding, + Unknown, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +#[allow(clippy::enum_variant_names)] +pub enum HetznerServerType { + // Shared + #[serde(rename = "cx11")] + SharedIntel1Core2Ram20Disk, + #[serde(rename = "cpx11")] + SharedAmd2Core2Ram40Disk, + #[serde(rename = "cax11")] + SharedArm2Core4Ram40Disk, + #[serde(rename = "cx21")] + SharedIntel2Core4Ram40Disk, + #[serde(rename = "cpx21")] + SharedAmd3Core4Ram80Disk, + #[serde(rename = "cax21")] + SharedArm4Core8Ram80Disk, + #[serde(rename = "cx31")] + SharedIntel2Core8Ram80Disk, + #[serde(rename = "cpx31")] + SharedAmd4Core8Ram160Disk, + #[serde(rename = "cax31")] + SharedArm8Core16Ram160Disk, + #[serde(rename = "cx41")] + SharedIntel4Core16Ram160Disk, + #[serde(rename = "cpx41")] + SharedAmd8Core16Ram240Disk, + #[serde(rename = "cax41")] + SharedArm16Core32Ram320Disk, + #[serde(rename = "cx51")] + SharedIntel8Core32Ram240Disk, + #[serde(rename = "cpx51")] + SharedAmd16Core32Ram360Disk, + // Dedicated + #[serde(rename = "ccx13")] + DedicatedAmd2Core8Ram80Disk, + #[serde(rename = "ccx23")] + DedicatedAmd4Core16Ram160Disk, + #[serde(rename = "ccx33")] + DedicatedAmd8Core32Ram240Disk, + #[serde(rename = "ccx43")] + DedicatedAmd16Core64Ram360Disk, + #[serde(rename = "ccx53")] + DedicatedAmd32Core128Ram600Disk, + #[serde(rename = "ccx63")] + DedicatedAmd48Core192Ram960Disk, +} diff --git a/bin/core/src/cloud/hetzner/create_server.rs b/bin/core/src/cloud/hetzner/create_server.rs index 13be129f6..8d220368f 100644 --- a/bin/core/src/cloud/hetzner/create_server.rs +++ b/bin/core/src/cloud/hetzner/create_server.rs @@ -2,38 +2,47 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use super::common::{HetznerAction, HetznerServer}; +use super::common::{ + HetznerAction, HetznerDatacenter, HetznerLocation, HetznerServer, + HetznerServerType, +}; #[derive(Debug, Clone, Serialize)] pub struct CreateServerBody { + /// Name of the Server to create (must be unique per Project and a valid hostname as per RFC 1123) + pub name: String, /// Auto-mount Volumes after attach - pub automount: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub automount: Option, /// ID or name of Datacenter to create Server in (must not be used together with location) - pub datacenter: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub datacenter: Option, /// ID or name of Location to create Server in (must not be used together with datacenter) - pub location: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, /// Firewalls which should be applied on the Server's public network interface at creation time pub firewalls: Vec, /// ID or name of the Image the Server is created from pub image: String, /// User-defined labels (key-value pairs) for the Resource pub labels: HashMap, - /// Name of the Server to create (must be unique per Project and a valid hostname as per RFC 1123) - pub name: String, /// Network IDs which should be attached to the Server private network interface at the creation time pub networks: Vec, /// ID of the Placement Group the server should be in - pub placement_group: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub placement_group: Option, /// Public Network options - pub public_net: PublicNet, + #[serde(skip_serializing_if = "Option::is_none")] + pub public_net: Option, /// ID or name of the Server type this Server should be created with - pub server_type: String, + pub server_type: HetznerServerType, /// SSH key IDs ( integer ) or names ( string ) which should be injected into the Server at creation time pub ssh_keys: Vec, /// This automatically triggers a Power on a Server-Server Action after the creation is finished and is returned in the next_actions response object. pub start_after_create: bool, /// Cloud-Init user data to use during Server creation. This field is limited to 32KiB. - pub user_data: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub user_data: Option, /// Volume IDs which should be attached to the Server at the creation time. Volumes must be in the same Location. pub volumes: Vec, } @@ -51,8 +60,10 @@ pub struct PublicNet { /// Attach an IPv6 on the public NIC. If false, no IPv6 address will be attached. pub enable_ipv6: bool, /// ID of the ipv4 Primary IP to use. If omitted and enable_ipv4 is true, a new ipv4 Primary IP will automatically be created. + #[serde(skip_serializing_if = "Option::is_none")] pub ipv4: Option, /// ID of the ipv6 Primary IP to use. If omitted and enable_ipv6 is true, a new ipv6 Primary IP will automatically be created. + #[serde(skip_serializing_if = "Option::is_none")] pub ipv6: Option, } diff --git a/bin/core/src/cloud/hetzner/create_volume.rs b/bin/core/src/cloud/hetzner/create_volume.rs new file mode 100644 index 000000000..79dc57e39 --- /dev/null +++ b/bin/core/src/cloud/hetzner/create_volume.rs @@ -0,0 +1,36 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use super::common::{ + HetznerAction, HetznerLocation, HetznerVolume, HetznerVolumeFormat, +}; + +#[derive(Debug, Clone, Serialize)] +pub struct CreateVolumeBody { + /// Name of the volume + pub name: String, + /// Auto-mount Volume after attach. server must be provided. + #[serde(skip_serializing_if = "Option::is_none")] + pub automount: Option, + /// Format Volume after creation. One of: xfs, ext4 + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + /// User-defined labels (key-value pairs) for the Resource + pub labels: HashMap, + /// Location to create the Volume in (can be omitted if Server is specified) + #[serde(skip_serializing_if = "Option::is_none")] + pub location: Option, + /// Server to which to attach the Volume once it's created (Volume will be created in the same Location as the server) + #[serde(skip_serializing_if = "Option::is_none")] + pub server: Option, + /// Size of the Volume in GB + pub size: i64, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct CreateVolumeResponse { + pub action: HetznerAction, + pub next_actions: Vec, + pub volume: HetznerVolume, +} diff --git a/bin/core/src/cloud/hetzner/mod.rs b/bin/core/src/cloud/hetzner/mod.rs index bfcbd6e82..00d141747 100644 --- a/bin/core/src/cloud/hetzner/mod.rs +++ b/bin/core/src/cloud/hetzner/mod.rs @@ -1,60 +1,308 @@ -use std::sync::OnceLock; +use std::{sync::OnceLock, time::Duration}; use anyhow::{anyhow, Context}; -use reqwest::StatusCode; -use serde::{de::DeserializeOwned, Serialize}; +use monitor_client::entities::{ + alert::{Alert, AlertData, AlertDataVariant}, + monitor_timestamp, + server::stats::SeverityLevel, + server_template::hetzner::{ + HetznerDatacenter, HetznerServerTemplateConfig, + HetznerServerType, HetznerVolumeFormat, + }, + update::ResourceTarget, +}; -use self::create_server::{CreateServerBody, CreateServerResponse}; +use crate::{ + cloud::hetzner::{ + common::HetznerServerStatus, create_server::CreateServerBody, + create_volume::CreateVolumeBody, + }, + config::core_config, + helpers::alert::send_alerts, +}; -pub mod common; -pub mod create_server; +use self::{ + client::HetznerClient, + common::{HetznerAction, HetznerActionResponse}, +}; -const BASE_URL: &str = "https://api.hetzner.cloud/v1"; +mod client; +mod common; +mod create_server; +mod create_volume; -pub struct HetznerClient { - client: reqwest::Client, - token: String, +fn hetzner() -> Option<&'static HetznerClient> { + static HETZNER_CLIENT: OnceLock> = + OnceLock::new(); + HETZNER_CLIENT + .get_or_init(|| { + let token = &core_config().hetzner.token; + (!token.is_empty()).then(|| HetznerClient::new(token)) + }) + .as_ref() } -impl HetznerClient { - fn new(token: &str) -> HetznerClient { - HetznerClient { - client: Default::default(), - token: format!("Bearer {token}"), - } - } +pub struct HetznerServerMinimal { + pub id: i64, + pub ip: String, +} - pub async fn create_server( - &self, - body: &CreateServerBody, - ) -> anyhow::Result { - self.post("/servers", body).await - } +const POLL_RATE_SECS: u64 = 2; +const MAX_POLL_TRIES: usize = 30; - async fn post( - &self, - path: &str, - body: &Body, - ) -> anyhow::Result { - let res = self - .client - .post(format!("{BASE_URL}{path}")) - .json(&body) - .header("authorization", &self.token) - .send() +#[instrument] +pub async fn launch_hetzner_server( + name: &str, + config: HetznerServerTemplateConfig, +) -> anyhow::Result { + let hetzner = + *hetzner().as_ref().context("Hetzner token not configured")?; + let HetznerServerTemplateConfig { + image, + automount, + datacenter, + private_network_ids, + placement_group, + enable_public_ipv4, + enable_public_ipv6, + firewall_ids, + server_type, + ssh_keys, + user_data, + use_public_ip, + labels, + volumes, + port: _, + } = config; + let datacenter = hetzner_datacenter(datacenter); + + // Create volumes and get their ids + let mut volume_ids = Vec::new(); + for volume in volumes { + let body = CreateVolumeBody { + name: volume.name, + format: Some(hetzner_format(volume.format)), + location: Some(datacenter.into()), + labels: volume.labels, + size: volume.size_gb, + automount: None, + server: None, + }; + let id = hetzner + .create_volume(&body) .await - .context("failed to make request to Hetzner")?; + .context("failed to create hetzner volume")? + .volume + .id; + volume_ids.push(id); + } - let status = res.status(); + let body = CreateServerBody { + name: name.to_string(), + automount: Some(automount), + datacenter: Some(datacenter), + location: None, + firewalls: firewall_ids + .into_iter() + .map(|firewall| create_server::Firewall { firewall }) + .collect(), + image, + labels, + networks: private_network_ids, + placement_group: (placement_group > 0).then_some(placement_group), + public_net: (enable_public_ipv4 || enable_public_ipv6).then_some( + create_server::PublicNet { + enable_ipv4: enable_public_ipv4, + enable_ipv6: enable_public_ipv6, + ipv4: None, + ipv6: None, + }, + ), + server_type: hetzner_server_type(server_type), + ssh_keys, + start_after_create: true, + user_data: (!user_data.is_empty()).then_some(user_data), + volumes: volume_ids, + }; - if status == StatusCode::OK { - res.json().await.context("failed to parse response to json") - } else { - let text = res - .text() - .await - .context("failed to get response body as text")?; - Err(anyhow!("FAILED | {path} | {status} | {text}")) + let server = hetzner + .create_server(&body) + .await + .context("failed to create hetnzer server")? + .server; + + let ip = if use_public_ip { + server.public_net.ipv4.context("instance ")?.ip + } else { + server + .private_net + .first() + .context("no private networks attached")? + .ip + .to_string() + }; + let server = HetznerServerMinimal { id: server.id, ip }; + + for _ in 0..MAX_POLL_TRIES { + tokio::time::sleep(Duration::from_secs(POLL_RATE_SECS)).await; + let Ok(res) = hetzner.get_server(server.id).await else { + continue; + }; + if matches!(res.server.status, HetznerServerStatus::Running) { + return Ok(server); + } + } + + Err(anyhow!( + "failed to verify server running after polling status" + )) +} + +const MAX_TERMINATION_TRIES: usize = 5; +const TERMINATION_WAIT_SECS: u64 = 15; + +pub async fn terminate_hetzner_server_with_retry( + id: i64, +) -> anyhow::Result<()> { + let hetzner = + *hetzner().as_ref().context("Hetzner token not configured")?; + + for i in 0..MAX_TERMINATION_TRIES { + let message = match hetzner.delete_server(id).await { + Ok(HetznerActionResponse { + action: HetznerAction { error: None, .. }, + }) => return Ok(()), + Ok(HetznerActionResponse { + action: HetznerAction { error: Some(e), .. }, + }) => (i == MAX_TERMINATION_TRIES - 1).then(|| { + format!( + "failed to terminate instance | code: {} | {}", + e.code, e.message + ) + }), + Err(e) => { + (i == MAX_TERMINATION_TRIES - 1).then(|| format!("{e:#}")) + } + }; + if let Some(message) = message { + error!("failed to terminate hetzner server {id} | {message}"); + let alert = Alert { + id: Default::default(), + ts: monitor_timestamp(), + resolved: false, + level: SeverityLevel::Critical, + target: ResourceTarget::system(), + variant: AlertDataVariant::HetznerBuilderTerminationFailed, + data: AlertData::HetznerBuilderTerminationFailed { + server_id: id, + message: message.clone(), + }, + resolved_ts: None, + }; + send_alerts(&[alert]).await; + return Err(anyhow::Error::msg(message)); + } + tokio::time::sleep(Duration::from_secs(TERMINATION_WAIT_SECS)) + .await; + } + + Ok(()) +} + +fn hetzner_format( + format: HetznerVolumeFormat, +) -> common::HetznerVolumeFormat { + match format { + HetznerVolumeFormat::Xfs => common::HetznerVolumeFormat::Xfs, + HetznerVolumeFormat::Ext4 => common::HetznerVolumeFormat::Ext4, + } +} + +fn hetzner_datacenter( + datacenter: HetznerDatacenter, +) -> common::HetznerDatacenter { + match datacenter { + HetznerDatacenter::Nuremberg1Dc3 => { + common::HetznerDatacenter::Nuremberg1Dc3 + } + HetznerDatacenter::Helsinki1Dc2 => { + common::HetznerDatacenter::Helsinki1Dc2 + } + HetznerDatacenter::Falkenstein1Dc14 => { + common::HetznerDatacenter::Falkenstein1Dc14 + } + HetznerDatacenter::AshburnDc1 => { + common::HetznerDatacenter::AshburnDc1 + } + HetznerDatacenter::HillsboroDc1 => { + common::HetznerDatacenter::HillsboroDc1 + } + } +} + +fn hetzner_server_type( + server_type: HetznerServerType, +) -> common::HetznerServerType { + match server_type { + HetznerServerType::SharedIntel1Core2Ram20Disk => { + common::HetznerServerType::SharedIntel1Core2Ram20Disk + } + HetznerServerType::SharedAmd2Core2Ram40Disk => { + common::HetznerServerType::SharedAmd2Core2Ram40Disk + } + HetznerServerType::SharedArm2Core4Ram40Disk => { + common::HetznerServerType::SharedArm2Core4Ram40Disk + } + HetznerServerType::SharedIntel2Core4Ram40Disk => { + common::HetznerServerType::SharedIntel2Core4Ram40Disk + } + HetznerServerType::SharedAmd3Core4Ram80Disk => { + common::HetznerServerType::SharedAmd3Core4Ram80Disk + } + HetznerServerType::SharedArm4Core8Ram80Disk => { + common::HetznerServerType::SharedArm4Core8Ram80Disk + } + HetznerServerType::SharedIntel2Core8Ram80Disk => { + common::HetznerServerType::SharedIntel2Core8Ram80Disk + } + HetznerServerType::SharedAmd4Core8Ram160Disk => { + common::HetznerServerType::SharedAmd4Core8Ram160Disk + } + HetznerServerType::SharedArm8Core16Ram160Disk => { + common::HetznerServerType::SharedArm8Core16Ram160Disk + } + HetznerServerType::SharedIntel4Core16Ram160Disk => { + common::HetznerServerType::SharedIntel4Core16Ram160Disk + } + HetznerServerType::SharedAmd8Core16Ram240Disk => { + common::HetznerServerType::SharedAmd8Core16Ram240Disk + } + HetznerServerType::SharedArm16Core32Ram320Disk => { + common::HetznerServerType::SharedArm16Core32Ram320Disk + } + HetznerServerType::SharedIntel8Core32Ram240Disk => { + common::HetznerServerType::SharedIntel8Core32Ram240Disk + } + HetznerServerType::SharedAmd16Core32Ram360Disk => { + common::HetznerServerType::SharedAmd16Core32Ram360Disk + } + HetznerServerType::DedicatedAmd2Core8Ram80Disk => { + common::HetznerServerType::DedicatedAmd2Core8Ram80Disk + } + HetznerServerType::DedicatedAmd4Core16Ram160Disk => { + common::HetznerServerType::DedicatedAmd4Core16Ram160Disk + } + HetznerServerType::DedicatedAmd8Core32Ram240Disk => { + common::HetznerServerType::DedicatedAmd8Core32Ram240Disk + } + HetznerServerType::DedicatedAmd16Core64Ram360Disk => { + common::HetznerServerType::DedicatedAmd16Core64Ram360Disk + } + HetznerServerType::DedicatedAmd32Core128Ram600Disk => { + common::HetznerServerType::DedicatedAmd32Core128Ram600Disk + } + HetznerServerType::DedicatedAmd48Core192Ram960Disk => { + common::HetznerServerType::DedicatedAmd48Core192Ram960Disk } } } diff --git a/bin/core/src/helpers/alert.rs b/bin/core/src/helpers/alert.rs index e98bb9397..b0c14e7f2 100644 --- a/bin/core/src/helpers/alert.rs +++ b/bin/core/src/helpers/alert.rs @@ -210,13 +210,33 @@ async fn send_slack_alert( ]; (text, blocks.into()) } - AlertData::AwsBuilderTerminationFailed { instance_id } => { + AlertData::AwsBuilderTerminationFailed { + instance_id, + message, + } => { let text = format!( "{level} | Failed to terminated AWS builder instance" ); let blocks = vec![ Block::header(text.clone()), - Block::section(format!("instance id: {instance_id}")), + Block::section(format!( + "instance id: **{instance_id}**\n{message}" + )), + ]; + (text, blocks.into()) + } + AlertData::HetznerBuilderTerminationFailed { + server_id, + message, + } => { + let text = format!( + "{level} | Failed to terminated Hetzner builder instance" + ); + let blocks = vec![ + Block::header(text.clone()), + Block::section(format!( + "server id: **{server_id}**\n{message}" + )), ]; (text, blocks.into()) } diff --git a/bin/core/src/resource/server_template.rs b/bin/core/src/resource/server_template.rs index 1f32adf53..53da0e030 100644 --- a/bin/core/src/resource/server_template.rs +++ b/bin/core/src/resource/server_template.rs @@ -43,6 +43,10 @@ impl super::MonitorResource for ServerTemplate { ServerTemplateConfigVariant::Aws.to_string(), Some(config.instance_type), ), + ServerTemplateConfig::Hetzner(config) => ( + ServerTemplateConfigVariant::Hetzner.to_string(), + Some(config.server_type.as_ref().to_string()), + ), }; ServerTemplateListItem { name: server_template.name, diff --git a/client/core/rs/src/entities/alert.rs b/client/core/rs/src/entities/alert.rs index eb65cf677..3d3c09002 100644 --- a/client/core/rs/src/entities/alert.rs +++ b/client/core/rs/src/entities/alert.rs @@ -147,7 +147,18 @@ pub enum AlertData { AwsBuilderTerminationFailed { /// The id of the aws instance which failed to terminate instance_id: String, + /// A reason for the failure + message: String, }, + + /// A Hetzner builder failed to terminate. + HetznerBuilderTerminationFailed { + /// The id of the server which failed to terminate + server_id: I64, + /// A reason for the failure + message: String, + }, + None {}, } diff --git a/client/core/rs/src/entities/config/core.rs b/client/core/rs/src/entities/config/core.rs index 1e04720c6..3360576eb 100644 --- a/client/core/rs/src/entities/config/core.rs +++ b/client/core/rs/src/entities/config/core.rs @@ -104,6 +104,9 @@ pub struct Env { pub monitor_aws_access_key_id: Option, /// Override `aws.secret_access_key` pub monitor_aws_secret_access_key: Option, + + /// Override `hetzner.token` + pub monitor_hetzner_token: Option, } fn default_config_path() -> String { @@ -188,7 +191,7 @@ fn default_config_path() -> String { /// ## if empty, the "docker organization" config option will not be shown. /// ## default: empty /// # docker_organizations = ["your_docker_org1", "your_docker_org_2"] -/// +/// /// ## allows all users to have read access on all resources /// # transparent_mode = true /// @@ -222,6 +225,9 @@ fn default_config_path() -> String { /// # aws.access_key_id = "your_aws_key_id" /// # aws.secret_access_key = "your_aws_secret_key" /// +/// ## provide hetzner api token for ephemeral builders +/// # hetzner.token = "your_hetzner_token" +/// /// ## provide core-base secrets /// [secrets] /// # SECRET_1 = "value_1" @@ -328,6 +334,10 @@ pub struct CoreConfig { #[serde(default)] pub aws: AwsCredentials, + /// Configure Hetzner credentials to use with Hetzner builds / server launches. + #[serde(default)] + pub hetzner: HetznerCredentials, + /// Configure core-based secrets. These will be preferentially interpolated into /// values if they contain a matching secret. Otherwise, the periphery will have to have the /// secret configured. @@ -471,3 +481,9 @@ pub struct AwsCredentials { /// The aws SECRET_ACCESS_KEY pub secret_access_key: String, } + +/// Provide Hetzner credentials for monitor to use. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HetznerCredentials { + pub token: String, +} diff --git a/client/core/rs/src/entities/server_template/aws.rs b/client/core/rs/src/entities/server_template/aws.rs new file mode 100644 index 000000000..7f3d3b554 --- /dev/null +++ b/client/core/rs/src/entities/server_template/aws.rs @@ -0,0 +1,178 @@ +use derive_builder::Builder; +use partial_derive2::Partial; +use serde::{Deserialize, Serialize}; +use strum::{AsRefStr, Display}; +use typeshare::typeshare; + +use crate::entities::builder::AwsBuilderConfig; + +#[typeshare(serialized_as = "Partial")] +pub type _PartialAwsServerTemplateConfig = + PartialAwsServerTemplateConfig; + +/// Aws EC2 instance config. +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, Builder, Partial)] +#[partial_derive(Debug, Clone, Default, Serialize, Deserialize)] +#[partial(skip_serializing_none, from, diff)] +pub struct AwsServerTemplateConfig { + /// The aws region to launch the server in, eg. us-east-1 + #[serde(default = "default_region")] + #[builder(default = "default_region()")] + #[partial_default(default_region())] + pub region: String, + /// The instance type to launch, eg. c5.2xlarge + #[serde(default = "default_instance_type")] + #[builder(default = "default_instance_type()")] + #[partial_default(default_instance_type())] + pub instance_type: String, + /// Specify the ami id to use. Must be set up to start the periphery binary on startup. + pub ami_id: String, + /// The subnet to assign to the instance. + pub subnet_id: String, + /// The key pair name to give to the instance in case SSH access required. + pub key_pair_name: String, + /// Assign a public ip to the instance. Depending on how your network is + /// setup, this may be required for the instance to reach the public internet. + #[serde(default = "default_assign_public_ip")] + #[builder(default = "default_assign_public_ip()")] + #[partial_default(default_assign_public_ip())] + pub assign_public_ip: bool, + /// Use the instances public ip as the address for the server. + /// Could be used when build instances are created in another non-interconnected network to the core api. + #[serde(default = "default_use_public_ip")] + #[builder(default = "default_use_public_ip()")] + #[partial_default(default_use_public_ip())] + pub use_public_ip: bool, + /// The port periphery will be running on in AMI. + /// Default: `8120` + #[serde(default = "default_port")] + #[builder(default = "default_port()")] + #[partial_default(default_port())] + pub port: i32, + /// The user data to deploy the instance with. + #[serde(default)] + #[builder(default)] + pub user_data: String, + /// The security groups to give to the instance. + #[serde(default)] + #[builder(default)] + pub security_group_ids: Vec, + /// Specify the EBS volumes to attach. + #[serde(default = "default_volumes")] + #[builder(default = "default_volumes()")] + #[partial_default(default_volumes())] + pub volumes: Vec, +} + +fn default_region() -> String { + String::from("us-east-1") +} + +fn default_instance_type() -> String { + String::from("t3.small") +} + +fn default_assign_public_ip() -> bool { + true +} + +fn default_use_public_ip() -> bool { + false +} + +fn default_volumes() -> Vec { + vec![AwsVolume { + device_name: "/dev/sda1".to_string(), + size_gb: 20, + volume_type: AwsVolumeType::Gp2, + iops: 0, + throughput: 0, + }] +} + +fn default_port() -> i32 { + 8120 +} + +impl Default for AwsServerTemplateConfig { + fn default() -> Self { + Self { + region: default_region(), + instance_type: default_instance_type(), + assign_public_ip: default_assign_public_ip(), + use_public_ip: default_use_public_ip(), + port: default_port(), + volumes: default_volumes(), + ami_id: Default::default(), + subnet_id: Default::default(), + key_pair_name: Default::default(), + user_data: Default::default(), + security_group_ids: Default::default(), + } + } +} + +/// For information on AWS volumes, see +/// ``. +#[typeshare] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AwsVolume { + /// The device name (for example, `/dev/sda1` or `xvdh`). + pub device_name: String, + /// The size of the volume in GB + pub size_gb: i32, + /// The type of volume. Options: gp2, gp3, io1, io2. + pub volume_type: AwsVolumeType, + /// The iops of the volume, or 0 for AWS default. + pub iops: i32, + /// The throughput of the volume, or 0 for AWS default. + pub throughput: i32, +} + +#[typeshare] +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Eq, + Serialize, + Deserialize, + Display, + AsRefStr, +)] +#[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] +pub enum AwsVolumeType { + #[default] + Gp2, + Gp3, + Io1, + Io2, +} + +impl AwsServerTemplateConfig { + pub fn from_builder_config(value: &AwsBuilderConfig) -> Self { + Self { + region: value.region.clone(), + instance_type: value.instance_type.clone(), + volumes: vec![AwsVolume { + device_name: "/dev/sda1".to_string(), + size_gb: value.volume_gb, + volume_type: AwsVolumeType::Gp2, + iops: 0, + throughput: 0, + }], + ami_id: value.ami_id.clone(), + subnet_id: value.subnet_id.clone(), + security_group_ids: value.security_group_ids.clone(), + key_pair_name: value.key_pair_name.clone(), + assign_public_ip: value.assign_public_ip, + use_public_ip: value.use_public_ip, + port: value.port, + user_data: Default::default(), + } + } +} diff --git a/client/core/rs/src/entities/server_template/hetzner.rs b/client/core/rs/src/entities/server_template/hetzner.rs new file mode 100644 index 000000000..41c83b45d --- /dev/null +++ b/client/core/rs/src/entities/server_template/hetzner.rs @@ -0,0 +1,193 @@ +use std::collections::HashMap; + +use derive_builder::Builder; +use partial_derive2::Partial; +use serde::{Deserialize, Serialize}; +use strum::AsRefStr; +use typeshare::typeshare; + +use crate::entities::I64; + +#[typeshare(serialized_as = "Partial")] +pub type _PartialHetznerServerTemplateConfig = + PartialHetznerServerTemplateConfig; + +/// Hetzner server config. +#[typeshare] +#[derive(Debug, Clone, Serialize, Deserialize, Builder, Partial)] +#[partial_derive(Debug, Clone, Default, Serialize, Deserialize)] +#[partial(skip_serializing_none, from, diff)] +pub struct HetznerServerTemplateConfig { + /// ID or name of the Image the Server is created from + #[serde(default)] + #[builder(default)] + pub image: String, + /// Auto-mount Volumes after attach + #[serde(default = "default_automount")] + #[builder(default = "default_automount()")] + #[partial_default(default_automount())] + pub automount: bool, + /// ID or name of Datacenter to create Server in + #[serde(default)] + #[builder(default)] + pub datacenter: HetznerDatacenter, + /// Network IDs which should be attached to the Server private network interface at the creation time + #[serde(default)] + #[builder(default)] + pub private_network_ids: Vec, + /// ID of the Placement Group the server should be in, + /// Or 0 to not use placement group. + #[serde(default)] + #[builder(default)] + pub placement_group: I64, + /// Attach an IPv4 on the public NIC. If false, no IPv4 address will be attached. + #[serde(default)] + #[builder(default)] + pub enable_public_ipv4: bool, + /// Attach an IPv6 on the public NIC. If false, no IPv6 address will be attached. + #[serde(default)] + #[builder(default)] + pub enable_public_ipv6: bool, + /// The firewalls to attach to the instance + #[serde(default)] + #[builder(default)] + pub firewall_ids: Vec, + /// ID or name of the Server type this Server should be created with + #[serde(default)] + #[builder(default)] + pub server_type: HetznerServerType, + /// SSH key IDs ( integer ) or names ( string ) which should be injected into the Server at creation time + #[serde(default)] + #[builder(default)] + pub ssh_keys: Vec, + /// Cloud-Init user data to use during Server creation. This field is limited to 32KiB. + #[serde(default)] + #[builder(default)] + pub user_data: String, + /// Connect to the instance using it's public ip. + #[serde(default)] + #[builder(default)] + pub use_public_ip: bool, + /// Labels for the server + #[serde(default)] + #[builder(default)] + pub labels: HashMap, + /// Specs for volumes to attach + #[serde(default)] + #[builder(default)] + pub volumes: Vec, + /// The port periphery will be running on in AMI. + /// Default: `8120` + #[serde(default = "default_port")] + #[builder(default = "default_port()")] + #[partial_default(default_port())] + pub port: i32, +} + +fn default_automount() -> bool { + true +} + +fn default_port() -> i32 { + 8120 +} + +impl Default for HetznerServerTemplateConfig { + fn default() -> Self { + Self { + automount: default_automount(), + port: default_port(), + image: Default::default(), + datacenter: Default::default(), + private_network_ids: Default::default(), + placement_group: Default::default(), + enable_public_ipv4: Default::default(), + enable_public_ipv6: Default::default(), + firewall_ids: Default::default(), + server_type: Default::default(), + ssh_keys: Default::default(), + user_data: Default::default(), + use_public_ip: Default::default(), + labels: Default::default(), + volumes: Default::default(), + } + } +} + +#[typeshare] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct HetznerVolumeSpecs { + /// A name for the volume + pub name: String, + /// The format for the volume + pub format: HetznerVolumeFormat, + /// Labels for the volume + pub labels: HashMap, + /// Size of the volume in GB + pub size_gb: I64, +} + +#[typeshare] +#[derive( + Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize, +)] +pub enum HetznerVolumeFormat { + #[default] + Xfs, + Ext4, +} + +#[typeshare] +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Serialize, + Deserialize, + AsRefStr, +)] +#[allow(clippy::enum_variant_names)] +pub enum HetznerServerType { + #[default] + SharedIntel1Core2Ram20Disk, + SharedAmd2Core2Ram40Disk, + SharedArm2Core4Ram40Disk, + SharedIntel2Core4Ram40Disk, + SharedAmd3Core4Ram80Disk, + SharedArm4Core8Ram80Disk, + SharedIntel2Core8Ram80Disk, + SharedAmd4Core8Ram160Disk, + SharedArm8Core16Ram160Disk, + SharedIntel4Core16Ram160Disk, + SharedAmd8Core16Ram240Disk, + SharedArm16Core32Ram320Disk, + SharedIntel8Core32Ram240Disk, + SharedAmd16Core32Ram360Disk, + DedicatedAmd2Core8Ram80Disk, + DedicatedAmd4Core16Ram160Disk, + DedicatedAmd8Core32Ram240Disk, + DedicatedAmd16Core64Ram360Disk, + DedicatedAmd32Core128Ram600Disk, + DedicatedAmd48Core192Ram960Disk, +} + +#[derive( + Debug, + Clone, + Copy, + Default, + PartialEq, + Serialize, + Deserialize, + AsRefStr, +)] +pub enum HetznerDatacenter { + #[default] + Nuremberg1Dc3, + Helsinki1Dc2, + Falkenstein1Dc14, + AshburnDc1, + HillsboroDc1, +} diff --git a/client/core/rs/src/entities/server_template.rs b/client/core/rs/src/entities/server_template/mod.rs similarity index 51% rename from client/core/rs/src/entities/server_template.rs rename to client/core/rs/src/entities/server_template/mod.rs index c1bd1e442..9e430d628 100644 --- a/client/core/rs/src/entities/server_template.rs +++ b/client/core/rs/src/entities/server_template/mod.rs @@ -1,18 +1,23 @@ use bson::{doc, Document}; -use derive_builder::Builder; use derive_default_builder::DefaultBuilder; use derive_variants::EnumVariants; -use partial_derive2::{Diff, MaybeNone, Partial, PartialDiff}; +use partial_derive2::{Diff, MaybeNone, PartialDiff}; use serde::{Deserialize, Serialize}; use strum::{AsRefStr, Display, EnumString}; use typeshare::typeshare; +use self::{ + aws::AwsServerTemplateConfig, hetzner::HetznerServerTemplateConfig, +}; + use super::{ - builder::AwsBuilderConfig, resource::{AddFilters, Resource, ResourceListItem, ResourceQuery}, MergePartial, }; +pub mod aws; +pub mod hetzner; + #[typeshare] pub type ServerTemplate = Resource; @@ -44,7 +49,9 @@ pub struct ServerTemplateListItemInfo { #[serde(tag = "type", content = "params")] pub enum ServerTemplateConfig { /// Template to launch an AWS EC2 instance - Aws(AwsServerTemplateConfig), + Aws(aws::AwsServerTemplateConfig), + /// Template to launch a Hetzner server + Hetzner(hetzner::HetznerServerTemplateConfig), } #[typeshare] @@ -61,20 +68,25 @@ pub enum ServerTemplateConfig { )] #[serde(tag = "type", content = "params")] pub enum PartialServerTemplateConfig { - Aws(_PartialAwsServerTemplateConfig), + Aws(aws::_PartialAwsServerTemplateConfig), + Hetzner(hetzner::_PartialHetznerServerTemplateConfig), } impl MaybeNone for PartialServerTemplateConfig { fn is_none(&self) -> bool { match self { PartialServerTemplateConfig::Aws(config) => config.is_none(), + PartialServerTemplateConfig::Hetzner(config) => { + config.is_none() + } } } } #[derive(Debug, Clone, Serialize, Deserialize)] pub enum ServerTemplateConfigDiff { - Aws(AwsServerTemplateConfigDiff), + Aws(aws::AwsServerTemplateConfigDiff), + Hetzner(hetzner::HetznerServerTemplateConfigDiff), } impl From for PartialServerTemplateConfig { @@ -83,6 +95,9 @@ impl From for PartialServerTemplateConfig { ServerTemplateConfigDiff::Aws(diff) => { PartialServerTemplateConfig::Aws(diff.into()) } + ServerTemplateConfigDiff::Hetzner(diff) => { + PartialServerTemplateConfig::Hetzner(diff.into()) + } } } } @@ -92,7 +107,12 @@ impl Diff for ServerTemplateConfigDiff { &self, ) -> impl Iterator { match self { - ServerTemplateConfigDiff::Aws(diff) => diff.iter_field_diffs(), + ServerTemplateConfigDiff::Aws(diff) => { + diff.iter_field_diffs().collect::>().into_iter() + } + ServerTemplateConfigDiff::Hetzner(diff) => { + diff.iter_field_diffs().collect::>().into_iter() + } } } } @@ -112,6 +132,23 @@ impl original.partial_diff(partial), ) } + PartialServerTemplateConfig::Hetzner(partial) => { + let default = HetznerServerTemplateConfig::default(); + ServerTemplateConfigDiff::Hetzner( + default.partial_diff(partial), + ) + } + }, + ServerTemplateConfig::Hetzner(original) => match partial { + PartialServerTemplateConfig::Hetzner(partial) => { + ServerTemplateConfigDiff::Hetzner( + original.partial_diff(partial), + ) + } + PartialServerTemplateConfig::Aws(partial) => { + let default = AwsServerTemplateConfig::default(); + ServerTemplateConfigDiff::Aws(default.partial_diff(partial)) + } }, } } @@ -121,6 +158,7 @@ impl MaybeNone for ServerTemplateConfigDiff { fn is_none(&self) -> bool { match self { ServerTemplateConfigDiff::Aws(config) => config.is_none(), + ServerTemplateConfigDiff::Hetzner(config) => config.is_none(), } } } @@ -133,6 +171,9 @@ impl From for ServerTemplateConfig { PartialServerTemplateConfig::Aws(config) => { ServerTemplateConfig::Aws(config.into()) } + PartialServerTemplateConfig::Hetzner(config) => { + ServerTemplateConfig::Hetzner(config.into()) + } } } } @@ -143,6 +184,9 @@ impl From for PartialServerTemplateConfig { ServerTemplateConfig::Aws(config) => { PartialServerTemplateConfig::Aws(config.into()) } + ServerTemplateConfig::Hetzner(config) => { + PartialServerTemplateConfig::Hetzner(config.into()) + } } } } @@ -156,7 +200,7 @@ impl MergePartial for ServerTemplateConfig { match partial { PartialServerTemplateConfig::Aws(partial) => match self { ServerTemplateConfig::Aws(config) => { - let config = AwsServerTemplateConfig { + let config = aws::AwsServerTemplateConfig { region: partial.region.unwrap_or(config.region), instance_type: partial .instance_type @@ -181,160 +225,51 @@ impl MergePartial for ServerTemplateConfig { }; ServerTemplateConfig::Aws(config) } + ServerTemplateConfig::Hetzner(_) => { + ServerTemplateConfig::Aws(partial.into()) + } + }, + PartialServerTemplateConfig::Hetzner(partial) => match self { + ServerTemplateConfig::Hetzner(config) => { + let config = hetzner::HetznerServerTemplateConfig { + image: partial.image.unwrap_or(config.image), + automount: partial.automount.unwrap_or(config.automount), + datacenter: partial + .datacenter + .unwrap_or(config.datacenter), + private_network_ids: partial + .private_network_ids + .unwrap_or(config.private_network_ids), + placement_group: partial + .placement_group + .unwrap_or(config.placement_group), + enable_public_ipv4: partial + .enable_public_ipv4 + .unwrap_or(config.enable_public_ipv4), + enable_public_ipv6: partial + .enable_public_ipv6 + .unwrap_or(config.enable_public_ipv6), + firewall_ids: partial + .firewall_ids + .unwrap_or(config.firewall_ids), + server_type: partial + .server_type + .unwrap_or(config.server_type), + ssh_keys: partial.ssh_keys.unwrap_or(config.ssh_keys), + user_data: partial.user_data.unwrap_or(config.user_data), + use_public_ip: partial + .use_public_ip + .unwrap_or(config.use_public_ip), + labels: partial.labels.unwrap_or(config.labels), + volumes: partial.volumes.unwrap_or(config.volumes), + port: partial.port.unwrap_or(config.port), + }; + ServerTemplateConfig::Hetzner(config) + } + ServerTemplateConfig::Aws(_) => { + ServerTemplateConfig::Hetzner(partial.into()) + } }, - } - } -} - -#[typeshare(serialized_as = "Partial")] -pub type _PartialAwsServerTemplateConfig = - PartialAwsServerTemplateConfig; - -/// Aws EC2 instance config. -#[typeshare] -#[derive(Debug, Clone, Serialize, Deserialize, Builder, Partial)] -#[partial_derive(Debug, Clone, Default, Serialize, Deserialize)] -#[partial(skip_serializing_none, from, diff)] -pub struct AwsServerTemplateConfig { - /// The aws region to launch the server in, eg. us-east-1 - #[serde(default = "default_region")] - #[builder(default = "default_region()")] - #[partial_default(default_region())] - pub region: String, - /// The instance type to launch, eg. c5.2xlarge - #[serde(default = "default_instance_type")] - #[builder(default = "default_instance_type()")] - #[partial_default(default_instance_type())] - pub instance_type: String, - /// Specify the ami id to use. Must be set up to start the periphery binary on startup. - pub ami_id: String, - /// The subnet to assign to the instance. - pub subnet_id: String, - /// The key pair name to give to the instance in case SSH access required. - pub key_pair_name: String, - /// Assign a public ip to the instance. Depending on how your network is - /// setup, this may be required for the instance to reach the public internet. - #[serde(default = "default_assign_public_ip")] - #[builder(default = "default_assign_public_ip()")] - #[partial_default(default_assign_public_ip())] - pub assign_public_ip: bool, - /// Use the instances public ip as the address for the server. - /// Could be used when build instances are created in another non-interconnected network to the core api. - #[serde(default = "default_use_public_ip")] - #[builder(default = "default_use_public_ip()")] - #[partial_default(default_use_public_ip())] - pub use_public_ip: bool, - /// The port periphery will be running on in AMI. - /// Default: `8120` - #[serde(default = "default_port")] - #[builder(default = "default_port()")] - #[partial_default(default_port())] - pub port: i32, - /// The user data to deploy the instance with. - #[serde(default)] - #[builder(default)] - pub user_data: String, - /// The security groups to give to the instance. - #[serde(default)] - #[builder(default)] - pub security_group_ids: Vec, - /// Specify the EBS volumes to attach. - #[serde(default = "default_volumes")] - #[builder(default = "default_volumes()")] - #[partial_default(default_volumes())] - pub volumes: Vec, -} - -fn default_region() -> String { - String::from("us-east-1") -} - -fn default_instance_type() -> String { - String::from("t3.small") -} - -fn default_assign_public_ip() -> bool { - true -} - -fn default_use_public_ip() -> bool { - false -} - -fn default_volumes() -> Vec { - vec![AwsVolume { - device_name: "/dev/sda1".to_string(), - size_gb: 20, - volume_type: AwsVolumeType::Gp2, - iops: 0, - throughput: 0, - }] -} - -fn default_port() -> i32 { - 8120 -} - -/// For information on AWS volumes, see -/// ``. -#[typeshare] -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AwsVolume { - /// The device name (for example, `/dev/sda1` or `xvdh`). - pub device_name: String, - /// The size of the volume in GB - pub size_gb: i32, - /// The type of volume. Options: gp2, gp3, io1, io2. - pub volume_type: AwsVolumeType, - /// The iops of the volume, or 0 for AWS default. - pub iops: i32, - /// The throughput of the volume, or 0 for AWS default. - pub throughput: i32, -} - -#[typeshare] -#[derive( - Debug, - Clone, - Copy, - Default, - PartialEq, - Eq, - Serialize, - Deserialize, - Display, - AsRefStr, -)] -#[serde(rename_all = "lowercase")] -#[strum(serialize_all = "lowercase")] -pub enum AwsVolumeType { - #[default] - Gp2, - Gp3, - Io1, - Io2, -} - -impl AwsServerTemplateConfig { - pub fn from_builder_config(value: &AwsBuilderConfig) -> Self { - Self { - region: value.region.clone(), - instance_type: value.instance_type.clone(), - volumes: vec![AwsVolume { - device_name: "/dev/sda1".to_string(), - size_gb: value.volume_gb, - volume_type: AwsVolumeType::Gp2, - iops: 0, - throughput: 0, - }], - ami_id: value.ami_id.clone(), - subnet_id: value.subnet_id.clone(), - security_group_ids: value.security_group_ids.clone(), - key_pair_name: value.key_pair_name.clone(), - assign_public_ip: value.assign_public_ip, - use_public_ip: value.use_public_ip, - port: value.port, - user_data: Default::default(), } } } diff --git a/config_example/core.config.example.toml b/config_example/core.config.example.toml index 58108b7c4..08ba2f014 100644 --- a/config_example/core.config.example.toml +++ b/config_example/core.config.example.toml @@ -89,10 +89,13 @@ mongo.address = "localhost:27017" ## default: monitor_core. this is the assigned app_name of the mongo client # mongo.app_name = "monitor_core" -## provide aws api keys for ephemeral builders +## provide aws api keys for ephemeral builders / server launch # aws.access_key_id = "your_aws_key_id" # aws.secret_access_key = "your_aws_secret_key" +## provide hetzner api token for ephemeral builders / server launch +# hetzner.token = "your_hetzner_token" + ## provide core-base secrets # [secrets] # SECRET_1 = "value_1"