From 4fd3e606fd74772b977dab9df341c79931fbb9a7 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Fri, 11 Aug 2023 03:17:58 -0400 Subject: [PATCH 1/3] LaunchServer --- bin/core/src/cloud/aws.rs | 63 +++++++++++++---------- bin/core/src/requests/execute/build.rs | 2 +- bin/core/src/requests/write/launch.rs | 69 ++++++++++++++++++++++++++ bin/core/src/requests/write/mod.rs | 2 + client/ts/src/responses.ts | 1 + client/ts/src/types.ts | 29 +++++++++++ lib/types/src/entities/mod.rs | 1 + lib/types/src/requests/write/launch.rs | 64 ++++++++++++++++++++++++ lib/types/src/requests/write/mod.rs | 2 + 9 files changed, 206 insertions(+), 27 deletions(-) create mode 100644 bin/core/src/requests/write/launch.rs create mode 100644 lib/types/src/requests/write/launch.rs diff --git a/bin/core/src/cloud/aws.rs b/bin/core/src/cloud/aws.rs index 0b7f1ca49..c7aa661a7 100644 --- a/bin/core/src/cloud/aws.rs +++ b/bin/core/src/cloud/aws.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::{str::FromStr, time::Duration}; use anyhow::{anyhow, Context}; use aws_sdk_ec2::{ @@ -6,11 +6,11 @@ use aws_sdk_ec2::{ types::{ BlockDeviceMapping, EbsBlockDevice, InstanceNetworkInterfaceSpecification, InstanceStateChange, InstanceStateName, InstanceStatus, InstanceType, ResourceType, Tag, - TagSpecification, + TagSpecification, VolumeType, }, Client, }; -use monitor_types::entities::builder::AwsBuilderConfig; +use monitor_types::requests::write::LaunchAwsServerConfig; use crate::state::State; @@ -32,45 +32,34 @@ impl State { Client::new(&config) } - pub async fn create_ec2_instance( + pub async fn launch_ec2_instance( &self, - instance_name: &str, - AwsBuilderConfig { + name: &str, + config: impl Into, + ) -> anyhow::Result { + let LaunchAwsServerConfig { region, instance_type, - volume_gb, + volumes, ami_id, subnet_id, security_group_ids, key_pair_name, assign_public_ip, - .. - }: &AwsBuilderConfig, - ) -> anyhow::Result { + } = config.into(); let instance_type = InstanceType::from(instance_type.as_str()); if let InstanceType::Unknown(t) = instance_type { return Err(anyhow!("unknown instance type {t:?}")); } let client = self.create_ec2_client(region.clone()).await; - let res = client + let mut req = client .run_instances() .image_id(ami_id) .instance_type(instance_type) - .block_device_mappings( - BlockDeviceMapping::builder() - .set_device_name(String::from("/dev/sda1").into()) - .set_ebs( - EbsBlockDevice::builder() - .volume_size(*volume_gb) - .build() - .into(), - ) - .build(), - ) .network_interfaces( InstanceNetworkInterfaceSpecification::builder() .subnet_id(subnet_id) - .associate_public_ip_address(*assign_public_ip) + .associate_public_ip_address(assign_public_ip) .set_groups(security_group_ids.to_vec().into()) .device_index(0) .build(), @@ -78,28 +67,50 @@ impl State { .key_name(key_pair_name) .tag_specifications( TagSpecification::builder() - .tags(Tag::builder().key("Name").value(instance_name).build()) + .tags(Tag::builder().key("Name").value(name).build()) .resource_type(ResourceType::Instance) .build(), ) .min_count(1) - .max_count(1) + .max_count(1); + + for volume in volumes { + let mut ebs = EbsBlockDevice::builder() + .volume_size(volume.size_gb) + .set_iops(volume.iops) + .set_throughput(volume.throughput); + if let Some(volume_type) = &volume.volume_type { + ebs = ebs + .volume_type(VolumeType::from_str(volume_type).context("invalid volume type")?); + } + req = req.block_device_mappings( + BlockDeviceMapping::builder() + .set_device_name(volume.device_name.clone().into()) + .set_ebs(ebs.build().into()) + .build(), + ) + } + + let res = req .send() .await .context("failed to start builder ec2 instance")?; + let instance = res .instances() .ok_or(anyhow!("got None for created instances"))? .get(0) .ok_or(anyhow!("instances array is empty"))?; + let instance_id = instance .instance_id() .ok_or(anyhow!("instance does not have instance_id"))? .to_string(); + for _ in 0..MAX_POLL_TRIES { let state_name = get_ec2_instance_state_name(&client, &instance_id).await?; if state_name == Some(InstanceStateName::Running) { - let ip = if *assign_public_ip { + let ip = if assign_public_ip { get_ec2_instance_public_ip(&client, &instance_id).await? } else { instance diff --git a/bin/core/src/requests/execute/build.rs b/bin/core/src/requests/execute/build.rs index 70c715fbc..f5d8df555 100644 --- a/bin/core/src/requests/execute/build.rs +++ b/bin/core/src/requests/execute/build.rs @@ -254,7 +254,7 @@ impl State { build.config.version.to_string() ); let Ec2Instance { instance_id, ip } = - self.create_ec2_instance(&instance_name, &config).await?; + self.launch_ec2_instance(&instance_name, &config).await?; let readable_sec_group_ids = config.security_group_ids.join(", "); let AwsBuilderConfig { diff --git a/bin/core/src/requests/write/launch.rs b/bin/core/src/requests/write/launch.rs new file mode 100644 index 000000000..af6f0864f --- /dev/null +++ b/bin/core/src/requests/write/launch.rs @@ -0,0 +1,69 @@ +use anyhow::anyhow; +use async_trait::async_trait; +use monitor_types::{ + entities::{ + server::PartialServerConfig, + update::{ResourceTarget, Update}, + Operation, + }, + requests::write::{self, LaunchServer, LaunchServerConfig}, +}; +use resolver_api::Resolve; + +use crate::{auth::RequestUser, helpers::make_update, state::State}; + +#[async_trait] +impl Resolve for State { + async fn resolve( + &self, + LaunchServer { name, config }: LaunchServer, + user: RequestUser, + ) -> anyhow::Result { + if !user.is_admin { + return Err(anyhow!("only admins can launch servers")); + } + let mut update = make_update( + ResourceTarget::System("system".to_string()), + Operation::LaunchServer, + &user, + ); + update.push_simple_log("launching server", format!("{:#?}", config)); + update.id = self.add_update(update.clone()).await?; + match config { + LaunchServerConfig::Aws(config) => { + let region = config.region.clone(); + let instance = self.launch_ec2_instance(&name, config).await; + if let Err(e) = &instance { + update.push_error_log( + "launch server", + format!("failed to launch aws instance\n\n{e:#?}"), + ); + update.finalize(); + self.update_update(update.clone()).await?; + return Ok(update); + } + let instance = instance.unwrap(); + update.push_simple_log( + "launch server", + format!("successfully launched server {name} on ip {}", instance.ip), + ); + let _ = self + .resolve( + write::CreateServer { + name, + config: PartialServerConfig { + address: format!("http://{}:8000", instance.ip).into(), + region: region.into(), + ..Default::default() + }, + }, + user, + ) + .await; + } + } + update.finalize(); + self.update_update(update.clone()).await?; + Ok(update) + } +} diff --git a/bin/core/src/requests/write/mod.rs b/bin/core/src/requests/write/mod.rs index d8def03a1..83e9508d8 100644 --- a/bin/core/src/requests/write/mod.rs +++ b/bin/core/src/requests/write/mod.rs @@ -20,6 +20,7 @@ mod build; mod builder; mod deployment; mod description; +mod launch; mod permissions; mod repo; mod secret; @@ -49,6 +50,7 @@ enum WriteRequest { UpdateDescription(UpdateDescription), // ==== SERVER ==== + LaunchServer(LaunchServer), CreateServer(CreateServer), DeleteServer(DeleteServer), UpdateServer(UpdateServer), diff --git a/client/ts/src/responses.ts b/client/ts/src/responses.ts index 60243b33d..611e1dfdd 100644 --- a/client/ts/src/responses.ts +++ b/client/ts/src/responses.ts @@ -100,6 +100,7 @@ export type WriteResponses = { UpdateDescription: Types.UpdateDescriptionResponse; // ==== SERVER ==== + LaunchServer: Types.Update; CreateServer: Types.Server; DeleteServer: Types.Server; UpdateServer: Types.Server; diff --git a/client/ts/src/types.ts b/client/ts/src/types.ts index 2d386b668..ab6e731b6 100644 --- a/client/ts/src/types.ts +++ b/client/ts/src/types.ts @@ -559,6 +559,7 @@ export interface CustomTag { export enum Operation { None = "None", + LaunchServer = "LaunchServer", CreateServer = "CreateServer", UpdateServer = "UpdateServer", DeleteServer = "DeleteServer", @@ -1173,6 +1174,33 @@ export interface UpdateDescription { export interface UpdateDescriptionResponse { } +export type LaunchServerConfig = + | { type: "Aws", params: LaunchAwsServerConfig }; + +export interface LaunchServer { + name: string; + config: LaunchServerConfig; +} + +export interface AwsVolume { + device_name: string; + size_gb: number; + volume_type?: string; + iops?: number; + throughput?: number; +} + +export interface LaunchAwsServerConfig { + region: string; + instance_type: string; + volumes: AwsVolume[]; + ami_id: string; + subnet_id: string; + security_group_ids: string[]; + key_pair_name: string; + assign_public_ip: boolean; +} + export interface UpdateUserPermissionsOnTarget { user_id: string; permission: PermissionLevel; @@ -1369,6 +1397,7 @@ export type WriteRequest = | { type: "UpdateUserPerimissions", params: UpdateUserPermissions } | { type: "UpdateUserPermissionsOnTarget", params: UpdateUserPermissionsOnTarget } | { type: "UpdateDescription", params: UpdateDescription } + | { type: "LaunchServer", params: LaunchServer } | { type: "CreateServer", params: CreateServer } | { type: "DeleteServer", params: DeleteServer } | { type: "UpdateServer", params: UpdateServer } diff --git a/lib/types/src/entities/mod.rs b/lib/types/src/entities/mod.rs index 9cd07cefb..717a01f10 100644 --- a/lib/types/src/entities/mod.rs +++ b/lib/types/src/entities/mod.rs @@ -267,6 +267,7 @@ pub enum Operation { None, // server + LaunchServer, CreateServer, UpdateServer, DeleteServer, diff --git a/lib/types/src/requests/write/launch.rs b/lib/types/src/requests/write/launch.rs new file mode 100644 index 000000000..91eb65e8d --- /dev/null +++ b/lib/types/src/requests/write/launch.rs @@ -0,0 +1,64 @@ +use resolver_api::derive::Request; +use serde::{Deserialize, Serialize}; +use typeshare::typeshare; + +use crate::entities::{builder::AwsBuilderConfig, update::Update}; + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(Update)] +pub struct LaunchServer { + pub name: String, + pub config: LaunchServerConfig, +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "params")] +pub enum LaunchServerConfig { + Aws(LaunchAwsServerConfig), +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct LaunchAwsServerConfig { + pub region: String, + pub instance_type: String, + pub volumes: Vec, + pub ami_id: String, + pub subnet_id: String, + pub security_group_ids: Vec, + pub key_pair_name: String, + pub assign_public_ip: bool, +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct AwsVolume { + pub device_name: String, + pub size_gb: i32, + pub volume_type: Option, + pub iops: Option, + pub throughput: Option, +} + +impl From<&AwsBuilderConfig> for LaunchAwsServerConfig { + fn from(value: &AwsBuilderConfig) -> Self { + Self { + region: value.region.clone(), + instance_type: value.instance_type.clone(), + volumes: vec![AwsVolume { + size_gb: value.volume_gb, + device_name: "/dev/sda1".to_string(), + volume_type: None, + iops: None, + throughput: None, + }], + ami_id: value.ami_id.clone(), + subnet_id: value.subnet_id.clone(), + security_group_ids: value.security_group_ids.clone(), + key_pair_name: value.key_pair_name.clone(), + assign_public_ip: value.assign_public_ip, + } + } +} diff --git a/lib/types/src/requests/write/mod.rs b/lib/types/src/requests/write/mod.rs index d59ce273c..41f34556c 100644 --- a/lib/types/src/requests/write/mod.rs +++ b/lib/types/src/requests/write/mod.rs @@ -3,6 +3,7 @@ mod build; mod builder; mod deployment; mod description; +mod launch; mod permissions; mod repo; mod secret; @@ -15,6 +16,7 @@ pub use build::*; pub use builder::*; pub use deployment::*; pub use description::*; +pub use launch::*; pub use permissions::*; pub use repo::*; pub use secret::*; From b80589ea630290068b50a6988820287e0b2b009a Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Fri, 11 Aug 2023 03:49:35 -0400 Subject: [PATCH 2/3] get available accounts --- bin/core/src/requests/read/builder.rs | 16 +++++++++++++++- bin/core/src/requests/read/mod.rs | 3 ++- bin/core/src/requests/read/server.rs | 10 +++++----- client/ts/src/responses.ts | 3 ++- client/ts/src/types.ts | 18 ++++++++++++++---- lib/types/src/requests/read/builder.rs | 16 ++++++++++++++++ lib/types/src/requests/read/server.rs | 8 ++++---- 7 files changed, 58 insertions(+), 16 deletions(-) diff --git a/bin/core/src/requests/read/builder.rs b/bin/core/src/requests/read/builder.rs index 7f1027844..a3df4fc95 100644 --- a/bin/core/src/requests/read/builder.rs +++ b/bin/core/src/requests/read/builder.rs @@ -1,7 +1,10 @@ use anyhow::Context; use async_trait::async_trait; use monitor_types::{ - entities::{builder::{Builder, BuilderListItem}, PermissionLevel}, + entities::{ + builder::{Builder, BuilderListItem}, + PermissionLevel, + }, requests::read::*, }; use mungos::mongodb::bson::doc; @@ -60,3 +63,14 @@ impl Resolve for State { Ok(res) } } + +#[async_trait] +impl Resolve for State { + async fn resolve( + &self, + GetBuilderAvailableAccounts { id }: GetBuilderAvailableAccounts, + user: RequestUser, + ) -> anyhow::Result { + todo!() + } +} diff --git a/bin/core/src/requests/read/mod.rs b/bin/core/src/requests/read/mod.rs index a7be9d7e7..480e96d76 100644 --- a/bin/core/src/requests/read/mod.rs +++ b/bin/core/src/requests/read/mod.rs @@ -52,7 +52,7 @@ enum ReadRequest { GetDockerNetworks(GetDockerNetworks), GetServerActionState(GetServerActionState), GetHistoricalServerStats(GetHistoricalServerStats), - GetAvailableAccounts(GetAvailableAccounts), + GetServerAvailableAccounts(GetServerAvailableAccounts), GetAvailableNetworks(GetAvailableNetworks), // ==== DEPLOYMENT ==== @@ -83,6 +83,7 @@ enum ReadRequest { GetBuildersSummary(GetBuildersSummary), GetBuilder(GetBuilder), ListBuilders(ListBuilders), + GetBuilderAvailableAccounts(GetBuilderAvailableAccounts), // ==== ALERTER ==== GetAlertersSummary(GetAlertersSummary), diff --git a/bin/core/src/requests/read/server.rs b/bin/core/src/requests/read/server.rs index d6a4b939c..f2dd525ac 100644 --- a/bin/core/src/requests/read/server.rs +++ b/bin/core/src/requests/read/server.rs @@ -405,21 +405,21 @@ impl Resolve for State { } #[async_trait] -impl Resolve for State { +impl Resolve for State { async fn resolve( &self, - GetAvailableAccounts { server_id }: GetAvailableAccounts, + GetServerAvailableAccounts { id }: GetServerAvailableAccounts, user: RequestUser, - ) -> anyhow::Result { + ) -> anyhow::Result { let server: Server = self - .get_resource_check_permissions(&server_id, &user, PermissionLevel::Read) + .get_resource_check_permissions(&id, &user, PermissionLevel::Read) .await?; let GetAccountsResponse { github, docker } = self .periphery_client(&server) .request(requests::GetAccounts {}) .await .context("failed to get accounts from periphery")?; - let res = GetAvailableAccountsResponse { github, docker }; + let res = GetServerAvailableAccountsResponse { github, docker }; Ok(res) } } diff --git a/client/ts/src/responses.ts b/client/ts/src/responses.ts index 611e1dfdd..3e5534153 100644 --- a/client/ts/src/responses.ts +++ b/client/ts/src/responses.ts @@ -28,7 +28,7 @@ export type ReadResponses = { GetDockerNetworks: Types.DockerNetwork[]; GetServerActionState: Types.ServerActionState; GetHistoricalServerStats: Types.GetHistoricalServerStatsResponse; - GetAvailableAccounts: Types.GetAvailableAccountsResponse; + GetServerAvailableAccounts: Types.GetServerAvailableAccountsResponse; GetAvailableNetworks: Types.GetAvailableNetworksResponse; // ==== DEPLOYMENT ==== @@ -59,6 +59,7 @@ export type ReadResponses = { GetBuildersSummary: Types.GetBuildersSummaryResponse; GetBuilder: Types.Builder; ListBuilders: Types.BuilderListItem[]; + GetBuilderAvailableAccounts: Types.GetBuilderAvailableAccountsResponse; // ==== ALERTER ==== GetAlertersSummary: Types.GetAlertersSummaryResponse; diff --git a/client/ts/src/types.ts b/client/ts/src/types.ts index ab6e731b6..28f28a595 100644 --- a/client/ts/src/types.ts +++ b/client/ts/src/types.ts @@ -847,6 +847,15 @@ export interface GetBuildersSummaryResponse { total: number; } +export interface GetBuilderAvailableAccounts { + id: string; +} + +export interface GetBuilderAvailableAccountsResponse { + github: string[]; + docker: string[]; +} + export interface GetDeployment { id: string; } @@ -1038,11 +1047,11 @@ export interface GetServersSummaryResponse { disabled: I64; } -export interface GetAvailableAccounts { - server_id: string; +export interface GetServerAvailableAccounts { + id: string; } -export interface GetAvailableAccountsResponse { +export interface GetServerAvailableAccountsResponse { github: string[]; docker: string[]; } @@ -1351,7 +1360,7 @@ export type ReadRequest = | { type: "GetDockerNetworks", params: GetDockerNetworks } | { type: "GetServerActionState", params: GetServerActionState } | { type: "GetHistoricalServerStats", params: GetHistoricalServerStats } - | { type: "GetAvailableAccounts", params: GetAvailableAccounts } + | { type: "GetServerAvailableAccounts", params: GetServerAvailableAccounts } | { type: "GetAvailableNetworks", params: GetAvailableNetworks } | { type: "GetDeploymentsSummary", params: GetDeploymentsSummary } | { type: "GetDeployment", params: GetDeployment } @@ -1374,6 +1383,7 @@ export type ReadRequest = | { type: "GetBuildersSummary", params: GetBuildersSummary } | { type: "GetBuilder", params: GetBuilder } | { type: "ListBuilders", params: ListBuilders } + | { type: "GetBuilderAvailableAccounts", params: GetBuilderAvailableAccounts } | { type: "GetAlertersSummary", params: GetAlertersSummary } | { type: "GetAlerter", params: GetAlerter } | { type: "ListAlerters", params: ListAlerters } diff --git a/lib/types/src/requests/read/builder.rs b/lib/types/src/requests/read/builder.rs index 00d7a82ea..acb7ead84 100644 --- a/lib/types/src/requests/read/builder.rs +++ b/lib/types/src/requests/read/builder.rs @@ -34,3 +34,19 @@ pub struct GetBuildersSummary {} pub struct GetBuildersSummaryResponse { pub total: u32, } + +// + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone, Request)] +#[response(GetBuilderAvailableAccountsResponse)] +pub struct GetBuilderAvailableAccounts { + pub id: String, +} + +#[typeshare] +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct GetBuilderAvailableAccountsResponse { + pub github: Vec, + pub docker: Vec, +} \ No newline at end of file diff --git a/lib/types/src/requests/read/server.rs b/lib/types/src/requests/read/server.rs index e691c7336..ea3048e41 100644 --- a/lib/types/src/requests/read/server.rs +++ b/lib/types/src/requests/read/server.rs @@ -214,14 +214,14 @@ pub struct GetServersSummaryResponse { #[typeshare] #[derive(Serialize, Deserialize, Debug, Clone, Request)] -#[response(GetAvailableAccountsResponse)] -pub struct GetAvailableAccounts { - pub server_id: String, +#[response(GetServerAvailableAccountsResponse)] +pub struct GetServerAvailableAccounts { + pub id: String, } #[typeshare] #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct GetAvailableAccountsResponse { +pub struct GetServerAvailableAccountsResponse { pub github: Vec, pub docker: Vec, } From 97c03d97adb185916e3349f41346c83bfccac685 Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Fri, 11 Aug 2023 03:54:35 -0400 Subject: [PATCH 3/3] implement get builder avaialabel accounts --- bin/core/src/requests/read/builder.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/bin/core/src/requests/read/builder.rs b/bin/core/src/requests/read/builder.rs index a3df4fc95..796512b36 100644 --- a/bin/core/src/requests/read/builder.rs +++ b/bin/core/src/requests/read/builder.rs @@ -2,10 +2,10 @@ use anyhow::Context; use async_trait::async_trait; use monitor_types::{ entities::{ - builder::{Builder, BuilderListItem}, + builder::{Builder, BuilderConfig, BuilderListItem}, PermissionLevel, }, - requests::read::*, + requests::read::{self, *}, }; use mungos::mongodb::bson::doc; use resolver_api::Resolve; @@ -71,6 +71,23 @@ impl Resolve for State { GetBuilderAvailableAccounts { id }: GetBuilderAvailableAccounts, user: RequestUser, ) -> anyhow::Result { - todo!() + let builder: Builder = self + .get_resource_check_permissions(&id, &user, PermissionLevel::Read) + .await?; + match builder.config { + BuilderConfig::Aws(config) => Ok(GetBuilderAvailableAccountsResponse { + github: config.github_accounts, + docker: config.docker_accounts, + }), + BuilderConfig::Server(config) => { + let res = self + .resolve(read::GetServerAvailableAccounts { id: config.id }, user) + .await?; + Ok(GetBuilderAvailableAccountsResponse { + github: res.github, + docker: res.docker, + }) + } + } } }