From b200088093b5d00e89dc00d3093ef08de317a7fe Mon Sep 17 00:00:00 2001 From: mbecker20 Date: Sun, 24 Mar 2024 06:35:56 -0700 Subject: [PATCH] monrun sync --- Cargo.lock | 1 + bin/monrun/Cargo.toml | 3 +- bin/monrun/src/main.rs | 11 +- bin/monrun/src/maps.rs | 50 +++- bin/monrun/src/resource.rs | 173 ------------- bin/monrun/src/sync/mod.rs | 44 ++++ bin/monrun/src/sync/resource_file.rs | 78 ++++++ bin/monrun/src/sync/sync_trait.rs | 360 +++++++++++++++++++++++++++ runfile.toml | 4 +- 9 files changed, 539 insertions(+), 185 deletions(-) delete mode 100644 bin/monrun/src/resource.rs create mode 100644 bin/monrun/src/sync/mod.rs create mode 100644 bin/monrun/src/sync/resource_file.rs create mode 100644 bin/monrun/src/sync/sync_trait.rs diff --git a/Cargo.lock b/Cargo.lock index 0313195a5..aa8970826 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2206,6 +2206,7 @@ name = "monrun" version = "1.0.1" dependencies = [ "anyhow", + "async-trait", "clap", "futures", "monitor_client", diff --git a/bin/monrun/Cargo.toml b/bin/monrun/Cargo.toml index fc750c9d3..c2d4859fd 100644 --- a/bin/monrun/Cargo.toml +++ b/bin/monrun/Cargo.toml @@ -19,4 +19,5 @@ toml.workspace = true clap.workspace = true futures.workspace = true tracing.workspace = true -tracing-subscriber.workspace = true \ No newline at end of file +tracing-subscriber.workspace = true +async-trait.workspace = true \ No newline at end of file diff --git a/bin/monrun/src/main.rs b/bin/monrun/src/main.rs index fd14c39ad..405efd531 100644 --- a/bin/monrun/src/main.rs +++ b/bin/monrun/src/main.rs @@ -10,7 +10,7 @@ use serde::{de::DeserializeOwned, Deserialize}; mod execution; mod maps; -mod resource; +mod sync; #[derive(Parser, Debug)] #[command(version, about, long_about = None)] @@ -29,9 +29,7 @@ fn cli_args() -> &'static CliArgs { #[derive(Debug, Clone, Subcommand)] enum Command { /// Runs syncs on resource files - Resource { - /// The resource action to take - action: resource::SyncDirection, + Sync { /// The path of the resource folder / file /// Folder paths will recursively incorporate all the resources it finds under the folder #[arg(default_value_t = String::from("./resources"))] @@ -72,9 +70,8 @@ async fn main() -> anyhow::Result<()> { match &cli_args().command { Command::Exec { path } => execution::run_execution(path).await?, - Command::Resource { action, path } => { - resource::run_resource(*action, &PathBuf::from_str(path)?) - .await? + Command::Sync { path } => { + sync::run_sync(&PathBuf::from_str(path)?).await? } } diff --git a/bin/monrun/src/maps.rs b/bin/monrun/src/maps.rs index fdcfd5ab7..d14a3b8b3 100644 --- a/bin/monrun/src/maps.rs +++ b/bin/monrun/src/maps.rs @@ -4,8 +4,10 @@ use anyhow::Context; use monitor_client::{ api::read, entities::{ - build::BuildListItem, deployment::DeploymentListItem, - resource::ResourceListItem, server::ServerListItem, + alerter::AlerterListItem, build::BuildListItem, + builder::BuilderListItem, deployment::DeploymentListItem, + repo::RepoListItem, resource::ResourceListItem, + server::ServerListItem, }, }; @@ -69,3 +71,47 @@ pub fn name_to_server() -> &'static HashMap { .collect() }) } + +pub fn name_to_builder() -> &'static HashMap +{ + static NAME_TO_BUILDER: OnceLock> = + OnceLock::new(); + NAME_TO_BUILDER.get_or_init(|| { + futures::executor::block_on( + monitor_client().read(read::ListBuilders::default()), + ) + .expect("failed to get builders from monitor") + .into_iter() + .map(|builder| (builder.name.clone(), builder)) + .collect() + }) +} + +pub fn name_to_alerter() -> &'static HashMap +{ + static NAME_TO_ALERTER: OnceLock> = + OnceLock::new(); + NAME_TO_ALERTER.get_or_init(|| { + futures::executor::block_on( + monitor_client().read(read::ListAlerters::default()), + ) + .expect("failed to get alerters from monitor") + .into_iter() + .map(|alerter| (alerter.name.clone(), alerter)) + .collect() + }) +} + +pub fn name_to_repo() -> &'static HashMap { + static NAME_TO_ALERTER: OnceLock> = + OnceLock::new(); + NAME_TO_ALERTER.get_or_init(|| { + futures::executor::block_on( + monitor_client().read(read::ListRepos::default()), + ) + .expect("failed to get repos from monitor") + .into_iter() + .map(|repo| (repo.name.clone(), repo)) + .collect() + }) +} diff --git a/bin/monrun/src/resource.rs b/bin/monrun/src/resource.rs deleted file mode 100644 index f0b9b2e1c..000000000 --- a/bin/monrun/src/resource.rs +++ /dev/null @@ -1,173 +0,0 @@ -use std::{fs, path::Path}; - -use anyhow::{anyhow, Context}; -use clap::ValueEnum; -use monitor_client::{ - api::write, - entities::{ - build::PartialBuildConfig, deployment::PartialDeploymentConfig, - resource::Resource, server::PartialServerConfig, - }, -}; -use serde::Deserialize; - -use crate::{maps::name_to_server, monitor_client, wait_for_enter}; - -pub async fn run_resource( - action: SyncDirection, - path: &Path, -) -> anyhow::Result<()> { - info!("action: {action:?} | path: {path:?}"); - - let resources = read_resources(path)?; - - match action { - SyncDirection::Up => run_resource_up(resources).await, - SyncDirection::Down => { - todo!() - } - } -} - -async fn run_resource_up( - resources: ResourceFile, -) -> anyhow::Result<()> { - let servers = name_to_server(); - - // (name, partial config) - let mut to_update = - Vec::<(String, Resource)>::new(); - let mut to_create = Vec::>::new(); - - for server in resources.servers { - match servers.get(&server.name).map(|s| s.id.clone()) { - Some(id) => { - to_update.push((id, server)); - } - None => { - to_create.push(server); - } - } - } - - if !to_create.is_empty() { - println!( - "\nTO CREATE: {}", - to_create - .iter() - .map(|server| server.name.as_str()) - .collect::>() - .join(", ") - ); - } - - if !to_update.is_empty() { - println!( - "\nTO UPDATE: {}", - to_update - .iter() - .map(|(_, server)| server.name.as_str()) - .collect::>() - .join(", ") - ); - } - - wait_for_enter("CONTINUE")?; - - for (id, server) in to_update { - if let Err(e) = monitor_client() - .write(write::UpdateServer { - id, - config: server.config, - }) - .await - { - warn!("failed to update server {} | {e:#}", server.name) - } - } - - for server in to_create { - if let Err(e) = monitor_client() - .write(write::CreateServer { - name: server.name.clone(), - config: server.config, - }) - .await - { - warn!("failed to create server {} | {e:#}", server.name) - } - } - - Ok(()) -} - -/// Specifies resources to sync on monitor -#[derive(Debug, Clone, Default, Deserialize)] -struct ResourceFile { - #[serde(default, rename = "server")] - servers: Vec>, - #[serde(default, rename = "build")] - builds: Vec>, - #[serde(default, rename = "deployment")] - deployments: Vec>, - // #[serde(rename = "builder")] - // builders: (), - // #[serde(rename = "repo")] - // repos: (), -} - -#[derive(Debug, Clone, Copy, ValueEnum)] -pub enum SyncDirection { - /// Brings up resources / updates - Up, - /// Takes down / deletes resources - Down, -} - -fn read_resources(path: &Path) -> anyhow::Result { - let mut res = ResourceFile::default(); - read_resources_recursive(path, &mut res)?; - Ok(res) -} - -fn read_resources_recursive( - path: &Path, - resources: &mut ResourceFile, -) -> anyhow::Result<()> { - let res = - fs::metadata(path).context("failed to get path metadata")?; - if res.is_file() { - if !path - .extension() - .map(|ext| ext == "toml") - .unwrap_or_default() - { - return Ok(()); - } - let more = match crate::parse_toml_file::(path) { - Ok(res) => res, - Err(e) => { - warn!("failed to parse {:?}. skipping file | {e:#}", path); - return Ok(()); - } - }; - info!("adding resources from {path:?}"); - resources.servers.extend(more.servers); - resources.builds.extend(more.builds); - resources.deployments.extend(more.deployments); - Ok(()) - } else if res.is_dir() { - let directory = fs::read_dir(path) - .context("failed to read directory contents")?; - for entry in directory.into_iter().flatten() { - if let Err(e) = - read_resources_recursive(&entry.path(), resources) - { - warn!("failed to read additional resources at path | {e:#}"); - } - } - Ok(()) - } else { - Err(anyhow!("resources path is neither file nor directory")) - } -} diff --git a/bin/monrun/src/sync/mod.rs b/bin/monrun/src/sync/mod.rs new file mode 100644 index 000000000..ff9e1a6e6 --- /dev/null +++ b/bin/monrun/src/sync/mod.rs @@ -0,0 +1,44 @@ +use std::path::Path; + +use monitor_client::entities::{ + alerter::Alerter, build::Build, builder::Builder, + deployment::Deployment, repo::Repo, server::Server, +}; + +use crate::wait_for_enter; + +mod resource_file; +mod sync_trait; + +use sync_trait::Sync; + +pub async fn run_sync(path: &Path) -> anyhow::Result<()> { + info!("path: {path:?}"); + + let resources = resource_file::read_resources(path)?; + + let (server_updates, server_creates) = + Server::get_updates(resources.servers)?; + let (deployment_updates, deployment_creates) = + Deployment::get_updates(resources.deployments)?; + let (build_updates, build_creates) = + Build::get_updates(resources.builds)?; + let (builder_updates, builder_creates) = + Builder::get_updates(resources.builders)?; + let (alerter_updates, alerter_creates) = + Alerter::get_updates(resources.alerters)?; + let (repo_updates, repo_creates) = + Repo::get_updates(resources.repos)?; + + wait_for_enter("CONTINUE")?; + + Build::run_updates(build_updates, build_creates).await; + Server::run_updates(server_updates, server_creates).await; + Deployment::run_updates(deployment_updates, deployment_creates) + .await; + Builder::run_updates(builder_updates, builder_creates).await; + Alerter::run_updates(alerter_updates, alerter_creates).await; + Repo::run_updates(repo_updates, repo_creates).await; + + Ok(()) +} diff --git a/bin/monrun/src/sync/resource_file.rs b/bin/monrun/src/sync/resource_file.rs new file mode 100644 index 000000000..6fe0e15a0 --- /dev/null +++ b/bin/monrun/src/sync/resource_file.rs @@ -0,0 +1,78 @@ +use std::{fs, path::Path}; + +use anyhow::{anyhow, Context}; +use monitor_client::entities::{ + alerter::PartialAlerterConfig, build::PartialBuildConfig, + builder::PartialBuilderConfig, deployment::PartialDeploymentConfig, + repo::PartialRepoConfig, resource::Resource, + server::PartialServerConfig, +}; +use serde::Deserialize; + +/// Specifies resources to sync on monitor +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ResourceFile { + #[serde(default, rename = "server")] + pub servers: Vec>, + #[serde(default, rename = "build")] + pub builds: Vec>, + #[serde(default, rename = "deployment")] + pub deployments: Vec>, + #[serde(rename = "builder")] + pub builders: Vec>, + #[serde(rename = "repo")] + pub repos: Vec>, + #[serde(rename = "alerter")] + pub alerters: Vec>, +} + +pub fn read_resources(path: &Path) -> anyhow::Result { + let mut res = ResourceFile::default(); + read_resources_recursive(path, &mut res)?; + Ok(res) +} + +fn read_resources_recursive( + path: &Path, + resources: &mut ResourceFile, +) -> anyhow::Result<()> { + let res = + fs::metadata(path).context("failed to get path metadata")?; + if res.is_file() { + if !path + .extension() + .map(|ext| ext == "toml") + .unwrap_or_default() + { + return Ok(()); + } + let more = match crate::parse_toml_file::(path) { + Ok(res) => res, + Err(e) => { + warn!("failed to parse {:?}. skipping file | {e:#}", path); + return Ok(()); + } + }; + info!("adding resources from {path:?}"); + resources.servers.extend(more.servers); + resources.builds.extend(more.builds); + resources.deployments.extend(more.deployments); + resources.builders.extend(more.builders); + resources.repos.extend(more.repos); + resources.alerters.extend(more.alerters); + Ok(()) + } else if res.is_dir() { + let directory = fs::read_dir(path) + .context("failed to read directory contents")?; + for entry in directory.into_iter().flatten() { + if let Err(e) = + read_resources_recursive(&entry.path(), resources) + { + warn!("failed to read additional resources at path | {e:#}"); + } + } + Ok(()) + } else { + Err(anyhow!("resources path is neither file nor directory")) + } +} diff --git a/bin/monrun/src/sync/sync_trait.rs b/bin/monrun/src/sync/sync_trait.rs new file mode 100644 index 000000000..36c42981e --- /dev/null +++ b/bin/monrun/src/sync/sync_trait.rs @@ -0,0 +1,360 @@ +use std::collections::HashMap; + +use async_trait::async_trait; +use monitor_client::{ + api::write, + entities::{ + alerter::{Alerter, AlerterListItemInfo, PartialAlerterConfig}, + build::{Build, BuildListItemInfo, PartialBuildConfig}, + builder::{Builder, BuilderListItemInfo, PartialBuilderConfig}, + deployment::{ + Deployment, DeploymentListItemInfo, PartialDeploymentConfig, + }, + repo::{PartialRepoConfig, Repo, RepoInfo}, + resource::{Resource, ResourceListItem}, + server::{PartialServerConfig, Server, ServerListItemInfo}, + }, +}; + +use crate::{ + maps::{ + name_to_alerter, name_to_build, name_to_builder, + name_to_deployment, name_to_repo, name_to_server, + }, + monitor_client, +}; + +#[async_trait] +impl Sync for Server { + type ListItemInfo = ServerListItemInfo; + type PartialConfig = PartialServerConfig; + + fn display() -> &'static str { + "server" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_server() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateServer { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateServer { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +#[async_trait] +impl Sync for Deployment { + type PartialConfig = PartialDeploymentConfig; + type ListItemInfo = DeploymentListItemInfo; + + fn display() -> &'static str { + "deployment" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_deployment() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateDeployment { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateDeployment { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +#[async_trait] +impl Sync for Build { + type PartialConfig = PartialBuildConfig; + type ListItemInfo = BuildListItemInfo; + + fn display() -> &'static str { + "build" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_build() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateBuild { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateBuild { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +#[async_trait] +impl Sync for Builder { + type PartialConfig = PartialBuilderConfig; + type ListItemInfo = BuilderListItemInfo; + + fn display() -> &'static str { + "builder" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_builder() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateBuilder { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateBuilder { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +#[async_trait] +impl Sync for Alerter { + type PartialConfig = PartialAlerterConfig; + type ListItemInfo = AlerterListItemInfo; + + fn display() -> &'static str { + "alerter" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_alerter() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateAlerter { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateAlerter { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +#[async_trait] +impl Sync for Repo { + type PartialConfig = PartialRepoConfig; + type ListItemInfo = RepoInfo; + + fn display() -> &'static str { + "repo" + } + + fn name_to_resource( + ) -> &'static HashMap> + { + name_to_repo() + } + + async fn create( + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::CreateRepo { + name: resource.name, + config: resource.config, + }) + .await?; + Ok(()) + } + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()> { + monitor_client() + .write(write::UpdateRepo { + id, + config: resource.config, + }) + .await?; + Ok(()) + } +} + +type ToUpdate = Vec<(String, Resource)>; +type ToCreate = Vec>; +type UpdatesResult = (ToUpdate, ToCreate); + +#[async_trait] +pub trait Sync { + type PartialConfig: Send + 'static; + type ListItemInfo: 'static; + + fn display() -> &'static str; + + fn name_to_resource( + ) -> &'static HashMap>; + + async fn create( + resource: Resource, + ) -> anyhow::Result<()>; + + async fn update( + id: String, + resource: Resource, + ) -> anyhow::Result<()>; + + fn get_updates( + resources: Vec>, + ) -> anyhow::Result> { + let map = Self::name_to_resource(); + + // (name, partial config) + let mut to_update = + Vec::<(String, Resource)>::new(); + let mut to_create = Vec::>::new(); + + for resource in resources { + match map.get(&resource.name).map(|s| s.id.clone()) { + Some(id) => { + to_update.push((id, resource)); + } + None => { + to_create.push(resource); + } + } + } + + if !to_create.is_empty() { + println!( + "\nTO CREATE: {}", + to_create + .iter() + .map(|item| item.name.as_str()) + .collect::>() + .join(", ") + ); + } + + if !to_update.is_empty() { + println!( + "\nTO UPDATE: {}", + to_update + .iter() + .map(|(_, item)| item.name.as_str()) + .collect::>() + .join(", ") + ); + } + + Ok((to_update, to_create)) + } + + async fn run_updates( + to_update: ToUpdate, + to_create: ToCreate, + ) { + for (id, resource) in to_update { + let name = resource.name.clone(); + if let Err(e) = Self::update(id, resource).await { + warn!("failed to update {} {name} | {e:#}", Self::display(),) + } + } + + for resource in to_create { + let name = resource.name.clone(); + if let Err(e) = Self::create(resource).await { + warn!("failed to create {} {name} | {e:#}", Self::display(),) + } + } + } +} diff --git a/runfile.toml b/runfile.toml index 9631b74f4..ab48854c9 100644 --- a/runfile.toml +++ b/runfile.toml @@ -7,6 +7,6 @@ cmd = "node ./client/ts/generate_types.mjs" path = "frontend" cmd = "yarn dev" -[monrun-resource] +[monrun-sync] path = "bin/monrun" -cmd = "cargo run -- resource up" \ No newline at end of file +cmd = "cargo run -- sync" \ No newline at end of file