From faa8929ae1cc22c4ea6157e7aec46dc95eda8151 Mon Sep 17 00:00:00 2001 From: Patrick Sullivan Date: Tue, 26 Nov 2024 21:22:40 -0600 Subject: [PATCH] feat: migrate dashboards out of `meta` table and into their own table (#5146) ## Summary by CodeRabbit ## Release Notes - **New Features** - Enhanced dashboard management with new methods for retrieving properties like ID, owner, title, and description. - Added functionality for creating, updating, listing, and deleting dashboards through a new service module. - Introduced a new `dashboards` table in the database, with migrations for creation and population. - **Bug Fixes** - Improved error handling for dashboard retrieval and management operations, providing clearer responses for various conditions. - **Documentation** - Updated documentation to reflect new methods and error handling improvements. - **Refactor** - Streamlined the dashboard handling logic in service and database modules, reducing complexity and enhancing modularity. - Enhanced error handling logic in dashboard management functions for better clarity and control flow. - Refined the dashboard retrieval process to improve clarity in control flow and error handling. - Updated the logic for dashboard operations to simplify version handling and reduce redundancy. - Improved the overall structure and organization of the dashboard management code. --- src/config/src/meta/dashboards/mod.rs | 136 ++++++++ src/config/src/meta/dashboards/v1/mod.rs | 23 ++ src/config/src/meta/dashboards/v2/mod.rs | 23 ++ src/config/src/meta/dashboards/v3/mod.rs | 22 ++ src/config/src/meta/dashboards/v4/mod.rs | 23 ++ src/config/src/meta/dashboards/v5/mod.rs | 23 ++ src/infra/src/errors/mod.rs | 38 ++ src/infra/src/table/dashboards.rs | 328 ++++++++++++++++++ src/infra/src/table/entity/dashboards.rs | 40 +++ src/infra/src/table/entity/folders.rs | 11 +- src/infra/src/table/entity/mod.rs | 1 + src/infra/src/table/entity/prelude.rs | 2 +- src/infra/src/table/folders.rs | 2 +- ...20241119_000001_create_dashboards_table.rs | 204 +++++++++++ ...241119_000002_populate_dashboards_table.rs | 233 +++++++++++++ .../m20241119_000003_delete_metas.rs | 68 ++++ src/infra/src/table/migration/mod.rs | 6 + src/infra/src/table/mod.rs | 1 + src/service/dashboards/mod.rs | 52 ++- src/service/dashboards/reports.rs | 4 +- src/service/db/dashboards/mod.rs | 287 +++------------ tests/api-testing/tests/test_folders.py | 8 +- 22 files changed, 1279 insertions(+), 256 deletions(-) create mode 100644 src/infra/src/table/dashboards.rs create mode 100644 src/infra/src/table/entity/dashboards.rs create mode 100644 src/infra/src/table/migration/m20241119_000001_create_dashboards_table.rs create mode 100644 src/infra/src/table/migration/m20241119_000002_populate_dashboards_table.rs create mode 100644 src/infra/src/table/migration/m20241119_000003_delete_metas.rs diff --git a/src/config/src/meta/dashboards/mod.rs b/src/config/src/meta/dashboards/mod.rs index 07f12e004d..0ab5d9372e 100644 --- a/src/config/src/meta/dashboards/mod.rs +++ b/src/config/src/meta/dashboards/mod.rs @@ -40,6 +40,142 @@ pub struct Dashboard { pub hash: String, } +impl Dashboard { + pub fn dashboard_id(&self) -> Option<&str> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.dashboard_id.as_str()), + 2 => self.v2.as_ref().map(|inner| inner.dashboard_id.as_str()), + 3 => self.v3.as_ref().map(|inner| inner.dashboard_id.as_str()), + 4 => self.v4.as_ref().map(|inner| inner.dashboard_id.as_str()), + 5 => self.v5.as_ref().map(|inner| inner.dashboard_id.as_str()), + _ => None, + } + } + + pub fn set_dashboard_id(&mut self, dashboard_id: String) { + match self { + Self { + version: 1, + v1: Some(inner), + .. + } => inner.dashboard_id = dashboard_id, + Self { + version: 2, + v2: Some(inner), + .. + } => inner.dashboard_id = dashboard_id, + Self { + version: 3, + v3: Some(inner), + .. + } => inner.dashboard_id = dashboard_id, + Self { + version: 4, + v4: Some(inner), + .. + } => inner.dashboard_id = dashboard_id, + Self { + version: 5, + v5: Some(inner), + .. + } => inner.dashboard_id = dashboard_id, + _ => {} + }; + } + + pub fn owner(&self) -> Option<&str> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.owner.as_str()), + 2 => self.v2.as_ref().map(|inner| inner.owner.as_str()), + 3 => self.v3.as_ref().map(|inner| inner.owner.as_str()), + 4 => self.v4.as_ref().map(|inner| inner.owner.as_str()), + 5 => self.v5.as_ref().map(|inner| inner.owner.as_str()), + _ => None, + } + } + + pub fn title(&self) -> Option<&str> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.title.as_str()), + 2 => self.v2.as_ref().map(|inner| inner.title.as_str()), + 3 => self.v3.as_ref().map(|inner| inner.title.as_str()), + 4 => self.v4.as_ref().map(|inner| inner.title.as_str()), + 5 => self.v5.as_ref().map(|inner| inner.title.as_str()), + _ => None, + } + } + + pub fn set_title(&mut self, title: String) { + match self { + Self { + version: 1, + v1: Some(inner), + .. + } => inner.title = title, + Self { + version: 2, + v2: Some(inner), + .. + } => inner.title = title, + Self { + version: 3, + v3: Some(inner), + .. + } => inner.title = title, + Self { + version: 4, + v4: Some(inner), + .. + } => inner.title = title, + Self { + version: 5, + v5: Some(inner), + .. + } => inner.title = title, + _ => {} + }; + } + + pub fn description(&self) -> Option<&str> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.description.as_str()), + 2 => self.v2.as_ref().map(|inner| inner.description.as_str()), + 3 => self.v3.as_ref().map(|inner| inner.description.as_str()), + 4 => self.v4.as_ref().map(|inner| inner.description.as_str()), + 5 => self.v5.as_ref().map(|inner| inner.description.as_str()), + _ => None, + } + } + + pub fn role(&self) -> Option<&str> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.role.as_str()), + 2 => self.v2.as_ref().map(|inner| inner.role.as_str()), + 3 => self.v3.as_ref().map(|inner| inner.role.as_str()), + 4 => self.v4.as_ref().map(|inner| inner.role.as_str()), + 5 => self.v5.as_ref().map(|inner| inner.role.as_str()), + _ => None, + } + } + + /// Returns the timestamp with timezone of the time at which the dashboard + /// was created. + /// + /// This value is stored in JSON for versions 1-5 of the dashboard. However + /// future versions of the dashboard should utilize the `created_at` Unix + /// timestamp field in the database to represent the creation timestamp. + pub fn created_at_deprecated(&self) -> Option> { + match self.version { + 1 => self.v1.as_ref().map(|inner| inner.created), + 2 => self.v2.as_ref().map(|inner| inner.created), + 3 => self.v3.as_ref().map(|inner| inner.created), + 4 => self.v4.as_ref().map(|inner| inner.created), + 5 => self.v5.as_ref().map(|inner| inner.created), + _ => None, + } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, ToSchema)] pub struct Dashboards { pub dashboards: Vec, diff --git a/src/config/src/meta/dashboards/v1/mod.rs b/src/config/src/meta/dashboards/v1/mod.rs index 2433799820..451cb39c75 100644 --- a/src/config/src/meta/dashboards/v1/mod.rs +++ b/src/config/src/meta/dashboards/v1/mod.rs @@ -13,6 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::hash::{Hash, Hasher}; + use chrono::{DateTime, FixedOffset, Utc}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -40,6 +42,27 @@ pub struct Dashboard { pub variables: Option, } +impl From for super::Dashboard { + fn from(value: Dashboard) -> Self { + let version: i32 = 1; + + let mut hasher = std::hash::DefaultHasher::new(); + hasher.write_i32(version); + value.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + Self { + v1: Some(value), + v2: None, + v3: None, + v4: None, + v5: None, + version, + hash, + } + } +} + fn datetime_now() -> DateTime { Utc::now().with_timezone(&FixedOffset::east_opt(0).expect( "BUG", // This can't possibly fail. Can it? diff --git a/src/config/src/meta/dashboards/v2/mod.rs b/src/config/src/meta/dashboards/v2/mod.rs index 4098e5cd19..4082431bd8 100644 --- a/src/config/src/meta/dashboards/v2/mod.rs +++ b/src/config/src/meta/dashboards/v2/mod.rs @@ -13,6 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::hash::{Hash, Hasher}; + use chrono::{DateTime, FixedOffset, Utc}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -41,6 +43,27 @@ pub struct Dashboard { pub variables: Option, } +impl From for super::Dashboard { + fn from(value: Dashboard) -> Self { + let version: i32 = 2; + + let mut hasher = std::hash::DefaultHasher::new(); + hasher.write_i32(version); + value.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + Self { + v1: None, + v2: Some(value), + v3: None, + v4: None, + v5: None, + version, + hash, + } + } +} + fn datetime_now() -> DateTime { Utc::now().with_timezone(&FixedOffset::east_opt(0).expect( "BUG", // This can't possibly fail. Can it? diff --git a/src/config/src/meta/dashboards/v3/mod.rs b/src/config/src/meta/dashboards/v3/mod.rs index fed54560af..6d72417c4a 100644 --- a/src/config/src/meta/dashboards/v3/mod.rs +++ b/src/config/src/meta/dashboards/v3/mod.rs @@ -13,6 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::hash::{Hash, Hasher}; + use chrono::{DateTime, FixedOffset}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -43,6 +45,26 @@ pub struct Dashboard { pub default_datetime_duration: Option, } +impl From for super::Dashboard { + fn from(value: Dashboard) -> Self { + let version: i32 = 3; + + let mut hasher = std::hash::DefaultHasher::new(); + hasher.write_i32(version); + value.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + Self { + v1: None, + v2: None, + v3: Some(value), + v4: None, + v5: None, + version, + hash, + } + } +} #[derive(Debug, Clone, PartialEq, Hash, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Layout { diff --git a/src/config/src/meta/dashboards/v4/mod.rs b/src/config/src/meta/dashboards/v4/mod.rs index 70eb7f40ca..80f4fe220c 100644 --- a/src/config/src/meta/dashboards/v4/mod.rs +++ b/src/config/src/meta/dashboards/v4/mod.rs @@ -13,6 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::hash::{Hash, Hasher}; + use chrono::{DateTime, FixedOffset}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -43,6 +45,27 @@ pub struct Dashboard { pub default_datetime_duration: Option, } +impl From for super::Dashboard { + fn from(value: Dashboard) -> Self { + let version: i32 = 4; + + let mut hasher = std::hash::DefaultHasher::new(); + hasher.write_i32(version); + value.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + Self { + v1: None, + v2: None, + v3: None, + v4: Some(value), + v5: None, + version, + hash, + } + } +} + #[derive(Debug, Clone, PartialEq, Hash, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Layout { diff --git a/src/config/src/meta/dashboards/v5/mod.rs b/src/config/src/meta/dashboards/v5/mod.rs index f0171478af..72596fa8fd 100644 --- a/src/config/src/meta/dashboards/v5/mod.rs +++ b/src/config/src/meta/dashboards/v5/mod.rs @@ -13,6 +13,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +use std::hash::{Hash, Hasher}; + use chrono::{DateTime, FixedOffset}; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; @@ -43,6 +45,27 @@ pub struct Dashboard { pub default_datetime_duration: Option, } +impl From for super::Dashboard { + fn from(value: Dashboard) -> Self { + let version: i32 = 5; + + let mut hasher = std::hash::DefaultHasher::new(); + hasher.write_i32(version); + value.hash(&mut hasher); + let hash = hasher.finish().to_string(); + + Self { + v1: None, + v2: None, + v3: None, + v4: None, + v5: Some(value), + version, + hash, + } + } +} + #[derive(Debug, Clone, PartialEq, Hash, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] pub struct Layout { diff --git a/src/infra/src/errors/mod.rs b/src/infra/src/errors/mod.rs index 73fad1941a..0a26bb2e3b 100644 --- a/src/infra/src/errors/mod.rs +++ b/src/infra/src/errors/mod.rs @@ -88,6 +88,32 @@ pub enum DbError { UniqueViolation, #[error("SeaORMError# {0}")] SeaORMError(String), + #[error("error getting dashboard")] + GetDashboardError(#[from] GetDashboardError), + #[error("PutDashbord# {0}")] + PutDashboard(#[from] PutDashboardError), +} + +#[derive(ThisError, Debug)] +pub enum GetDashboardError { + #[error("dashboard in DB has version {0} which cannot be deserialized")] + UnsupportedVersion(i32), +} + +#[derive(ThisError, Debug)] +pub enum PutDashboardError { + #[error("error putting dashboard with folder that does not exist")] + FolderDoesNotExist, + #[error("error putting dashboard with missing dashboard_id")] + MissingDashboardId, + #[error("error putting dashboard with missing title")] + MissingTitle, + #[error("error putting dashboard with missing owner")] + MissingOwner, + #[error("error putting dashboard with missing inner data for version {0}")] + MissingInnerData(i32), + #[error("error converting created timestamp with timezone to Unix timestamp")] + ConvertingCreatedTimestamp, } #[derive(ThisError, Debug)] @@ -111,6 +137,18 @@ impl From for Error { } } +impl From for Error { + fn from(value: GetDashboardError) -> Self { + Error::DbError(value.into()) + } +} + +impl From for Error { + fn from(value: PutDashboardError) -> Self { + Error::DbError(value.into()) + } +} + impl std::fmt::Display for ErrorCodes { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( diff --git a/src/infra/src/table/dashboards.rs b/src/infra/src/table/dashboards.rs new file mode 100644 index 0000000000..703e822dfc --- /dev/null +++ b/src/infra/src/table/dashboards.rs @@ -0,0 +1,328 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use config::meta::dashboards::{ + v1::Dashboard as DashboardV1, v2::Dashboard as DashboardV2, v3::Dashboard as DashboardV3, + v4::Dashboard as DashboardV4, v5::Dashboard as DashboardV5, Dashboard, +}; +use sea_orm::{ + ActiveModelTrait, ActiveValue::NotSet, ColumnTrait, DatabaseConnection, EntityTrait, + ModelTrait, QueryFilter, QueryOrder, Set, TryIntoModel, +}; +use serde_json::Value as JsonValue; + +use super::{ + entity::{dashboards, folders}, + folders::FolderType, +}; +use crate::{ + db::{connect_to_orm, ORM_CLIENT}, + errors::{self, GetDashboardError, PutDashboardError}, +}; + +impl TryFrom for Dashboard { + type Error = errors::Error; + + fn try_from(mut value: dashboards::Model) -> Result { + if let Some(obj) = value.data.as_object_mut() { + // The domain model JSON deserialization logic for v1-v5 expects + // some or all these fields to be present in the JSON even though we + // store them in DB columns. Therefore we add these values back into + // the JSON object so that deserializing the JSON can succeed. + obj.insert("dashboardId".to_owned(), value.dashboard_id.into()); + obj.insert("version".to_owned(), value.version.into()); + obj.insert("owner".to_owned(), value.owner.into()); + obj.insert("role".to_owned(), value.role.unwrap_or_default().into()); + obj.insert("title".to_owned(), value.title.into()); + obj.insert( + "description".to_owned(), + value.description.unwrap_or_default().into(), + ); + } + + match value.version { + 1 => { + let inner: DashboardV1 = serde_json::from_value(value.data)?; + let dash = inner.into(); + Ok(dash) + } + 2 => { + let inner: DashboardV2 = serde_json::from_value(value.data)?; + let dash = inner.into(); + Ok(dash) + } + 3 => { + let inner: DashboardV3 = serde_json::from_value(value.data)?; + let dash = inner.into(); + Ok(dash) + } + 4 => { + let inner: DashboardV4 = serde_json::from_value(value.data)?; + let dash = inner.into(); + Ok(dash) + } + 5 => { + let inner: DashboardV5 = serde_json::from_value(value.data)?; + let dash = inner.into(); + Ok(dash) + } + _ => Err(GetDashboardError::UnsupportedVersion(value.version).into()), + } + } +} + +/// Gets a dashboard. +pub async fn get( + org_id: &str, + folder_id: &str, + dashboard_id: &str, +) -> Result, errors::Error> { + let client = ORM_CLIENT.get_or_init(connect_to_orm).await; + let model = get_model(client, org_id, folder_id, dashboard_id) + .await? + .and_then(|(_folder, maybe_dash)| maybe_dash); + + if let Some(model) = model { + let dash = model.try_into()?; + Ok(Some(dash)) + } else { + Ok(None) + } +} + +/// Lists all dashboards belonging to the given org and folder. +pub async fn list(org_id: &str, folder_id: &str) -> Result, errors::Error> { + let client = ORM_CLIENT.get_or_init(connect_to_orm).await; + let dashboards = list_models(client, org_id, folder_id) + .await? + .into_iter() + .map(Dashboard::try_from) + .collect::>()?; + Ok(dashboards) +} + +/// Creates a new dashboard or updates an existing dashboard in the database. +/// Returns the new or updated dashboard. +pub async fn put( + org_id: &str, + folder_id: &str, + dashboard: Dashboard, +) -> Result { + let client = ORM_CLIENT.get_or_init(connect_to_orm).await; + + // Get the fields that will be inserted into or updated in the database. + let dashboard_id = dashboard + .dashboard_id() + .and_then(|s| if s.is_empty() { None } else { Some(s) }) + .ok_or(errors::PutDashboardError::MissingDashboardId)? + .to_owned(); + let owner = dashboard + .owner() + .and_then(|s| if s.is_empty() { None } else { Some(s) }) + .ok_or(errors::PutDashboardError::MissingOwner)? + .to_owned(); + let title = dashboard + .title() + .and_then(|s| if s.is_empty() { None } else { Some(s) }) + .ok_or(errors::PutDashboardError::MissingTitle)? + .to_owned(); + let role = dashboard + .role() + .and_then(|s| if s.is_empty() { None } else { Some(s) }) + .map(|r| r.to_owned()); + let description = dashboard + .description() + .and_then(|s| if s.is_empty() { None } else { Some(s) }) + .map(|d| d.to_owned()); + let version = dashboard.version; + let created_at_depricated = dashboard.created_at_deprecated(); + + let data = inner_data_as_json(dashboard)?; + + let (folder_m, mut dash_am) = match get_model(client, org_id, folder_id, &dashboard_id).await? { + None => { + // Destination folder does not exist so the dashboard can neither be + // created nor updated. + Err(errors::PutDashboardError::FolderDoesNotExist) + } + Some((folder_m, Some(dash_m))) => { + // Destination folder exists and dashboard already exists, so + // convert the dashboard model to an active model that will be + // updated. + Ok((folder_m, dash_m.into())) + } + Some((folder_m, None)) => { + // Destination folder exists but dashboard does not exist, so create + // a new dashboard active model that will be inserted. + let created_at_unix: i64 = if let Some(created_at_tz) = created_at_depricated { + created_at_tz.timestamp() + } else { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| PutDashboardError::ConvertingCreatedTimestamp)? + .as_secs() + .try_into() + .map_err(|_| PutDashboardError::ConvertingCreatedTimestamp)? + }; + + let dash_am = dashboards::ActiveModel { + id: NotSet, // Set by DB. + dashboard_id: Set(dashboard_id.to_owned()), + folder_id: NotSet, // Can be updated, so it is set below. + owner: NotSet, // Can be updated, so it is set below. + role: NotSet, // Can be updated, so it is set below. + title: NotSet, // Can be updated, so it is set below. + description: NotSet, // Can be updated, so it is set below. + data: NotSet, // Can be updated, so it is set below. + version: NotSet, // Can be updated, so it is set below. + created_at: Set(created_at_unix), + }; + Ok((folder_m, dash_am)) + } + }?; + + // All of the following fields will be set on creation or updated. + dash_am.folder_id = Set(folder_m.id); + dash_am.owner = Set(owner); + dash_am.role = Set(role); + dash_am.title = Set(title); + dash_am.description = Set(description); + dash_am.data = Set(data); + dash_am.version = Set(version); + + let model: dashboards::Model = dash_am.save(client).await?.try_into_model()?; + let dash = model.try_into()?; + Ok(dash) +} + +/// Deletes a dashboard with the given `folder_id` and `dashboard_id` surrogate +/// keys. +pub async fn delete( + org_id: &str, + folder_id: &str, + dashboard_id: &str, +) -> Result<(), errors::Error> { + let client = ORM_CLIENT.get_or_init(connect_to_orm).await; + let model = get_model(client, org_id, folder_id, dashboard_id) + .await? + .and_then(|(_folder, maybe_dash)| maybe_dash); + + if let Some(model) = model { + let _ = model.delete(client).await?; + } + + Ok(()) +} + +/// Deletes all dashboards. +pub async fn delete_all() -> Result<(), errors::Error> { + let client = ORM_CLIENT.get_or_init(connect_to_orm).await; + dashboards::Entity::delete_many().exec(client).await?; + Ok(()) +} + +/// Tries to get a dashboard ORM entity and its parent folder ORM entity. +async fn get_model( + db: &DatabaseConnection, + org_id: &str, + folder_id: &str, + dashboard_id: &str, +) -> Result)>, sea_orm::DbErr> { + let select_folders = folders::Entity::find() + .filter(folders::Column::Org.eq(org_id)) + .filter(folders::Column::Type.eq::(FolderType::Dashboards.into())) + .filter(folders::Column::FolderId.eq(folder_id)); + + let Some(folder) = select_folders.one(db).await? else { + return Ok(None); + }; + + let maybe_dashboard = folder + .find_related(dashboards::Entity) + .filter(dashboards::Column::DashboardId.eq(dashboard_id)) + .one(db) + .await?; + + Ok(Some((folder, maybe_dashboard))) +} + +/// Lists all dashboard ORM models that belong to the given org and folder. +async fn list_models( + db: &DatabaseConnection, + org_id: &str, + folder_id: &str, +) -> Result, sea_orm::DbErr> { + let rslt = folders::Entity::find() + .filter(folders::Column::Org.eq(org_id)) + .filter(folders::Column::Type.eq::(FolderType::Dashboards.into())) + .filter(folders::Column::FolderId.eq(folder_id)) + .find_with_related(dashboards::Entity) + .order_by_asc(dashboards::Column::Title) + .all(db) + .await?; + let dashboards = rslt.into_iter().flat_map(|(_, ds)| ds).collect(); + Ok(dashboards) +} + +/// Converts the [Dashboard] into the JSON represention of the inner data that +/// will be stored in the data column of the database. +/// +/// The returned JSON object should exclude any field that will be stored in +/// database columns. The schema of the JSON object will vary depending on the +/// dashboard version. +fn inner_data_as_json(dashboard: Dashboard) -> Result { + let mut data: JsonValue = match dashboard { + Dashboard { + version: 1, + v1: Some(inner), + .. + } => serde_json::to_value(inner).map_err(errors::Error::SerdeJsonError), + Dashboard { + version: 2, + v2: Some(inner), + .. + } => serde_json::to_value(inner).map_err(errors::Error::SerdeJsonError), + Dashboard { + version: 3, + v3: Some(inner), + .. + } => serde_json::to_value(inner).map_err(errors::Error::SerdeJsonError), + Dashboard { + version: 4, + v4: Some(inner), + .. + } => serde_json::to_value(inner).map_err(errors::Error::SerdeJsonError), + Dashboard { + version: 5, + v5: Some(inner), + .. + } => serde_json::to_value(inner).map_err(errors::Error::SerdeJsonError), + Dashboard { version: v, .. } => Err(errors::PutDashboardError::MissingInnerData(v).into()), + }?; + + // Remove fields that will be saved in DB columns from the JSON blob so that + // we are not storing the same value in two different places and risking + // desynchronization. + if let Some(obj) = data.as_object_mut() { + obj.remove("dashboard_id"); + obj.remove("version"); + obj.remove("owner"); + obj.remove("role"); + obj.remove("title"); + obj.remove("description"); + } + + Ok(data) +} diff --git a/src/infra/src/table/entity/dashboards.rs b/src/infra/src/table/entity/dashboards.rs new file mode 100644 index 0000000000..93fc45a6ef --- /dev/null +++ b/src/infra/src/table/entity/dashboards.rs @@ -0,0 +1,40 @@ +//! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "dashboards")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + pub dashboard_id: String, + pub folder_id: i64, + pub owner: String, + pub role: Option, + pub title: String, + #[sea_orm(column_type = "Text", nullable)] + pub description: Option, + pub data: Json, + pub version: i32, + pub created_at: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::folders::Entity", + from = "Column::FolderId", + to = "super::folders::Column::Id", + on_update = "NoAction", + on_delete = "NoAction" + )] + Folders, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Folders.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/infra/src/table/entity/folders.rs b/src/infra/src/table/entity/folders.rs index f68868e08c..ae925ccb41 100644 --- a/src/infra/src/table/entity/folders.rs +++ b/src/infra/src/table/entity/folders.rs @@ -16,6 +16,15 @@ pub struct Model { } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} +pub enum Relation { + #[sea_orm(has_many = "super::dashboards::Entity")] + Dashboards, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Dashboards.def() + } +} impl ActiveModelBehavior for ActiveModel {} diff --git a/src/infra/src/table/entity/mod.rs b/src/infra/src/table/entity/mod.rs index e44bf65f68..7b2a6a2931 100644 --- a/src/infra/src/table/entity/mod.rs +++ b/src/infra/src/table/entity/mod.rs @@ -2,4 +2,5 @@ pub mod prelude; +pub mod dashboards; pub mod folders; diff --git a/src/infra/src/table/entity/prelude.rs b/src/infra/src/table/entity/prelude.rs index 2c8a895a94..ca648cff5a 100644 --- a/src/infra/src/table/entity/prelude.rs +++ b/src/infra/src/table/entity/prelude.rs @@ -1,3 +1,3 @@ //! `SeaORM` Entity, @generated by sea-orm-codegen 1.1.0 -pub use super::folders::Entity as Folders; +pub use super::{dashboards::Entity as Dashboards, folders::Entity as Folders}; diff --git a/src/infra/src/table/folders.rs b/src/infra/src/table/folders.rs index 6b06568d2d..db87d57922 100644 --- a/src/infra/src/table/folders.rs +++ b/src/infra/src/table/folders.rs @@ -27,7 +27,7 @@ use crate::{ /// Indicates the type of data that the folder can contain. #[derive(Debug, Clone, Copy)] -enum FolderType { +pub(crate) enum FolderType { Dashboards, } diff --git a/src/infra/src/table/migration/m20241119_000001_create_dashboards_table.rs b/src/infra/src/table/migration/m20241119_000001_create_dashboards_table.rs new file mode 100644 index 0000000000..d19197f967 --- /dev/null +++ b/src/infra/src/table/migration/m20241119_000001_create_dashboards_table.rs @@ -0,0 +1,204 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +const DASHBOARDS_FOLDERS_FK: &str = "dashboards_folders_fk"; +const DASHBOARDS_FOLDER_ID_DASHBOARD_ID_IDX: &str = "dashboards_folder_id_dashboard_id_idx"; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table(create_dashboards_table_statement()) + .await?; + manager + .create_index(create_dashboards_folder_id_dashboard_id_idx_stmnt()) + .await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_index( + Index::drop() + .name(DASHBOARDS_FOLDER_ID_DASHBOARD_ID_IDX) + .to_owned(), + ) + .await?; + manager + .drop_table(Table::drop().table(Dashboards::Table).to_owned()) + .await?; + Ok(()) + } +} + +/// Statement to create the dashboards table. +fn create_dashboards_table_statement() -> TableCreateStatement { + Table::create() + .table(Dashboards::Table) + .if_not_exists() + .col( + ColumnDef::new(Dashboards::Id) + .big_integer() + .not_null() + .auto_increment() + .primary_key(), + ) + // A user-facing ID for the folder. This value can be a 64-bit signed + // integer "snowflake". + .col(ColumnDef::new(Dashboards::DashboardId).string_len(256).not_null()) + // Foreign key to the folders table. + .col(ColumnDef::new(Dashboards::FolderId).big_integer().not_null()) + // Identifier of the user that owns the dashboard. + .col(ColumnDef::new(Dashboards::Owner).string_len(256).not_null()) + .col(ColumnDef::new(Dashboards::Role).string_len(256).null()) + .col(ColumnDef::new(Dashboards::Title).string_len(256).not_null()) + .col(ColumnDef::new(Dashboards::Description).text().null()) + .col(ColumnDef::new(Dashboards::Data).json().not_null()) + .col(ColumnDef::new(Dashboards::Version).integer().not_null()) + .col( + ColumnDef::new(Dashboards::CreatedAt) + .big_integer() + .not_null(), + ) + .foreign_key( + sea_query::ForeignKey::create() + .name(DASHBOARDS_FOLDERS_FK) + .from(Dashboards::Table, Dashboards::FolderId) + .to(Folders::Table, Folders::Id) + ) + .to_owned() +} + +/// Statement to create unique index on dashboard_id. +fn create_dashboards_folder_id_dashboard_id_idx_stmnt() -> IndexCreateStatement { + sea_query::Index::create() + .if_not_exists() + .name(DASHBOARDS_FOLDER_ID_DASHBOARD_ID_IDX) + .table(Dashboards::Table) + .col(Dashboards::FolderId) + .col(Dashboards::DashboardId) + .unique() + .to_owned() +} + +/// Identifiers used in queries on the dashboards table. +#[derive(DeriveIden)] +enum Dashboards { + Table, + Id, + DashboardId, + FolderId, + Owner, + Role, + Title, + Description, + Data, + Version, + CreatedAt, +} + +/// Identifiers used in queries on the folders table. +#[derive(DeriveIden)] +enum Folders { + Table, + Id, +} + +#[cfg(test)] +mod tests { + use collapse::*; + + use super::*; + + #[test] + fn postgres() { + collapsed_eq!( + &create_dashboards_table_statement().to_string(PostgresQueryBuilder), + r#" + CREATE TABLE IF NOT EXISTS "dashboards" ( + "id" bigserial NOT NULL PRIMARY KEY, + "dashboard_id" varchar(256) NOT NULL, + "folder_id" bigint NOT NULL, + "owner" varchar(256) NOT NULL, + "role" varchar(256) NULL, + "title" varchar(256) NOT NULL, + "description" text NULL, + "data" json NOT NULL, + "version" integer NOT NULL, + "created_at" bigint NOT NULL, + CONSTRAINT "dashboards_folders_fk" FOREIGN KEY ("folder_id") REFERENCES "folders" ("id") + )"# + ); + assert_eq!( + &create_dashboards_folder_id_dashboard_id_idx_stmnt().to_string(PostgresQueryBuilder), + r#"CREATE UNIQUE INDEX IF NOT EXISTS "dashboards_folder_id_dashboard_id_idx" ON "dashboards" ("folder_id", "dashboard_id")"# + ); + } + + #[test] + fn mysql() { + collapsed_eq!( + &create_dashboards_table_statement().to_string(MysqlQueryBuilder), + r#" + CREATE TABLE IF NOT EXISTS `dashboards` ( + `id` bigint NOT NULL AUTO_INCREMENT PRIMARY KEY, + `dashboard_id` varchar(256) NOT NULL, + `folder_id` bigint NOT NULL, + `owner` varchar(256) NOT NULL, + `role` varchar(256) NULL, + `title` varchar(256) NOT NULL, + `description` text NULL, + `data` json NOT NULL, + `version` int NOT NULL, + `created_at` bigint NOT NULL, + CONSTRAINT `dashboards_folders_fk` FOREIGN KEY (`folder_id`) REFERENCES `folders` (`id`) + )"# + ); + assert_eq!( + &create_dashboards_folder_id_dashboard_id_idx_stmnt().to_string(MysqlQueryBuilder), + r#"CREATE UNIQUE INDEX `dashboards_folder_id_dashboard_id_idx` ON `dashboards` (`folder_id`, `dashboard_id`)"# + ); + } + + #[test] + fn sqlite() { + collapsed_eq!( + &create_dashboards_table_statement().to_string(SqliteQueryBuilder), + r#" + CREATE TABLE IF NOT EXISTS "dashboards" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "dashboard_id" varchar(256) NOT NULL, + "folder_id" bigint NOT NULL, + "owner" varchar(256) NOT NULL, + "role" varchar(256) NULL, + "title" varchar(256) NOT NULL, + "description" text NULL, + "data" json_text NOT NULL, + "version" integer NOT NULL, + "created_at" bigint NOT NULL, + FOREIGN KEY ("folder_id") REFERENCES "folders" ("id") + )"# + ); + assert_eq!( + &create_dashboards_folder_id_dashboard_id_idx_stmnt().to_string(SqliteQueryBuilder), + r#"CREATE UNIQUE INDEX IF NOT EXISTS "dashboards_folder_id_dashboard_id_idx" ON "dashboards" ("folder_id", "dashboard_id")"# + ); + } +} diff --git a/src/infra/src/table/migration/m20241119_000002_populate_dashboards_table.rs b/src/infra/src/table/migration/m20241119_000002_populate_dashboards_table.rs new file mode 100644 index 0000000000..a45d57a2a8 --- /dev/null +++ b/src/infra/src/table/migration/m20241119_000002_populate_dashboards_table.rs @@ -0,0 +1,233 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Populates the dashboards table by transforming unstructured dashboard +//! records from the meta table. + +use chrono::{DateTime, FixedOffset}; +use sea_orm::{ + ActiveValue::NotSet, EntityTrait, FromQueryResult, Paginator, PaginatorTrait, SelectModel, Set, + Statement, TransactionTrait, +}; +use sea_orm_migration::prelude::*; +use serde_json::Value as JsonValue; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let txn = manager.get_connection().begin().await?; + + // Migrate pages of 100 records at a time to avoid loading too many + // records into memory. + let mut meta_pages = MetaDashboard::paginate(&txn, 100); + + while let Some(metas) = meta_pages.fetch_and_next().await? { + let dashboards_rslt: Result, DbErr> = metas + .into_iter() + .map(|m| { + let m: dashboards::ActiveModel = m.try_into().map_err(DbErr::Migration)?; + Ok(m) + }) + .collect(); + let dashboards = dashboards_rslt?; + dashboards::Entity::insert_many(dashboards) + .exec(&txn) + .await?; + } + + txn.commit().await?; + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let db = manager.get_connection(); + dashboards::Entity::delete_many().exec(db).await?; + Ok(()) + } +} + +/// A result from querying for dashboards from the meta table and joining on the +/// folders table. +#[derive(Debug, FromQueryResult)] +pub struct MetaDashboard { + folder_id: Option, + dashboard_id: String, + value: String, +} + +impl MetaDashboard { + /// Paginate through the results of querying for dashboards from the meta + /// table and joining on the folders table. + fn paginate(db: &C, page_size: u64) -> Paginator<'_, C, SelectModel> + where + C: ConnectionTrait, + { + let backend = db.get_database_backend(); + let sql = match backend { + sea_orm::DatabaseBackend::MySql => { + r#" + SELECT + f.id AS folder_id, + SUBSTRING_INDEX(m.key2,'/', -1) AS dashboard_id, + m.value AS value + FROM meta AS m + JOIN folders f ON + SUBSTRING_INDEX(m.key2,'/', 1) = f.folder_id AND + m.key1 = f.org + WHERE m.module = 'dashboard' + ORDER BY m.id ASC + "# + } + sea_orm::DatabaseBackend::Postgres => { + r#" + SELECT + f.id AS folder_id, + SPLIT_PART(m.key2, '/', 2) AS dashboard_id, + m.value AS value + FROM meta AS m + JOIN folders f ON + SPLIT_PART(m.key2, '/', 1) = f.folder_id AND + m.key1 = f.org + WHERE m.module = 'dashboard' + ORDER BY m.id ASC + "# + } + sea_orm::DatabaseBackend::Sqlite => { + r#" + SELECT + f.id AS folder_id, + SUBSTR(m.key2, INSTR(m.key2, '/') + 1) AS dashboard_id, + m.value AS value + FROM meta AS m + INNER JOIN folders f ON + SUBSTR(m.key2, 1, INSTR(m.key2, '/') - 1) = f.folder_id AND + m.key1 = f.org + WHERE m.module = 'dashboard' + ORDER BY m.id ASC + "# + } + }; + + Self::find_by_statement(Statement::from_sql_and_values(backend, sql, [])) + .paginate(db, page_size) + } +} + +impl TryFrom for dashboards::ActiveModel { + type Error = String; + + fn try_from(m: MetaDashboard) -> Result { + let folder_id = m + .folder_id + .ok_or("Dashboard in meta table references folder that does not exist")?; + let dashboard_id = if m.dashboard_id.is_empty() { + Err("Dashboard in meta table is missing a dasbhoard ID") + } else { + Ok(m.dashboard_id) + }?; + + let mut value: JsonValue = serde_json::from_str(&m.value) + .map_err(|_| "Dashboard in meta table has \"value\" field that is not valid JSON")?; + let obj = value + .as_object_mut() + .ok_or("Dashboard in meta table has \"value\" field that is not a JSON object")?; + + // Remove each of the following fields from the inner JSON since they + // will now be stored in table columns and we don't want to keep + // duplicate sources of the data that we would need to synchronize. + obj.remove("dashboardId"); + let version: i32 = obj + .remove("version") + .and_then(|v| v.as_i64()) + .and_then(|v: i64| v.try_into().ok()) + .ok_or("Dashboard JSON does not have i32 \"version\" field")?; + let owner = obj + .remove("owner") + .and_then(|v| v.as_str().map(|s| s.to_string())) + .ok_or("Dashboard JSON does not have string \"owner\" field")?; + let title = obj + .remove("title") + .and_then(|v| v.as_str().map(|s| s.to_string())) + .ok_or("Dashboard JSON does not have string \"title\" field")?; + let role = obj + .remove("role") + .and_then(|v| v.as_str().map(|s| s.to_string())) + .and_then(|s| if s.is_empty() { None } else { Some(s) }); + let description = obj + .remove("description") + .and_then(|v| v.as_str().map(|s| s.to_string())) + .and_then(|s| if s.is_empty() { None } else { Some(s) }); + + // Since the created field includes a timezone that cannot be + // represented in all databases and since the UI expects to receieve a + // timestamp with a timezone, we will leave the created field in the + // JSON while also migrating its value into the created_at Unix timestamp column. + let created_at_tz: DateTime = obj + .get("created") + .and_then(|v| v.as_str().map(|s| s.to_string())) + .ok_or("Dashboard JSON does not have string \"created\" field".to_string()) + .and_then(|s| DateTime::parse_from_rfc3339(&s).map_err(|e| e.to_string()))?; + let created_at_unix = created_at_tz.timestamp(); + + Ok(dashboards::ActiveModel { + id: NotSet, // Set automatically by DB. + dashboard_id: Set(dashboard_id), + folder_id: Set(folder_id), + owner: Set(owner), + role: Set(role), + title: Set(title), + description: Set(description), + data: Set(value), + version: Set(version), + created_at: Set(created_at_unix), + }) + } +} + +// The schemas of tables might change after subsequent migrations. Therefore +// this migration only references ORM models in private submodules that should +// remain unchanged rather than ORM models in the `entity` module that will be +// updated to reflect the latest changes to table schemas. + +/// Representation of the dashboards table at the time this migration executes. +mod dashboards { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] + #[sea_orm(table_name = "dashboards")] + pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + pub dashboard_id: String, + pub folder_id: i64, + pub owner: String, + pub role: Option, + pub title: String, + #[sea_orm(column_type = "Text", nullable)] + pub description: Option, + pub data: Json, + pub version: i32, + pub created_at: i64, + } + + // There are relations but they are not important to this migration. + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} diff --git a/src/infra/src/table/migration/m20241119_000003_delete_metas.rs b/src/infra/src/table/migration/m20241119_000003_delete_metas.rs new file mode 100644 index 0000000000..a6e4237be8 --- /dev/null +++ b/src/infra/src/table/migration/m20241119_000003_delete_metas.rs @@ -0,0 +1,68 @@ +// Copyright 2024 OpenObserve Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! Deletes dashboard records from the meta table. + +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, TransactionTrait}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let txn = manager.get_connection().begin().await?; + meta::Entity::delete_many() + .filter(meta::Column::Module.eq("dashboard")) + .exec(&txn) + .await?; + txn.commit().await?; + Ok(()) + } + + async fn down(&self, _: &SchemaManager) -> Result<(), DbErr> { + // The deletion of records from the meta table is not reversable. + Ok(()) + } +} + +// The schemas of tables might change after subsequent migrations. Therefore +// this migration only references ORM models in private submodules that should +// remain unchanged rather than ORM models in the `entity` module that will be +// updated to reflect the latest changes to table schemas. + +/// Representation of the meta table at the time this migration executes. +mod meta { + use sea_orm::entity::prelude::*; + + #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] + #[sea_orm(table_name = "meta")] + pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + pub module: String, + pub key1: String, + pub key2: String, + pub start_dt: i64, + #[sea_orm(column_type = "Text")] + pub value: String, + } + + #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] + pub enum Relation {} + + impl ActiveModelBehavior for ActiveModel {} +} diff --git a/src/infra/src/table/migration/mod.rs b/src/infra/src/table/migration/mod.rs index 2a056e7fba..4e5d51fb6a 100644 --- a/src/infra/src/table/migration/mod.rs +++ b/src/infra/src/table/migration/mod.rs @@ -19,6 +19,9 @@ mod m20241114_000001_create_folders_table; mod m20241115_150000_populate_folders_table; mod m20241116_000001_delete_metas; mod m20241116_000002_drop_folders_created_at_column; +mod m20241119_000001_create_dashboards_table; +mod m20241119_000002_populate_dashboards_table; +mod m20241119_000003_delete_metas; pub struct Migrator; @@ -30,6 +33,9 @@ impl MigratorTrait for Migrator { Box::new(m20241115_150000_populate_folders_table::Migration), Box::new(m20241116_000001_delete_metas::Migration), Box::new(m20241116_000002_drop_folders_created_at_column::Migration), + Box::new(m20241119_000001_create_dashboards_table::Migration), + Box::new(m20241119_000002_populate_dashboards_table::Migration), + Box::new(m20241119_000003_delete_metas::Migration), ] } } diff --git a/src/infra/src/table/mod.rs b/src/infra/src/table/mod.rs index 93dd067fbc..581885833c 100644 --- a/src/infra/src/table/mod.rs +++ b/src/infra/src/table/mod.rs @@ -19,6 +19,7 @@ use sea_orm_migration::MigratorTrait; use crate::db::{connect_to_orm, sqlite::CLIENT_RW, ORM_CLIENT, SQLITE_STORE}; +pub mod dashboards; #[allow(unused_imports)] mod entity; pub mod folders; diff --git a/src/service/dashboards/mod.rs b/src/service/dashboards/mod.rs index 485ac83d39..1940f2bae4 100644 --- a/src/service/dashboards/mod.rs +++ b/src/service/dashboards/mod.rs @@ -119,9 +119,14 @@ pub async fn update_dashboard( #[tracing::instrument] pub async fn list_dashboards(org_id: &str, folder_id: &str) -> Result { - Ok(HttpResponse::Ok().json(Dashboards { - dashboards: db::dashboards::list(org_id, folder_id).await.unwrap(), - })) + let resp = match db::dashboards::list(org_id, folder_id).await { + Ok(dashboards) => HttpResponse::Ok().json(Dashboards { dashboards }), + Err(error) => HttpResponse::InternalServerError().json(MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + }; + Ok(resp) } #[tracing::instrument] @@ -130,10 +135,13 @@ pub async fn get_dashboard( dashboard_id: &str, folder_id: &str, ) -> Result { - let resp = if let Ok(dashboard) = db::dashboards::get(org_id, dashboard_id, folder_id).await { - HttpResponse::Ok().json(dashboard) - } else { - return Ok(Response::NotFound("Dashboard".to_string()).into()); + let resp = match db::dashboards::get(org_id, dashboard_id, folder_id).await { + Ok(Some(dash)) => HttpResponse::Ok().json(dash), + Ok(None) => Response::NotFound("Dashboard".to_string()).into(), + Err(error) => HttpResponse::InternalServerError().json(MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), }; Ok(resp) } @@ -144,15 +152,24 @@ pub async fn delete_dashboard( dashboard_id: &str, folder_id: &str, ) -> Result { - if db::dashboards::get(org_id, dashboard_id, folder_id) - .await - .is_err() - { - return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( - http::StatusCode::NOT_FOUND.into(), - "Dashboard not found".to_string(), - ))); - } + match db::dashboards::get(org_id, dashboard_id, folder_id).await { + Ok(Some(_)) => {} // Dashboard exists. Continue with deleting. + Ok(None) => { + return Ok(HttpResponse::NotFound().json(MetaHttpResponse::error( + http::StatusCode::NOT_FOUND.into(), + "Dashboard not found".to_string(), + ))); + } + Err(error) => { + return Ok( + HttpResponse::InternalServerError().json(MetaHttpResponse::error( + http::StatusCode::INTERNAL_SERVER_ERROR.into(), + error.to_string(), + )), + ) + } + }; + match db::dashboards::delete(org_id, dashboard_id, folder_id).await { Ok(_) => { remove_ownership( @@ -209,11 +226,12 @@ pub async fn move_dashboard( from_folder: &str, to_folder: &str, ) -> Result { - if let Ok(dashboard) = db::dashboards::get(org_id, dashboard_id, from_folder).await { + if let Ok(Some(dashboard)) = db::dashboards::get(org_id, dashboard_id, from_folder).await { // make sure the destination folder exists if !db::folders::exists(org_id, to_folder).await? { return Ok(Response::NotFound("Destination Folder".to_string()).into()); } + let dash = if dashboard.version == 1 { json::to_vec(&dashboard.v1.unwrap()).unwrap() } else if dashboard.version == 2 { diff --git a/src/service/dashboards/reports.rs b/src/service/dashboards/reports.rs index 7a04018a0e..5d9e65e6a4 100644 --- a/src/service/dashboards/reports.rs +++ b/src/service/dashboards/reports.rs @@ -141,9 +141,9 @@ pub async fn save( // Supports only one tab for now let tab_id = &dashboard.tabs[0]; tasks.push(async move { - let dashboard = db::dashboards::get(org_id, dash_id, folder).await?; + let maybe_dashboard = db::dashboards::get(org_id, dash_id, folder).await?; // Check if the tab_id exists - if let Some(dashboard) = dashboard.v3 { + if let Some(dashboard) = maybe_dashboard.and_then(|d| d.v3) { let mut tab_found = false; for tab in dashboard.tabs { if &tab.tab_id == tab_id { diff --git a/src/service/db/dashboards/mod.rs b/src/service/db/dashboards/mod.rs index 7b3db77252..3b6b5eb4f1 100644 --- a/src/service/db/dashboards/mod.rs +++ b/src/service/db/dashboards/mod.rs @@ -13,15 +13,12 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -use std::hash::{Hash, Hasher}; - use actix_web::web; use config::{ meta::dashboards::{v1, v2, v3, v4, v5, Dashboard, DashboardVersion}, utils::json, }; - -use crate::service::db; +use infra::table::dashboards; pub mod reports; @@ -29,77 +26,21 @@ pub mod reports; pub(crate) async fn get( org_id: &str, dashboard_id: &str, - folder: &str, -) -> Result { - let key = format!("/dashboard/{org_id}/{folder}/{dashboard_id}"); - let bytes = db::get(&key).await?; - let d_version: DashboardVersion = json::from_slice(&bytes)?; - let mut hasher = std::hash::DefaultHasher::new(); - hasher.write_i32(d_version.version); - - if d_version.version == 1 { - let dash: v1::Dashboard = json::from_slice(&bytes)?; - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v1: Some(dash), - version: 1, - hash, - ..Default::default() - }) - } else if d_version.version == 2 { - let dash: v2::Dashboard = json::from_slice(&bytes)?; - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v2: Some(dash), - version: 2, - hash, - ..Default::default() - }) - } else if d_version.version == 3 { - let dash: v3::Dashboard = json::from_slice(&bytes)?; - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v3: Some(dash), - version: 3, - hash, - ..Default::default() - }) - } else if d_version.version == 4 { - let dash: v4::Dashboard = json::from_slice(&bytes)?; - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v4: Some(dash), - version: 4, - hash, - ..Default::default() - }) - } else { - let dash: v5::Dashboard = json::from_slice(&bytes)?; - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v5: Some(dash), - version: 5, - hash, - ..Default::default() - }) - } + folder_id: &str, +) -> Result, anyhow::Error> { + let dash = dashboards::get(org_id, folder_id, dashboard_id).await?; + Ok(dash) } #[tracing::instrument(skip(body))] pub(crate) async fn put( org_id: &str, dashboard_id: &str, - folder: &str, + folder_id: &str, body: web::Bytes, hash: Option<&str>, ) -> Result { - let key = format!("/dashboard/{org_id}/{folder}/{}", dashboard_id); - if let Ok(existing_dash) = get(org_id, dashboard_id, folder).await { + if let Some(existing_dash) = get(org_id, dashboard_id, folder_id).await? { let existing_dash_hash = existing_dash.hash; let Some(Ok(hash_val)) = hash.map(|hash_str| hash_str.parse::()) else { @@ -115,186 +56,68 @@ pub(crate) async fn put( }; let d_version: DashboardVersion = json::from_slice(&body)?; - let mut hasher = std::hash::DefaultHasher::new(); - hasher.write_i32(d_version.version); + let mut dash = parse_dashboard(body, d_version.version)?; - if d_version.version == 1 { - let mut dash: v1::Dashboard = json::from_slice(&body)?; - dash.title = dash.title.trim().to_string(); - if dash.title.is_empty() { - return Err(anyhow::anyhow!("Dashboard should have title")); - }; - dash.dashboard_id = dashboard_id.to_string(); - let value: bytes::Bytes = json::to_vec(&dash)?.into(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); + let title = dash + .title() + .map(|t| t.trim().to_string()) + .and_then(|t| if t.is_empty() { None } else { Some(t) }) + .ok_or_else(|| anyhow::anyhow!("Dashboard should have title"))?; + dash.set_title(title); - match db::put(&key, value, db::NO_NEED_WATCH, None).await { - Ok(_) => Ok(Dashboard { - v1: Some(dash), - version: 1, - hash, - ..Default::default() - }), - Err(_) => Err(anyhow::anyhow!("Failed to save Dashboard")), - } - } else if d_version.version == 2 { - let mut dash: v2::Dashboard = json::from_slice(&body)?; - dash.title = dash.title.trim().to_string(); - if dash.title.is_empty() { - return Err(anyhow::anyhow!("Dashboard should have title")); - }; - dash.dashboard_id = dashboard_id.to_string(); - let value: bytes::Bytes = json::to_vec(&dash)?.into(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - match db::put(&key, value, db::NO_NEED_WATCH, None).await { - Ok(_) => Ok(Dashboard { - v2: Some(dash), - version: 2, - hash, - ..Default::default() - }), - Err(_) => Err(anyhow::anyhow!("Failed to save Dashboard")), - } - } else if d_version.version == 3 { - let mut dash: v3::Dashboard = json::from_slice(&body)?; - dash.title = dash.title.trim().to_string(); - if dash.title.is_empty() { - return Err(anyhow::anyhow!("Dashboard should have title")); - }; - dash.dashboard_id = dashboard_id.to_string(); - let value: bytes::Bytes = json::to_vec(&dash)?.into(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - match db::put(&key, value, db::NO_NEED_WATCH, None).await { - Ok(_) => Ok(Dashboard { - v3: Some(dash), - version: 3, - hash, - ..Default::default() - }), - Err(_) => Err(anyhow::anyhow!("Failed to save Dashboard")), - } - } else if d_version.version == 4 { - let mut dash: v4::Dashboard = json::from_slice(&body)?; - dash.title = dash.title.trim().to_string(); - if dash.title.is_empty() { - return Err(anyhow::anyhow!("Dashboard should have title")); - }; - dash.dashboard_id = dashboard_id.to_string(); - let value: bytes::Bytes = json::to_vec(&dash)?.into(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - match db::put(&key, value, db::NO_NEED_WATCH, None).await { - Ok(_) => Ok(Dashboard { - v4: Some(dash), - version: 4, - hash, - ..Default::default() - }), - Err(_) => Err(anyhow::anyhow!("Failed to save Dashboard")), - } - } else { - let mut dash: v5::Dashboard = json::from_slice(&body)?; - dash.title = dash.title.trim().to_string(); - if dash.title.is_empty() { - return Err(anyhow::anyhow!("Dashboard should have title")); - }; - dash.dashboard_id = dashboard_id.to_string(); - let value: bytes::Bytes = json::to_vec(&dash)?.into(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - match db::put(&key, value, db::NO_NEED_WATCH, None).await { - Ok(_) => Ok(Dashboard { - v5: Some(dash), - version: 5, - hash, - ..Default::default() - }), - Err(_) => Err(anyhow::anyhow!("Failed to save Dashboard")), - } - } + dash.set_dashboard_id(dashboard_id.to_owned()); + let dash = dashboards::put(org_id, folder_id, dash).await?; + Ok(dash) } #[tracing::instrument] -pub(crate) async fn list(org_id: &str, folder: &str) -> Result, anyhow::Error> { - let db_key = format!("/dashboard/{org_id}/{folder}/"); - db::list(&db_key) - .await? - .into_values() - .map(|val| { - let d_version: DashboardVersion = json::from_slice(&val).unwrap(); - let mut hasher = std::hash::DefaultHasher::new(); - hasher.write_i32(d_version.version); - - if d_version.version == 1 { - let dash: v1::Dashboard = json::from_slice(&val).unwrap(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v1: Some(dash), - version: 1, - hash, - ..Default::default() - }) - } else if d_version.version == 2 { - let dash: v2::Dashboard = json::from_slice(&val).unwrap(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v2: Some(dash), - version: 2, - hash, - ..Default::default() - }) - } else if d_version.version == 3 { - let dash: v3::Dashboard = json::from_slice(&val).unwrap(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v3: Some(dash), - version: 3, - hash, - ..Default::default() - }) - } else if d_version.version == 4 { - let dash: v4::Dashboard = json::from_slice(&val).unwrap(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v4: Some(dash), - version: 4, - hash, - ..Default::default() - }) - } else { - let dash: v5::Dashboard = json::from_slice(&val).unwrap(); - dash.hash(&mut hasher); - let hash = hasher.finish().to_string(); - Ok(Dashboard { - v5: Some(dash), - version: 5, - hash, - ..Default::default() - }) - } - }) - .collect() +pub(crate) async fn list(org_id: &str, folder_id: &str) -> Result, anyhow::Error> { + let ds = dashboards::list(org_id, folder_id).await?; + Ok(ds) } #[tracing::instrument] pub(crate) async fn delete( org_id: &str, dashboard_id: &str, - folder: &str, + folder_id: &str, ) -> Result<(), anyhow::Error> { - let key = format!("/dashboard/{org_id}/{folder}/{dashboard_id}"); - Ok(db::delete(&key, false, db::NO_NEED_WATCH, None).await?) + dashboards::delete(org_id, folder_id, dashboard_id).await?; + Ok(()) } #[tracing::instrument] pub async fn reset() -> Result<(), anyhow::Error> { - let key = "/dashboard/"; - Ok(db::delete(key, true, db::NO_NEED_WATCH, None).await?) + dashboards::delete_all().await?; + Ok(()) +} + +/// Parses the bytes into a dashboard with the given version. +pub(crate) fn parse_dashboard( + bytes: web::Bytes, + version: i32, +) -> Result { + let dash = match version { + 1 => { + let inner: v1::Dashboard = json::from_slice(&bytes)?; + inner.into() + } + 2 => { + let inner: v2::Dashboard = json::from_slice(&bytes)?; + inner.into() + } + 3 => { + let inner: v3::Dashboard = json::from_slice(&bytes)?; + inner.into() + } + 4 => { + let inner: v4::Dashboard = json::from_slice(&bytes)?; + inner.into() + } + _ => { + let inner: v5::Dashboard = json::from_slice(&bytes)?; + inner.into() + } + }; + Ok(dash) } diff --git a/tests/api-testing/tests/test_folders.py b/tests/api-testing/tests/test_folders.py index f7e9e56a9d..87133d13e1 100644 --- a/tests/api-testing/tests/test_folders.py +++ b/tests/api-testing/tests/test_folders.py @@ -102,7 +102,11 @@ def test_e2e_createdeletedashboard(create_session, base_url): session = create_session # Create a dashboard org_id = "default" - payload = {"description": "pydashboards", "title": "pytestdashboard"} + payload = { + "description": "pydashboards", + "title": "pytestdashboard", + "owner": "user1@alpha1.com", + } resp_create_dashboard = session.post( f"{base_url}api/{org_id}/dashboards", json=payload @@ -120,7 +124,7 @@ def test_e2e_createdeletedashboard(create_session, base_url): "dashboardId": "", "description": "updatedashboard", "role": "", - "owner": "root@alpha1.com", + "owner": "user2@alpha1.com", "created": "2023-10-19T14:08:48.090Z", "panels": [], }