From 2c82ded97560b1035785b43a51521fb4de24cba0 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Sat, 6 May 2023 14:59:49 +0800 Subject: [PATCH] feat: table metrics (#1469) * feat: Statistic * add todo * fmt: cargo fmt * feat: some simple impl for MemTable * chore: a try on adding statistics * Update src/table/src/stats.rs Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * docs: fix typo * newlines unnecessary --------- Co-authored-by: Ruihang Xia Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- src/catalog/src/system.rs | 4 +++ src/mito/src/table.rs | 4 +-- src/store-api/src/storage.rs | 2 +- src/store-api/src/storage/region.rs | 13 ++++++++ src/table/src/lib.rs | 4 +++ src/table/src/stats.rs | 47 +++++++++++++++++++++++++++++ src/table/src/table.rs | 13 ++++---- src/table/src/test_util/memtable.rs | 26 +++++++++++++++- 8 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 src/table/src/stats.rs diff --git a/src/catalog/src/system.rs b/src/catalog/src/system.rs index 7a51c9af532..7e2c905e538 100644 --- a/src/catalog/src/system.rs +++ b/src/catalog/src/system.rs @@ -80,6 +80,10 @@ impl Table for SystemCatalogTable { async fn delete(&self, request: DeleteRequest) -> table::Result { self.0.delete(request).await } + + fn statistics(&self) -> Option { + self.0.statistics() + } } impl SystemCatalogTable { diff --git a/src/mito/src/table.rs b/src/mito/src/table.rs index 87dcced42f0..620807b43d0 100644 --- a/src/mito/src/table.rs +++ b/src/mito/src/table.rs @@ -38,7 +38,6 @@ use store_api::storage::{ AddColumn, AlterOperation, AlterRequest, ChunkReader, FlushContext, ReadContext, Region, RegionMeta, RegionNumber, ScanRequest, SchemaRef, Snapshot, WriteContext, WriteRequest, }; -use table::error as table_error; use table::error::{ InvalidTableSnafu, RegionSchemaMismatchSnafu, Result as TableResult, TableOperationSnafu, }; @@ -49,7 +48,8 @@ use table::requests::{ AddColumnRequest, AlterKind, AlterTableRequest, DeleteRequest, InsertRequest, }; use table::table::scan::SimpleTableScan; -use table::table::{AlterContext, RegionStat, Table}; +use table::table::{AlterContext, Table}; +use table::{error as table_error, RegionStat}; use tokio::sync::Mutex; use crate::error; diff --git a/src/store-api/src/storage.rs b/src/store-api/src/storage.rs index 93013032292..aba97a3e3b7 100644 --- a/src/store-api/src/storage.rs +++ b/src/store-api/src/storage.rs @@ -34,7 +34,7 @@ pub use self::chunk::{Chunk, ChunkReader}; pub use self::descriptors::*; pub use self::engine::{CreateOptions, EngineContext, OpenOptions, StorageEngine}; pub use self::metadata::RegionMeta; -pub use self::region::{FlushContext, Region, WriteContext}; +pub use self::region::{FlushContext, Region, RegionStat, WriteContext}; pub use self::requests::{ AddColumn, AlterOperation, AlterRequest, GetRequest, ScanRequest, WriteRequest, }; diff --git a/src/store-api/src/storage/region.rs b/src/store-api/src/storage/region.rs index d4dd2c0e1e6..de3904bd2e9 100644 --- a/src/store-api/src/storage/region.rs +++ b/src/store-api/src/storage/region.rs @@ -77,10 +77,23 @@ pub trait Region: Send + Sync + Clone + std::fmt::Debug + 'static { fn disk_usage_bytes(&self) -> u64; + fn region_stat(&self) -> RegionStat { + RegionStat { + region_id: self.id(), + disk_usage_bytes: self.disk_usage_bytes(), + } + } + /// Flush memtable of the region to disk. async fn flush(&self, ctx: &FlushContext) -> Result<(), Self::Error>; } +#[derive(Default, Debug)] +pub struct RegionStat { + pub region_id: u64, + pub disk_usage_bytes: u64, +} + /// Context for write operations. #[derive(Debug, Clone, Default)] pub struct WriteContext {} diff --git a/src/table/src/lib.rs b/src/table/src/lib.rs index 2e9ceb71687..fa2fb5d5b1b 100644 --- a/src/table/src/lib.rs +++ b/src/table/src/lib.rs @@ -18,8 +18,12 @@ pub mod error; pub mod metadata; pub mod predicate; pub mod requests; +pub mod stats; pub mod table; pub mod test_util; +pub use store_api::storage::RegionStat; + pub use crate::error::{Error, Result}; +pub use crate::stats::{ColumnStatistics, TableStatistics}; pub use crate::table::{Table, TableRef}; diff --git a/src/table/src/stats.rs b/src/table/src/stats.rs new file mode 100644 index 00000000000..23b19ddcea4 --- /dev/null +++ b/src/table/src/stats.rs @@ -0,0 +1,47 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use datatypes::value::Value; + +/// Statistics for a relation +/// Fields are optional and can be inexact because the sources +/// sometimes provide approximate estimates for performance reasons +/// and the transformations output are not always predictable. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TableStatistics { + /// The number of table rows + pub num_rows: Option, + /// total bytes of the table rows + pub total_byte_size: Option, + /// Statistics on a column level + pub column_statistics: Option>, + /// If true, any field that is `Some(..)` is the actual value in the data provided by the operator (it is not + /// an estimation). Any or all other fields might still be None, in which case no information is known. + /// if false, any field that is `Some(..)` may contain an inexact estimate and may not be the actual value. + pub is_exact: bool, +} + +/// Statistics for a column within a relation +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ColumnStatistics { + /// Number of null values on column + pub null_count: Option, + /// Maximum value of column + pub max_value: Option, + /// Minimum value of column + pub min_value: Option, + /// Number of distinct values + pub distinct_count: Option, + // TODO(discord9): histogram of values +} diff --git a/src/table/src/table.rs b/src/table/src/table.rs index 9f48dead0ff..110113630e9 100644 --- a/src/table/src/table.rs +++ b/src/table/src/table.rs @@ -28,6 +28,8 @@ use store_api::storage::RegionNumber; use crate::error::{Result, UnsupportedSnafu}; use crate::metadata::{FilterPushDownType, TableId, TableInfoRef, TableType}; use crate::requests::{AlterTableRequest, DeleteRequest, InsertRequest}; +use crate::stats::TableStatistics; +use crate::RegionStat; pub type AlterContext = anymap::Map; @@ -117,6 +119,11 @@ pub trait Table: Send + Sync { } .fail()? } + + /// Get statistics for this table, if available + fn statistics(&self) -> Option { + None + } } pub type TableRef = Arc; @@ -127,9 +134,3 @@ pub trait TableIdProvider { } pub type TableIdProviderRef = Arc; - -#[derive(Default, Debug)] -pub struct RegionStat { - pub region_id: u64, - pub disk_usage_bytes: u64, -} diff --git a/src/table/src/test_util/memtable.rs b/src/table/src/test_util/memtable.rs index c1fb41d5d80..8aa5f3dd2d1 100644 --- a/src/table/src/test_util/memtable.rs +++ b/src/table/src/test_util/memtable.rs @@ -35,7 +35,7 @@ use crate::metadata::{ TableId, TableInfoBuilder, TableInfoRef, TableMetaBuilder, TableType, TableVersion, }; use crate::table::scan::SimpleTableScan; -use crate::Table; +use crate::{ColumnStatistics, Table, TableStatistics}; #[derive(Debug, Clone)] pub struct MemTable { @@ -172,6 +172,30 @@ impl Table for MemTable { recordbatch: Some(recordbatch), })))) } + + fn statistics(&self) -> Option { + let df_recordbatch = self.recordbatch.df_record_batch(); + let num_rows = df_recordbatch.num_rows(); + let total_byte_size = df_recordbatch.get_array_memory_size(); + let column_statistics: Vec<_> = df_recordbatch + .columns() + .iter() + .map(|col| { + let null_count = col.null_count(); + ColumnStatistics { + null_count: Some(null_count), + // TODO(discord9): implement more statistics + ..Default::default() + } + }) + .collect(); + Some(TableStatistics { + num_rows: Some(num_rows), + total_byte_size: Some(total_byte_size), + column_statistics: Some(column_statistics), + is_exact: true, + }) + } } impl RecordBatchStream for MemtableStream {