rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-01-17 10:42:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(index): distinguish different types of index metrics (#4337)

Browse Source 
Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
This commit is contained in:
Zhenchi
2024-07-14 16:03:09 +08:00
committed by GitHub
parent 5a1732279b
commit 377a513690
5 changed files with 67 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/mito2/src/metrics.rs
View File

@@ -209,26 +209,29 @@ lazy_static! {
pub static ref INDEX_CREATE_ELAPSED: HistogramVec = register_histogram_vec!(
"greptime_index_create_elapsed",
"index create elapsed",
&[STAGE_LABEL],
&[STAGE_LABEL, TYPE_LABEL],
vec![0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 60.0, 300.0]
)
.unwrap();
/// Counter of rows indexed.
pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounter = register_int_counter!(
pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
"greptime_index_create_rows_total",
"index create rows total",
&[TYPE_LABEL],
)
.unwrap();
/// Counter of created index bytes.
pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounter = register_int_counter!(
pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
"greptime_index_create_bytes_total",
"index create bytes total",
&[TYPE_LABEL],
)
.unwrap();
/// Gauge of index create memory usage.
pub static ref INDEX_CREATE_MEMORY_USAGE: IntGauge = register_int_gauge!(
pub static ref INDEX_CREATE_MEMORY_USAGE: IntGaugeVec = register_int_gauge_vec!(
"greptime_index_create_memory_usage",
"index create memory usage",
&[TYPE_LABEL],
).unwrap();
/// Counter of r/w bytes on index related IO operations.
pub static ref INDEX_IO_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(

46
src/mito2/src/sst/index.rs
View File

@@ -38,6 +38,9 @@ use crate::sst::index::fulltext_index::creator::SstIndexCreator as FulltextIndex
use crate::sst::index::intermediate::IntermediateManager;
use crate::sst::index::inverted_index::creator::SstIndexCreator as InvertedIndexer;
pub(crate) const TYPE_INVERTED_INDEX: &str = "inverted_index";
pub(crate) const TYPE_FULLTEXT_INDEX: &str = "fulltext_index";
/// Output of the index creation.
#[derive(Debug, Clone, Default)]
pub struct IndexOutput {
@@ -89,11 +92,12 @@ pub struct Indexer {
file_id: FileId,
file_path: String,
region_id: RegionId,
last_memory_usage: usize,
puffin_manager: Option<SstPuffinManager>,
inverted_indexer: Option<InvertedIndexer>,
last_mem_inverted_index: usize,
fulltext_indexer: Option<FulltextIndexer>,
last_mem_fulltext_index: usize,
}
impl Indexer {
@@ -101,35 +105,42 @@ impl Indexer {
pub async fn update(&mut self, batch: &Batch) {
self.do_update(batch).await;
let memory_usage = self.memory_usage();
INDEX_CREATE_MEMORY_USAGE.add(memory_usage as i64 - self.last_memory_usage as i64);
self.last_memory_usage = memory_usage;
self.flush_mem_metrics();
}
/// Finalizes the index creation.
pub async fn finish(&mut self) -> IndexOutput {
INDEX_CREATE_MEMORY_USAGE.sub(self.last_memory_usage as i64);
self.last_memory_usage = 0;
let output = self.do_finish().await;
self.do_finish().await
self.flush_mem_metrics();
output
}
/// Aborts the index creation.
pub async fn abort(&mut self) {
INDEX_CREATE_MEMORY_USAGE.sub(self.last_memory_usage as i64);
self.last_memory_usage = 0;
self.do_abort().await;
self.flush_mem_metrics();
}
fn memory_usage(&self) -> usize {
self.inverted_indexer
fn flush_mem_metrics(&mut self) {
let inverted_mem = self
.inverted_indexer
.as_ref()
.map_or(0, |creator| creator.memory_usage())
+ self
.fulltext_indexer
.as_ref()
.map_or(0, |creator| creator.memory_usage())
.map_or(0, |creator| creator.memory_usage());
INDEX_CREATE_MEMORY_USAGE
.with_label_values(&[TYPE_INVERTED_INDEX])
.add(inverted_mem as i64 - self.last_mem_inverted_index as i64);
self.last_mem_inverted_index = inverted_mem;
let fulltext_mem = self
.fulltext_indexer
.as_ref()
.map_or(0, |creator| creator.memory_usage());
INDEX_CREATE_MEMORY_USAGE
.with_label_values(&[TYPE_FULLTEXT_INDEX])
.add(fulltext_mem as i64 - self.last_mem_fulltext_index as i64);
self.last_mem_fulltext_index = fulltext_mem;
}
}
@@ -153,7 +164,6 @@ impl<'a> IndexerBuilder<'a> {
file_id: self.file_id,
file_path: self.file_path.clone(),
region_id: self.metadata.region_id,
last_memory_usage: 0,
..Default::default()
};

3
src/mito2/src/sst/index/fulltext_index/creator.rs
View File

@@ -36,6 +36,7 @@ use crate::sst::index::fulltext_index::INDEX_BLOB_TYPE;
use crate::sst::index::intermediate::IntermediateManager;
use crate::sst::index::puffin_manager::SstPuffinWriter;
use crate::sst::index::statistics::{ByteCount, RowCount, Statistics};
use crate::sst::index::TYPE_FULLTEXT_INDEX;
/// `SstIndexCreator` is responsible for creating fulltext indexes for SST files.
pub struct SstIndexCreator {
@@ -104,7 +105,7 @@ impl SstIndexCreator {
Ok(Self {
creators,
aborted: false,
stats: Statistics::default(),
stats: Statistics::new(TYPE_FULLTEXT_INDEX),
})
}

3
src/mito2/src/sst/index/inverted_index/creator.rs
View File

@@ -43,6 +43,7 @@ use crate::sst::index::inverted_index::creator::temp_provider::TempFileProvider;
use crate::sst::index::inverted_index::INDEX_BLOB_TYPE;
use crate::sst::index::puffin_manager::SstPuffinWriter;
use crate::sst::index::statistics::{ByteCount, RowCount, Statistics};
use crate::sst::index::TYPE_INVERTED_INDEX;
/// The minimum memory usage threshold for one column.
const MIN_MEMORY_USAGE_THRESHOLD_PER_COLUMN: usize = 1024 * 1024; // 1MB
@@ -114,7 +115,7 @@ impl SstIndexCreator {
index_creator,
temp_file_provider,
value_buf: vec![],
stats: Statistics::default(),
stats: Statistics::new(TYPE_INVERTED_INDEX),
aborted: false,
memory_usage,
column_ids,

38
src/mito2/src/sst/index/statistics.rs
View File

@@ -27,8 +27,9 @@ enum Stage {
}
/// Statistics for index creation. Flush metrics when dropped.
#[derive(Default)]
pub(crate) struct Statistics {
/// Index type.
index_type: &'static str,
/// Accumulated elapsed time for the index update stage.
update_elapsed: Duration,
/// Accumulated elapsed time for the index finish stage.
@@ -42,6 +43,17 @@ pub(crate) struct Statistics {
}
impl Statistics {
pub fn new(index_type: &'static str) -> Self {
Self {
index_type,
update_elapsed: Duration::default(),
finish_elapsed: Duration::default(),
cleanup_eplased: Duration::default(),
row_count: 0,
byte_count: 0,
}
}
/// Starts timing the update stage, returning a `TimerGuard` to automatically record duration.
#[must_use]
pub fn record_update(&mut self) -> TimerGuard<'_> {
@@ -74,20 +86,26 @@ impl Statistics {
impl Drop for Statistics {
fn drop(&mut self) {
INDEX_CREATE_ELAPSED
.with_label_values(&["update"])
.with_label_values(&["update", self.index_type])
.observe(self.update_elapsed.as_secs_f64());
INDEX_CREATE_ELAPSED
.with_label_values(&["finish"])
.with_label_values(&["finish", self.index_type])
.observe(self.finish_elapsed.as_secs_f64());
INDEX_CREATE_ELAPSED
.with_label_values(&["cleanup"])
.with_label_values(&["cleanup", self.index_type])
.observe(self.cleanup_eplased.as_secs_f64());
INDEX_CREATE_ELAPSED.with_label_values(&["total"]).observe(
(self.update_elapsed + self.finish_elapsed + self.cleanup_eplased).as_secs_f64(),
);
INDEX_CREATE_ELAPSED
.with_label_values(&["total", self.index_type])
.observe(
(self.update_elapsed + self.finish_elapsed + self.cleanup_eplased).as_secs_f64(),
);
INDEX_CREATE_ROWS_TOTAL.inc_by(self.row_count as _);
INDEX_CREATE_BYTES_TOTAL.inc_by(self.byte_count as _);
INDEX_CREATE_ROWS_TOTAL
.with_label_values(&[self.index_type])
.inc_by(self.row_count as _);
INDEX_CREATE_BYTES_TOTAL
.with_label_values(&[self.index_type])
.inc_by(self.byte_count as _);
}
}
@@ -142,7 +160,7 @@ mod tests {
#[test]
fn test_statistics_basic() {
let mut stats = Statistics::default();
let mut stats = Statistics::new("test");
{
let mut guard = stats.record_update();
guard.inc_byte_count(100);