mito2/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use lazy_static::lazy_static;
18use prometheus::*;
19use puffin::puffin_manager::stager::StagerNotifier;
20
21/// Stage label.
22pub const STAGE_LABEL: &str = "stage";
23/// Type label.
24pub const TYPE_LABEL: &str = "type";
25const CACHE_EVICTION_CAUSE: &str = "cause";
26/// Reason to flush.
27pub const FLUSH_REASON: &str = "reason";
28/// File type label.
29pub const FILE_TYPE_LABEL: &str = "file_type";
30/// Region worker id label.
31pub const WORKER_LABEL: &str = "worker";
32/// Partition label.
33pub const PARTITION_LABEL: &str = "partition";
34/// Staging dir type label.
35pub const STAGING_TYPE: &str = "index_staging";
36/// Recycle bin type label.
37pub const RECYCLE_TYPE: &str = "recycle_bin";
38
39// Write metrics.
40lazy_static! {
41    /// Global write buffer size in bytes.
42    pub static ref WRITE_BUFFER_BYTES: IntGauge =
43        register_int_gauge!("greptime_mito_write_buffer_bytes", "mito write buffer bytes").unwrap();
44    /// Global memtable dictionary size in bytes.
45    pub static ref MEMTABLE_DICT_BYTES: IntGauge =
46        register_int_gauge!("greptime_mito_memtable_dict_bytes", "mito memtable dictionary size in bytes").unwrap();
47    /// Gauge for open regions in each worker.
48    pub static ref REGION_COUNT: IntGaugeVec =
49        register_int_gauge_vec!(
50            "greptime_mito_region_count",
51            "mito region count in each worker",
52            &[WORKER_LABEL],
53        ).unwrap();
54    /// Elapsed time to handle requests.
55    pub static ref HANDLE_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!(
56            "greptime_mito_handle_request_elapsed",
57            "mito handle request elapsed",
58            &[TYPE_LABEL],
59            // 0.01 ~ 10000
60            exponential_buckets(0.01, 10.0, 7).unwrap(),
61        )
62        .unwrap();
63
64    // ------ Flush related metrics
65    /// Counter of scheduled flush requests.
66    /// Note that the flush scheduler may merge some flush requests.
67    pub static ref FLUSH_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
68            "greptime_mito_flush_requests_total",
69            "mito flush requests total",
70            &[FLUSH_REASON]
71        )
72        .unwrap();
73    /// Counter of scheduled failed flush jobs.
74    pub static ref FLUSH_FAILURE_TOTAL: IntCounter =
75        register_int_counter!("greptime_mito_flush_failure_total", "mito flush failure total").unwrap();
76    /// Elapsed time of a flush job.
77    pub static ref FLUSH_ELAPSED: HistogramVec = register_histogram_vec!(
78            "greptime_mito_flush_elapsed",
79            "mito flush elapsed",
80            &[TYPE_LABEL],
81            // 1 ~ 625
82            exponential_buckets(1.0, 5.0, 6).unwrap(),
83        )
84        .unwrap();
85    /// Histogram of flushed bytes.
86    pub static ref FLUSH_BYTES_TOTAL: IntCounter =
87        register_int_counter!("greptime_mito_flush_bytes_total", "mito flush bytes total").unwrap();
88    /// Gauge for inflight flush tasks.
89    pub static ref INFLIGHT_FLUSH_COUNT: IntGauge =
90        register_int_gauge!(
91            "greptime_mito_inflight_flush_count",
92            "inflight flush count",
93        ).unwrap();
94    // ------ End of flush related metrics
95
96
97    // ------ Write related metrics
98    //
99    /// Counter of rejected write requests.
100    pub static ref WRITE_REJECT_TOTAL: IntCounter =
101        register_int_counter!("greptime_mito_write_reject_total", "mito write reject total").unwrap();
102    /// Elapsed time of each write stage.
103    pub static ref WRITE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
104            "greptime_mito_write_stage_elapsed",
105            "mito write stage elapsed",
106            &[STAGE_LABEL],
107            // 0.01 ~ 1000
108            exponential_buckets(0.01, 10.0, 6).unwrap(),
109        )
110        .unwrap();
111    /// Counter of rows to write.
112    pub static ref WRITE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
113        "greptime_mito_write_rows_total",
114        "mito write rows total",
115        &[TYPE_LABEL]
116    )
117    .unwrap();
118}
119
120// Compaction metrics.
121lazy_static! {
122    /// Timer of different stages in compaction.
123    /// - pick
124    /// - merge (in parallel)
125    ///   - iter_source
126    ///   - write_batch
127    ///   - update_index
128    ///   - upload_parquet
129    ///   - upload puffin
130    /// - write_manifest
131    pub static ref COMPACTION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
132        "greptime_mito_compaction_stage_elapsed",
133        "mito compaction stage elapsed",
134        &[STAGE_LABEL],
135        // 1 ~ 100000
136        exponential_buckets(1.0, 10.0, 6).unwrap(),
137    )
138    .unwrap();
139    /// Elapsed time of each successfully published compaction unit, labeled by `type`.
140    /// - local: one dependency unit, from task start through worker apply acknowledgement
141    /// - remote: one remote job, from request start through publication
142    pub static ref COMPACTION_ELAPSED_TOTAL: HistogramVec =
143        register_histogram_vec!(
144        "greptime_mito_compaction_total_elapsed",
145        "mito compaction elapsed per published unit",
146        &[TYPE_LABEL],
147        // 1 ~ 100000
148        exponential_buckets(1.0, 10.0, 6).unwrap(),
149    ).unwrap();
150    /// Counter of all requested compaction task.
151    pub static ref COMPACTION_REQUEST_COUNT: IntCounter =
152        register_int_counter!("greptime_mito_compaction_requests_total", "mito compaction requests total").unwrap();
153    /// Counter of failed compaction task.
154    pub static ref COMPACTION_FAILURE_COUNT: IntCounter =
155        register_int_counter!("greptime_mito_compaction_failure_total", "mito compaction failure total").unwrap();
156
157    /// Gauge for inflight compaction tasks: local units and remote jobs.
158    pub static ref INFLIGHT_COMPACTION_COUNT: IntGauge =
159        register_int_gauge!(
160            "greptime_mito_inflight_compaction_count",
161            "inflight compaction count",
162        ).unwrap();
163
164    /// Bytes reserved by compaction memory manager.
165    pub static ref COMPACTION_MEMORY_IN_USE: IntGauge =
166        register_int_gauge!(
167            "greptime_mito_compaction_memory_in_use_bytes",
168            "bytes currently reserved for compaction tasks",
169        )
170        .unwrap();
171    /// Configured compaction memory limit.
172    pub static ref COMPACTION_MEMORY_LIMIT: IntGauge =
173        register_int_gauge!(
174            "greptime_mito_compaction_memory_limit_bytes",
175            "maximum bytes allowed for compaction tasks",
176        )
177        .unwrap();
178    /// Wait time to obtain compaction memory.
179    pub static ref COMPACTION_MEMORY_WAIT: Histogram = register_histogram!(
180        "greptime_mito_compaction_memory_wait_seconds",
181        "time waiting for compaction memory",
182        // 0.01s ~ ~10s
183        exponential_buckets(0.01, 2.0, 10).unwrap(),
184    ).unwrap();
185    /// Counter of rejected compaction memory allocations.
186    pub static ref COMPACTION_MEMORY_REJECTED: IntCounterVec =
187        register_int_counter_vec!(
188            "greptime_mito_compaction_memory_rejected_total",
189            "number of compaction tasks rejected due to memory limit",
190            &[TYPE_LABEL]
191        ).unwrap();
192}
193
194// Query metrics.
195lazy_static! {
196    /// Timer of different stages in query.
197    pub static ref READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
198        "greptime_mito_read_stage_elapsed",
199        "mito read stage elapsed",
200        &[STAGE_LABEL],
201        // 0.01 ~ 10000
202        exponential_buckets(0.01, 10.0, 7).unwrap(),
203    )
204    .unwrap();
205    pub static ref READ_STAGE_FETCH_PAGES: Histogram = READ_STAGE_ELAPSED.with_label_values(&["fetch_pages"]);
206    /// Number of in-progress scan per partition.
207    pub static ref IN_PROGRESS_SCAN: IntGaugeVec = register_int_gauge_vec!(
208        "greptime_mito_in_progress_scan",
209        "mito in progress scan per partition",
210        &[TYPE_LABEL, PARTITION_LABEL]
211    )
212    .unwrap();
213    /// Counter of rows read from different source.
214    pub static ref READ_ROWS_TOTAL: IntCounterVec =
215        register_int_counter_vec!("greptime_mito_read_rows_total", "mito read rows total", &[TYPE_LABEL]).unwrap();
216    /// Counter of filtered rows during merge.
217    pub static ref MERGE_FILTER_ROWS_TOTAL: IntCounterVec =
218        register_int_counter_vec!("greptime_mito_merge_filter_rows_total", "mito merge filter rows total", &[TYPE_LABEL]).unwrap();
219    /// Counter of row groups read.
220    pub static ref READ_ROW_GROUPS_TOTAL: IntCounterVec =
221        register_int_counter_vec!("greptime_mito_read_row_groups_total", "mito read row groups total", &[TYPE_LABEL]).unwrap();
222    /// Counter of filtered rows by precise filter.
223    pub static ref PRECISE_FILTER_ROWS_TOTAL: IntCounterVec =
224        register_int_counter_vec!("greptime_mito_precise_filter_rows_total", "mito precise filter rows total", &[TYPE_LABEL]).unwrap();
225    pub static ref READ_ROWS_IN_ROW_GROUP_TOTAL: IntCounterVec =
226        register_int_counter_vec!("greptime_mito_read_rows_in_row_group_total", "mito read rows in row group total", &[TYPE_LABEL]).unwrap();
227    /// Histogram for the number of SSTs to scan per query.
228    pub static ref READ_SST_COUNT: Histogram = register_histogram!(
229        "greptime_mito_read_sst_count",
230        "Number of SSTs to scan in a scan task",
231        vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 256.0, 1024.0],
232    ).unwrap();
233    /// Histogram for the number of rows returned per query.
234    pub static ref READ_ROWS_RETURN: Histogram = register_histogram!(
235        "greptime_mito_read_rows_return",
236        "Number of rows returned in a scan task",
237        exponential_buckets(100.0, 10.0, 8).unwrap(),
238    ).unwrap();
239    /// Histogram for the number of batches returned per query.
240    pub static ref READ_BATCHES_RETURN: Histogram = register_histogram!(
241        "greptime_mito_read_batches_return",
242        "Number of rows returned in a scan task",
243        exponential_buckets(100.0, 10.0, 7).unwrap(),
244    ).unwrap();
245    /// Gauge for scan memory usage in bytes.
246    pub static ref SCAN_MEMORY_USAGE_BYTES: IntGauge = register_int_gauge!(
247        "greptime_mito_scan_memory_usage_bytes",
248        "current scan memory usage in bytes"
249    ).unwrap();
250    /// Counter of scan allocation attempts that could not acquire memory immediately.
251    pub static ref SCAN_MEMORY_EXHAUSTED_TOTAL: IntCounter = register_int_counter!(
252        "greptime_mito_scan_memory_exhausted_total",
253        "total number of times scan memory was unavailable for immediate acquisition"
254    ).unwrap();
255    /// Counter of scan requests that ultimately failed due to memory pressure.
256    pub static ref SCAN_REQUESTS_REJECTED_TOTAL: IntCounter = register_int_counter!(
257        "greptime_mito_scan_requests_rejected_total",
258        "total number of scan requests that ultimately failed due to memory limit"
259    ).unwrap();
260    /// Gauge for active file range builders in the pruner.
261    pub static ref PRUNER_ACTIVE_BUILDERS: IntGauge = register_int_gauge!(
262        "greptime_mito_pruner_active_builders",
263        "number of active file range builders in the pruner"
264    ).unwrap();
265}
266
267// Cache metrics.
268lazy_static! {
269    /// Cache hit counter.
270    pub static ref CACHE_HIT: IntCounterVec = register_int_counter_vec!(
271        "greptime_mito_cache_hit",
272        "mito cache hit",
273        &[TYPE_LABEL]
274    )
275    .unwrap();
276    /// Cache miss counter.
277    pub static ref CACHE_MISS: IntCounterVec = register_int_counter_vec!(
278        "greptime_mito_cache_miss",
279        "mito cache miss",
280        &[TYPE_LABEL]
281    )
282    .unwrap();
283    /// Cache size in bytes.
284    pub static ref CACHE_BYTES: IntGaugeVec = register_int_gauge_vec!(
285        "greptime_mito_cache_bytes",
286        "mito cache bytes",
287        &[TYPE_LABEL]
288    )
289    .unwrap();
290    /// Download bytes counter in the write cache.
291    pub static ref WRITE_CACHE_DOWNLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
292        "mito_write_cache_download_bytes_total",
293        "mito write cache download bytes total",
294    ).unwrap();
295    /// Timer of the downloading task in the write cache.
296    pub static ref WRITE_CACHE_DOWNLOAD_ELAPSED: HistogramVec = register_histogram_vec!(
297        "mito_write_cache_download_elapsed",
298        "mito write cache download elapsed",
299        &[TYPE_LABEL],
300        // 0.1 ~ 10000
301        exponential_buckets(0.1, 10.0, 6).unwrap(),
302    ).unwrap();
303    /// Number of inflight download tasks.
304    pub static ref WRITE_CACHE_INFLIGHT_DOWNLOAD: IntGauge = register_int_gauge!(
305        "mito_write_cache_inflight_download_count",
306        "mito write cache inflight download tasks",
307    ).unwrap();
308    /// Upload bytes counter.
309    pub static ref UPLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
310        "mito_upload_bytes_total",
311        "mito upload bytes total",
312    )
313    .unwrap();
314    /// Cache eviction counter, labeled with cache type and eviction reason.
315    pub static ref CACHE_EVICTION: IntCounterVec = register_int_counter_vec!(
316        "greptime_mito_cache_eviction",
317        "mito cache eviction",
318        &[TYPE_LABEL, CACHE_EVICTION_CAUSE]
319    ).unwrap();
320}
321
322// Index metrics.
323lazy_static! {
324    /// Approximate published index bytes in open regions, refreshed by maintenance.
325    pub static ref SERIES_INDEX_DISK_BYTES: IntGauge = register_int_gauge!(
326        "greptime_mito_series_index_disk_bytes", "estimated series and range index bytes in open regions"
327    ).unwrap();
328    /// Maintenance passes deferred by the estimated disk usage.
329    pub static ref SERIES_INDEX_CAPACITY_DEFERRED: IntCounter = register_int_counter!(
330        "greptime_mito_series_index_capacity_deferred_total", "series-index capacity deferrals"
331    ).unwrap();
332    // Index metrics.
333    /// Outcomes of series-index reconciliation passes.
334    pub static ref SERIES_INDEX_RECONCILE_TOTAL: IntCounterVec = register_int_counter_vec!(
335        "greptime_mito_series_index_reconcile_total",
336        "series-index reconciliation passes",
337        &["result"],
338    ).unwrap();
339    /// Elapsed time of series-index reconciliation phases.
340    pub static ref SERIES_INDEX_RECONCILE_ELAPSED: HistogramVec = register_histogram_vec!(
341        "greptime_mito_series_index_reconcile_elapsed",
342        "series-index reconciliation elapsed time",
343        &["phase"],
344        exponential_buckets(0.01, 10.0, 7).unwrap(),
345    ).unwrap();
346    /// Series and range index file operations.
347    pub static ref SERIES_INDEX_FILE_OPERATION_TOTAL: IntCounterVec = register_int_counter_vec!(
348        "greptime_mito_series_index_file_operation_total",
349        "series-index file operations",
350        &["index_type", "operation", "result"],
351    ).unwrap();
352    /// Number of stale index publications rejected at each publication stage.
353    pub static ref INDEX_PUBLICATION_STALE_TOTAL: IntCounterVec =
354        register_int_counter_vec!(
355            "greptime_mito_index_publication_stale_total",
356            "stale index publications rejected",
357            &[STAGE_LABEL],
358        ).unwrap();
359    /// Number of failures while cleaning local state for stale index artifacts.
360    pub static ref INDEX_ARTIFACT_CLEANUP_FAILURE_TOTAL: IntCounter =
361        register_int_counter!(
362            "greptime_mito_index_artifact_cleanup_failure_total",
363            "failures while cleaning local state for stale index artifacts",
364        ).unwrap();
365    /// Timer of index application.
366    pub static ref INDEX_APPLY_ELAPSED: HistogramVec = register_histogram_vec!(
367        "greptime_index_apply_elapsed",
368        "index apply elapsed",
369        &[TYPE_LABEL],
370        // 0.01 ~ 1000
371        exponential_buckets(0.01, 10.0, 6).unwrap(),
372    )
373    .unwrap();
374    /// Gauge of index apply memory usage.
375    pub static ref INDEX_APPLY_MEMORY_USAGE: IntGauge = register_int_gauge!(
376        "greptime_index_apply_memory_usage",
377        "index apply memory usage",
378    )
379    .unwrap();
380    /// Timer of index creation.
381    pub static ref INDEX_CREATE_ELAPSED: HistogramVec = register_histogram_vec!(
382        "greptime_index_create_elapsed",
383        "index create elapsed",
384        &[STAGE_LABEL, TYPE_LABEL],
385        // 0.1 ~ 10000
386        exponential_buckets(0.1, 10.0, 6).unwrap(),
387    )
388    .unwrap();
389    /// Counter of rows indexed.
390    pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
391        "greptime_index_create_rows_total",
392        "index create rows total",
393        &[TYPE_LABEL],
394    )
395    .unwrap();
396    /// Counter of created index bytes.
397    pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
398        "greptime_index_create_bytes_total",
399        "index create bytes total",
400        &[TYPE_LABEL],
401    )
402    .unwrap();
403    /// Gauge of index create memory usage.
404    pub static ref INDEX_CREATE_MEMORY_USAGE: IntGaugeVec = register_int_gauge_vec!(
405        "greptime_index_create_memory_usage",
406        "index create memory usage",
407        &[TYPE_LABEL],
408    ).unwrap();
409    /// Counter of r/w bytes on index related IO operations.
410    pub static ref INDEX_IO_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
411        "greptime_index_io_bytes_total",
412        "index io bytes total",
413        &[TYPE_LABEL, FILE_TYPE_LABEL]
414    )
415    .unwrap();
416    /// Counter of read bytes on puffin files.
417    pub static ref INDEX_PUFFIN_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
418        .with_label_values(&["read", "puffin"]);
419    /// Counter of write bytes on puffin files.
420    pub static ref INDEX_PUFFIN_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
421        .with_label_values(&["write", "puffin"]);
422    /// Counter of read bytes on intermediate files.
423    pub static ref INDEX_INTERMEDIATE_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
424        .with_label_values(&["read", "intermediate"]);
425    /// Counter of write bytes on intermediate files.
426    pub static ref INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
427        .with_label_values(&["write", "intermediate"]);
428
429    /// Counter of r/w operations on index related IO operations, e.g. read, write, seek and flush.
430    pub static ref INDEX_IO_OP_TOTAL: IntCounterVec = register_int_counter_vec!(
431        "greptime_index_io_op_total",
432        "index io op total",
433        &[TYPE_LABEL, FILE_TYPE_LABEL]
434    )
435    .unwrap();
436    /// Counter of read operations on puffin files.
437    pub static ref INDEX_PUFFIN_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
438        .with_label_values(&["read", "puffin"]);
439    /// Counter of seek operations on puffin files.
440    pub static ref INDEX_PUFFIN_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
441        .with_label_values(&["seek", "puffin"]);
442    /// Counter of write operations on puffin files.
443    pub static ref INDEX_PUFFIN_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
444        .with_label_values(&["write", "puffin"]);
445    /// Counter of flush operations on puffin files.
446    pub static ref INDEX_PUFFIN_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
447        .with_label_values(&["flush", "puffin"]);
448    /// Counter of read operations on intermediate files.
449    pub static ref INDEX_INTERMEDIATE_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
450        .with_label_values(&["read", "intermediate"]);
451    /// Counter of seek operations on intermediate files.
452    pub static ref INDEX_INTERMEDIATE_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
453        .with_label_values(&["seek", "intermediate"]);
454    /// Counter of write operations on intermediate files.
455    pub static ref INDEX_INTERMEDIATE_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
456        .with_label_values(&["write", "intermediate"]);
457    /// Counter of flush operations on intermediate files.
458    pub static ref INDEX_INTERMEDIATE_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
459        .with_label_values(&["flush", "intermediate"]);
460}
461
462lazy_static! {
463    // Manifest related metrics:
464
465    /// Elapsed time of manifest operation. Labeled with "op".
466    pub static ref MANIFEST_OP_ELAPSED: HistogramVec = register_histogram_vec!(
467        "greptime_manifest_op_elapsed",
468        "mito manifest operation elapsed",
469        &["op"],
470        // 0.01 ~ 1000
471        exponential_buckets(0.01, 10.0, 6).unwrap(),
472    ).unwrap();
473
474
475    pub static ref REGION_WORKER_HANDLE_WRITE_ELAPSED: HistogramVec = register_histogram_vec!(
476        "greptime_region_worker_handle_write",
477        "elapsed time for handling writes in region worker loop",
478        &["stage"],
479        exponential_buckets(0.001, 10.0, 5).unwrap()
480    ).unwrap();
481
482}
483
484lazy_static! {
485    /// Counter for compaction input file size.
486    pub static ref COMPACTION_INPUT_BYTES: Counter = register_counter!(
487        "greptime_mito_compaction_input_bytes",
488        "mito compaction input file size",
489        ).unwrap();
490
491    /// Counter for compaction output file size.
492    pub static ref COMPACTION_OUTPUT_BYTES: Counter = register_counter!(
493        "greptime_mito_compaction_output_bytes",
494        "mito compaction output file size",
495        ).unwrap();
496
497    /// Active series count in TimeSeriesMemtable
498    pub static ref MEMTABLE_ACTIVE_SERIES_COUNT: IntGauge = register_int_gauge!(
499        "greptime_mito_memtable_active_series_count",
500        "active time series count in TimeSeriesMemtable",
501        ).unwrap();
502
503    /// Active field builder count in TimeSeriesMemtable
504    pub static ref MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT: IntGauge = register_int_gauge!(
505        "greptime_mito_memtable_field_builder_count",
506        "active field builder count in TimeSeriesMemtable",
507        ).unwrap();
508
509    /// Number of stalling write requests in each worker.
510    pub static ref WRITE_STALLING: IntGaugeVec = register_int_gauge_vec!(
511            "greptime_mito_write_stalling_count",
512            "mito stalled write request in each worker",
513            &[WORKER_LABEL]
514        ).unwrap();
515    /// Number of ref files
516    pub static ref GC_REF_FILE_CNT: IntGauge = register_int_gauge!(
517            "greptime_gc_ref_file_count",
518            "gc ref file count",
519        ).unwrap();
520    /// Total number of stalled write requests.
521    pub static ref WRITE_STALL_TOTAL: IntCounter = register_int_counter!(
522        "greptime_mito_write_stall_total",
523        "Total number of stalled write requests"
524    ).unwrap();
525    /// Time waiting for requests to be handled by the region worker.
526    pub static ref REQUEST_WAIT_TIME: HistogramVec = register_histogram_vec!(
527            "greptime_mito_request_wait_time",
528            "mito request wait time before being handled by region worker",
529            &[WORKER_LABEL],
530            // 0.001 ~ 10000
531            exponential_buckets(0.001, 10.0, 8).unwrap(),
532        )
533        .unwrap();
534
535    /// Counter for the number of files deleted by the GC worker.
536    pub static ref GC_DELETE_FILE_CNT: IntCounter =
537        register_int_counter!(
538            "greptime_mito_gc_delete_file_count",
539            "mito gc deleted file count",
540        ).unwrap();
541
542    /// Counter for the number of unparsable files skipped by GC.
543    pub static ref GC_SKIPPED_UNPARSABLE_FILES: IntCounter =
544        register_int_counter!(
545            "greptime_mito_gc_skipped_unparsable_files",
546            "mito gc skipped unparsable files count",
547        ).unwrap();
548
549    /// Counter for the number of orphaned index files found by GC.
550    pub static ref GC_ORPHANED_INDEX_FILES: IntCounter =
551        register_int_counter!(
552            "greptime_mito_gc_orphaned_index_files",
553            "mito gc orphaned index files count",
554        ).unwrap();
555
556    /// Histogram for GC operation duration by stage.
557    pub static ref GC_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
558        "greptime_mito_gc_duration_seconds",
559        "GC operation duration by stage",
560        &[STAGE_LABEL],
561        exponential_buckets(0.01, 10.0, 6).unwrap(),
562    ).unwrap();
563
564    /// Counter for GC runs by mode.
565    pub static ref GC_RUNS_TOTAL: IntCounterVec = register_int_counter_vec!(
566        "greptime_mito_gc_runs_total",
567        "Total GC runs by mode",
568        &["mode"],
569    ).unwrap();
570
571    /// Counter for GC errors by type.
572    pub static ref GC_ERRORS_TOTAL: IntCounterVec = register_int_counter_vec!(
573        "greptime_mito_gc_errors_total",
574        "Total GC errors by type",
575        &["error_type"],
576    ).unwrap();
577
578    /// Counter for total files deleted by GC, labeled by file type.
579    pub static ref GC_FILES_DELETED_TOTAL: IntCounterVec = register_int_counter_vec!(
580        "greptime_mito_gc_files_deleted_total",
581        "Total files deleted by GC",
582        &[FILE_TYPE_LABEL],
583    ).unwrap();
584
585    /// Total number of files downloaded during cache fill on region open.
586    pub static ref CACHE_FILL_DOWNLOADED_FILES: IntCounter = register_int_counter!(
587        "mito_cache_fill_downloaded_files",
588        "mito cache fill downloaded files count",
589    ).unwrap();
590
591    /// Number of files pending download during cache fill on region open.
592    pub static ref CACHE_FILL_PENDING_FILES: IntGauge = register_int_gauge!(
593        "mito_cache_fill_pending_files",
594        "mito cache fill pending files count",
595    ).unwrap();
596
597    /// Counter of flush files.
598    pub static ref FLUSH_FILE_TOTAL: IntCounter =
599        register_int_counter!("greptime_mito_flush_file_total", "mito flushed file count").unwrap();
600}
601
602/// Stager notifier to collect metrics.
603pub struct StagerMetrics {
604    cache_hit: IntCounter,
605    cache_miss: IntCounter,
606    staging_cache_bytes: IntGauge,
607    recycle_cache_bytes: IntGauge,
608    cache_eviction: IntCounter,
609    staging_miss_read: Histogram,
610}
611
612impl StagerMetrics {
613    /// Creates a new stager notifier.
614    pub fn new() -> Self {
615        Self {
616            cache_hit: CACHE_HIT.with_label_values(&[STAGING_TYPE]),
617            cache_miss: CACHE_MISS.with_label_values(&[STAGING_TYPE]),
618            staging_cache_bytes: CACHE_BYTES.with_label_values(&[STAGING_TYPE]),
619            recycle_cache_bytes: CACHE_BYTES.with_label_values(&[RECYCLE_TYPE]),
620            cache_eviction: CACHE_EVICTION.with_label_values(&[STAGING_TYPE, "size"]),
621            staging_miss_read: READ_STAGE_ELAPSED.with_label_values(&["staging_miss_read"]),
622        }
623    }
624}
625
626impl Default for StagerMetrics {
627    fn default() -> Self {
628        Self::new()
629    }
630}
631
632impl StagerNotifier for StagerMetrics {
633    fn on_cache_hit(&self, _size: u64) {
634        self.cache_hit.inc();
635    }
636
637    fn on_cache_miss(&self, _size: u64) {
638        self.cache_miss.inc();
639    }
640
641    fn on_cache_insert(&self, size: u64) {
642        self.staging_cache_bytes.add(size as i64);
643    }
644
645    fn on_load_dir(&self, duration: Duration) {
646        self.staging_miss_read.observe(duration.as_secs_f64());
647    }
648
649    fn on_load_blob(&self, duration: Duration) {
650        self.staging_miss_read.observe(duration.as_secs_f64());
651    }
652
653    fn on_cache_evict(&self, size: u64) {
654        self.cache_eviction.inc();
655        self.staging_cache_bytes.sub(size as i64);
656    }
657
658    fn on_recycle_insert(&self, size: u64) {
659        self.recycle_cache_bytes.add(size as i64);
660    }
661
662    fn on_recycle_clear(&self, size: u64) {
663        self.recycle_cache_bytes.sub(size as i64);
664    }
665}
mito2/metrics.rs

mito2/
metrics.rs
