log_store/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use lazy_static::lazy_static;
16use prometheus::*;
17
18/// Logstore label.
19pub const LOGSTORE_LABEL: &str = "logstore";
20/// Operation type label.
21pub const OPTYPE_LABEL: &str = "optype";
22/// Kafka topic label.
23pub const TOPIC_LABEL: &str = "topic";
24/// Kafka partition label.
25pub const PARTITION_LABEL: &str = "partition";
26
27lazy_static! {
28    /// Counters of bytes of each operation on a logstore.
29    pub static ref METRIC_LOGSTORE_OP_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
30        "greptime_logstore_op_bytes_total",
31        "logstore operation bytes total",
32        &[LOGSTORE_LABEL, OPTYPE_LABEL],
33    )
34    .unwrap();
35    /// Counter of bytes of the append_batch operation on the kafka logstore.
36    pub static ref METRIC_KAFKA_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
37        &["kafka", "append_batch"],
38    );
39    /// Counter of bytes of the read operation on the kafka logstore.
40    pub static ref METRIC_KAFKA_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
41        &["kafka", "read"],
42    );
43    /// Counter of bytes of the append_batch operation on the raft-engine logstore.
44    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
45        &["raft-engine", "append_batch"],
46    );
47    /// Counter of bytes of the read operation on the raft-engine logstore.
48    pub static ref METRIC_RAFT_ENGINE_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
49        &["raft-engine", "read"],
50    );
51
52    /// Timer of operations on a logstore.
53    pub static ref METRIC_LOGSTORE_OP_ELAPSED: HistogramVec = register_histogram_vec!(
54        "greptime_logstore_op_elapsed",
55        "logstore operation elapsed",
56        &[LOGSTORE_LABEL, OPTYPE_LABEL],
57        vec![
58            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
59            45.0, 60.0,
60        ],
61    )
62    .unwrap();
63    /// Timer of the append_batch operation on the kafka logstore.
64    pub static ref METRIC_KAFKA_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "append_batch"]);
65    /// Timer of the append_batch operation on the kafka logstore.
66    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
67    pub static ref METRIC_KAFKA_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "read"]);
68    /// Timer of the append_batch operation on the raft-engine logstore.
69    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "append_batch"]);
70    /// Timer of the append_batch operation on the raft-engine logstore.
71    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
72    pub static ref METRIC_RAFT_ENGINE_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "read"]);
73
74    pub static ref METRIC_KAFKA_CLIENT_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
75        "greptime_logstore_kafka_client_bytes_total",
76        "kafka logstore's bytes traffic total",
77        &[LOGSTORE_LABEL, PARTITION_LABEL],
78    )
79    .unwrap();
80    pub static ref METRIC_KAFKA_CLIENT_TRAFFIC_TOTAL: IntCounterVec = register_int_counter_vec!(
81        "greptime_logstore_kafka_client_traffic_total",
82        "kafka logstore's request count traffic total",
83        &[LOGSTORE_LABEL, PARTITION_LABEL],
84    )
85    .unwrap();
86    pub static ref METRIC_KAFKA_CLIENT_PRODUCE_ELAPSED: HistogramVec = register_histogram_vec!(
87        "greptime_logstore_kafka_client_produce_elapsed",
88        "kafka logstore produce operation elapsed",
89        &[LOGSTORE_LABEL, PARTITION_LABEL],
90    )
91    .unwrap();
92
93    /// Counter of segments a read of the object store logstore skipped because
94    /// they did not decode.
95    pub static ref METRIC_OBJECT_STORE_WAL_SKIPPED_SEGMENTS_TOTAL: IntCounter = register_int_counter!(
96        "greptime_logstore_object_store_wal_skipped_segments_total",
97        "object store logstore skipped corrupted segments total",
98    )
99    .unwrap();
100    /// Counter of objects the object store logstore deleted because no
101    /// region needs them any more.
102    pub static ref METRIC_OBJECT_STORE_WAL_DELETED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
103        "greptime_logstore_object_store_wal_deleted_objects_total",
104        "object store logstore deleted objects total",
105    )
106    .unwrap();
107    /// Counter of object deletes of the object store logstore that failed and
108    /// are attempted again at the next collection.
109    pub static ref METRIC_OBJECT_STORE_WAL_FAILED_DELETES_TOTAL: IntCounter = register_int_counter!(
110        "greptime_logstore_object_store_wal_failed_deletes_total",
111        "object store logstore failed object deletes total",
112    )
113    .unwrap();
114
115    /// How long an object of the object store logstore takes to become durable
116    /// once its batch is sealed, sampled for every conditional create that
117    /// succeeds, an identical retry included. The time runs from the seal, so
118    /// it includes any wait for a create slot and, in the `enqueued` mode,
119    /// every repeated attempt and the delay before it.
120    ///
121    /// A create is a few milliseconds on a local object store and tens of
122    /// milliseconds on a cloud one, so the buckets start at 1ms and reach 8s.
123    pub static ref METRIC_OBJECT_STORE_WAL_SEAL_TO_DURABLE_SECONDS: Histogram = register_histogram!(
124        "greptime_logstore_object_store_wal_seal_to_durable_seconds",
125        "object store logstore seconds from sealing a batch to the success of its conditional create, slot waits and retries included",
126        exponential_buckets(0.001, 2.0, 14).unwrap(),
127    )
128    .unwrap();
129
130    /// How long an append of the object store logstore waits for its
131    /// acknowledgement in the `durable` mode, which is the write latency a
132    /// caller sees.
133    ///
134    /// The wait is one seal plus every earlier object plus one create, so with
135    /// the default 100ms flush interval it sits in the tens of milliseconds;
136    /// the buckets straddle that and leave room for a stalled uploader.
137    pub static ref METRIC_OBJECT_STORE_WAL_APPEND_ACK_SECONDS: Histogram = register_histogram!(
138        "greptime_logstore_object_store_wal_append_ack_seconds",
139        "object store logstore seconds an append waits for its acknowledgement in the durable mode",
140        exponential_buckets(0.002, 2.0, 14).unwrap(),
141    )
142    .unwrap();
143
144    /// How large the objects of the object store logstore are, which says
145    /// whether batches seal on size or on the flush interval.
146    ///
147    /// The buckets are powers of two from 1KiB, the size of a timer-sealed
148    /// batch of one small entry, to 16MiB, twice the 8MiB default batch size,
149    /// which is a threshold rather than an upper bound.
150    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_BYTES: Histogram = register_histogram!(
151        "greptime_logstore_object_store_wal_object_bytes",
152        "object store logstore bytes of each object sealed",
153        exponential_buckets(1024.0, 2.0, 15).unwrap(),
154    )
155    .unwrap();
156
157    /// How many entries the objects of the object store logstore hold, which
158    /// says how many writes one request amortises.
159    ///
160    /// The largest finite bucket is 2^20, the position limit of one region in
161    /// one object. An object holds the entries of many regions, so its total
162    /// can pass that; such an object falls in the overflow bucket.
163    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_ENTRIES: Histogram = register_histogram!(
164        "greptime_logstore_object_store_wal_object_entries",
165        "object store logstore entries of each object sealed",
166        exponential_buckets(1.0, 4.0, 11).unwrap(),
167    )
168    .unwrap();
169
170    /// Counter of conditional creates of the object store logstore that
171    /// succeeded, an identical retry that found its own object included. A
172    /// create that failed is counted in the create failure or conflict counter
173    /// instead.
174    pub static ref METRIC_OBJECT_STORE_WAL_CREATED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
175        "greptime_logstore_object_store_wal_created_objects_total",
176        "object store logstore conditional creates that succeeded, identical retries included",
177    )
178    .unwrap();
179
180    /// Number of objects the object store logstore has indexed, which is what
181    /// a recovery of the prefix would have to read.
182    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_OBJECTS: IntGauge = register_int_gauge!(
183        "greptime_logstore_object_store_wal_indexed_objects",
184        "object store logstore objects currently indexed",
185    )
186    .unwrap();
187
188    /// Bytes of the objects the object store logstore has indexed, which is
189    /// what the WAL occupies under its prefix.
190    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_BYTES: IntGauge = register_int_gauge!(
191        "greptime_logstore_object_store_wal_indexed_bytes",
192        "object store logstore bytes currently indexed",
193    )
194    .unwrap();
195
196    /// How long a recovery of the object store logstore that succeeded took to
197    /// rebuild its catalog, which is the part of a restart the WAL is
198    /// responsible for. A recovery that was abandoned records nothing.
199    ///
200    /// Recovery reads a few small ranges per object, so the buckets run from
201    /// 10ms, an empty prefix, to minutes on a prefix with many objects.
202    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERY_SECONDS: Histogram = register_histogram!(
203        "greptime_logstore_object_store_wal_recovery_seconds",
204        "object store logstore seconds spent by a recovery that succeeded",
205        exponential_buckets(0.01, 3.0, 10).unwrap(),
206    )
207    .unwrap();
208
209    /// Counter of objects a recovery of the object store logstore that
210    /// succeeded read footers from, which is what its time is spent on. It
211    /// counts the objects of the same recoveries the timer above measures, so
212    /// an abandoned one contributes to neither.
213    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
214        "greptime_logstore_object_store_wal_recovered_objects_total",
215        "object store logstore objects read by a recovery that succeeded total",
216    )
217    .unwrap();
218
219    /// How long a read stream of the object store logstore took to serve the
220    /// replay of one region, recorded once the stream has been read to its
221    /// end; one that fails or is dropped part way records nothing.
222    ///
223    /// A read fetches one segment per object of the region, so the buckets run
224    /// from 5ms, a region with nothing to replay, to minutes.
225    pub static ref METRIC_OBJECT_STORE_WAL_READ_SECONDS: Histogram = register_histogram!(
226        "greptime_logstore_object_store_wal_read_seconds",
227        "object store logstore seconds spent serving a region read that ran to its end",
228        exponential_buckets(0.005, 3.0, 10).unwrap(),
229    )
230    .unwrap();
231
232    /// Counter of conditional creates of the object store logstore that found
233    /// an object with different content. In the `durable` mode an object of an
234    /// earlier epoch fails the batch like a transient failure; any other
235    /// conflict poisons the store.
236    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_CONFLICTS_TOTAL: IntCounter = register_int_counter!(
237        "greptime_logstore_object_store_wal_create_conflicts_total",
238        "object store logstore conflicting conditional creates total",
239    )
240    .unwrap();
241
242    /// Counter of conditional creates of the object store logstore the object
243    /// store did not confirm, which fail the appends behind them in the
244    /// `durable` mode and are repeated in the `enqueued` mode.
245    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_FAILURES_TOTAL: IntCounter = register_int_counter!(
246        "greptime_logstore_object_store_wal_create_failures_total",
247        "object store logstore transient conditional create failures total",
248    )
249    .unwrap();
250
251    /// Counter of object store logstores that hit an error they cannot recover
252    /// from, after which every operation of the store fails until it reopens.
253    pub static ref METRIC_OBJECT_STORE_WAL_POISONED_TOTAL: IntCounter = register_int_counter!(
254        "greptime_logstore_object_store_wal_poisoned_total",
255        "object store logstores poisoned total",
256    )
257    .unwrap();
258
259    /// Counter of appends the object store logstore held back because the
260    /// unpersisted backlog reached a threshold in the `enqueued` mode, which
261    /// says the write rate is above what the uploads sustain.
262    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPENDS_TOTAL: IntCounter = register_int_counter!(
263        "greptime_logstore_object_store_wal_stalled_appends_total",
264        "object store logstore appends stalled by the backlog thresholds total",
265    )
266    .unwrap();
267
268    /// How long such an append stays held back, until it is admitted or until
269    /// it is refused, a stop, a terminal error or the teardown of the store
270    /// releasing it included, which is the latency the `enqueued` mode pays
271    /// once the backlog is at a threshold.
272    ///
273    /// A stalled append waits for one upload to complete, so the buckets run
274    /// from 5ms to half a minute of a struggling object store.
275    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPEND_SECONDS: Histogram = register_histogram!(
276        "greptime_logstore_object_store_wal_stalled_append_seconds",
277        "object store logstore seconds a stalled append is held back before it is admitted or refused",
278        exponential_buckets(0.005, 3.0, 9).unwrap(),
279    )
280    .unwrap();
281}
log_store/metrics.rs

log_store/
metrics.rs
