log_store/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use lazy_static::lazy_static;
16use prometheus::*;
17
18/// Logstore label.
19pub const LOGSTORE_LABEL: &str = "logstore";
20/// Operation type label.
21pub const OPTYPE_LABEL: &str = "optype";
22/// Kafka topic label.
23pub const TOPIC_LABEL: &str = "topic";
24/// Kafka partition label.
25pub const PARTITION_LABEL: &str = "partition";
26
27lazy_static! {
28    /// Counters of bytes of each operation on a logstore.
29    pub static ref METRIC_LOGSTORE_OP_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
30        "greptime_logstore_op_bytes_total",
31        "logstore operation bytes total",
32        &[LOGSTORE_LABEL, OPTYPE_LABEL],
33    )
34    .unwrap();
35    /// Counter of bytes of the append_batch operation on the kafka logstore.
36    pub static ref METRIC_KAFKA_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
37        &["kafka", "append_batch"],
38    );
39    /// Counter of bytes of the read operation on the kafka logstore.
40    pub static ref METRIC_KAFKA_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
41        &["kafka", "read"],
42    );
43    /// Counter of bytes of the append_batch operation on the raft-engine logstore.
44    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
45        &["raft-engine", "append_batch"],
46    );
47    /// Counter of bytes of the read operation on the raft-engine logstore.
48    pub static ref METRIC_RAFT_ENGINE_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
49        &["raft-engine", "read"],
50    );
51
52    /// Timer of operations on a logstore.
53    pub static ref METRIC_LOGSTORE_OP_ELAPSED: HistogramVec = register_histogram_vec!(
54        "greptime_logstore_op_elapsed",
55        "logstore operation elapsed",
56        &[LOGSTORE_LABEL, OPTYPE_LABEL],
57        vec![
58            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
59            45.0, 60.0,
60        ],
61    )
62    .unwrap();
63    /// Timer of the append_batch operation on the kafka logstore.
64    pub static ref METRIC_KAFKA_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "append_batch"]);
65    /// Timer of the append_batch operation on the kafka logstore.
66    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
67    pub static ref METRIC_KAFKA_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "read"]);
68    /// Timer of the append_batch operation on the raft-engine logstore.
69    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "append_batch"]);
70    /// Timer of the append_batch operation on the raft-engine logstore.
71    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
72    pub static ref METRIC_RAFT_ENGINE_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "read"]);
73
74    pub static ref METRIC_KAFKA_CLIENT_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
75        "greptime_logstore_kafka_client_bytes_total",
76        "kafka logstore's bytes traffic total",
77        &[LOGSTORE_LABEL, PARTITION_LABEL],
78    )
79    .unwrap();
80    pub static ref METRIC_KAFKA_CLIENT_TRAFFIC_TOTAL: IntCounterVec = register_int_counter_vec!(
81        "greptime_logstore_kafka_client_traffic_total",
82        "kafka logstore's request count traffic total",
83        &[LOGSTORE_LABEL, PARTITION_LABEL],
84    )
85    .unwrap();
86    pub static ref METRIC_KAFKA_CLIENT_PRODUCE_ELAPSED: HistogramVec = register_histogram_vec!(
87        "greptime_logstore_kafka_client_produce_elapsed",
88        "kafka logstore produce operation elapsed",
89        &[LOGSTORE_LABEL, PARTITION_LABEL],
90    )
91    .unwrap();
92
93    /// Counter of segments a read of the object store logstore skipped because
94    /// they did not decode.
95    pub static ref METRIC_OBJECT_STORE_WAL_SKIPPED_SEGMENTS_TOTAL: IntCounter = register_int_counter!(
96        "greptime_logstore_object_store_wal_skipped_segments_total",
97        "object store logstore skipped corrupted segments total",
98    )
99    .unwrap();
100    /// Counter of objects the object store logstore deleted because no
101    /// region needs them any more.
102    pub static ref METRIC_OBJECT_STORE_WAL_DELETED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
103        "greptime_logstore_object_store_wal_deleted_objects_total",
104        "object store logstore deleted objects total",
105    )
106    .unwrap();
107    /// Counter of object deletes of the object store logstore that failed and
108    /// are attempted again at the next collection.
109    pub static ref METRIC_OBJECT_STORE_WAL_FAILED_DELETES_TOTAL: IntCounter = register_int_counter!(
110        "greptime_logstore_object_store_wal_failed_deletes_total",
111        "object store logstore failed object deletes total",
112    )
113    .unwrap();
114    /// Counter of release records in the objects the object store logstore
115    /// indexed, including records carried over from batches that failed.
116    pub static ref METRIC_OBJECT_STORE_WAL_RELEASE_RECORDS_TOTAL: IntCounter = register_int_counter!(
117        "greptime_logstore_object_store_wal_release_records_total",
118        "object store logstore release records made durable total",
119    )
120    .unwrap();
121
122    /// How long an object of the object store logstore takes to become durable
123    /// once its batch is sealed, sampled for every conditional create that
124    /// succeeds, an identical retry included. The time runs from the seal, so
125    /// it includes any wait for a create slot and, in the `enqueued` mode,
126    /// every repeated attempt and the delay before it.
127    ///
128    /// A create is a few milliseconds on a local object store and tens of
129    /// milliseconds on a cloud one, so the buckets start at 1ms and reach 8s.
130    pub static ref METRIC_OBJECT_STORE_WAL_SEAL_TO_DURABLE_SECONDS: Histogram = register_histogram!(
131        "greptime_logstore_object_store_wal_seal_to_durable_seconds",
132        "object store logstore seconds from sealing a batch to the success of its conditional create, slot waits and retries included",
133        exponential_buckets(0.001, 2.0, 14).unwrap(),
134    )
135    .unwrap();
136
137    /// How long an append of the object store logstore waits for its
138    /// acknowledgement in the `durable` mode, which is the write latency a
139    /// caller sees.
140    ///
141    /// The wait is one seal plus every earlier object plus one create, so with
142    /// the default 100ms flush interval it sits in the tens of milliseconds;
143    /// the buckets straddle that and leave room for a stalled uploader.
144    pub static ref METRIC_OBJECT_STORE_WAL_APPEND_ACK_SECONDS: Histogram = register_histogram!(
145        "greptime_logstore_object_store_wal_append_ack_seconds",
146        "object store logstore seconds an append waits for its acknowledgement in the durable mode",
147        exponential_buckets(0.002, 2.0, 14).unwrap(),
148    )
149    .unwrap();
150
151    /// How large the objects of the object store logstore are, which says
152    /// whether batches seal on size or on the flush interval.
153    ///
154    /// The buckets are powers of two from 1KiB, the size of a timer-sealed
155    /// batch of one small entry, to 16MiB, twice the 8MiB default batch size,
156    /// which is a threshold rather than an upper bound.
157    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_BYTES: Histogram = register_histogram!(
158        "greptime_logstore_object_store_wal_object_bytes",
159        "object store logstore bytes of each object sealed",
160        exponential_buckets(1024.0, 2.0, 15).unwrap(),
161    )
162    .unwrap();
163
164    /// How many entries the objects of the object store logstore hold, which
165    /// says how many writes one request amortises.
166    ///
167    /// The largest finite bucket is 2^20, the position limit of one region in
168    /// one object. An object holds the entries of many regions, so its total
169    /// can pass that; such an object falls in the overflow bucket.
170    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_ENTRIES: Histogram = register_histogram!(
171        "greptime_logstore_object_store_wal_object_entries",
172        "object store logstore entries of each object sealed",
173        exponential_buckets(1.0, 4.0, 11).unwrap(),
174    )
175    .unwrap();
176
177    /// Counter of conditional creates of the object store logstore that
178    /// succeeded, an identical retry that found its own object included. A
179    /// create that failed is counted in the create failure or conflict counter
180    /// instead.
181    pub static ref METRIC_OBJECT_STORE_WAL_CREATED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
182        "greptime_logstore_object_store_wal_created_objects_total",
183        "object store logstore conditional creates that succeeded, identical retries included",
184    )
185    .unwrap();
186
187    /// Number of objects the object store logstore has indexed, which is what
188    /// a recovery of the prefix would have to read.
189    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_OBJECTS: IntGauge = register_int_gauge!(
190        "greptime_logstore_object_store_wal_indexed_objects",
191        "object store logstore objects currently indexed",
192    )
193    .unwrap();
194
195    /// Bytes of the objects the object store logstore has indexed, which is
196    /// what the WAL occupies under its prefix.
197    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_BYTES: IntGauge = register_int_gauge!(
198        "greptime_logstore_object_store_wal_indexed_bytes",
199        "object store logstore bytes currently indexed",
200    )
201    .unwrap();
202
203    /// How long a recovery of the object store logstore that succeeded took to
204    /// rebuild its catalog, which is the part of a restart the WAL is
205    /// responsible for. A recovery that was abandoned records nothing.
206    ///
207    /// Recovery reads a few small ranges per object, so the buckets run from
208    /// 10ms, an empty prefix, to minutes on a prefix with many objects.
209    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERY_SECONDS: Histogram = register_histogram!(
210        "greptime_logstore_object_store_wal_recovery_seconds",
211        "object store logstore seconds spent by a recovery that succeeded",
212        exponential_buckets(0.01, 3.0, 10).unwrap(),
213    )
214    .unwrap();
215
216    /// Counter of objects a recovery of the object store logstore that
217    /// succeeded read footers from, which is what its time is spent on. It
218    /// counts the objects of the same recoveries the timer above measures, so
219    /// an abandoned one contributes to neither.
220    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
221        "greptime_logstore_object_store_wal_recovered_objects_total",
222        "object store logstore objects read by a recovery that succeeded total",
223    )
224    .unwrap();
225
226    /// How long a read stream of the object store logstore took to serve the
227    /// replay of one region, recorded once the stream has been read to its
228    /// end; one that fails or is dropped part way records nothing.
229    ///
230    /// A read fetches one segment per object of the region, so the buckets run
231    /// from 5ms, a region with nothing to replay, to minutes.
232    pub static ref METRIC_OBJECT_STORE_WAL_READ_SECONDS: Histogram = register_histogram!(
233        "greptime_logstore_object_store_wal_read_seconds",
234        "object store logstore seconds spent serving a region read that ran to its end",
235        exponential_buckets(0.005, 3.0, 10).unwrap(),
236    )
237    .unwrap();
238
239    /// Counter of conditional creates of the object store logstore that found
240    /// an object with different content. In the `durable` mode an object of an
241    /// earlier epoch fails the batch like a transient failure; any other
242    /// conflict poisons the store.
243    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_CONFLICTS_TOTAL: IntCounter = register_int_counter!(
244        "greptime_logstore_object_store_wal_create_conflicts_total",
245        "object store logstore conflicting conditional creates total",
246    )
247    .unwrap();
248
249    /// Counter of conditional creates of the object store logstore the object
250    /// store did not confirm, which fail the appends behind them in the
251    /// `durable` mode and are repeated in the `enqueued` mode.
252    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_FAILURES_TOTAL: IntCounter = register_int_counter!(
253        "greptime_logstore_object_store_wal_create_failures_total",
254        "object store logstore conditional creates the object store did not confirm total",
255    )
256    .unwrap();
257
258    /// Counter of object store logstores that hit an error they cannot recover
259    /// from, after which every operation of the store fails until it reopens.
260    pub static ref METRIC_OBJECT_STORE_WAL_POISONED_TOTAL: IntCounter = register_int_counter!(
261        "greptime_logstore_object_store_wal_poisoned_total",
262        "object store logstores poisoned total",
263    )
264    .unwrap();
265
266    /// Counter of appends the object store logstore held back until an upload
267    /// completed: in either mode because the sealed batches waiting to become
268    /// durable left no room for another, and in the `enqueued` mode also
269    /// because the unpersisted backlog reached a threshold. Either says the
270    /// write rate is above what the uploads sustain.
271    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPENDS_TOTAL: IntCounter = register_int_counter!(
272        "greptime_logstore_object_store_wal_stalled_appends_total",
273        "object store logstore appends stalled by full sealed batches or the backlog thresholds total",
274    )
275    .unwrap();
276
277    /// How long such an append stays held back, until it is admitted or until
278    /// it is refused, a stop, a terminal error or the teardown of the store
279    /// releasing it included, which is the latency the `enqueued` mode pays
280    /// once the backlog is at a threshold.
281    ///
282    /// A stalled append waits for one upload to complete, so the buckets run
283    /// from 5ms to half a minute of a struggling object store.
284    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPEND_SECONDS: Histogram = register_histogram!(
285        "greptime_logstore_object_store_wal_stalled_append_seconds",
286        "object store logstore seconds a stalled append is held back before it is admitted or refused",
287        exponential_buckets(0.005, 3.0, 9).unwrap(),
288    )
289    .unwrap();
290
291    /// Number of object store logstores in the `enqueued` mode whose backlog
292    /// the object store refuses with an error that repeating the create
293    /// cannot settle, such as denied access or a missing bucket. Such a store
294    /// keeps its backlog, attempts it again with backoff and rejects appends
295    /// until a create succeeds.
296    pub static ref METRIC_OBJECT_STORE_WAL_FAILING: IntGauge = register_int_gauge!(
297        "greptime_logstore_object_store_wal_failing",
298        "object store logstores rejecting appends because the object store refuses their backlog with an error that requires intervention",
299    )
300    .unwrap();
301
302    /// Counter of appends the object store logstore rejected while its
303    /// backlog is refused, see the gauge above. They take no entry id and
304    /// fail with a retryable error. Releases of dropped regions failed in
305    /// that state count too; their records wait for the next object.
306    pub static ref METRIC_OBJECT_STORE_WAL_REJECTED_APPENDS_TOTAL: IntCounter = register_int_counter!(
307        "greptime_logstore_object_store_wal_rejected_appends_total",
308        "object store logstore appends rejected while the object store refuses the backlog total",
309    )
310    .unwrap();
311}
log_store/metrics.rs

log_store/
metrics.rs
