diff --git a/src/datanode/src/metrics.rs b/src/datanode/src/metrics.rs index 7573490c747..e9f947ef9da 100644 --- a/src/datanode/src/metrics.rs +++ b/src/datanode/src/metrics.rs @@ -85,6 +85,16 @@ lazy_static! { ) .unwrap(); + /// Failed region open attempts, including individual and batch requests. + /// Counts every request in an engine group when no individual results are available. + /// Failed retries count separately; explicitly ignored missing regions do not count. + pub static ref REGION_OPEN_FAILURES_TOTAL: IntCounterVec = register_int_counter_vec!( + "greptime_datanode_region_open_failures_total", + "Failed region open attempts, including retries and all requests in failed engine groups, excluding ignored missing regions", + &["engine", "status_code"] + ) + .unwrap(); + /// Total count of failed insert requests to region server, labeled with request /// type and status code. pub static ref REGION_SERVER_INSERT_FAIL_COUNT: IntCounterVec = register_int_counter_vec!( diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index 527ab0491c0..aba09de8797 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -1285,13 +1285,17 @@ impl RegionServerInner { requests: Vec<(RegionId, RegionOpenRequest)>, ignore_nonexistent_region: bool, ) -> Result> { + let request_count = requests.len(); let region_changes = requests .iter() .map(|(region_id, open)| { let attribute = parse_region_attribute(&open.engine, &open.options)?; Ok((*region_id, RegionChange::Register(attribute))) }) - .collect::>>()?; + .collect::>>() + .inspect_err(|err| { + record_region_open_failures(engine.name(), err.status_code(), request_count); + })?; for (®ion_id, region_change) in ®ion_changes { self.set_region_status_not_ready(region_id, &engine, region_change) @@ -1314,6 +1318,7 @@ impl RegionServerInner { .await { error!(e; "Failed to set region to ready: {}", region_id); + record_region_open_failures(engine.name(), e.status_code(), 1); errors.push(BoxedError::new(e)); } else { open_regions.push(region_id) @@ -1327,6 +1332,7 @@ impl RegionServerInner { warn!("Region {} not found, ignore it, source: {:?}", region_id, e); } else { error!(e; "Failed to open region: {}", region_id); + record_region_open_failures(engine.name(), e.status_code(), 1); errors.push(e); } } @@ -1338,6 +1344,7 @@ impl RegionServerInner { self.unset_region_status(region_id, &engine, *region_change); } error!(e; "Failed to open batch regions"); + record_region_open_failures(engine.name(), e.status_code(), request_count); errors.push(BoxedError::new(e)); } } @@ -1373,7 +1380,10 @@ impl RegionServerInner { .read() .unwrap() .get(&engine) - .with_context(|| RegionEngineNotFoundSnafu { name: &engine })? + .with_context(|| RegionEngineNotFoundSnafu { name: &engine }) + .inspect_err(|err| { + record_region_open_failures(&engine, err.status_code(), requests.len()); + })? .clone(); results.push( self.handle_batch_open_requests_inner( @@ -1574,6 +1584,24 @@ impl RegionServerInner { &self, region_id: RegionId, request: RegionRequest, + ) -> Result { + let open_engine = match &request { + RegionRequest::Open(open) => Some(open.engine.clone()), + _ => None, + }; + self.handle_request_inner(region_id, request) + .await + .inspect_err(|err| { + if let Some(engine) = open_engine { + record_region_open_failures(&engine, err.status_code(), 1); + } + }) + } + + async fn handle_request_inner( + &self, + region_id: RegionId, + request: RegionRequest, ) -> Result { let request_type = request.request_type(); let _timer = crate::metrics::HANDLE_REGION_REQUEST_ELAPSED @@ -1946,6 +1974,17 @@ fn is_metric_engine(engine: &str) -> bool { engine == METRIC_ENGINE_NAME } +/// Records failed open attempts without using arbitrary engine names as metric labels. +fn record_region_open_failures(engine: &str, status_code: StatusCode, count: usize) { + let engine = match engine { + MITO_ENGINE_NAME | METRIC_ENGINE_NAME | FILE_ENGINE_NAME => engine, + _ => "unknown", + }; + crate::metrics::REGION_OPEN_FAILURES_TOTAL + .with_label_values(&[engine, status_code.as_ref()]) + .inc_by(count as u64); +} + fn parse_region_attribute( engine: &str, options: &HashMap,