meta_srv/handler/
failure_handler.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use api::v1::meta::{HeartbeatRequest, Role};
16use async_trait::async_trait;
17use common_telemetry::info;
18
19use crate::error::Result;
20use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
21use crate::metasrv::Context;
22use crate::region::supervisor::{DatanodeHeartbeat, HeartbeatAcceptor, RegionSupervisor};
23
24pub struct RegionFailureHandler {
25    heartbeat_acceptor: HeartbeatAcceptor,
26}
27
28impl RegionFailureHandler {
29    pub(crate) fn new(
30        mut region_supervisor: RegionSupervisor,
31        heartbeat_acceptor: HeartbeatAcceptor,
32    ) -> Self {
33        info!("Starting region supervisor");
34        common_runtime::spawn_global(async move { region_supervisor.run().await });
35        Self { heartbeat_acceptor }
36    }
37}
38
39#[async_trait]
40impl HeartbeatHandler for RegionFailureHandler {
41    fn is_acceptable(&self, role: Role) -> bool {
42        role == Role::Datanode
43    }
44
45    async fn handle(
46        &self,
47        _: &HeartbeatRequest,
48        _ctx: &mut Context,
49        acc: &mut HeartbeatAccumulator,
50    ) -> Result<HandleControl> {
51        let Some(stat) = acc.stat.as_ref() else {
52            return Ok(HandleControl::Continue);
53        };
54
55        // Never blocks: the region lease has already been decided earlier in the
56        // chain, so a busy supervisor must not withhold the heartbeat response.
57        self.heartbeat_acceptor
58            .accept(DatanodeHeartbeat::from(stat));
59
60        Ok(HandleControl::Continue)
61    }
62}
63
64#[cfg(test)]
65mod tests {
66    use std::time::Duration;
67
68    use api::v1::meta::HeartbeatRequest;
69    use common_catalog::consts::default_engine;
70    use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
71    use store_api::region_engine::RegionRole;
72    use store_api::storage::RegionId;
73    use tokio::sync::oneshot;
74
75    use crate::handler::failure_handler::RegionFailureHandler;
76    use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
77    use crate::metasrv::builder::MetasrvBuilder;
78    use crate::region::supervisor::tests::new_test_supervisor;
79    use crate::region::supervisor::{DatanodeHeartbeat, Event, HeartbeatAcceptor};
80
81    fn new_region_stat(region_id: u64) -> RegionStat {
82        RegionStat {
83            id: RegionId::from_u64(region_id),
84            rcus: 0,
85            wcus: 0,
86            approximate_bytes: 0,
87            engine: default_engine().to_string(),
88            role: RegionRole::Follower,
89            num_rows: 0,
90            memtable_size: 0,
91            manifest_size: 0,
92            sst_size: 0,
93            sst_num: 0,
94            index_size: 0,
95            region_manifest: RegionManifestInfo::Mito {
96                manifest_version: 0,
97                flushed_entry_id: 0,
98                file_removed_cnt: 0,
99            },
100            data_topic_latest_entry_id: 0,
101            metadata_topic_latest_entry_id: 0,
102            written_bytes: 0,
103            query_cpu_time: 0,
104            query_scanned_bytes: 0,
105            min_timestamp: None,
106            max_timestamp: None,
107        }
108    }
109
110    fn new_stat() -> Stat {
111        Stat {
112            id: 42,
113            region_stats: vec![new_region_stat(1), new_region_stat(2), new_region_stat(3)],
114            timestamp_millis: 1000,
115            ..Default::default()
116        }
117    }
118
119    #[tokio::test]
120    async fn test_handle_heartbeat() {
121        let (supervisor, sender) = new_test_supervisor();
122        let heartbeat_acceptor = HeartbeatAcceptor::new(sender.clone());
123        let handler = RegionFailureHandler::new(supervisor, heartbeat_acceptor);
124        let req = &HeartbeatRequest::default();
125        let builder = MetasrvBuilder::new();
126        let metasrv = builder.build().await.unwrap();
127        let mut ctx = metasrv.new_ctx();
128        let acc = &mut HeartbeatAccumulator::default();
129        acc.stat = Some(new_stat());
130
131        handler.handle(req, &mut ctx, acc).await.unwrap();
132        let (tx, rx) = oneshot::channel();
133        sender.send(Event::Dump(tx)).await.unwrap();
134        let detector = rx.await.unwrap();
135        assert_eq!(detector.iter().collect::<Vec<_>>().len(), 3);
136    }
137
138    #[tokio::test]
139    async fn test_handle_heartbeat_does_not_block_on_full_supervisor_queue() {
140        // Regression test for #9419: a full queue used to park the handler.
141        // Keep the supervisor alive, or the queue reports Closed.
142        let (_supervisor, sender) = new_test_supervisor();
143        // Built directly, so the supervisor stays undrained.
144        let handler = RegionFailureHandler {
145            heartbeat_acceptor: HeartbeatAcceptor::new(sender.clone()),
146        };
147
148        let stat = new_stat();
149        let mut queued = 0;
150        while sender
151            .try_send(Event::HeartbeatArrived(DatanodeHeartbeat::from(&stat)))
152            .is_ok()
153        {
154            queued += 1;
155        }
156        assert_eq!(queued, sender.max_capacity());
157
158        let builder = MetasrvBuilder::new();
159        let metasrv = builder.build().await.unwrap();
160        let mut ctx = metasrv.new_ctx();
161        let acc = &mut HeartbeatAccumulator::default();
162        // Without a stat the handler returns before the acceptor.
163        acc.stat = Some(stat);
164
165        let result = tokio::time::timeout(
166            Duration::from_secs(5),
167            handler.handle(&HeartbeatRequest::default(), &mut ctx, acc),
168        )
169        .await;
170        assert!(result.is_ok());
171    }
172}
meta_srv/handler/failure_handler.rs

meta_srv/handler/
failure_handler.rs
