datanode/heartbeat/
handler.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use async_trait::async_trait;
16use common_meta::error::{InvalidHeartbeatResponseSnafu, Result as MetaResult};
17use common_meta::heartbeat::handler::{
18    HandleControl, HeartbeatResponseHandler, HeartbeatResponseHandlerContext,
19};
20use common_meta::instruction::{Instruction, InstructionReply};
21use common_meta::kv_backend::KvBackendRef;
22use common_telemetry::error;
23use common_telemetry::tracing_context::FutureExt;
24use snafu::OptionExt;
25use store_api::storage::GcReport;
26use strum::AsRefStr;
27
28mod apply_staging_manifest;
29mod close_region;
30mod downgrade_region;
31mod enter_staging;
32mod file_ref;
33mod flush_region;
34mod gc_worker;
35mod open_region;
36mod remap_manifest;
37mod sync_region;
38mod upgrade_region;
39
40use crate::heartbeat::handler::apply_staging_manifest::ApplyStagingManifestsHandler;
41use crate::heartbeat::handler::close_region::CloseRegionsHandler;
42use crate::heartbeat::handler::downgrade_region::DowngradeRegionsHandler;
43use crate::heartbeat::handler::enter_staging::EnterStagingRegionsHandler;
44use crate::heartbeat::handler::file_ref::{GetFileRefsHandler, GetPackedFileRefsHandler};
45use crate::heartbeat::handler::flush_region::FlushRegionsHandler;
46use crate::heartbeat::handler::gc_worker::{GcRegionsHandler, PackedGcRegionsHandler};
47use crate::heartbeat::handler::open_region::OpenRegionsHandler;
48use crate::heartbeat::handler::remap_manifest::RemapManifestHandler;
49use crate::heartbeat::handler::sync_region::SyncRegionHandler;
50use crate::heartbeat::handler::upgrade_region::UpgradeRegionsHandler;
51use crate::heartbeat::task_tracker::TaskTracker;
52use crate::region_server::RegionServer;
53
54/// The handler for [`Instruction`]s.
55#[derive(Clone)]
56pub struct RegionHeartbeatResponseHandler {
57    region_server: RegionServer,
58    downgrade_tasks: TaskTracker<()>,
59    flush_tasks: TaskTracker<()>,
60    open_region_parallelism: usize,
61    gc_tasks: TaskTracker<GcReport>,
62    kv_backend: KvBackendRef,
63}
64
65#[async_trait::async_trait]
66pub trait InstructionHandler: Send + Sync {
67    type Instruction;
68    async fn handle(
69        &self,
70        ctx: &HandlerContext,
71        instruction: Self::Instruction,
72    ) -> Option<InstructionReply>;
73}
74
75#[derive(Clone)]
76pub struct HandlerContext {
77    pub region_server: RegionServer,
78    pub downgrade_tasks: TaskTracker<()>,
79    pub flush_tasks: TaskTracker<()>,
80    pub gc_tasks: TaskTracker<GcReport>,
81    pub kv_backend: KvBackendRef,
82}
83
84impl HandlerContext {
85    #[cfg(test)]
86    pub fn new_for_test(region_server: RegionServer, kv_backend: KvBackendRef) -> Self {
87        Self {
88            region_server,
89            downgrade_tasks: TaskTracker::new(),
90            flush_tasks: TaskTracker::new(),
91            gc_tasks: TaskTracker::new(),
92            kv_backend,
93        }
94    }
95}
96
97impl RegionHeartbeatResponseHandler {
98    /// Returns the [RegionHeartbeatResponseHandler].
99    pub fn new(region_server: RegionServer, kv_backend: KvBackendRef) -> Self {
100        Self {
101            region_server,
102            downgrade_tasks: TaskTracker::new(),
103            flush_tasks: TaskTracker::new(),
104            // Default to half of the number of CPUs.
105            open_region_parallelism: (num_cpus::get() / 2).max(1),
106            gc_tasks: TaskTracker::new(),
107            kv_backend,
108        }
109    }
110
111    /// Sets the parallelism for opening regions.
112    pub fn with_open_region_parallelism(mut self, parallelism: usize) -> Self {
113        self.open_region_parallelism = parallelism;
114        self
115    }
116
117    fn build_handler(
118        &self,
119        instruction: &Instruction,
120    ) -> MetaResult<Option<Box<InstructionHandlers>>> {
121        match instruction {
122            Instruction::CloseRegions(_) => Ok(Some(Box::new(CloseRegionsHandler.into()))),
123            Instruction::OpenRegions(_) => Ok(Some(Box::new(
124                OpenRegionsHandler {
125                    open_region_parallelism: self.open_region_parallelism,
126                }
127                .into(),
128            ))),
129            Instruction::FlushRegions(_) => Ok(Some(Box::new(FlushRegionsHandler.into()))),
130            Instruction::DowngradeRegions(_) => Ok(Some(Box::new(DowngradeRegionsHandler.into()))),
131            Instruction::UpgradeRegions(_) => Ok(Some(Box::new(
132                UpgradeRegionsHandler {
133                    upgrade_region_parallelism: self.open_region_parallelism,
134                }
135                .into(),
136            ))),
137            Instruction::GetFileRefs(_) => Ok(Some(Box::new(GetFileRefsHandler.into()))),
138            Instruction::GetPackedFileRefs(_) => {
139                Ok(Some(Box::new(GetPackedFileRefsHandler.into())))
140            }
141            Instruction::GcRegions(_) => Ok(Some(Box::new(GcRegionsHandler.into()))),
142            Instruction::PackedGcRegions(_) => Ok(Some(Box::new(PackedGcRegionsHandler.into()))),
143            Instruction::InvalidateCaches(_) => InvalidHeartbeatResponseSnafu.fail(),
144            Instruction::Suspend => Ok(None),
145            Instruction::EnterStagingRegions(_) => {
146                Ok(Some(Box::new(EnterStagingRegionsHandler.into())))
147            }
148            Instruction::SyncRegions(_) => Ok(Some(Box::new(SyncRegionHandler.into()))),
149            Instruction::RemapManifest(_) => Ok(Some(Box::new(RemapManifestHandler.into()))),
150            Instruction::ApplyStagingManifests(_) => {
151                Ok(Some(Box::new(ApplyStagingManifestsHandler.into())))
152            }
153        }
154    }
155}
156
157#[allow(clippy::enum_variant_names)]
158#[derive(AsRefStr)]
159pub enum InstructionHandlers {
160    CloseRegions(CloseRegionsHandler),
161    OpenRegions(OpenRegionsHandler),
162    FlushRegions(FlushRegionsHandler),
163    DowngradeRegions(DowngradeRegionsHandler),
164    UpgradeRegions(UpgradeRegionsHandler),
165    GetFileRefs(GetFileRefsHandler),
166    GetPackedFileRefs(GetPackedFileRefsHandler),
167    GcRegions(GcRegionsHandler),
168    PackedGcRegions(PackedGcRegionsHandler),
169    EnterStagingRegions(EnterStagingRegionsHandler),
170    SyncRegions(SyncRegionHandler),
171    RemapManifest(RemapManifestHandler),
172    ApplyStagingManifests(ApplyStagingManifestsHandler),
173}
174
175impl InstructionHandlers {
176    // Returns the string representation of the instruction handler.
177    pub fn as_ref_str(&self) -> &str {
178        self.as_ref()
179    }
180}
181
182macro_rules! impl_from_handler {
183    ($($handler:ident => $variant:ident),*) => {
184        $(
185            impl From<$handler> for InstructionHandlers {
186                fn from(handler: $handler) -> Self {
187                    InstructionHandlers::$variant(handler)
188                }
189            }
190        )*
191    };
192}
193
194impl_from_handler!(
195    CloseRegionsHandler => CloseRegions,
196    OpenRegionsHandler => OpenRegions,
197    FlushRegionsHandler => FlushRegions,
198    DowngradeRegionsHandler => DowngradeRegions,
199    UpgradeRegionsHandler => UpgradeRegions,
200    GetFileRefsHandler => GetFileRefs,
201    GetPackedFileRefsHandler => GetPackedFileRefs,
202    GcRegionsHandler => GcRegions,
203    PackedGcRegionsHandler => PackedGcRegions,
204    EnterStagingRegionsHandler => EnterStagingRegions,
205    SyncRegionHandler => SyncRegions,
206    RemapManifestHandler => RemapManifest,
207    ApplyStagingManifestsHandler => ApplyStagingManifests
208);
209
210macro_rules! dispatch_instr {
211    (
212        $( $instr_variant:ident => $handler_variant:ident ),* $(,)?
213    ) => {
214        impl InstructionHandlers {
215            pub async fn handle(
216                &self,
217                ctx: &HandlerContext,
218                instruction: Instruction,
219            ) -> Option<InstructionReply> {
220                match (self, instruction) {
221                    $(
222                        (
223                            InstructionHandlers::$handler_variant(handler),
224                            Instruction::$instr_variant(instr),
225                        ) => handler.handle(ctx, instr).await,
226                    )*
227                    // Safety: must be used in pairs with `build_handler`.
228                    _ => unreachable!(),
229                }
230            }
231            /// Check whether this instruction is acceptable by any handler.
232            pub fn is_acceptable(instruction: &Instruction) -> bool {
233                matches!(
234                    instruction,
235                    $(
236                        Instruction::$instr_variant { .. }
237                    )|*
238                )
239            }
240        }
241    };
242}
243
244dispatch_instr!(
245    CloseRegions => CloseRegions,
246    OpenRegions => OpenRegions,
247    FlushRegions => FlushRegions,
248    DowngradeRegions => DowngradeRegions,
249    UpgradeRegions => UpgradeRegions,
250    GetFileRefs => GetFileRefs,
251    GetPackedFileRefs => GetPackedFileRefs,
252    GcRegions => GcRegions,
253    PackedGcRegions => PackedGcRegions,
254    EnterStagingRegions => EnterStagingRegions,
255    SyncRegions => SyncRegions,
256    RemapManifest => RemapManifest,
257    ApplyStagingManifests => ApplyStagingManifests,
258);
259
260#[async_trait]
261impl HeartbeatResponseHandler for RegionHeartbeatResponseHandler {
262    fn is_acceptable(&self, ctx: &HeartbeatResponseHandlerContext) -> bool {
263        if let Some((_, _, instruction)) = ctx.incoming_message.as_ref() {
264            return InstructionHandlers::is_acceptable(instruction);
265        }
266        false
267    }
268
269    async fn handle(&self, ctx: &mut HeartbeatResponseHandlerContext) -> MetaResult<HandleControl> {
270        let (meta, tracing_ctx, instruction) = ctx
271            .incoming_message
272            .take()
273            .context(InvalidHeartbeatResponseSnafu)?;
274
275        let mailbox = ctx.mailbox.clone();
276        if let Some(handler) = self.build_handler(&instruction)? {
277            let context = HandlerContext {
278                region_server: self.region_server.clone(),
279                downgrade_tasks: self.downgrade_tasks.clone(),
280                flush_tasks: self.flush_tasks.clone(),
281                gc_tasks: self.gc_tasks.clone(),
282                kv_backend: self.kv_backend.clone(),
283            };
284            let span = tracing_ctx.attach(tracing::info_span!(
285                "RegionHeartbeatResponseHandler::handle",
286                from = %meta.from,
287                to = %meta.to,
288                handler = %handler.as_ref_str(),
289            ));
290            let _handle = common_runtime::spawn_global(async move {
291                let reply = handler.handle(&context, instruction).trace(span).await;
292                if let Some(reply) = reply
293                    && let Err(e) = mailbox.send((meta, reply)).await
294                {
295                    let error = e.to_string();
296                    let (meta, reply) = e.0;
297                    error!("Failed to send reply {reply} to {meta:?}: {error}");
298                }
299            });
300        }
301
302        Ok(HandleControl::Continue)
303    }
304}
305
306#[cfg(test)]
307mod tests {
308    use std::assert_matches;
309    use std::collections::HashMap;
310    use std::sync::Arc;
311    use std::time::Duration;
312
313    use common_meta::RegionIdent;
314    use common_meta::heartbeat::mailbox::{
315        HeartbeatMailbox, IncomingMessage, MailboxRef, MessageMeta,
316    };
317    use common_meta::instruction::{
318        DowngradeRegion, EnterStagingRegion, OpenRegion, StagingPartitionDirective, UpgradeRegion,
319    };
320    use common_meta::kv_backend::memory::MemoryKvBackend;
321    use mito2::config::MitoConfig;
322    use mito2::engine::MITO_ENGINE_NAME;
323    use mito2::test_util::{CreateRequestBuilder, TestEnv};
324    use store_api::path_utils::table_dir;
325    use store_api::region_engine::RegionRole;
326    use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements};
327    use store_api::storage::RegionId;
328    use tokio::sync::mpsc::{self, Receiver};
329
330    use super::*;
331    use crate::error;
332    use crate::tests::mock_region_server;
333
334    pub struct HeartbeatResponseTestEnv {
335        pub(crate) mailbox: MailboxRef,
336        pub(crate) receiver: Receiver<(MessageMeta, InstructionReply)>,
337    }
338
339    impl HeartbeatResponseTestEnv {
340        pub fn new() -> Self {
341            let (tx, rx) = mpsc::channel(8);
342            let mailbox = Arc::new(HeartbeatMailbox::new(tx));
343
344            HeartbeatResponseTestEnv {
345                mailbox,
346                receiver: rx,
347            }
348        }
349
350        pub fn create_handler_ctx(
351            &self,
352            incoming_message: IncomingMessage,
353        ) -> HeartbeatResponseHandlerContext {
354            HeartbeatResponseHandlerContext {
355                mailbox: self.mailbox.clone(),
356                response: Default::default(),
357                incoming_message: Some(incoming_message),
358            }
359        }
360    }
361
362    #[test]
363    fn test_is_acceptable() {
364        common_telemetry::init_default_ut_logging();
365        let region_server = mock_region_server();
366        let kv_backend = Arc::new(MemoryKvBackend::new());
367        let heartbeat_handler =
368            RegionHeartbeatResponseHandler::new(region_server.clone(), kv_backend);
369        let heartbeat_env = HeartbeatResponseTestEnv::new();
370        let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
371
372        // Open region
373        let region_id = RegionId::new(1024, 1);
374        let storage_path = "test";
375        let instruction = open_region_instruction(region_id, storage_path);
376        assert!(
377            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((
378                meta.clone(),
379                Default::default(),
380                instruction
381            )))
382        );
383
384        // Close region
385        let instruction = close_region_instruction(region_id);
386        assert!(
387            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((
388                meta.clone(),
389                Default::default(),
390                instruction
391            )))
392        );
393
394        // Downgrade region
395        let instruction = Instruction::DowngradeRegions(vec![DowngradeRegion {
396            region_id: RegionId::new(2048, 1),
397            flush_timeout: Some(Duration::from_secs(1)),
398        }]);
399        assert!(
400            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((
401                meta.clone(),
402                Default::default(),
403                instruction
404            )))
405        );
406
407        // Upgrade region
408        let instruction = Instruction::UpgradeRegions(vec![UpgradeRegion {
409            region_id,
410            ..Default::default()
411        }]);
412        assert!(
413            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((
414                meta.clone(),
415                Default::default(),
416                instruction
417            )))
418        );
419
420        // Enter staging region
421        let instruction = Instruction::EnterStagingRegions(vec![EnterStagingRegion {
422            region_id,
423            partition_directive: StagingPartitionDirective::UpdatePartitionExpr("".to_string()),
424        }]);
425        assert!(
426            heartbeat_handler.is_acceptable(&heartbeat_env.create_handler_ctx((
427                meta,
428                Default::default(),
429                instruction
430            )))
431        );
432    }
433
434    fn close_region_instruction(region_id: RegionId) -> Instruction {
435        Instruction::CloseRegions(vec![RegionIdent {
436            table_id: region_id.table_id(),
437            region_number: region_id.region_number(),
438            datanode_id: 2,
439            engine: MITO_ENGINE_NAME.to_string(),
440        }])
441    }
442
443    fn open_region_instruction(region_id: RegionId, path: &str) -> Instruction {
444        Instruction::OpenRegions(vec![OpenRegion::new(
445            RegionIdent {
446                table_id: region_id.table_id(),
447                region_number: region_id.region_number(),
448                datanode_id: 2,
449                engine: MITO_ENGINE_NAME.to_string(),
450            },
451            path,
452            HashMap::new(),
453            HashMap::new(),
454            false,
455            None,
456            RegionRequirements::empty(),
457        )])
458    }
459
460    #[tokio::test]
461    async fn test_close_region() {
462        common_telemetry::init_default_ut_logging();
463
464        let mut region_server = mock_region_server();
465        let kv_backend = Arc::new(MemoryKvBackend::new());
466        let heartbeat_handler =
467            RegionHeartbeatResponseHandler::new(region_server.clone(), kv_backend);
468
469        let mut engine_env = TestEnv::with_prefix("close-region").await;
470        let engine = engine_env.create_engine(MitoConfig::default()).await;
471        region_server.register_engine(Arc::new(engine));
472        let region_id = RegionId::new(1024, 1);
473
474        let builder = CreateRequestBuilder::new();
475        let create_req = builder.build();
476        region_server
477            .handle_request(region_id, RegionRequest::Create(create_req))
478            .await
479            .unwrap();
480
481        let mut heartbeat_env = HeartbeatResponseTestEnv::new();
482
483        // Should be ok, if we try to close it twice.
484        for _ in 0..2 {
485            let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
486            let instruction = close_region_instruction(region_id);
487
488            let mut ctx = heartbeat_env.create_handler_ctx((meta, Default::default(), instruction));
489            let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
490            assert_matches!(control, HandleControl::Continue);
491
492            let (_, reply) = heartbeat_env.receiver.recv().await.unwrap();
493
494            if let InstructionReply::CloseRegions(reply) = reply {
495                assert!(reply.result);
496                assert!(reply.error.is_none());
497            } else {
498                unreachable!()
499            }
500
501            assert_matches!(
502                region_server
503                    .set_region_role(region_id, RegionRole::Leader)
504                    .unwrap_err(),
505                error::Error::RegionNotFound { .. }
506            );
507        }
508    }
509
510    #[tokio::test]
511    async fn test_open_region_ok() {
512        common_telemetry::init_default_ut_logging();
513
514        let mut region_server = mock_region_server();
515        let kv_backend = Arc::new(MemoryKvBackend::new());
516        let heartbeat_handler =
517            RegionHeartbeatResponseHandler::new(region_server.clone(), kv_backend);
518
519        let mut engine_env = TestEnv::with_prefix("open-region").await;
520        let engine = engine_env.create_engine(MitoConfig::default()).await;
521        region_server.register_engine(Arc::new(engine));
522        let region_id = RegionId::new(1024, 1);
523
524        let builder = CreateRequestBuilder::new();
525        let mut create_req = builder.build();
526        let storage_path = "test";
527        create_req.table_dir = table_dir(storage_path, region_id.table_id());
528
529        region_server
530            .handle_request(region_id, RegionRequest::Create(create_req))
531            .await
532            .unwrap();
533
534        region_server
535            .handle_request(
536                region_id,
537                RegionRequest::Close(RegionCloseRequest::default()),
538            )
539            .await
540            .unwrap();
541        let mut heartbeat_env = HeartbeatResponseTestEnv::new();
542
543        // Should be ok, if we try to open it twice.
544        for _ in 0..2 {
545            let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
546            let instruction = open_region_instruction(region_id, storage_path);
547
548            let mut ctx = heartbeat_env.create_handler_ctx((meta, Default::default(), instruction));
549            let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
550            assert_matches!(control, HandleControl::Continue);
551
552            let (_, reply) = heartbeat_env.receiver.recv().await.unwrap();
553
554            if let InstructionReply::OpenRegions(reply) = reply {
555                assert!(reply.result);
556                assert!(reply.error.is_none());
557            } else {
558                unreachable!()
559            }
560        }
561    }
562
563    #[tokio::test]
564    async fn test_open_not_exists_region() {
565        common_telemetry::init_default_ut_logging();
566
567        let mut region_server = mock_region_server();
568        let kv_backend = Arc::new(MemoryKvBackend::new());
569        let heartbeat_handler =
570            RegionHeartbeatResponseHandler::new(region_server.clone(), kv_backend);
571
572        let mut engine_env = TestEnv::with_prefix("open-not-exists-region").await;
573        let engine = engine_env.create_engine(MitoConfig::default()).await;
574        region_server.register_engine(Arc::new(engine));
575        let region_id = RegionId::new(1024, 1);
576        let storage_path = "test";
577
578        let mut heartbeat_env = HeartbeatResponseTestEnv::new();
579
580        let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
581        let instruction = open_region_instruction(region_id, storage_path);
582
583        let mut ctx = heartbeat_env.create_handler_ctx((meta, Default::default(), instruction));
584        let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
585        assert_matches!(control, HandleControl::Continue);
586
587        let (_, reply) = heartbeat_env.receiver.recv().await.unwrap();
588
589        if let InstructionReply::OpenRegions(reply) = reply {
590            assert!(!reply.result);
591            assert!(reply.error.is_some());
592        } else {
593            unreachable!()
594        }
595    }
596
597    #[tokio::test]
598    async fn test_downgrade_region() {
599        common_telemetry::init_default_ut_logging();
600
601        let mut region_server = mock_region_server();
602        let kv_backend = Arc::new(MemoryKvBackend::new());
603        let heartbeat_handler =
604            RegionHeartbeatResponseHandler::new(region_server.clone(), kv_backend);
605
606        let mut engine_env = TestEnv::with_prefix("downgrade-region").await;
607        let engine = engine_env.create_engine(MitoConfig::default()).await;
608        region_server.register_engine(Arc::new(engine));
609        let region_id = RegionId::new(1024, 1);
610
611        let builder = CreateRequestBuilder::new();
612        let mut create_req = builder.build();
613        let storage_path = "test";
614        create_req.table_dir = table_dir(storage_path, region_id.table_id());
615
616        region_server
617            .handle_request(region_id, RegionRequest::Create(create_req))
618            .await
619            .unwrap();
620
621        let mut heartbeat_env = HeartbeatResponseTestEnv::new();
622
623        // Should be ok, if we try to downgrade it twice.
624        for _ in 0..2 {
625            let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
626            let instruction = Instruction::DowngradeRegions(vec![DowngradeRegion {
627                region_id,
628                flush_timeout: Some(Duration::from_secs(1)),
629            }]);
630
631            let mut ctx = heartbeat_env.create_handler_ctx((meta, Default::default(), instruction));
632            let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
633            assert_matches!(control, HandleControl::Continue);
634
635            let (_, reply) = heartbeat_env.receiver.recv().await.unwrap();
636
637            let reply = &reply.expect_downgrade_regions_reply()[0];
638            assert!(reply.exists);
639            assert!(reply.error.is_none());
640            assert_eq!(reply.last_entry_id.unwrap(), 0);
641        }
642
643        // Downgrades a not exists region.
644        let meta = MessageMeta::new_test(1, "test", "dn-1", "me-0");
645        let instruction = Instruction::DowngradeRegions(vec![DowngradeRegion {
646            region_id: RegionId::new(2048, 1),
647            flush_timeout: Some(Duration::from_secs(1)),
648        }]);
649        let mut ctx = heartbeat_env.create_handler_ctx((meta, Default::default(), instruction));
650        let control = heartbeat_handler.handle(&mut ctx).await.unwrap();
651        assert_matches!(control, HandleControl::Continue);
652
653        let (_, reply) = heartbeat_env.receiver.recv().await.unwrap();
654
655        let reply = reply.expect_downgrade_regions_reply();
656        assert!(!reply[0].exists);
657        assert!(reply[0].error.is_none());
658        assert!(reply[0].last_entry_id.is_none());
659    }
660}
datanode/heartbeat/handler.rs

datanode/heartbeat/
handler.rs
