Skip to main content

metric_engine/
engine.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod alter;
16mod bulk_insert;
17mod catchup;
18mod close;
19mod create;
20mod drop;
21mod flush;
22mod open;
23mod options;
24mod put;
25mod read;
26mod region_metadata;
27mod staging;
28mod state;
29mod sync;
30
31use std::any::Any;
32use std::collections::HashMap;
33use std::sync::{Arc, RwLock};
34
35use api::region::RegionResponse;
36use async_trait::async_trait;
37use common_error::ext::{BoxedError, ErrorExt};
38use common_error::status_code::StatusCode;
39use common_runtime::RepeatedTask;
40use mito2::engine::MitoEngine;
41pub(crate) use options::IndexOptions;
42use snafu::{OptionExt, ResultExt};
43pub(crate) use state::MetricEngineState;
44use store_api::metadata::RegionMetadataRef;
45use store_api::metric_engine_consts::METRIC_ENGINE_NAME;
46use store_api::region_engine::{
47    BatchResponses, RegionEngine, RegionRole, RegionScannerRef, RegionStatistic,
48    RemapManifestsRequest, RemapManifestsResponse, SetRegionRoleStateResponse,
49    SetRegionRoleStateSuccess, SettableRegionRoleState, SyncRegionFromRequest,
50    SyncRegionFromResponse,
51};
52use store_api::region_request::{
53    AffectedRows, BatchRegionDdlRequest, RegionCatchupRequest, RegionOpenRequest, RegionPutRequest,
54    RegionRequest,
55};
56use store_api::storage::{RegionId, ScanRequest, SequenceNumber};
57
58use crate::config::EngineConfig;
59use crate::data_region::DataRegion;
60use crate::error::{
61    self, Error, Result, StartRepeatedTaskSnafu, UnsupportedRegionRequestSnafu,
62    UnsupportedRemapManifestsRequestSnafu,
63};
64use crate::metadata_region::MetadataRegion;
65use crate::repeated_task::FlushMetadataRegionTask;
66use crate::row_modifier::RowModifier;
67use crate::utils::{self, get_region_statistic};
68
69#[cfg_attr(doc, aquamarine::aquamarine)]
70/// # Metric Engine
71///
72/// ## Regions
73///
74/// Regions in this metric engine has several roles. There is `PhysicalRegion`,
75/// which refer to the region that actually stores the data. And `LogicalRegion`
76/// that is "simulated" over physical regions. Each logical region is associated
77/// with one physical region group, which is a group of two physical regions.
78/// Their relationship is illustrated below:
79///
80/// ```mermaid
81/// erDiagram
82///     LogicalRegion ||--o{ PhysicalRegionGroup : corresponds
83///     PhysicalRegionGroup ||--|| DataRegion : contains
84///     PhysicalRegionGroup ||--|| MetadataRegion : contains
85/// ```
86///
87/// Metric engine uses two region groups. One is for data region
88/// ([METRIC_DATA_REGION_GROUP](crate::consts::METRIC_DATA_REGION_GROUP)), and the
89/// other is for metadata region ([METRIC_METADATA_REGION_GROUP](crate::consts::METRIC_METADATA_REGION_GROUP)).
90/// From the definition of [`RegionId`], we can convert between these two physical
91/// region ids easily. Thus in the code base we usually refer to one "physical
92/// region id", and convert it to the other one when necessary.
93///
94/// The logical region, in contrast, is a virtual region. It doesn't has dedicated
95/// storage or region group. Only a region id that is allocated by meta server.
96/// And all other things is shared with other logical region that are associated
97/// with the same physical region group.
98///
99/// For more document about physical regions, please refer to [`MetadataRegion`]
100/// and [`DataRegion`].
101///
102/// ## Operations
103///
104/// Both physical and logical region are accessible to user. But the operation
105/// they support are different. List below:
106///
107/// | Operations | Logical Region | Physical Region |
108/// | ---------- | -------------- | --------------- |
109/// |   Create   |       ✅        |        ✅        |
110/// |    Drop    |       ✅        |        ❓*       |
111/// |   Write    |       ✅        |        ❌        |
112/// |    Read    |       ✅        |        ✅        |
113/// |   Close    |       ✅        |        ✅        |
114/// |    Open    |       ✅        |        ✅        |
115/// |   Alter    |       ✅        |        ❓*       |
116///
117/// *: Physical region can be dropped only when all related logical regions are dropped.
118/// *: Alter: Physical regions only support altering region options.
119///
120/// ## Internal Columns
121///
122/// The physical data region contains two internal columns. Should
123/// mention that "internal" here is for metric engine itself. Mito
124/// engine will add it's internal columns to the region as well.
125///
126/// Their column id is registered in [`ReservedColumnId`]. And column name is
127/// defined in [`DATA_SCHEMA_TSID_COLUMN_NAME`] and [`DATA_SCHEMA_TABLE_ID_COLUMN_NAME`].
128///
129/// Tsid is generated by hashing all tags. And table id is retrieved from logical region
130/// id to distinguish data from different logical tables.
131#[derive(Clone)]
132pub struct MetricEngine {
133    inner: Arc<MetricEngineInner>,
134}
135
136#[async_trait]
137impl RegionEngine for MetricEngine {
138    /// Name of this engine
139    fn name(&self) -> &str {
140        METRIC_ENGINE_NAME
141    }
142
143    async fn handle_batch_open_requests(
144        &self,
145        parallelism: usize,
146        requests: Vec<(RegionId, RegionOpenRequest)>,
147    ) -> Result<BatchResponses, BoxedError> {
148        self.inner
149            .handle_batch_open_requests(parallelism, requests)
150            .await
151            .map_err(BoxedError::new)
152    }
153
154    async fn handle_batch_catchup_requests(
155        &self,
156        parallelism: usize,
157        requests: Vec<(RegionId, RegionCatchupRequest)>,
158    ) -> Result<BatchResponses, BoxedError> {
159        self.inner
160            .handle_batch_catchup_requests(parallelism, requests)
161            .await
162            .map_err(BoxedError::new)
163    }
164
165    async fn handle_batch_ddl_requests(
166        &self,
167        batch_request: BatchRegionDdlRequest,
168    ) -> Result<RegionResponse, BoxedError> {
169        match batch_request {
170            BatchRegionDdlRequest::Create(requests) => {
171                let mut extension_return_value = HashMap::new();
172                let rows = self
173                    .inner
174                    .create_regions(requests, &mut extension_return_value)
175                    .await
176                    .map_err(BoxedError::new)?;
177
178                Ok(RegionResponse {
179                    affected_rows: rows,
180                    extensions: extension_return_value,
181                    metadata: Vec::new(),
182                })
183            }
184            BatchRegionDdlRequest::Alter(requests) => {
185                let mut extension_return_value = HashMap::new();
186                let rows = self
187                    .inner
188                    .alter_regions(requests, &mut extension_return_value)
189                    .await
190                    .map_err(BoxedError::new)?;
191
192                Ok(RegionResponse {
193                    affected_rows: rows,
194                    extensions: extension_return_value,
195                    metadata: Vec::new(),
196                })
197            }
198            BatchRegionDdlRequest::Drop(requests) => {
199                self.handle_requests(
200                    requests
201                        .into_iter()
202                        .map(|(region_id, req)| (region_id, RegionRequest::Drop(req))),
203                )
204                .await
205            }
206        }
207    }
208
209    /// Handles non-query request to the region. Returns the count of affected rows.
210    async fn handle_request(
211        &self,
212        region_id: RegionId,
213        request: RegionRequest,
214    ) -> Result<RegionResponse, BoxedError> {
215        let mut extension_return_value = HashMap::new();
216
217        let result = match request {
218            RegionRequest::EnterStaging(_) => {
219                if self.inner.is_physical_region(region_id) {
220                    self.handle_enter_staging_request(region_id, request).await
221                } else {
222                    UnsupportedRegionRequestSnafu { request }.fail()
223                }
224            }
225            RegionRequest::ApplyStagingManifest(_) => {
226                if self.inner.is_physical_region(region_id) {
227                    return self.inner.mito.handle_request(region_id, request).await;
228                } else {
229                    UnsupportedRegionRequestSnafu { request }.fail()
230                }
231            }
232            RegionRequest::Put(put) => self.inner.put_region(region_id, put).await,
233            RegionRequest::Create(create) => {
234                self.inner
235                    .create_regions(vec![(region_id, create)], &mut extension_return_value)
236                    .await
237            }
238            RegionRequest::Drop(drop) => self.inner.drop_region(region_id, drop).await,
239            RegionRequest::Open(open) => self.inner.open_region(region_id, open).await,
240            RegionRequest::CleanUp(clean_up) => {
241                self.inner.clean_up_region(region_id, clean_up).await
242            }
243            RegionRequest::Close(close) => self.inner.close_region(region_id, close).await,
244            RegionRequest::Alter(alter) => {
245                self.inner
246                    .alter_regions(vec![(region_id, alter)], &mut extension_return_value)
247                    .await
248            }
249            RegionRequest::Compact(_) => {
250                if self.inner.is_physical_region(region_id) {
251                    self.inner
252                        .mito
253                        .handle_request(region_id, request)
254                        .await
255                        .context(error::MitoFlushOperationSnafu)
256                        .map(|response| response.affected_rows)
257                } else {
258                    UnsupportedRegionRequestSnafu { request }.fail()
259                }
260            }
261            RegionRequest::Flush(req) => self.inner.flush_region(region_id, req).await,
262            RegionRequest::BuildIndex(_) => {
263                if self.inner.is_physical_region(region_id) {
264                    self.inner
265                        .mito
266                        .handle_request(region_id, request)
267                        .await
268                        .context(error::MitoFlushOperationSnafu)
269                        .map(|response| response.affected_rows)
270                } else {
271                    UnsupportedRegionRequestSnafu { request }.fail()
272                }
273            }
274            RegionRequest::Truncate(_) => UnsupportedRegionRequestSnafu { request }.fail(),
275            RegionRequest::Delete(delete) => self.inner.delete_region(region_id, delete).await,
276            RegionRequest::Catchup(_) => {
277                let mut response = self
278                    .inner
279                    .handle_batch_catchup_requests(
280                        1,
281                        vec![(region_id, RegionCatchupRequest::default())],
282                    )
283                    .await
284                    .map_err(BoxedError::new)?;
285                debug_assert_eq!(response.len(), 1);
286                let (resp_region_id, response) = response
287                    .pop()
288                    .context(error::UnexpectedRequestSnafu {
289                        reason: "expected 1 response, but got zero responses",
290                    })
291                    .map_err(BoxedError::new)?;
292                debug_assert_eq!(region_id, resp_region_id);
293                return response;
294            }
295            RegionRequest::BulkInserts(bulk) => {
296                self.inner.bulk_insert_region(region_id, bulk).await
297            }
298        };
299
300        result.map_err(BoxedError::new).map(|rows| RegionResponse {
301            affected_rows: rows,
302            extensions: extension_return_value,
303            metadata: Vec::new(),
304        })
305    }
306
307    async fn handle_query(
308        &self,
309        region_id: RegionId,
310        request: ScanRequest,
311    ) -> Result<RegionScannerRef, BoxedError> {
312        self.handle_query(region_id, request).await
313    }
314
315    async fn get_committed_sequence(
316        &self,
317        region_id: RegionId,
318    ) -> Result<SequenceNumber, BoxedError> {
319        self.inner
320            .get_last_seq_num(region_id)
321            .await
322            .map_err(BoxedError::new)
323    }
324
325    /// Retrieves region's metadata.
326    async fn get_metadata(&self, region_id: RegionId) -> Result<RegionMetadataRef, BoxedError> {
327        self.inner
328            .load_region_metadata(region_id)
329            .await
330            .map_err(BoxedError::new)
331    }
332
333    /// Retrieves region's disk usage.
334    ///
335    /// Note: Returns `None` if it's a logical region.
336    fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
337        if self.inner.is_physical_region(region_id) {
338            get_region_statistic(&self.inner.mito, region_id)
339        } else {
340            None
341        }
342    }
343
344    /// Stops the engine
345    async fn stop(&self) -> Result<(), BoxedError> {
346        // don't need to stop the underlying mito engine
347        Ok(())
348    }
349
350    fn set_region_role(&self, region_id: RegionId, role: RegionRole) -> Result<(), BoxedError> {
351        // ignore the region not found error
352        for x in [
353            utils::to_metadata_region_id(region_id),
354            utils::to_data_region_id(region_id),
355        ] {
356            if let Err(e) = self.inner.mito.set_region_role(x, role)
357                && e.status_code() != StatusCode::RegionNotFound
358            {
359                return Err(e);
360            }
361        }
362        Ok(())
363    }
364
365    async fn sync_region(
366        &self,
367        region_id: RegionId,
368        request: SyncRegionFromRequest,
369    ) -> Result<SyncRegionFromResponse, BoxedError> {
370        match request {
371            SyncRegionFromRequest::FromManifest(manifest_info) => self
372                .inner
373                .sync_region_from_manifest(region_id, manifest_info)
374                .await
375                .map_err(BoxedError::new),
376            SyncRegionFromRequest::FromRegion {
377                source_region_id,
378                parallelism,
379            } => {
380                if self.inner.is_physical_region(region_id) {
381                    self.inner
382                        .sync_region_from_region(region_id, source_region_id, parallelism)
383                        .await
384                        .map_err(BoxedError::new)
385                } else {
386                    Err(BoxedError::new(
387                        error::UnsupportedSyncRegionFromRequestSnafu { region_id }.build(),
388                    ))
389                }
390            }
391        }
392    }
393
394    async fn remap_manifests(
395        &self,
396        request: RemapManifestsRequest,
397    ) -> Result<RemapManifestsResponse, BoxedError> {
398        let region_id = request.region_id;
399        if self.inner.is_physical_region(region_id) {
400            self.inner.mito.remap_manifests(request).await
401        } else {
402            Err(BoxedError::new(
403                UnsupportedRemapManifestsRequestSnafu { region_id }.build(),
404            ))
405        }
406    }
407
408    async fn set_region_role_state_gracefully(
409        &self,
410        region_id: RegionId,
411        region_role_state: SettableRegionRoleState,
412    ) -> std::result::Result<SetRegionRoleStateResponse, BoxedError> {
413        let metadata_result = match self
414            .inner
415            .mito
416            .set_region_role_state_gracefully(
417                utils::to_metadata_region_id(region_id),
418                region_role_state,
419            )
420            .await?
421        {
422            SetRegionRoleStateResponse::Success(success) => success,
423            SetRegionRoleStateResponse::NotFound => {
424                return Ok(SetRegionRoleStateResponse::NotFound);
425            }
426            SetRegionRoleStateResponse::InvalidTransition(error) => {
427                return Ok(SetRegionRoleStateResponse::InvalidTransition(error));
428            }
429        };
430
431        let data_result = match self
432            .inner
433            .mito
434            .set_region_role_state_gracefully(region_id, region_role_state)
435            .await?
436        {
437            SetRegionRoleStateResponse::Success(success) => success,
438            SetRegionRoleStateResponse::NotFound => {
439                return Ok(SetRegionRoleStateResponse::NotFound);
440            }
441            SetRegionRoleStateResponse::InvalidTransition(error) => {
442                return Ok(SetRegionRoleStateResponse::InvalidTransition(error));
443            }
444        };
445
446        Ok(SetRegionRoleStateResponse::success(
447            SetRegionRoleStateSuccess::metric(
448                data_result.last_entry_id().unwrap_or_default(),
449                metadata_result.last_entry_id().unwrap_or_default(),
450            ),
451        ))
452    }
453
454    /// Returns the physical region role.
455    ///
456    /// Note: Returns `None` if it's a logical region.
457    fn role(&self, region_id: RegionId) -> Option<RegionRole> {
458        if self.inner.is_physical_region(region_id) {
459            self.inner.mito.role(region_id)
460        } else {
461            None
462        }
463    }
464
465    fn as_any(&self) -> &dyn Any {
466        self
467    }
468}
469
470impl MetricEngine {
471    pub fn try_new(mito: MitoEngine, mut config: EngineConfig) -> Result<Self> {
472        let metadata_region = MetadataRegion::new(mito.clone());
473        let data_region = DataRegion::new(mito.clone());
474        let state = Arc::new(RwLock::default());
475        config.sanitize();
476        let flush_interval = config.flush_metadata_region_interval;
477        let inner = Arc::new(MetricEngineInner {
478            mito: mito.clone(),
479            metadata_region,
480            data_region,
481            state: state.clone(),
482            row_modifier: RowModifier::default(),
483            flush_task: RepeatedTask::new(
484                flush_interval,
485                Box::new(FlushMetadataRegionTask {
486                    state: state.clone(),
487                    mito: mito.clone(),
488                }),
489            ),
490        });
491        inner
492            .flush_task
493            .start(common_runtime::global_runtime())
494            .context(StartRepeatedTaskSnafu { name: "flush_task" })?;
495        Ok(Self { inner })
496    }
497
498    pub fn mito(&self) -> MitoEngine {
499        self.inner.mito.clone()
500    }
501
502    /// Batch put operation for multiple logical regions.
503    /// Requests are grouped by physical region; a failure can leave earlier
504    /// physical-region groups committed.
505    pub async fn put_regions_batch(
506        &self,
507        requests: impl ExactSizeIterator<Item = (RegionId, RegionPutRequest)>,
508    ) -> Result<AffectedRows> {
509        self.inner.put_regions_batch(requests).await
510    }
511
512    /// Returns all logical regions associated with the physical region.
513    pub async fn logical_regions(&self, physical_region_id: RegionId) -> Result<Vec<RegionId>> {
514        self.inner
515            .metadata_region
516            .logical_regions(physical_region_id)
517            .await
518    }
519
520    /// Handles substrait query and return a stream of record batches
521    async fn handle_query(
522        &self,
523        region_id: RegionId,
524        request: ScanRequest,
525    ) -> Result<RegionScannerRef, BoxedError> {
526        self.inner
527            .read_region(region_id, request)
528            .await
529            .map_err(BoxedError::new)
530    }
531
532    async fn handle_requests(
533        &self,
534        requests: impl IntoIterator<Item = (RegionId, RegionRequest)>,
535    ) -> Result<RegionResponse, BoxedError> {
536        let mut affected_rows = 0;
537        let mut extensions = HashMap::new();
538        for (region_id, request) in requests {
539            let response = self.handle_request(region_id, request).await?;
540            affected_rows += response.affected_rows;
541            extensions.extend(response.extensions);
542        }
543
544        Ok(RegionResponse {
545            affected_rows,
546            extensions,
547            metadata: Vec::new(),
548        })
549    }
550}
551
552#[cfg(test)]
553impl MetricEngine {
554    pub async fn scan_to_stream(
555        &self,
556        region_id: RegionId,
557        request: ScanRequest,
558    ) -> Result<common_recordbatch::SendableRecordBatchStream, BoxedError> {
559        self.inner.scan_to_stream(region_id, request).await
560    }
561}
562
563struct MetricEngineInner {
564    mito: MitoEngine,
565    metadata_region: MetadataRegion,
566    data_region: DataRegion,
567    state: Arc<RwLock<MetricEngineState>>,
568    row_modifier: RowModifier,
569    flush_task: RepeatedTask<Error>,
570}
571
572#[cfg(test)]
573mod test {
574    use std::assert_matches;
575    use std::collections::HashMap;
576
577    use common_telemetry::info;
578    use common_wal::options::{KafkaWalOptions, WalOptions};
579    use mito2::sst::location::region_dir_from_table_dir;
580    use mito2::test_util::{kafka_log_store_factory, prepare_test_for_kafka_log_store};
581    use store_api::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY;
582    use store_api::mito_engine_options::WAL_OPTIONS_KEY;
583    use store_api::region_request::{
584        PathType, RegionCleanUpRequest, RegionCloseRequest, RegionDropRequest, RegionFlushRequest,
585        RegionOpenRequest, RegionRequest,
586    };
587
588    use super::*;
589    use crate::maybe_skip_kafka_log_store_integration_test;
590    use crate::test_util::{TestEnv, create_logical_region_request};
591
592    #[tokio::test]
593    async fn close_open_regions() {
594        let env = TestEnv::new().await;
595        env.init_metric_region().await;
596        let engine = env.metric();
597
598        // close physical region
599        let physical_region_id = env.default_physical_region_id();
600        engine
601            .handle_request(
602                physical_region_id,
603                RegionRequest::Close(RegionCloseRequest::default()),
604            )
605            .await
606            .unwrap();
607
608        // reopen physical region
609        let physical_region_option = [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
610            .into_iter()
611            .collect();
612        let open_request = RegionOpenRequest {
613            engine: METRIC_ENGINE_NAME.to_string(),
614            table_dir: TestEnv::default_table_dir(),
615            path_type: PathType::Bare, // Use Bare path type for engine regions
616            options: physical_region_option,
617            skip_wal_replay: false,
618            checkpoint: None,
619            requirements: Default::default(),
620        };
621        engine
622            .handle_request(physical_region_id, RegionRequest::Open(open_request))
623            .await
624            .unwrap();
625
626        // close nonexistent region won't report error
627        let nonexistent_region_id = RegionId::new(12313, 12);
628        engine
629            .handle_request(
630                nonexistent_region_id,
631                RegionRequest::Close(RegionCloseRequest::default()),
632            )
633            .await
634            .unwrap();
635
636        // open nonexistent region won't report error
637        let invalid_open_request = RegionOpenRequest {
638            engine: METRIC_ENGINE_NAME.to_string(),
639            table_dir: TestEnv::default_table_dir(),
640            path_type: PathType::Bare, // Use Bare path type for engine regions
641            options: HashMap::new(),
642            skip_wal_replay: false,
643            checkpoint: None,
644            requirements: Default::default(),
645        };
646        engine
647            .handle_request(
648                nonexistent_region_id,
649                RegionRequest::Open(invalid_open_request),
650            )
651            .await
652            .unwrap();
653    }
654
655    #[tokio::test]
656    async fn test_offline_cleanup_physical_region() {
657        let env = TestEnv::new().await;
658        env.init_metric_region().await;
659        let engine = env.metric();
660        let mito = env.mito();
661        let physical_region_id = env.default_physical_region_id();
662        let metadata_region_id = crate::utils::to_metadata_region_id(physical_region_id);
663        let data_region_id = crate::utils::to_data_region_id(physical_region_id);
664
665        engine
666            .handle_request(
667                physical_region_id,
668                RegionRequest::Close(RegionCloseRequest::default()),
669            )
670            .await
671            .unwrap();
672
673        let object_store = env.get_object_store().unwrap();
674        let metadata_region_dir = region_dir_from_table_dir(
675            &TestEnv::default_table_dir(),
676            metadata_region_id,
677            PathType::Metadata,
678        );
679        let data_region_dir = region_dir_from_table_dir(
680            &TestEnv::default_table_dir(),
681            data_region_id,
682            PathType::Data,
683        );
684        assert!(object_store.exists(&metadata_region_dir).await.unwrap());
685        assert!(object_store.exists(&data_region_dir).await.unwrap());
686
687        let physical_region_option = [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
688            .into_iter()
689            .collect();
690        let clean_up_request = RegionCleanUpRequest {
691            engine: METRIC_ENGINE_NAME.to_string(),
692            table_dir: TestEnv::default_table_dir(),
693            path_type: PathType::Bare,
694            options: physical_region_option,
695        };
696        engine
697            .handle_request(physical_region_id, RegionRequest::CleanUp(clean_up_request))
698            .await
699            .unwrap();
700
701        assert!(!mito.is_region_exists(metadata_region_id));
702        assert!(!mito.is_region_exists(data_region_id));
703        assert!(!object_store.exists(&metadata_region_dir).await.unwrap());
704        assert!(!object_store.exists(&data_region_dir).await.unwrap());
705    }
706
707    #[tokio::test]
708    async fn test_role() {
709        let env = TestEnv::new().await;
710        env.init_metric_region().await;
711
712        let logical_region_id = env.default_logical_region_id();
713        let physical_region_id = env.default_physical_region_id();
714
715        assert!(env.metric().role(logical_region_id).is_none());
716        assert!(env.metric().role(physical_region_id).is_some());
717    }
718
719    #[tokio::test]
720    async fn test_region_disk_usage() {
721        let env = TestEnv::new().await;
722        env.init_metric_region().await;
723
724        let logical_region_id = env.default_logical_region_id();
725        let physical_region_id = env.default_physical_region_id();
726
727        assert!(env.metric().region_statistic(logical_region_id).is_none());
728        assert!(env.metric().region_statistic(physical_region_id).is_some());
729    }
730
731    #[tokio::test]
732    async fn test_open_region_failure() {
733        let env = TestEnv::new().await;
734        env.init_metric_region().await;
735        let physical_region_id = env.default_physical_region_id();
736
737        let metric_engine = env.metric();
738        metric_engine
739            .handle_request(
740                physical_region_id,
741                RegionRequest::Flush(RegionFlushRequest::default()),
742            )
743            .await
744            .unwrap();
745
746        let path = region_dir_from_table_dir(
747            &TestEnv::default_table_dir(),
748            physical_region_id,
749            PathType::Metadata,
750        );
751        let object_store = env.get_object_store().unwrap();
752        let list = object_store.list(&path).await.unwrap();
753        // Delete parquet files in metadata region
754        for entry in list {
755            if entry.metadata().is_dir() {
756                continue;
757            }
758            if entry.name().ends_with("parquet") {
759                info!("deleting {}", entry.path());
760                object_store.delete(entry.path()).await.unwrap();
761            }
762        }
763
764        let physical_region_option = [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
765            .into_iter()
766            .collect();
767        let open_request = RegionOpenRequest {
768            engine: METRIC_ENGINE_NAME.to_string(),
769            table_dir: TestEnv::default_table_dir(),
770            path_type: PathType::Bare,
771            options: physical_region_option,
772            skip_wal_replay: false,
773            checkpoint: None,
774            requirements: Default::default(),
775        };
776        // Opening an already opened region should succeed.
777        // Since the region is already open, no metadata recovery operations will be performed.
778        metric_engine
779            .handle_request(physical_region_id, RegionRequest::Open(open_request))
780            .await
781            .unwrap();
782
783        // Close the region
784        metric_engine
785            .handle_request(
786                physical_region_id,
787                RegionRequest::Close(RegionCloseRequest::default()),
788            )
789            .await
790            .unwrap();
791
792        // Try to reopen region.
793        let physical_region_option = [(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new())]
794            .into_iter()
795            .collect();
796        let open_request = RegionOpenRequest {
797            engine: METRIC_ENGINE_NAME.to_string(),
798            table_dir: TestEnv::default_table_dir(),
799            path_type: PathType::Bare,
800            options: physical_region_option,
801            skip_wal_replay: false,
802            checkpoint: None,
803            requirements: Default::default(),
804        };
805        let err = metric_engine
806            .handle_request(physical_region_id, RegionRequest::Open(open_request))
807            .await
808            .unwrap_err();
809        // Failed to open region because of missing parquet files.
810        assert_eq!(err.status_code(), StatusCode::StorageUnavailable);
811
812        let mito_engine = metric_engine.mito();
813        let data_region_id = utils::to_data_region_id(physical_region_id);
814        let metadata_region_id = utils::to_metadata_region_id(physical_region_id);
815        // The metadata/data region should be closed.
816        let err = mito_engine.get_metadata(data_region_id).await.unwrap_err();
817        assert_eq!(err.status_code(), StatusCode::RegionNotFound);
818        let err = mito_engine
819            .get_metadata(metadata_region_id)
820            .await
821            .unwrap_err();
822        assert_eq!(err.status_code(), StatusCode::RegionNotFound);
823    }
824
825    #[tokio::test]
826    async fn test_catchup_regions() {
827        common_telemetry::init_default_ut_logging();
828        maybe_skip_kafka_log_store_integration_test!();
829        let kafka_log_store_factory = kafka_log_store_factory().unwrap();
830        let mito_env = mito2::test_util::TestEnv::new()
831            .await
832            .with_log_store_factory(kafka_log_store_factory.clone());
833        let env = TestEnv::with_mito_env(mito_env).await;
834        let table_dir = |region_id| format!("table/{region_id}");
835        let mut physical_region_ids = vec![];
836        let mut logical_region_ids = vec![];
837
838        let num_topics = 3;
839        let num_physical_regions = 8;
840        let num_logical_regions = 16;
841        let parallelism = 2;
842        let mut topics = Vec::with_capacity(num_topics);
843        for _ in 0..num_topics {
844            let topic = prepare_test_for_kafka_log_store(&kafka_log_store_factory)
845                .await
846                .unwrap();
847            topics.push(topic);
848        }
849
850        let topic_idx = |id| (id as usize) % num_topics;
851        // Creates physical regions
852        for i in 0..num_physical_regions {
853            let physical_region_id = RegionId::new(1, i);
854            physical_region_ids.push(physical_region_id);
855
856            let wal_options = WalOptions::Kafka(KafkaWalOptions::new(topics[topic_idx(i)].clone()));
857            env.create_physical_region(
858                physical_region_id,
859                &table_dir(physical_region_id),
860                vec![(
861                    WAL_OPTIONS_KEY.to_string(),
862                    serde_json::to_string(&wal_options).unwrap(),
863                )],
864            )
865            .await;
866            // Creates logical regions for each physical region
867            for j in 0..num_logical_regions {
868                let logical_region_id = RegionId::new(1024 + i, j);
869                logical_region_ids.push(logical_region_id);
870                env.create_logical_region(physical_region_id, logical_region_id)
871                    .await;
872            }
873        }
874
875        let metric_engine = env.metric();
876        // Closes all regions
877        for region_id in logical_region_ids.iter().chain(physical_region_ids.iter()) {
878            metric_engine
879                .handle_request(
880                    *region_id,
881                    RegionRequest::Close(RegionCloseRequest::default()),
882                )
883                .await
884                .unwrap();
885        }
886
887        // Opens all regions and skip the wal
888        let requests = physical_region_ids
889            .iter()
890            .enumerate()
891            .map(|(idx, region_id)| {
892                let mut options = HashMap::new();
893                let wal_options =
894                    WalOptions::Kafka(KafkaWalOptions::new(topics[topic_idx(idx as u32)].clone()));
895                options.insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new());
896                options.insert(
897                    WAL_OPTIONS_KEY.to_string(),
898                    serde_json::to_string(&wal_options).unwrap(),
899                );
900                (
901                    *region_id,
902                    RegionOpenRequest {
903                        engine: METRIC_ENGINE_NAME.to_string(),
904                        table_dir: table_dir(*region_id),
905                        path_type: PathType::Bare,
906                        options: options.clone(),
907                        skip_wal_replay: true,
908                        checkpoint: None,
909                        requirements: Default::default(),
910                    },
911                )
912            })
913            .collect::<Vec<_>>();
914        info!("Open batch regions with parallelism: {parallelism}");
915        metric_engine
916            .handle_batch_open_requests(parallelism, requests)
917            .await
918            .unwrap();
919        {
920            let state = metric_engine.inner.state.read().unwrap();
921            for logical_region in &logical_region_ids {
922                assert!(!state.logical_regions().contains_key(logical_region));
923            }
924        }
925
926        let catch_requests = physical_region_ids
927            .iter()
928            .map(|region_id| {
929                (
930                    *region_id,
931                    RegionCatchupRequest {
932                        set_writable: true,
933                        ..Default::default()
934                    },
935                )
936            })
937            .collect::<Vec<_>>();
938        metric_engine
939            .handle_batch_catchup_requests(parallelism, catch_requests)
940            .await
941            .unwrap();
942        {
943            let state = metric_engine.inner.state.read().unwrap();
944            for logical_region in &logical_region_ids {
945                assert!(state.logical_regions().contains_key(logical_region));
946            }
947        }
948    }
949
950    #[tokio::test]
951    async fn test_drop_region() {
952        let env = TestEnv::new().await;
953        let engine = env.metric();
954        let physical_region_id1 = RegionId::new(1024, 0);
955        let logical_region_id1 = RegionId::new(1025, 0);
956        env.create_physical_region(physical_region_id1, "/test_dir1", vec![])
957            .await;
958        let region_create_request1 =
959            create_logical_region_request(&["job"], physical_region_id1, "logical1");
960        engine
961            .handle_batch_ddl_requests(BatchRegionDdlRequest::Create(vec![(
962                logical_region_id1,
963                region_create_request1,
964            )]))
965            .await
966            .unwrap();
967        let err = engine
968            .handle_request(
969                physical_region_id1,
970                RegionRequest::Drop(RegionDropRequest {
971                    fast_path: false,
972                    force: false,
973                    partial_drop: false,
974                }),
975            )
976            .await
977            .unwrap_err();
978        assert_matches!(
979            err.as_any().downcast_ref::<Error>().unwrap(),
980            &Error::PhysicalRegionBusy { .. }
981        );
982
983        engine
984            .handle_request(
985                physical_region_id1,
986                RegionRequest::Drop(RegionDropRequest {
987                    fast_path: false,
988                    force: true,
989                    partial_drop: false,
990                }),
991            )
992            .await
993            .unwrap();
994        assert!(
995            engine
996                .inner
997                .state
998                .read()
999                .unwrap()
1000                .physical_region_states()
1001                .get(&physical_region_id1)
1002                .is_none()
1003        );
1004        assert!(
1005            engine
1006                .inner
1007                .state
1008                .read()
1009                .unwrap()
1010                .logical_regions()
1011                .get(&logical_region_id1)
1012                .is_none()
1013        );
1014    }
1015}