metric_engine/engine/
drop.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Drop a metric region
16
17use common_telemetry::{debug, info};
18use snafu::ResultExt;
19use store_api::region_engine::RegionEngine;
20use store_api::region_request::{AffectedRows, RegionDropRequest, RegionRequest};
21use store_api::storage::RegionId;
22
23use crate::engine::MetricEngineInner;
24use crate::error::{
25    CloseMitoRegionSnafu, LogicalRegionNotFoundSnafu, PhysicalRegionBusySnafu, Result,
26};
27use crate::metrics::PHYSICAL_REGION_COUNT;
28use crate::utils;
29
30impl MetricEngineInner {
31    pub async fn drop_region(
32        &self,
33        region_id: RegionId,
34        req: RegionDropRequest,
35    ) -> Result<AffectedRows> {
36        let data_region_id = utils::to_data_region_id(region_id);
37        let fast_path = req.fast_path;
38        let force = req.force;
39
40        // enclose the guard in a block to prevent the guard from polluting the async context
41        let (is_physical_region, is_physical_region_busy) = {
42            if let Some(state) = self
43                .state
44                .read()
45                .unwrap()
46                .physical_region_states()
47                .get(&data_region_id)
48            {
49                debug!(
50                    "Physical region {} is busy, there are still some logical regions: {:?}",
51                    data_region_id,
52                    state
53                        .logical_regions()
54                        .iter()
55                        .map(|id| id.to_string())
56                        .collect::<Vec<_>>()
57                );
58                (true, !state.logical_regions().is_empty())
59            } else {
60                // the second argument is not used, just pass in a dummy value
61                (false, true)
62            }
63        };
64
65        if is_physical_region {
66            // check if there is no logical region relates to this physical region
67            if is_physical_region_busy && !force {
68                // reject if there is any present logical region
69                return Err(PhysicalRegionBusySnafu {
70                    region_id: data_region_id,
71                }
72                .build());
73            }
74            if is_physical_region_busy && force {
75                info!("Dropping physical region {} with force", data_region_id);
76            }
77            return self
78                .drop_physical_region(data_region_id, req.partial_drop)
79                .await;
80        }
81
82        if fast_path {
83            // for fast path, we don't delete the metadata in the metadata region.
84            // it only remove the logical region from the engine state.
85            //
86            // The drop database procedure will ensure the metadata region and data region are dropped eventually.
87            self.state
88                .write()
89                .unwrap()
90                .remove_logical_region(region_id)?;
91
92            Ok(0)
93        } else {
94            let metadata_region_id = self
95                .state
96                .read()
97                .unwrap()
98                .logical_regions()
99                .get(&region_id)
100                .copied();
101            if let Some(metadata_region_id) = metadata_region_id {
102                self.drop_logical_region(region_id, metadata_region_id)
103                    .await
104            } else {
105                Err(LogicalRegionNotFoundSnafu { region_id }.build())
106            }
107        }
108    }
109
110    async fn drop_physical_region(
111        &self,
112        region_id: RegionId,
113        partial_drop: bool,
114    ) -> Result<AffectedRows> {
115        let data_region_id = utils::to_data_region_id(region_id);
116        let metadata_region_id = utils::to_metadata_region_id(region_id);
117
118        // Drop mito regions.
119        // Since the physical regions are going to be dropped, we don't need to
120        // update the contents in metadata region.
121        self.mito
122            .handle_request(
123                data_region_id,
124                RegionRequest::Drop(RegionDropRequest {
125                    fast_path: false,
126                    force: false,
127                    partial_drop,
128                }),
129            )
130            .await
131            .with_context(|_| CloseMitoRegionSnafu { region_id })?;
132        self.mito
133            .handle_request(
134                metadata_region_id,
135                RegionRequest::Drop(RegionDropRequest {
136                    fast_path: false,
137                    force: false,
138                    partial_drop,
139                }),
140            )
141            .await
142            .with_context(|_| CloseMitoRegionSnafu { region_id })?;
143
144        PHYSICAL_REGION_COUNT.dec();
145
146        // Update engine state
147        self.state
148            .write()
149            .unwrap()
150            .remove_physical_region(data_region_id)?;
151
152        Ok(0)
153    }
154
155    async fn drop_logical_region(
156        &self,
157        logical_region_id: RegionId,
158        physical_region_id: RegionId,
159    ) -> Result<AffectedRows> {
160        // Update metadata
161        self.metadata_region
162            .remove_logical_region(physical_region_id, logical_region_id)
163            .await?;
164
165        // Update engine state
166        self.state
167            .write()
168            .unwrap()
169            .remove_logical_region(logical_region_id)?;
170
171        Ok(0)
172    }
173}
174
175#[cfg(test)]
176mod tests {
177    use api::v1::Rows;
178    use common_wal::config::object_store::{ObjectStoreWalConfig, STANDALONE_GENERATION};
179    use common_wal::options::{ObjectStoreWalOptions, WAL_OPTIONS_KEY, WalOptions};
180    use log_store::ObjectStoreLogStore;
181    use mito2::config::MitoConfig;
182    use mito2::test_util::TestEnv as MitoTestEnv;
183    use object_store::ObjectStore;
184    use object_store::services::Memory;
185    use store_api::logstore::LogStore;
186    use store_api::region_engine::RegionEngine;
187    use store_api::region_request::{RegionDropRequest, RegionPutRequest, RegionRequest};
188    use store_api::storage::RegionId;
189
190    use crate::test_util::{TestEnv, build_rows, row_schema_with_tags};
191    use crate::utils::{to_data_region_id, to_metadata_region_id};
192
193    fn drop_request() -> RegionRequest {
194        RegionRequest::Drop(RegionDropRequest {
195            fast_path: false,
196            force: false,
197            partial_drop: false,
198        })
199    }
200
201    /// How the WAL releases of the dropped regions fail.
202    #[derive(Clone, Copy, PartialEq)]
203    enum ReleaseFailure {
204        /// The release of the data region fails transiently.
205        Data,
206        /// The releases of the data and the metadata region fail transiently.
207        Both,
208        /// The release of the data region meets a conflicting object, which
209        /// poisons the WAL, so the cleanup of the metadata region fails too.
210        Conflict,
211    }
212
213    #[tokio::test]
214    async fn test_drop_physical_region_whose_wal_release_fails() {
215        for failure in [
216            ReleaseFailure::Data,
217            ReleaseFailure::Both,
218            ReleaseFailure::Conflict,
219        ] {
220            let object_store = ObjectStore::new(Memory::default()).unwrap();
221            let config = ObjectStoreWalConfig::default();
222            let open_wal = || {
223                ObjectStoreLogStore::try_new(
224                    object_store.clone(),
225                    &config,
226                    0,
227                    STANDALONE_GENERATION,
228                )
229            };
230            let wal = open_wal().await.unwrap();
231            let mut mito_env = MitoTestEnv::with_prefix("metric-drop-failed-release").await;
232            let mito = mito_env
233                .create_engine_with_log_store(MitoConfig::default(), wal.clone())
234                .await;
235            let env = TestEnv::with_mito_engine(mito_env, mito);
236            let region_id = RegionId::new(1, 1);
237            let wal_options = WalOptions::ObjectStore(ObjectStoreWalOptions::new(
238                "wal/datanodes/0/epochs/0".to_string(),
239            ));
240            env.create_physical_region(
241                region_id,
242                "test_metric_table",
243                vec![(
244                    WAL_OPTIONS_KEY.to_string(),
245                    serde_json::to_string(&wal_options).unwrap(),
246                )],
247            )
248            .await;
249            // Both mito regions write to the WAL: the data region the rows of a
250            // logical region, the metadata region its creation and its drop.
251            let logical_region_id = RegionId::new(1024, 1);
252            env.create_logical_region(region_id, logical_region_id)
253                .await;
254            let metric = env.metric();
255            metric
256                .handle_request(
257                    logical_region_id,
258                    RegionRequest::Put(RegionPutRequest {
259                        skip_wal: false,
260                        rows: Rows {
261                            schema: row_schema_with_tags(&["job"]),
262                            rows: build_rows(1, 10),
263                        },
264                        hint: None,
265                        partition_expr_version: None,
266                    }),
267                )
268                .await
269                .unwrap();
270            metric
271                .handle_request(logical_region_id, drop_request())
272                .await
273                .unwrap();
274
275            // The drop still removes both regions and the physical region state.
276            match failure {
277                ReleaseFailure::Data => wal.fail_next_create_after_write(),
278                ReleaseFailure::Both => wal.fail_creates(),
279                ReleaseFailure::Conflict => {
280                    // A copy of the last object, of the same epoch, takes the
281                    // sequence of the next one.
282                    let objects = "wal/datanodes/0/epochs/0/objects/";
283                    let last = object_store
284                        .list(objects)
285                        .await
286                        .unwrap()
287                        .iter()
288                        .filter_map(|entry| entry.name().strip_suffix(".wal")?.parse::<u64>().ok())
289                        .max()
290                        .unwrap();
291                    let path = |object_seq: u64| format!("{objects}{object_seq:020}.wal");
292                    let copy = object_store.read(&path(last)).await.unwrap();
293                    object_store.write(&path(last + 1), copy).await.unwrap();
294                }
295            }
296            metric
297                .handle_request(region_id, drop_request())
298                .await
299                .unwrap();
300            assert!(!env.mito().is_region_exists(to_data_region_id(region_id)));
301            assert!(
302                !env.mito()
303                    .is_region_exists(to_metadata_region_id(region_id))
304            );
305            assert!(
306                !metric
307                    .inner
308                    .state
309                    .read()
310                    .unwrap()
311                    .exist_physical_region(to_data_region_id(region_id))
312            );
313
314            // The stop writes the release records of failed releases. A store
315            // opened later finds both regions released and keeps no segment of
316            // either. A poisoned store writes nothing more.
317            if failure == ReleaseFailure::Conflict {
318                assert!(wal.list_namespaces().await.is_err());
319                continue;
320            }
321            wal.allow_creates();
322            wal.stop().await.unwrap();
323            let wal = open_wal().await.unwrap();
324            wal.wait_for_garbage_collection().await;
325            assert!(wal.list_namespaces().await.unwrap().is_empty());
326            wal.stop().await.unwrap();
327        }
328    }
329}
metric_engine/engine/drop.rs

metric_engine/engine/
drop.rs
