metric_engine/engine/
drop.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Drop a metric region
16
17use common_telemetry::{debug, info};
18use snafu::ResultExt;
19use store_api::region_engine::RegionEngine;
20use store_api::region_request::{AffectedRows, RegionDropRequest, RegionRequest};
21use store_api::storage::RegionId;
22
23use crate::engine::MetricEngineInner;
24use crate::error::{
25    CloseMitoRegionSnafu, LogicalRegionNotFoundSnafu, PhysicalRegionBusySnafu, Result,
26};
27use crate::metrics::PHYSICAL_REGION_COUNT;
28use crate::utils;
29
30impl MetricEngineInner {
31    pub async fn drop_region(
32        &self,
33        region_id: RegionId,
34        req: RegionDropRequest,
35    ) -> Result<AffectedRows> {
36        let data_region_id = utils::to_data_region_id(region_id);
37        let fast_path = req.fast_path;
38        let force = req.force;
39
40        // enclose the guard in a block to prevent the guard from polluting the async context
41        let (is_physical_region, is_physical_region_busy) = {
42            if let Some(state) = self
43                .state
44                .read()
45                .unwrap()
46                .physical_region_states()
47                .get(&data_region_id)
48            {
49                debug!(
50                    "Physical region {} is busy, there are still some logical regions: {:?}",
51                    data_region_id,
52                    state
53                        .logical_regions()
54                        .iter()
55                        .map(|id| id.to_string())
56                        .collect::<Vec<_>>()
57                );
58                (true, !state.logical_regions().is_empty())
59            } else {
60                // the second argument is not used, just pass in a dummy value
61                (false, true)
62            }
63        };
64
65        if is_physical_region {
66            // check if there is no logical region relates to this physical region
67            if is_physical_region_busy && !force {
68                // reject if there is any present logical region
69                return Err(PhysicalRegionBusySnafu {
70                    region_id: data_region_id,
71                }
72                .build());
73            }
74            if is_physical_region_busy && force {
75                info!("Dropping physical region {} with force", data_region_id);
76            }
77            return self
78                .drop_physical_region(data_region_id, req.partial_drop)
79                .await;
80        }
81
82        if fast_path {
83            // for fast path, we don't delete the metadata in the metadata region.
84            // it only remove the logical region from the engine state.
85            //
86            // The drop database procedure will ensure the metadata region and data region are dropped eventually.
87            self.state
88                .write()
89                .unwrap()
90                .remove_logical_region(region_id)?;
91
92            Ok(0)
93        } else {
94            let metadata_region_id = self
95                .state
96                .read()
97                .unwrap()
98                .logical_regions()
99                .get(&region_id)
100                .copied();
101            if let Some(metadata_region_id) = metadata_region_id {
102                self.drop_logical_region(region_id, metadata_region_id)
103                    .await
104            } else {
105                Err(LogicalRegionNotFoundSnafu { region_id }.build())
106            }
107        }
108    }
109
110    async fn drop_physical_region(
111        &self,
112        region_id: RegionId,
113        partial_drop: bool,
114    ) -> Result<AffectedRows> {
115        let data_region_id = utils::to_data_region_id(region_id);
116        let metadata_region_id = utils::to_metadata_region_id(region_id);
117
118        // Drop mito regions.
119        // Since the physical regions are going to be dropped, we don't need to
120        // update the contents in metadata region.
121        self.mito
122            .handle_request(
123                data_region_id,
124                RegionRequest::Drop(RegionDropRequest {
125                    fast_path: false,
126                    force: false,
127                    partial_drop,
128                }),
129            )
130            .await
131            .with_context(|_| CloseMitoRegionSnafu { region_id })?;
132        self.mito
133            .handle_request(
134                metadata_region_id,
135                RegionRequest::Drop(RegionDropRequest {
136                    fast_path: false,
137                    force: false,
138                    partial_drop,
139                }),
140            )
141            .await
142            .with_context(|_| CloseMitoRegionSnafu { region_id })?;
143
144        PHYSICAL_REGION_COUNT.dec();
145
146        // Update engine state
147        self.state
148            .write()
149            .unwrap()
150            .remove_physical_region(data_region_id)?;
151
152        Ok(0)
153    }
154
155    async fn drop_logical_region(
156        &self,
157        logical_region_id: RegionId,
158        physical_region_id: RegionId,
159    ) -> Result<AffectedRows> {
160        // A fill of the cached logical columns holds the read lock until it publishes them, so
161        // none publishes columns read before the delete after the invalidation below.
162        let _write_guard = self
163            .metadata_region
164            .write_lock_logical_region(logical_region_id)
165            .await?;
166        // Update metadata
167        let result = self
168            .metadata_region
169            .remove_logical_region(physical_region_id, logical_region_id)
170            .await;
171        // A failed delete may still be applied, e.g. when the region lease expires after the WAL
172        // append, so the cached logical columns are dropped either way.
173        self.state
174            .write()
175            .unwrap()
176            .invalid_logical_regions_cache([logical_region_id]);
177        result?;
178
179        // Update engine state
180        self.state
181            .write()
182            .unwrap()
183            .remove_logical_region(logical_region_id)?;
184
185        Ok(0)
186    }
187}
188
189#[cfg(test)]
190mod tests {
191    use api::v1::Rows;
192    use common_wal::config::object_store::{ObjectStoreWalConfig, STANDALONE_GENERATION};
193    use common_wal::options::{ObjectStoreWalOptions, WAL_OPTIONS_KEY, WalOptions};
194    use log_store::ObjectStoreLogStore;
195    use mito2::config::MitoConfig;
196    use mito2::test_util::TestEnv as MitoTestEnv;
197    use object_store::ObjectStore;
198    use object_store::services::Memory;
199    use store_api::logstore::LogStore;
200    use store_api::region_engine::RegionEngine;
201    use store_api::region_request::{RegionDropRequest, RegionPutRequest, RegionRequest};
202    use store_api::storage::RegionId;
203
204    use crate::test_util::{TestEnv, build_rows, row_schema_with_tags};
205    use crate::utils::{to_data_region_id, to_metadata_region_id};
206
207    fn drop_request() -> RegionRequest {
208        RegionRequest::Drop(RegionDropRequest {
209            fast_path: false,
210            force: false,
211            partial_drop: false,
212        })
213    }
214
215    /// How the WAL releases of the dropped regions fail.
216    #[derive(Clone, Copy, PartialEq)]
217    enum ReleaseFailure {
218        /// The release of the data region fails transiently.
219        Data,
220        /// The releases of the data and the metadata region fail transiently.
221        Both,
222        /// The release of the data region meets a conflicting object, which
223        /// poisons the WAL, so the cleanup of the metadata region fails too.
224        Conflict,
225    }
226
227    #[tokio::test]
228    async fn test_drop_physical_region_whose_wal_release_fails() {
229        for failure in [
230            ReleaseFailure::Data,
231            ReleaseFailure::Both,
232            ReleaseFailure::Conflict,
233        ] {
234            let object_store = ObjectStore::new(Memory::default()).unwrap();
235            let config = ObjectStoreWalConfig::default();
236            let open_wal = || {
237                ObjectStoreLogStore::try_new(
238                    object_store.clone(),
239                    &config,
240                    0,
241                    STANDALONE_GENERATION,
242                )
243            };
244            let wal = open_wal().await.unwrap();
245            let mut mito_env = MitoTestEnv::with_prefix("metric-drop-failed-release").await;
246            let mito = mito_env
247                .create_engine_with_log_store(MitoConfig::default(), wal.clone())
248                .await;
249            let env = TestEnv::with_mito_engine(mito_env, mito);
250            let region_id = RegionId::new(1, 1);
251            let wal_options = WalOptions::ObjectStore(ObjectStoreWalOptions::new(
252                "wal/datanodes/0/epochs/0".to_string(),
253            ));
254            env.create_physical_region(
255                region_id,
256                "test_metric_table",
257                vec![(
258                    WAL_OPTIONS_KEY.to_string(),
259                    serde_json::to_string(&wal_options).unwrap(),
260                )],
261            )
262            .await;
263            // Both mito regions write to the WAL: the data region the rows of a
264            // logical region, the metadata region its creation and its drop.
265            let logical_region_id = RegionId::new(1024, 1);
266            env.create_logical_region(region_id, logical_region_id)
267                .await;
268            let metric = env.metric();
269            metric
270                .handle_request(
271                    logical_region_id,
272                    RegionRequest::Put(RegionPutRequest {
273                        skip_wal: false,
274                        rows: Rows {
275                            schema: row_schema_with_tags(&["job"]),
276                            rows: build_rows(1, 10),
277                        },
278                        hint: None,
279                        partition_expr_version: None,
280                    }),
281                )
282                .await
283                .unwrap();
284            metric
285                .handle_request(logical_region_id, drop_request())
286                .await
287                .unwrap();
288
289            // The drop still removes both regions and the physical region state.
290            match failure {
291                ReleaseFailure::Data => wal.fail_next_create_after_write(),
292                ReleaseFailure::Both => wal.fail_creates(),
293                ReleaseFailure::Conflict => {
294                    // A copy of the last object, of the same epoch, takes the
295                    // sequence of the next one.
296                    let objects = "wal/datanodes/0/epochs/0/objects/";
297                    let last = object_store
298                        .list(objects)
299                        .await
300                        .unwrap()
301                        .iter()
302                        .filter_map(|entry| entry.name().strip_suffix(".wal")?.parse::<u64>().ok())
303                        .max()
304                        .unwrap();
305                    let path = |object_seq: u64| format!("{objects}{object_seq:020}.wal");
306                    let copy = object_store.read(&path(last)).await.unwrap();
307                    object_store.write(&path(last + 1), copy).await.unwrap();
308                }
309            }
310            metric
311                .handle_request(region_id, drop_request())
312                .await
313                .unwrap();
314            assert!(!env.mito().is_region_exists(to_data_region_id(region_id)));
315            assert!(
316                !env.mito()
317                    .is_region_exists(to_metadata_region_id(region_id))
318            );
319            assert!(
320                !metric
321                    .inner
322                    .state
323                    .read()
324                    .unwrap()
325                    .exist_physical_region(to_data_region_id(region_id))
326            );
327
328            // The stop writes the release records of failed releases. A store
329            // opened later finds both regions released and keeps no segment of
330            // either. A poisoned store writes nothing more.
331            if failure == ReleaseFailure::Conflict {
332                assert!(wal.list_namespaces().await.is_err());
333                continue;
334            }
335            wal.allow_creates();
336            wal.stop().await.unwrap();
337            let wal = open_wal().await.unwrap();
338            wal.wait_for_garbage_collection().await;
339            assert!(wal.list_namespaces().await.unwrap().is_empty());
340            wal.stop().await.unwrap();
341        }
342    }
343}
metric_engine/engine/drop.rs

metric_engine/engine/
drop.rs
