datanode/
store.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! object storage utilities
16
17use common_telemetry::{info, warn};
18use object_store::factory::new_raw_object_store;
19use object_store::util::{clean_temp_dir, join_dir, with_instrument_layers, with_retry_layers};
20use object_store::{ATOMIC_WRITE_DIR, ObjectStore};
21use snafu::prelude::*;
22
23use crate::config::ObjectStoreConfig;
24use crate::error::{self, Result};
25
26/// Cleans up old LRU read cache directories that were removed.
27fn clean_old_read_cache(store: &ObjectStoreConfig, data_home: &str) {
28    if !store.is_object_storage() {
29        return;
30    }
31
32    let Some(cache_config) = store.cache_config() else {
33        return;
34    };
35
36    // Only cleans if read cache was enabled
37    if !cache_config.enable_read_cache {
38        return;
39    }
40
41    let cache_base_dir = if cache_config.cache_path.is_empty() {
42        data_home
43    } else {
44        &cache_config.cache_path
45    };
46
47    // Cleans up the old read cache directory
48    let old_read_cache_dir = join_dir(cache_base_dir, "cache/object/read");
49    info!(
50        "Cleaning up old read cache directory: {}",
51        old_read_cache_dir
52    );
53    if let Err(e) = clean_temp_dir(&old_read_cache_dir) {
54        warn!(e; "Failed to clean old read cache directory {}", old_read_cache_dir);
55    }
56
57    // Cleans up the atomic temp dir used by the cache layer
58    let cache_atomic_temp_dir = join_dir(cache_base_dir, ATOMIC_WRITE_DIR);
59    info!(
60        "Cleaning up old cache atomic temp directory: {}",
61        cache_atomic_temp_dir
62    );
63    if let Err(e) = clean_temp_dir(&cache_atomic_temp_dir) {
64        warn!(e; "Failed to clean old cache atomic temp directory {}", cache_atomic_temp_dir);
65    }
66}
67
68pub async fn new_object_store(store: ObjectStoreConfig, data_home: &str) -> Result<ObjectStore> {
69    // Cleans up old LRU read cache directories.
70    // TODO: Remove this line after the 1.0 release.
71    clean_old_read_cache(&store, data_home);
72
73    let object_store = new_raw_object_store(&store, data_home)
74        .await
75        .context(error::ObjectStoreSnafu)?;
76    // Enables retry layer for non-fs object storages
77    let object_store = if store.is_object_storage() {
78        // Adds retry layer
79        with_retry_layers(object_store)
80    } else {
81        object_store
82    };
83
84    let object_store = with_instrument_layers(object_store, true);
85    Ok(object_store)
86}
datanode/store.rs

datanode/
store.rs
