feat: improve object storage cache (#2522)

* feat: refactor object storage cache with moka

* chore: minor fixes

* fix: concurrent issues and invalidate cache after write/delete

* chore: minor changes

* fix: cargo lock

* refactor: rename

* chore: change DEFAULT_OBJECT_STORE_CACHE_SIZE to 256Mib

* fix: typo

* chore: style

* fix: toml format

* chore: toml

* fix: toml format

* Update src/object-store/src/layers/lru_cache/read_cache.rs

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* chore: update Cargo.toml

Co-authored-by: Yingwen <realevenyag@gmail.com>

* chore: update src/object-store/Cargo.toml

Co-authored-by: Yingwen <realevenyag@gmail.com>

* chore: refactor and apply suggestions

* fix: typo

* feat: adds back allow list for caching

* chore: cr suggestion

Co-authored-by: Yingwen <realevenyag@gmail.com>

* chore: cr suggestion

Co-authored-by: Yingwen <realevenyag@gmail.com>

* refactor: wrap inner Accessor with Arc

* chore: remove run_pending_task in read and write path

* chore: the arc is unnecessary

---------

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>
Co-authored-by: Yingwen <realevenyag@gmail.com>

src/datanode/src/config.rs

@@ -37,7 +37,7 @@ use storage::config::{
 };
 use storage::scheduler::SchedulerConfig;
 
-pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize(1024);
+pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::mb(256);
 
 /// Default data home in file storage
 const DEFAULT_DATA_HOME: &str = "/tmp/greptimedb";
@@ -90,6 +90,15 @@ impl Default for StorageConfig {
 #[serde(default)]
 pub struct FileConfig {}
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+#[serde(default)]
+pub struct ObjectStorageCacheConfig {
+    /// The local file cache directory
+    pub cache_path: Option<String>,
+    /// The cache capacity in bytes
+    pub cache_capacity: Option<ReadableSize>,
+}
+
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(default)]
 pub struct S3Config {
@@ -101,8 +110,8 @@ pub struct S3Config {
     pub secret_access_key: SecretString,
     pub endpoint: Option<String>,
     pub region: Option<String>,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -115,8 +124,8 @@ pub struct OssConfig {
     #[serde(skip_serializing)]
     pub access_key_secret: SecretString,
     pub endpoint: String,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -130,8 +139,8 @@ pub struct AzblobConfig {
     pub account_key: SecretString,
     pub endpoint: String,
     pub sas_token: Option<String>,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -143,8 +152,8 @@ pub struct GcsConfig {
     #[serde(skip_serializing)]
     pub credential_path: SecretString,
     pub endpoint: String,
-    pub cache_path: Option<String>,
-    pub cache_capacity: Option<ReadableSize>,
+    #[serde(flatten)]
+    pub cache: ObjectStorageCacheConfig,
 }
 
 impl Default for S3Config {
@@ -156,8 +165,7 @@ impl Default for S3Config {
             secret_access_key: SecretString::from(String::default()),
             endpoint: Option::default(),
             region: Option::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -170,8 +178,7 @@ impl Default for OssConfig {
             access_key_id: SecretString::from(String::default()),
             access_key_secret: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -184,9 +191,8 @@ impl Default for AzblobConfig {
             account_name: SecretString::from(String::default()),
             account_key: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
             sas_token: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }
@@ -199,8 +205,7 @@ impl Default for GcsConfig {
             scope: String::default(),
             credential_path: SecretString::from(String::default()),
             endpoint: String::default(),
-            cache_path: Option::default(),
-            cache_capacity: Option::default(),
+            cache: ObjectStorageCacheConfig::default(),
         }
     }
 }

src/datanode/src/datanode.rs

@@ -19,7 +19,6 @@ use std::sync::Arc;
 
 use catalog::kvbackend::MetaKvBackend;
 use catalog::memory::MemoryCatalogManager;
-use common_base::readable_size::ReadableSize;
 use common_base::Plugins;
 use common_error::ext::BoxedError;
 use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
@@ -62,8 +61,6 @@ use crate::region_server::RegionServer;
 use crate::server::Services;
 use crate::store;
 
-pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize(1024);
-
 const OPEN_REGION_PARALLELISM: usize = 16;
 
 /// Datanode service.

src/datanode/src/store.rs

@@ -76,29 +76,33 @@ async fn create_object_store_with_cache(
 ) -> Result<ObjectStore> {
     let (cache_path, cache_capacity) = match store_config {
         ObjectStoreConfig::S3(s3_config) => {
-            let path = s3_config.cache_path.as_ref();
+            let path = s3_config.cache.cache_path.as_ref();
             let capacity = s3_config
+                .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
             (path, capacity)
         }
         ObjectStoreConfig::Oss(oss_config) => {
-            let path = oss_config.cache_path.as_ref();
+            let path = oss_config.cache.cache_path.as_ref();
             let capacity = oss_config
+                .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
             (path, capacity)
         }
         ObjectStoreConfig::Azblob(azblob_config) => {
-            let path = azblob_config.cache_path.as_ref();
+            let path = azblob_config.cache.cache_path.as_ref();
             let capacity = azblob_config
+                .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
             (path, capacity)
         }
         ObjectStoreConfig::Gcs(gcs_config) => {
-            let path = gcs_config.cache_path.as_ref();
+            let path = gcs_config.cache.cache_path.as_ref();
             let capacity = gcs_config
+                .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
             (path, capacity)
@@ -119,6 +123,12 @@ async fn create_object_store_with_cache(
         let cache_layer = LruCacheLayer::new(Arc::new(cache_store), cache_capacity.0 as usize)
             .await
             .context(error::InitBackendSnafu)?;
+
+        info!(
+            "Enabled local object storage cache, path: {}, capacity: {}.",
+            path, cache_capacity
+        );
+
         Ok(object_store.layer(cache_layer))
     } else {
         Ok(object_store)