feat!: enable read cache and write cache when using remote object stores (#5093)

* feat: enable read cache and write cache when using remote object stores

* feat: make read cache be aware of remote store names

* chore: docs

* chore: apply review suggestions

* chore: trim write cache path

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>

config/config.md

@@ -93,7 +93,7 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.<br/>The local file cache directory. |
+| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
 | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
@@ -131,9 +131,9 @@
 | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
-| `region_engine.mito.experimental_write_cache_size` | String | `1GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
+| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. |
+| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
 | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |
@@ -420,7 +420,7 @@
 | `storage` | -- | -- | The data storage options. |
 | `storage.data_home` | String | `/tmp/greptimedb/` | The working home directory. |
 | `storage.type` | String | `File` | The storage type used to store the data.<br/>- `File`: the data is stored in the local file system.<br/>- `S3`: the data is stored in the S3 object storage.<br/>- `Gcs`: the data is stored in the Google Cloud Storage.<br/>- `Azblob`: the data is stored in the Azure Blob Storage.<br/>- `Oss`: the data is stored in the Aliyun OSS. |
-| `storage.cache_path` | String | Unset | Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.<br/>The local file cache directory. |
+| `storage.cache_path` | String | Unset | Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.<br/>A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling. |
 | `storage.cache_capacity` | String | Unset | The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger. |
 | `storage.bucket` | String | Unset | The S3 bucket name.<br/>**It's only used when the storage type is `S3`, `Oss` and `Gcs`**. |
 | `storage.root` | String | Unset | The S3 data will be stored in the specified prefix, for example, `s3://${bucket}/${root}`.<br/>**It's only used when the storage type is `S3`, `Oss` and `Azblob`**. |
@@ -458,9 +458,9 @@
 | `region_engine.mito.vector_cache_size` | String | Auto | Cache size for vectors and arrow arrays. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
 | `region_engine.mito.page_cache_size` | String | Auto | Cache size for pages of SST row groups. Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/8 of OS memory. |
 | `region_engine.mito.selector_result_cache_size` | String | Auto | Cache size for time series selector (e.g. `last_value()`). Setting it to 0 to disable the cache.<br/>If not set, it's default to 1/16 of OS memory with a max limitation of 512MB. |
-| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance. |
-| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/write_cache`. |
-| `region_engine.mito.experimental_write_cache_size` | String | `1GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
+| `region_engine.mito.enable_experimental_write_cache` | Bool | `false` | Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance. |
+| `region_engine.mito.experimental_write_cache_path` | String | `""` | File system path for write cache, defaults to `{data_home}/object_cache/write`. |
+| `region_engine.mito.experimental_write_cache_size` | String | `5GiB` | Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger. |
 | `region_engine.mito.experimental_write_cache_ttl` | String | Unset | TTL for write cache. |
 | `region_engine.mito.sst_write_buffer_size` | String | `8MB` | Buffer size for SST writing. |
 | `region_engine.mito.parallel_scan_channel_size` | Integer | `32` | Capacity of the channel to send data from parallel scan tasks to the main task. |

config/datanode.example.toml

@@ -294,14 +294,14 @@ data_home = "/tmp/greptimedb/"
 ## - `Oss`: the data is stored in the Aliyun OSS.
 type = "File"
 
-## Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.
-## The local file cache directory.
+## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
+## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling.
 ## @toml2docs:none-default
-cache_path = "/path/local_cache"
+#+ cache_path = ""
 
 ## The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger.
 ## @toml2docs:none-default
-cache_capacity = "1GiB"
+cache_capacity = "5GiB"
 
 ## The S3 bucket name.
 ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**.
@@ -476,14 +476,14 @@ auto_flush_interval = "1h"
 ## @toml2docs:none-default="Auto"
 #+ selector_result_cache_size = "512MB"
 
-## Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance.
+## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
 enable_experimental_write_cache = false
 
-## File system path for write cache, defaults to `{data_home}/write_cache`.
+## File system path for write cache, defaults to `{data_home}/object_cache/write`.
 experimental_write_cache_path = ""
 
 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
-experimental_write_cache_size = "1GiB"
+experimental_write_cache_size = "5GiB"
 
 ## TTL for write cache.
 ## @toml2docs:none-default

config/standalone.example.toml

@@ -332,14 +332,14 @@ data_home = "/tmp/greptimedb/"
 ## - `Oss`: the data is stored in the Aliyun OSS.
 type = "File"
 
-## Cache configuration for object storage such as 'S3' etc. It is recommended to configure it when using object storage for better performance.
-## The local file cache directory.
+## Read cache configuration for object storage such as 'S3' etc, it's configured by default when using object storage. It is recommended to configure it when using object storage for better performance.
+## A local file directory, defaults to `{data_home}/object_cache/read`. An empty string means disabling.
 ## @toml2docs:none-default
-cache_path = "/path/local_cache"
+#+ cache_path = ""
 
 ## The local file cache capacity in bytes. If your disk space is sufficient, it is recommended to set it larger.
 ## @toml2docs:none-default
-cache_capacity = "1GiB"
+cache_capacity = "5GiB"
 
 ## The S3 bucket name.
 ## **It's only used when the storage type is `S3`, `Oss` and `Gcs`**.
@@ -514,14 +514,14 @@ auto_flush_interval = "1h"
 ## @toml2docs:none-default="Auto"
 #+ selector_result_cache_size = "512MB"
 
-## Whether to enable the experimental write cache. It is recommended to enable it when using object storage for better performance.
+## Whether to enable the experimental write cache, it's enabled by default when using object storage. It is recommended to enable it when using object storage for better performance.
 enable_experimental_write_cache = false
 
-## File system path for write cache, defaults to `{data_home}/write_cache`.
+## File system path for write cache, defaults to `{data_home}/object_cache/write`.
 experimental_write_cache_path = ""
 
 ## Capacity for write cache. If your disk space is sufficient, it is recommended to set it larger.
-experimental_write_cache_size = "1GiB"
+experimental_write_cache_size = "5GiB"
 
 ## TTL for write cache.
 ## @toml2docs:none-default

src/datanode/src/config.rs

@@ -32,7 +32,7 @@ use servers::heartbeat_options::HeartbeatOptions;
 use servers::http::HttpOptions;
 use servers::Mode;
 
-pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::gb(1);
+pub const DEFAULT_OBJECT_STORE_CACHE_SIZE: ReadableSize = ReadableSize::gb(5);
 
 /// Default data home in file storage
 const DEFAULT_DATA_HOME: &str = "/tmp/greptimedb";
@@ -60,6 +60,11 @@ impl ObjectStoreConfig {
         }
     }
 
+    /// Returns true when it's a remote object storage such as AWS s3 etc.
+    pub fn is_object_storage(&self) -> bool {
+        !matches!(self, Self::File(_))
+    }
+
     /// Returns the object storage configuration name, return the provider name if it's empty.
     pub fn config_name(&self) -> &str {
         let name = match self {
@@ -91,6 +96,13 @@ pub struct StorageConfig {
     pub providers: Vec<ObjectStoreConfig>,
 }
 
+impl StorageConfig {
+    /// Returns true when the default storage config is a remote object storage service such as AWS S3, etc.
+    pub fn is_object_storage(&self) -> bool {
+        self.store.is_object_storage()
+    }
+}
+
 impl Default for StorageConfig {
     fn default() -> Self {
         Self {
@@ -452,6 +464,20 @@ mod tests {
         assert_eq!("S3", s3_config.provider_name());
     }
 
+    #[test]
+    fn test_is_object_storage() {
+        let store = ObjectStoreConfig::default();
+        assert!(!store.is_object_storage());
+        let s3_config = ObjectStoreConfig::S3(S3Config::default());
+        assert!(s3_config.is_object_storage());
+        let oss_config = ObjectStoreConfig::Oss(OssConfig::default());
+        assert!(oss_config.is_object_storage());
+        let gcs_config = ObjectStoreConfig::Gcs(GcsConfig::default());
+        assert!(gcs_config.is_object_storage());
+        let azblob_config = ObjectStoreConfig::Azblob(AzblobConfig::default());
+        assert!(azblob_config.is_object_storage());
+    }
+
     #[test]
     fn test_secstr() {
         let toml_str = r#"

src/datanode/src/datanode.rs

@@ -428,10 +428,16 @@ impl DatanodeBuilder {
     async fn build_mito_engine(
         opts: &DatanodeOptions,
         object_store_manager: ObjectStoreManagerRef,
-        config: MitoConfig,
+        mut config: MitoConfig,
         schema_metadata_manager: SchemaMetadataManagerRef,
         plugins: Plugins,
     ) -> Result<MitoEngine> {
+        if opts.storage.is_object_storage() {
+            // Enable the write cache when setting object storage
+            config.enable_experimental_write_cache = true;
+            info!("Configured 'enable_experimental_write_cache=true' for mito engine.");
+        }
+
         let mito_engine = match &opts.wal {
             DatanodeWalConfig::RaftEngine(raft_engine_config) => MitoEngine::new(
                 &opts.storage.data_home,

src/datanode/src/lib.rs

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #![feature(assert_matches)]
+#![feature(let_chains)]
 
 pub mod alive_keeper;
 pub mod config;

src/datanode/src/store.rs

@@ -19,21 +19,20 @@ mod fs;
 mod gcs;
 mod oss;
 mod s3;
-
+use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 use std::{env, path};
 
-use common_base::readable_size::ReadableSize;
 use common_telemetry::{info, warn};
 use object_store::layers::{LruCacheLayer, RetryInterceptor, RetryLayer};
 use object_store::services::Fs;
 use object_store::util::{join_dir, normalize_dir, with_instrument_layers};
-use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder};
+use object_store::{Access, Error, HttpClient, ObjectStore, ObjectStoreBuilder, OBJECT_CACHE_DIR};
 use snafu::prelude::*;
 
 use crate::config::{HttpClientConfig, ObjectStoreConfig, DEFAULT_OBJECT_STORE_CACHE_SIZE};
-use crate::error::{self, Result};
+use crate::error::{self, CreateDirSnafu, Result};
 
 pub(crate) async fn new_raw_object_store(
     store: &ObjectStoreConfig,
@@ -68,7 +67,7 @@ pub(crate) async fn new_object_store_without_cache(
 ) -> Result<ObjectStore> {
     let object_store = new_raw_object_store(store, data_home).await?;
     // Enable retry layer and cache layer for non-fs object storages
-    let object_store = if !matches!(store, ObjectStoreConfig::File(..)) {
+    let object_store = if store.is_object_storage() {
         // Adds retry layer
         with_retry_layers(object_store)
     } else {
@@ -85,8 +84,8 @@ pub(crate) async fn new_object_store(
 ) -> Result<ObjectStore> {
     let object_store = new_raw_object_store(&store, data_home).await?;
     // Enable retry layer and cache layer for non-fs object storages
-    let object_store = if !matches!(store, ObjectStoreConfig::File(..)) {
-        let object_store = if let Some(cache_layer) = build_cache_layer(&store).await? {
+    let object_store = if store.is_object_storage() {
+        let object_store = if let Some(cache_layer) = build_cache_layer(&store, data_home).await? {
             // Adds cache layer
             object_store.layer(cache_layer)
         } else {
@@ -105,44 +104,72 @@ pub(crate) async fn new_object_store(
 
 async fn build_cache_layer(
     store_config: &ObjectStoreConfig,
+    data_home: &str,
 ) -> Result<Option<LruCacheLayer<impl Access>>> {
-    let (cache_path, cache_capacity) = match store_config {
+    let (name, mut cache_path, cache_capacity) = match store_config {
         ObjectStoreConfig::S3(s3_config) => {
-            let path = s3_config.cache.cache_path.as_ref();
+            let path = s3_config.cache.cache_path.clone();
+            let name = &s3_config.name;
             let capacity = s3_config
                 .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
-            (path, capacity)
+            (name, path, capacity)
         }
         ObjectStoreConfig::Oss(oss_config) => {
-            let path = oss_config.cache.cache_path.as_ref();
+            let path = oss_config.cache.cache_path.clone();
+            let name = &oss_config.name;
             let capacity = oss_config
                 .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
-            (path, capacity)
+            (name, path, capacity)
         }
         ObjectStoreConfig::Azblob(azblob_config) => {
-            let path = azblob_config.cache.cache_path.as_ref();
+            let path = azblob_config.cache.cache_path.clone();
+            let name = &azblob_config.name;
             let capacity = azblob_config
                 .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
-            (path, capacity)
+            (name, path, capacity)
         }
         ObjectStoreConfig::Gcs(gcs_config) => {
-            let path = gcs_config.cache.cache_path.as_ref();
+            let path = gcs_config.cache.cache_path.clone();
+            let name = &gcs_config.name;
             let capacity = gcs_config
                 .cache
                 .cache_capacity
                 .unwrap_or(DEFAULT_OBJECT_STORE_CACHE_SIZE);
-            (path, capacity)
+            (name, path, capacity)
         }
-        _ => (None, ReadableSize(0)),
+        _ => unreachable!("Already checked above"),
     };
 
-    if let Some(path) = cache_path {
+    // Enable object cache by default
+    // Set the cache_path to be `${data_home}/object_cache/read/{name}` by default
+    // if it's not present
+    if cache_path.is_none() {
+        let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR);
+        let read_cache_path = join_dir(&object_cache_path, "read");
+        let read_cache_path = join_dir(&read_cache_path, &name.to_lowercase());
+        tokio::fs::create_dir_all(Path::new(&read_cache_path))
+            .await
+            .context(CreateDirSnafu {
+                dir: &read_cache_path,
+            })?;
+
+        info!(
+            "The object storage cache path is not set for '{}', using the default path: '{}'",
+            name, &read_cache_path
+        );
+
+        cache_path = Some(read_cache_path);
+    }
+
+    if let Some(path) = cache_path.as_ref()
+        && !path.trim().is_empty()
+    {
         let atomic_temp_dir = join_dir(path, ".tmp/");
         clean_temp_dir(&atomic_temp_dir)?;
 

src/mito2/src/config.rs

@@ -21,6 +21,7 @@ use std::time::Duration;
 use common_base::readable_size::ReadableSize;
 use common_telemetry::warn;
 use object_store::util::join_dir;
+use object_store::OBJECT_CACHE_DIR;
 use serde::{Deserialize, Serialize};
 use serde_with::serde_as;
 
@@ -96,7 +97,7 @@ pub struct MitoConfig {
     pub selector_result_cache_size: ReadableSize,
     /// Whether to enable the experimental write cache.
     pub enable_experimental_write_cache: bool,
-    /// File system path for write cache, defaults to `{data_home}/write_cache`.
+    /// File system path for write cache, defaults to `{data_home}/object_cache/write`.
     pub experimental_write_cache_path: String,
     /// Capacity for write cache.
     pub experimental_write_cache_size: ReadableSize,
@@ -148,7 +149,7 @@ impl Default for MitoConfig {
             selector_result_cache_size: ReadableSize::mb(512),
             enable_experimental_write_cache: false,
             experimental_write_cache_path: String::new(),
-            experimental_write_cache_size: ReadableSize::gb(1),
+            experimental_write_cache_size: ReadableSize::gb(5),
             experimental_write_cache_ttl: None,
             sst_write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
             parallel_scan_channel_size: DEFAULT_SCAN_CHANNEL_SIZE,
@@ -232,8 +233,9 @@ impl MitoConfig {
         }
 
         // Sets write cache path if it is empty.
-        if self.experimental_write_cache_path.is_empty() {
-            self.experimental_write_cache_path = join_dir(data_home, "write_cache");
+        if self.experimental_write_cache_path.trim().is_empty() {
+            let object_cache_path = join_dir(data_home, OBJECT_CACHE_DIR);
+            self.experimental_write_cache_path = join_dir(&object_cache_path, "write");
         }
 
         self.index.sanitize(data_home, &self.inverted_index)?;

src/mito2/src/error.rs

@@ -677,6 +677,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to create directory {}", dir))]
+    CreateDir {
+        dir: String,
+        #[snafu(source)]
+        error: std::io::Error,
+    },
+
     #[snafu(display("Failed to filter record batch"))]
     FilterRecordBatch {
         source: common_recordbatch::error::Error,
@@ -955,6 +962,7 @@ impl ErrorExt for Error {
             | ComputeVector { .. }
             | SerializeField { .. }
             | EncodeMemtable { .. }
+            | CreateDir { .. }
             | ReadDataPart { .. }
             | CorruptedEntry { .. }
             | BuildEntry { .. } => StatusCode::Internal,

src/mito2/src/worker.rs

@@ -25,8 +25,8 @@ mod handle_manifest;
 mod handle_open;
 mod handle_truncate;
 mod handle_write;
-
 use std::collections::HashMap;
+use std::path::Path;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
@@ -50,7 +50,7 @@ use crate::cache::write_cache::{WriteCache, WriteCacheRef};
 use crate::cache::{CacheManager, CacheManagerRef};
 use crate::compaction::CompactionScheduler;
 use crate::config::MitoConfig;
-use crate::error::{JoinSnafu, Result, WorkerStoppedSnafu};
+use crate::error::{CreateDirSnafu, JoinSnafu, Result, WorkerStoppedSnafu};
 use crate::flush::{FlushScheduler, WriteBufferManagerImpl, WriteBufferManagerRef};
 use crate::memtable::MemtableBuilderProvider;
 use crate::metrics::{REGION_COUNT, WRITE_STALL_TOTAL};
@@ -373,6 +373,12 @@ async fn write_cache_from_config(
     // TODO(yingwen): Remove this and document the config once the write cache is ready.
     warn!("Write cache is an experimental feature");
 
+    tokio::fs::create_dir_all(Path::new(&config.experimental_write_cache_path))
+        .await
+        .context(CreateDirSnafu {
+            dir: &config.experimental_write_cache_path,
+        })?;
+
     let cache = WriteCache::new_fs(
         &config.experimental_write_cache_path,
         object_store_manager,

src/object-store/src/lib.rs

@@ -24,3 +24,5 @@ pub mod manager;
 mod metrics;
 pub mod test_util;
 pub mod util;
+/// The default object cache directory name.
+pub const OBJECT_CACHE_DIR: &str = "object_cache";

tests-integration/src/test_util.rs

@@ -233,6 +233,9 @@ pub fn get_test_store_config(store_type: &StorageType) -> (ObjectStoreConfig, Te
 
             if *store_type == StorageType::S3WithCache {
                 s3_config.cache.cache_path = Some("/tmp/greptimedb_cache".to_string());
+            } else {
+                // An empty string means disabling.
+                s3_config.cache.cache_path = Some("".to_string());
             }
 
             let mut builder = S3::default()

tests-integration/tests/http.rs

@@ -917,7 +917,7 @@ compress_manifest = false
 auto_flush_interval = "30m"
 enable_experimental_write_cache = false
 experimental_write_cache_path = ""
-experimental_write_cache_size = "1GiB"
+experimental_write_cache_size = "5GiB"
 sst_write_buffer_size = "8MiB"
 parallel_scan_channel_size = 32
 allow_stale_entries = false

tests-integration/tests/main.rs

@@ -22,8 +22,9 @@ mod sql;
 mod region_migration;
 
 grpc_tests!(File, S3, S3WithCache, Oss, Azblob, Gcs);
+
 http_tests!(File, S3, S3WithCache, Oss, Azblob, Gcs);
-// region_failover_tests!(File, S3, S3WithCache, Oss, Azblob);
+
 sql_tests!(File);
 
 region_migration_tests!(File);