fix: do not expose manifest compression algorithm (#2835)

* fix: don't allow to set manifest compression algorithm

* docs: update config examples

config/datanode.example.toml

@@ -64,8 +64,8 @@ worker_channel_size = 128
 worker_request_batch_size = 64
 # Number of meta action updated to trigger a new checkpoint for the manifest
 manifest_checkpoint_distance = 10
-# Manifest compression type
-manifest_compress_type = "uncompressed"
+# Whether to compress manifest and checkpoint file by gzip (default false).
+compress_manifest = false
 # Max number of running background jobs
 max_background_jobs = 4
 # Interval to auto flush a region if it has not flushed yet.

config/standalone.example.toml

@@ -133,8 +133,8 @@ worker_channel_size = 128
 worker_request_batch_size = 64
 # Number of meta action updated to trigger a new checkpoint for the manifest
 manifest_checkpoint_distance = 10
-# Manifest compression type
-manifest_compress_type = "uncompressed"
+# Whether to compress manifest and checkpoint file by gzip (default false).
+compress_manifest = false
 # Max number of running background jobs
 max_background_jobs = 4
 # Interval to auto flush a region if it has not flushed yet.

src/mito2/src/config.rs

@@ -17,7 +17,6 @@
 use std::time::Duration;
 
 use common_base::readable_size::ReadableSize;
-use common_datasource::compression::CompressionType;
 use common_telemetry::warn;
 use serde::{Deserialize, Serialize};
 
@@ -42,8 +41,8 @@ pub struct MitoConfig {
     /// Number of meta action updated to trigger a new checkpoint
     /// for the manifest (default 10).
     pub manifest_checkpoint_distance: u64,
-    /// Manifest compression type (default uncompressed).
-    pub manifest_compress_type: CompressionType,
+    /// Whether to compress manifest and checkpoint file by gzip (default false).
+    pub compress_manifest: bool,
 
     // Background job configs:
     /// Max number of running background jobs (default 4).
@@ -78,7 +77,7 @@ impl Default for MitoConfig {
             worker_channel_size: 128,
             worker_request_batch_size: 64,
             manifest_checkpoint_distance: 10,
-            manifest_compress_type: CompressionType::Uncompressed,
+            compress_manifest: false,
             max_background_jobs: DEFAULT_MAX_BG_JOB,
             auto_flush_interval: Duration::from_secs(30 * 60),
             global_write_buffer_size: ReadableSize::gb(1),

src/mito2/src/manifest/storage.rs

@@ -42,6 +42,7 @@ const DEFAULT_MANIFEST_COMPRESSION_TYPE: CompressionType = CompressionType::Gzip
 /// So when we encounter problems, we need to fall back to `FALL_BACK_COMPRESS_TYPE` for processing.
 const FALL_BACK_COMPRESS_TYPE: CompressionType = CompressionType::Uncompressed;
 
+/// Returns the [CompressionType] according to whether to compress manifest files.
 #[inline]
 pub const fn manifest_compress_type(compress: bool) -> CompressionType {
     if compress {

src/mito2/src/region/opener.rs

@@ -33,6 +33,7 @@ use crate::cache::CacheManagerRef;
 use crate::config::MitoConfig;
 use crate::error::{EmptyRegionDirSnafu, ObjectStoreNotFoundSnafu, RegionCorruptedSnafu, Result};
 use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions};
+use crate::manifest::storage::manifest_compress_type;
 use crate::memtable::MemtableBuilderRef;
 use crate::region::options::RegionOptions;
 use crate::region::version::{VersionBuilder, VersionControl, VersionControlRef};
@@ -259,7 +260,9 @@ impl RegionOpener {
         Ok(RegionManifestOptions {
             manifest_dir: new_manifest_dir(&self.region_dir),
             object_store,
-            compress_type: config.manifest_compress_type,
+            // We don't allow users to set the compression algorithm as we use it as a file suffix.
+            // Currently, the manifest storage doesn't have good support for changing compression algorithms.
+            compress_type: manifest_compress_type(config.compress_manifest),
             checkpoint_distance: config.manifest_checkpoint_distance,
         })
     }

tests-integration/tests/http.rs

@@ -720,7 +720,7 @@ num_workers = {}
 worker_channel_size = 128
 worker_request_batch_size = 64
 manifest_checkpoint_distance = 10
-manifest_compress_type = "uncompressed"
+compress_manifest = false
 max_background_jobs = 4
 auto_flush_interval = "30m"
 global_write_buffer_size = "1GiB"