From 9d179802b8ca52ec78cfcbc0c96dd32b499d19b1 Mon Sep 17 00:00:00 2001
From: Yingwen <realevenyag@gmail.com>
Date: Wed, 31 May 2023 15:36:25 +0800
Subject: [PATCH] feat: Add a global TTL option for all tables (#1679)

* feat: Add a global TTL option for all tables

* docs: update config examples

* chore: print start command and options when standalone/frontend starts
---
 config/datanode.example.toml   | 2 ++
 config/standalone.example.toml | 2 ++
 src/cmd/src/frontend.rs        | 4 ++++
 src/cmd/src/standalone.rs      | 1 +
 src/datanode/src/datanode.rs   | 8 ++++++++
 src/storage/src/config.rs      | 5 +++++
 src/storage/src/engine.rs      | 5 ++++-
 7 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 047201b1e4..4e610dc866 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -36,6 +36,8 @@ sync_write = false
 [storage]
 type = "File"
 data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index ec7bf77f78..1f39c6b4bc 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -97,6 +97,8 @@ sync_write = false
 type = "File"
 # Data directory, "/tmp/greptimedb/data" by default.
 data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options.
 [storage.compaction]
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 22555e9aa1..239ed68906 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 
 use clap::Parser;
 use common_base::Plugins;
+use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{InfluxdbOptions, PromOptions};
@@ -202,6 +203,9 @@ impl StartCommand {
     }
 
     async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+        logging::info!("Frontend start command: {:#?}", self);
+        logging::info!("Frontend options: {:#?}", opts);
+
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
 
         let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 93da9d6140..6e307aaad9 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -302,6 +302,7 @@ impl StartCommand {
     async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
 
+        info!("Standalone start command: {:#?}", self);
         info!(
             "Standalone frontend options: {:#?}, datanode options: {:#?}",
             fe_opts, dn_opts
diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
index 664f13aead..f7458be5a1 100644
--- a/src/datanode/src/datanode.rs
+++ b/src/datanode/src/datanode.rs
@@ -54,6 +54,13 @@ pub enum ObjectStoreConfig {
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 #[serde(default)]
 pub struct StorageConfig {
+    /// Retention period for all tables.
+    ///
+    /// Default value is `None`, which means no TTL.
+    ///
+    /// The precedence order is: ttl in table options > global ttl.
+    #[serde(with = "humantime_serde")]
+    pub global_ttl: Option<Duration>,
     #[serde(flatten)]
     pub store: ObjectStoreConfig,
     pub compaction: CompactionConfig,
@@ -300,6 +307,7 @@ impl From<&DatanodeOptions> for StorageEngineConfig {
             picker_schedule_interval: value.storage.flush.picker_schedule_interval,
             auto_flush_interval: value.storage.flush.auto_flush_interval,
             global_write_buffer_size: value.storage.flush.global_write_buffer_size,
+            global_ttl: value.storage.global_ttl,
         }
     }
 }
diff --git a/src/storage/src/config.rs b/src/storage/src/config.rs
index 324d45b0b6..3ec2c6c2e0 100644
--- a/src/storage/src/config.rs
+++ b/src/storage/src/config.rs
@@ -46,6 +46,10 @@ pub struct EngineConfig {
     pub auto_flush_interval: Duration,
     /// Limit for global write buffer size. Disabled by default.
     pub global_write_buffer_size: Option<ReadableSize>,
+    /// Global retention period for all regions.
+    ///
+    /// The precedence order is: region ttl > global ttl.
+    pub global_ttl: Option<Duration>,
 }
 
 impl Default for EngineConfig {
@@ -65,6 +69,7 @@ impl Default for EngineConfig {
             ),
             auto_flush_interval: Duration::from_millis(DEFAULT_AUTO_FLUSH_INTERVAL.into()),
             global_write_buffer_size: None,
+            global_ttl: None,
         }
     }
 }
diff --git a/src/storage/src/engine.rs b/src/storage/src/engine.rs
index 825b12cb48..7234ce92ac 100644
--- a/src/storage/src/engine.rs
+++ b/src/storage/src/engine.rs
@@ -465,7 +465,7 @@ impl<S: LogStore> EngineInner<S> {
         write_buffer_size: Option<usize>,
         region_name: &str,
         config: &EngineConfig,
-        ttl: Option<Duration>,
+        region_ttl: Option<Duration>,
         compaction_time_window: Option<i64>,
     ) -> Result<StoreConfig<S>> {
         let parent_dir = util::normalize_dir(parent_dir);
@@ -483,6 +483,9 @@ impl<S: LogStore> EngineInner<S> {
         manifest.start().await?;
         let flush_strategy = self.flush_strategy.clone();
 
+        // If region_ttl is `None`, the global ttl takes effect.
+        let ttl = region_ttl.or(self.config.global_ttl);
+
         Ok(StoreConfig {
             log_store: self.log_store.clone(),
             sst_layer,