From 0751cd74c059c138cc8adb20448261872aa10ca0 Mon Sep 17 00:00:00 2001
From: discord9 <discord9@163.com>
Date: Sun, 9 Mar 2025 20:36:10 +0800
Subject: [PATCH] feat: all in one cfg

---
 chore.md                 | 32 ++++++++++++++++++++++++++++++++
 ingester.toml            | 35 +++++++++++++++++++++++++++++++++++
 src/ingester/src/main.rs | 32 +++++++++++++++-----------------
 3 files changed, 82 insertions(+), 17 deletions(-)
 create mode 100644 chore.md
 create mode 100644 ingester.toml

diff --git a/chore.md b/chore.md
new file mode 100644
index 0000000000..8880ad57bb
--- /dev/null
+++ b/chore.md
@@ -0,0 +1,32 @@
+# first create table
+```bash
+mysql --host=127.0.0.1 --port=19195 --database=public;
+```
+
+```sql
+CREATE TABLE IF NOT EXISTS `public`.`logsbench` (
+  `greptime_timestamp` TimestampNanosecond NOT NULL TIME INDEX,
+  `app` STRING NULL INVERTED INDEX,
+  `cluster` STRING NULL INVERTED INDEX,
+  `message` STRING NULL,
+  `region` STRING NULL,
+  `cloud-provider` STRING NULL,
+  `environment` STRING NULL,
+  `product` STRING NULL,
+  `sub-product` STRING NULL,
+  `service` STRING NULL
+) WITH (
+  append_mode = 'true',
+  'compaction.type' = 'twcs',                        
+  'compaction.twcs.max_output_file_size' = '500MB',  
+  'compaction.twcs.max_active_window_files' = '16',   
+  'compaction.twcs.max_active_window_runs' = '4',    
+  'compaction.twcs.max_inactive_window_files' = '4', 
+  'compaction.twcs.max_inactive_window_runs' = '2',  
+);
+```
+
+# then ingest
+```bash
+cargo run --bin=ingester -- --input-dir="/home/discord9/greptimedb/parquet_store" --parquet-dir="." --cfg="ingester.toml" --db-http-addr="http://127.0.0.1:4000"
+```
\ No newline at end of file
diff --git a/ingester.toml b/ingester.toml
new file mode 100644
index 0000000000..4998387b07
--- /dev/null
+++ b/ingester.toml
@@ -0,0 +1,35 @@
+## The metasrv client options.
+[meta_client]
+## The addresses of the metasrv.
+metasrv_addrs = ["127.0.0.1:3002", "127.0.0.1:3003"]
+
+## Operation timeout.
+timeout = "3s"
+
+## Heartbeat timeout.
+heartbeat_timeout = "500ms"
+
+## DDL timeout.
+ddl_timeout = "10s"
+
+## Connect server timeout.
+connect_timeout = "1s"
+
+## `TCP_NODELAY` option for accepted connections.
+tcp_nodelay = true
+
+## The configuration about the cache of the metadata.
+metadata_cache_max_capacity = 100000
+
+## TTL of the metadata cache.
+metadata_cache_ttl = "10m"
+
+# TTI of the metadata cache.
+metadata_cache_tti = "5m"
+
+## The data storage options.
+[storage]
+## The working home directory.
+data_home = "/tmp/greptimedb/"
+type = "File"
+[mito]
\ No newline at end of file
diff --git a/src/ingester/src/main.rs b/src/ingester/src/main.rs
index 7952ffc147..7cd24cc17a 100644
--- a/src/ingester/src/main.rs
+++ b/src/ingester/src/main.rs
@@ -16,6 +16,7 @@ use clap::Parser;
 use common_time::timestamp::TimeUnit;
 use datanode::config::StorageConfig;
 use meta_client::MetaClientOptions;
+use mito2::config::MitoConfig;
 use mito2::sst::file::IndexType;
 use mito2::sst::parquet::SstInfo;
 use serde::{Deserialize, Serialize};
@@ -33,12 +34,9 @@ struct Args {
     /// Directory of input json files, relative to input_dir
     #[arg(short, long)]
     json_dir: Option<String>,
-    /// Output storage config file
+    /// Config file
     #[arg(short, long)]
-    output_storage_config_file: String,
-    /// Meta client config file
-    #[arg(short, long)]
-    meta_client_config_file: String,
+    cfg: String,
     /// DB HTTP address
     #[arg(short, long)]
     db_http_addr: String,
@@ -51,26 +49,26 @@ struct Args {
     sst_output_path: Option<String>,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+struct IngesterConfig {
+    meta_client: MetaClientOptions,
+    storage: StorageConfig,
+    mito: MitoConfig,
+}
+
 #[allow(unreachable_code)]
 #[tokio::main]
 async fn main() {
     let args = Args::parse();
 
-    let meta_client_config = std::fs::read_to_string(&args.meta_client_config_file)
-        .expect("Failed to read meta client config file");
-    let meta_options: MetaClientOptions =
-        toml::from_str(&meta_client_config).expect("Failed to parse meta client config");
+    let cfg_file = std::fs::read_to_string(&args.cfg).expect("Failed to read config file");
+    let cfg: IngesterConfig = toml::from_str(&cfg_file).expect("Failed to parse config");
 
-    let storage_config = std::fs::read_to_string(&args.output_storage_config_file)
-        .expect("Failed to read storage config file");
-    let storage_config: StorageConfig =
-        toml::from_str(&storage_config).expect("Failed to parse storage config");
-
-    // TODO: build sst converter
     let mut sst_converter = {
         let mut builder = SstConverterBuilder::new_fs(args.input_dir)
-            .with_meta_options(meta_options)
-            .with_storage_config(storage_config);
+            .with_meta_options(cfg.meta_client)
+            .with_storage_config(cfg.storage)
+            .with_config(cfg.mito);
 
         if let Some(output_path) = args.sst_output_path {
             builder = builder.with_output_path(output_path);