chore: add http metrics server in datanode node when greptime start in distributed mode (#1256)

* chore: add http metrics server in datanode node when greptime start in distributed mode

* chore: add some docs and license

* chore: change metrics_addr to resolve address already in use error

* chore add metrics for meta service

* chore: replace metrics exporter http server from hyper to axum

* chore: format

* fix: datanode mode branching error

* fix: sqlness test address already in use and start metrics in defualt config

* chore: change metrics location

* chore: use builder pattern to builder httpserver

* chore: remove useless debug_assert macro in httpserver builder

* chore: resolve conflicting build error

* chore: format code

src/datanode/src/datanode.rs

@@ -19,6 +19,7 @@ use common_base::readable_size::ReadableSize;
 use common_telemetry::info;
 use meta_client::MetaClientOptions;
 use serde::{Deserialize, Serialize};
+use servers::http::HttpOptions;
 use servers::Mode;
 use storage::config::EngineConfig as StorageEngineConfig;
 use storage::scheduler::SchedulerConfig;
@@ -224,6 +225,7 @@ pub struct DatanodeOptions {
     pub rpc_runtime_size: usize,
     pub mysql_addr: String,
     pub mysql_runtime_size: usize,
+    pub http_opts: HttpOptions,
     pub meta_client_options: Option<MetaClientOptions>,
     pub wal: WalConfig,
     pub storage: StorageConfig,
@@ -241,6 +243,7 @@ impl Default for DatanodeOptions {
             rpc_runtime_size: 8,
             mysql_addr: "127.0.0.1:4406".to_string(),
             mysql_runtime_size: 2,
+            http_opts: HttpOptions::default(),
             meta_client_options: None,
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
@@ -252,14 +255,17 @@ impl Default for DatanodeOptions {
 /// Datanode service.
 pub struct Datanode {
     opts: DatanodeOptions,
-    services: Services,
+    services: Option<Services>,
     instance: InstanceRef,
 }
 
 impl Datanode {
     pub async fn new(opts: DatanodeOptions) -> Result<Datanode> {
         let instance = Arc::new(Instance::new(&opts).await?);
-        let services = Services::try_new(instance.clone(), &opts).await?;
+        let services = match opts.mode {
+            Mode::Distributed => Some(Services::try_new(instance.clone(), &opts).await?),
+            Mode::Standalone => None,
+        };
         Ok(Self {
             opts,
             services,
@@ -280,7 +286,11 @@ impl Datanode {
 
     /// Start services of datanode. This method call will block until services are shutdown.
     pub async fn start_services(&mut self) -> Result<()> {
-        self.services.start(&self.opts).await
+        if let Some(service) = self.services.as_mut() {
+            service.start(&self.opts).await
+        } else {
+            Ok(())
+        }
     }
 
     pub fn get_instance(&self) -> InstanceRef {
@@ -292,7 +302,11 @@ impl Datanode {
     }
 
     async fn shutdown_services(&self) -> Result<()> {
-        self.services.shutdown().await
+        if let Some(service) = self.services.as_ref() {
+            service.shutdown().await
+        } else {
+            Ok(())
+        }
     }
 
     pub async fn shutdown(&self) -> Result<()> {

src/datanode/src/instance/script.rs

@@ -20,7 +20,7 @@ use common_telemetry::timer;
 use servers::query_handler::ScriptHandler;
 
 use crate::instance::Instance;
-use crate::metric;
+use crate::metrics;
 
 #[async_trait]
 impl ScriptHandler for Instance {
@@ -30,7 +30,7 @@ impl ScriptHandler for Instance {
         name: &str,
         script: &str,
     ) -> servers::error::Result<()> {
-        let _timer = timer!(metric::METRIC_HANDLE_SCRIPTS_ELAPSED);
+        let _timer = timer!(metrics::METRIC_HANDLE_SCRIPTS_ELAPSED);
         self.script_executor
             .insert_script(schema, name, script)
             .await
@@ -42,7 +42,7 @@ impl ScriptHandler for Instance {
         name: &str,
         params: HashMap<String, String>,
     ) -> servers::error::Result<Output> {
-        let _timer = timer!(metric::METRIC_RUN_SCRIPT_ELAPSED);
+        let _timer = timer!(metrics::METRIC_RUN_SCRIPT_ELAPSED);
         self.script_executor
             .execute_script(schema, name, params)
             .await

src/datanode/src/instance/sql.rs

@@ -37,7 +37,7 @@ use crate::error::{
     TableIdProviderNotFoundSnafu,
 };
 use crate::instance::Instance;
-use crate::metric;
+use crate::metrics;
 use crate::sql::{SqlHandler, SqlRequest};
 
 impl Instance {
@@ -190,7 +190,7 @@ impl Instance {
         promql: &PromQuery,
         query_ctx: QueryContextRef,
     ) -> Result<Output> {
-        let _timer = timer!(metric::METRIC_HANDLE_PROMQL_ELAPSED);
+        let _timer = timer!(metrics::METRIC_HANDLE_PROMQL_ELAPSED);
 
         let stmt = QueryLanguageParser::parse_promql(promql).context(ExecuteSqlSnafu)?;
 
@@ -294,7 +294,7 @@ impl StatementHandler for Instance {
 #[async_trait]
 impl PromHandler for Instance {
     async fn do_query(&self, query: &PromQuery) -> server_error::Result<Output> {
-        let _timer = timer!(metric::METRIC_HANDLE_PROMQL_ELAPSED);
+        let _timer = timer!(metrics::METRIC_HANDLE_PROMQL_ELAPSED);
 
         self.execute_promql(query, QueryContext::arc())
             .await

src/datanode/src/lib.rs

@@ -19,7 +19,7 @@ pub mod datanode;
 pub mod error;
 mod heartbeat;
 pub mod instance;
-pub mod metric;
+pub mod metrics;
 mod mock;
 mod script;
 pub mod server;

src/datanode/src/server.rs

@@ -18,9 +18,12 @@ use std::sync::Arc;
 
 use common_runtime::Builder as RuntimeBuilder;
 use servers::grpc::GrpcServer;
+use servers::http::{HttpServer, HttpServerBuilder};
+use servers::metrics_handler::MetricsHandler;
 use servers::query_handler::grpc::ServerGrpcQueryHandlerAdaptor;
 use servers::server::Server;
 use snafu::ResultExt;
+use tokio::select;
 
 use crate::datanode::DatanodeOptions;
 use crate::error::{
@@ -33,6 +36,7 @@ pub mod grpc;
 /// All rpc services.
 pub struct Services {
     grpc_server: GrpcServer,
+    http_server: HttpServer,
 }
 
 impl Services {
@@ -51,6 +55,9 @@ impl Services {
                 None,
                 grpc_runtime,
             ),
+            http_server: HttpServerBuilder::new(opts.http_opts.clone())
+                .with_metrics_handler(MetricsHandler)
+                .build(),
         })
     }
 
@@ -58,10 +65,15 @@ impl Services {
         let grpc_addr: SocketAddr = opts.rpc_addr.parse().context(ParseAddrSnafu {
             addr: &opts.rpc_addr,
         })?;
-        self.grpc_server
-            .start(grpc_addr)
-            .await
-            .context(StartServerSnafu)?;
+        let http_addr = opts.http_opts.addr.parse().context(ParseAddrSnafu {
+            addr: &opts.http_opts.addr,
+        })?;
+        let grpc = self.grpc_server.start(grpc_addr);
+        let http = self.http_server.start(http_addr);
+        select!(
+            v = grpc => v.context(StartServerSnafu)?,
+            v = http => v.context(StartServerSnafu)?,
+        );
         Ok(())
     }
 
@@ -69,6 +81,11 @@ impl Services {
         self.grpc_server
             .shutdown()
             .await
-            .context(ShutdownServerSnafu)
+            .context(ShutdownServerSnafu)?;
+        self.http_server
+            .shutdown()
+            .await
+            .context(ShutdownServerSnafu)?;
+        Ok(())
     }
 }