feat(http_body_limit): add initial support for DefaultBodyLimit (#1860)

* feat(http_body_limit): add initial support for DefaultBodyLimit

* fix: address CR suggestions

* fix: adjust the const for default http body limit

* fix: adjust the toml_str for the test

* fix: address CR suggestions

* fix: body_limit units in example config toml files

* fix: address clippy suggestions

config/frontend.example.toml

@@ -9,6 +9,7 @@ retry_interval_millis = 5000
 [http_options]
 addr = "127.0.0.1:4000"
 timeout = "30s"
+body_limit = "64MB"
 
 # gRPC server options, see `standalone.example.toml`.
 [grpc_options]

config/standalone.example.toml

@@ -9,6 +9,9 @@ enable_memory_catalog = false
 addr = "127.0.0.1:4000"
 # HTTP request timeout, 30s by default.
 timeout = "30s"
+# HTTP request body limit, 64Mb by default.
+# the following units are supported: B, KB, KiB, MB, MiB, GB, GiB, TB, TiB, PB, PiB
+body_limit = "64MB"
 
 # gRPC server options.
 [grpc_options]

src/cmd/src/frontend.rs

@@ -236,6 +236,7 @@ mod tests {
     use std::io::Write;
     use std::time::Duration;
 
+    use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use frontend::service_config::GrpcOptions;
     use servers::auth::{Identity, Password, UserProviderRef};
@@ -260,6 +261,10 @@ mod tests {
             command.load_options(TopLevelOptions::default()).unwrap() else { unreachable!() };
 
         assert_eq!(opts.http_options.as_ref().unwrap().addr, "127.0.0.1:1234");
+        assert_eq!(
+            ReadableSize::mb(64),
+            opts.http_options.as_ref().unwrap().body_limit
+        );
         assert_eq!(opts.mysql_options.as_ref().unwrap().addr, "127.0.0.1:5678");
         assert_eq!(
             opts.postgres_options.as_ref().unwrap().addr,
@@ -301,6 +306,7 @@ mod tests {
             [http_options]
             addr = "127.0.0.1:4000"
             timeout = "30s"
+            body_limit = "2GB"
 
             [logging]
             level = "debug"
@@ -326,6 +332,11 @@ mod tests {
             fe_opts.http_options.as_ref().unwrap().timeout
         );
 
+        assert_eq!(
+            ReadableSize::gb(2),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
+
         assert_eq!("debug", fe_opts.logging.level.as_ref().unwrap());
         assert_eq!("/tmp/greptimedb/test/logs".to_string(), fe_opts.logging.dir);
     }

src/cmd/src/standalone.rs

@@ -342,6 +342,7 @@ mod tests {
     use std::io::Write;
     use std::time::Duration;
 
+    use common_base::readable_size::ReadableSize;
     use common_test_util::temp_dir::create_named_temp_file;
     use servers::auth::{Identity, Password, UserProviderRef};
     use servers::Mode;
@@ -409,6 +410,7 @@ mod tests {
             [http_options]
             addr = "127.0.0.1:4000"
             timeout = "30s"
+            body_limit = "128MB"
 
             [logging]
             level = "debug"
@@ -434,6 +436,10 @@ mod tests {
             Duration::from_secs(30),
             fe_opts.http_options.as_ref().unwrap().timeout
         );
+        assert_eq!(
+            ReadableSize::mb(128),
+            fe_opts.http_options.as_ref().unwrap().body_limit
+        );
         assert_eq!(
             "127.0.0.1:4001".to_string(),
             fe_opts.grpc_options.unwrap().addr
@@ -560,6 +566,10 @@ mod tests {
                     opts.fe_opts.http_options.as_ref().unwrap().addr,
                     "127.0.0.1:14000"
                 );
+                assert_eq!(
+                    ReadableSize::mb(64),
+                    opts.fe_opts.http_options.as_ref().unwrap().body_limit
+                );
 
                 // Should be default value.
                 assert_eq!(

src/servers/src/http.rs

@@ -39,6 +39,7 @@ use axum::http::Request;
 use axum::middleware::{self, Next};
 use axum::response::{Html, IntoResponse, Json};
 use axum::{routing, BoxError, Extension, Router};
+use common_base::readable_size::ReadableSize;
 use common_error::prelude::ErrorExt;
 use common_error::status_code::StatusCode;
 use common_query::Output;
@@ -105,7 +106,7 @@ pub(crate) async fn query_context_from_db(
 pub const HTTP_API_VERSION: &str = "v1";
 pub const HTTP_API_PREFIX: &str = "/v1/";
 /// Default http body limit (64M).
-const DEFAULT_BODY_LIMIT: usize = 64 * 1024 * 1024;
+const DEFAULT_BODY_LIMIT: ReadableSize = ReadableSize::mb(64);
 
 // TODO(fys): This is a temporary workaround, it will be improved later
 pub static PUBLIC_APIS: [&str; 2] = ["/v1/influxdb/ping", "/v1/influxdb/health"];
@@ -135,6 +136,8 @@ pub struct HttpOptions {
 
     #[serde(skip)]
     pub disable_dashboard: bool,
+
+    pub body_limit: ReadableSize,
 }
 
 impl Default for HttpOptions {
@@ -143,6 +146,7 @@ impl Default for HttpOptions {
             addr: "127.0.0.1:4000".to_string(),
             timeout: Duration::from_secs(30),
             disable_dashboard: false,
+            body_limit: DEFAULT_BODY_LIMIT,
         }
     }
 }
@@ -546,7 +550,13 @@ impl HttpServer {
                     .layer(HandleErrorLayer::new(handle_error))
                     .layer(TraceLayer::new_for_http())
                     .layer(TimeoutLayer::new(self.options.timeout))
-                    .layer(DefaultBodyLimit::max(DEFAULT_BODY_LIMIT))
+                    .layer(DefaultBodyLimit::max(
+                        self.options
+                            .body_limit
+                            .0
+                            .try_into()
+                            .unwrap_or_else(|_| DEFAULT_BODY_LIMIT.as_bytes() as usize),
+                    ))
                     // custom layer
                     .layer(AsyncRequireAuthorizationLayer::new(
                         HttpAuth::<BoxBody>::new(self.user_provider.clone()),