feat: Exclude _all field for select * (#3808)

src/config/src/config.rs

@@ -512,6 +512,9 @@ pub struct Common {
     pub wal_line_mode_enabled: bool,
     #[env_config(name = "ZO_COLUMN_TIMESTAMP", default = "_timestamp")]
     pub column_timestamp: String,
+    // TODO: should rename to column_all
+    #[env_config(name = "ZO_CONCATENATED_SCHEMA_FIELD_NAME", default = "_all")]
+    pub column_all: String,
     #[env_config(name = "ZO_WIDENING_SCHEMA_EVOLUTION", default = true)]
     pub widening_schema_evolution: bool,
     #[env_config(name = "ZO_SKIP_SCHEMA_VALIDATION", default = false)]
@@ -530,6 +533,8 @@ pub struct Common {
     pub feature_query_partition_strategy: String,
     #[env_config(name = "ZO_FEATURE_QUERY_INFER_SCHEMA", default = false)]
     pub feature_query_infer_schema: bool,
+    #[env_config(name = "ZO_FEATURE_QUERY_EXCLUDE_ALL", default = true)]
+    pub feature_query_exclude_all: bool,
     #[env_config(name = "ZO_UI_ENABLED", default = true)]
     pub ui_enabled: bool,
     #[env_config(name = "ZO_UI_SQL_BASE64_ENABLED", default = false)]
@@ -678,8 +683,6 @@ pub struct Common {
     pub report_server_url: String,
     #[env_config(name = "ZO_REPORT_SERVER_SKIP_TLS_VERIFY", default = false)]
     pub report_server_skip_tls_verify: bool,
-    #[env_config(name = "ZO_CONCATENATED_SCHEMA_FIELD_NAME", default = "_all")]
-    pub all_fields_name: String,
     #[env_config(name = "ZO_SCHEMA_CACHE_COMPRESS_ENABLED", default = false)]
     pub schema_cache_compress_enabled: bool,
     #[env_config(name = "ZO_SKIP_FORMAT_BULK_STREAM_NAME", default = false)]

src/handler/http/request/status/mod.rs

@@ -267,7 +267,7 @@ pub async fn zo_config() -> Result<HttpResponse, Error> {
         meta_org: cfg.common.usage_org.to_string(),
         quick_mode_enabled: cfg.limit.quick_mode_enabled,
         user_defined_schemas_enabled: cfg.common.allow_user_defined_schemas,
-        all_fields_name: cfg.common.all_fields_name.to_string(),
+        all_fields_name: cfg.common.column_all.to_string(),
     }))
 }
 

src/service/compact/file_list_deleted.rs

@@ -58,7 +58,7 @@ pub async fn delete(
             if file.1 {
                 Some(format!(
                     "files{}/{}",
-                    config::get_config().common.all_fields_name,
+                    config::get_config().common.column_all,
                     file.0.strip_prefix("files/").unwrap()
                 ))
             } else {

src/service/compact/flatten.rs

@@ -188,7 +188,7 @@ pub async fn generate_file(file: &FileKey) -> Result<(), anyhow::Error> {
     let full_text_search_fields = stream_setting.full_text_search_keys;
     let new_file = format!(
         "files{}/{}",
-        get_config().common.all_fields_name,
+        get_config().common.column_all,
         file.key.strip_prefix("files/").unwrap()
     );
     let new_schema = new_batches.first().unwrap().schema();
@@ -227,7 +227,7 @@ fn generate_vertical_partition_recordbatch(
     if records_len == 0 {
         return Ok(Vec::new());
     }
-    let Ok(all_field_idx) = schema.index_of(&get_config().common.all_fields_name) else {
+    let Ok(all_field_idx) = schema.index_of(&get_config().common.column_all) else {
         return Ok(vec![batches]);
     };
 

src/service/logs/mod.rs

@@ -476,7 +476,7 @@ pub fn refactor_map(
 
     if has_elements {
         new_map.insert(
-            get_config().common.all_fields_name.to_string(),
+            get_config().common.column_all.to_string(),
             Value::String(String::from_utf8(non_schema_map).unwrap()),
         );
     }

src/service/schema.rs

@@ -317,8 +317,8 @@ pub fn generate_schema_for_defined_schema_fields(
     if !fields.contains(&cfg.common.column_timestamp) {
         fields.insert(&cfg.common.column_timestamp);
     }
-    if !fields.contains(&cfg.common.all_fields_name) {
-        fields.insert(&cfg.common.all_fields_name);
+    if !fields.contains(&cfg.common.column_all) {
+        fields.insert(&cfg.common.column_all);
     }
     let mut new_fields = Vec::with_capacity(fields.len());
     for field in fields {

src/service/search/grpc/mod.rs

@@ -430,8 +430,8 @@ fn generate_select_start_search_schema(
         if !fields.contains(&cfg.common.column_timestamp) {
             fields.insert(cfg.common.column_timestamp.to_string());
         }
-        if !fields.contains(&cfg.common.all_fields_name) {
-            fields.insert(cfg.common.all_fields_name.to_string());
+        if !cfg.common.feature_query_exclude_all && !fields.contains(&cfg.common.column_all) {
+            fields.insert(cfg.common.column_all.to_string());
         }
         let new_fields = fields
             .iter()