starred/openobserve
1
0
Fork 0
mirror of https://github.com/openobserve/openobserve.git synced 2026-11-06 00:09:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: increase datafusion target partitions (#3797)

Browse Source
This commit is contained in:
Hengfei Yang
2024-06-24 01:07:10 +08:00
committed by GitHub
parent bde4c8a952
commit d0a614702e
1 changed files with 2 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/service/search/datafusion/exec.rs
+2 -3
View File
@@ -1071,8 +1071,10 @@ pub async fn merge_parquet_files(
}
pub fn create_session_config(search_type: &SearchType) -> Result<SessionConfig> {
let cfg = get_config();
let mut config = SessionConfig::from_env()?
.with_batch_size(PARQUET_BATCH_SIZE)
.with_target_partitions(cfg.limit.query_thread_num)
.with_information_schema(true);
config = config.set_bool(
"datafusion.execution.listing_table_ignore_subdirectory",
@@ -1082,7 +1084,6 @@ pub fn create_session_config(search_type: &SearchType) -> Result<SessionConfig>
config = config.set_bool("datafusion.execution.parquet.pushdown_filters", true);
config = config.set_bool("datafusion.execution.parquet.reorder_filters", true);
}
let cfg = get_config();
if cfg.common.bloom_filter_enabled {
config = config.set_bool("datafusion.execution.parquet.bloom_filter_on_read", true);
}
@@ -1219,14 +1220,12 @@ pub async fn register_table(
let file_format = ParquetFormat::default();
ListingOptions::new(Arc::new(file_format))
.with_file_extension(FileType::PARQUET.get_ext())
.with_target_partitions(cfg.limit.cpu_num)
.with_collect_stat(true)
}
FileType::JSON => {
let file_format = JsonFormat::default();
ListingOptions::new(Arc::new(file_format))
.with_file_extension(FileType::JSON.get_ext())
.with_target_partitions(cfg.limit.cpu_num)
.with_collect_stat(true)
}
_ => {