perf: improve search limit logic (#3638)

- [x] If the search request size is 0 and in full sql mod and track
total hits, will not apply default limit `1000`.
- [x] change consistent hashing vnode default value to `16`

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Improvements**
- Enhanced performance and accuracy of data aggregation by updating
internal logic and default values.
- Improved configuration flexibility by allowing custom `history`
values.

These changes aim to provide a more robust and configurable experience
for end-users.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
Hengfei Yang
2024-05-31 10:56:50 +05:30
committed by GitHub
parent 05d0d23fbe
commit 5e2ea1aaed
4 changed files with 6 additions and 4 deletions
+1 -1
View File
@@ -146,7 +146,7 @@ pub enum AggregationFunc {
P50,
P90,
P95,
P99
P99,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
+3 -1
View File
@@ -784,7 +784,7 @@ pub struct Limit {
pub distinct_values_interval: u64,
#[env_config(name = "ZO_DISTINCT_VALUES_HOURLY", default = false)]
pub distinct_values_hourly: bool,
#[env_config(name = "ZO_CONSISTENT_HASH_VNODES", default = 3)]
#[env_config(name = "ZO_CONSISTENT_HASH_VNODES", default = 16)]
pub consistent_hash_vnodes: usize,
#[env_config(name = "ZO_DATAFUSION_FILE_STAT_CACHE_MAX_ENTRIES", default = 100000)]
pub datafusion_file_stat_cache_max_entries: usize,
@@ -949,6 +949,8 @@ pub struct Nats {
pub password: String,
#[env_config(name = "ZO_NATS_REPLICAS", default = 3)]
pub replicas: usize,
#[env_config(name = "ZO_NATS_HISTORY", default = 3)]
pub history: i64,
#[env_config(name = "ZO_NATS_CONNECT_TIMEOUT", default = 5)]
pub connect_timeout: u64,
#[env_config(name = "ZO_NATS_COMMAND_TIMEOUT", default = 10)]
+1 -1
View File
@@ -58,7 +58,7 @@ async fn get_bucket_by_key<'a>(
let mut bucket = jetstream::kv::Config {
bucket: format!("{}{}", prefix, bucket_name),
num_replicas: cfg.nats.replicas,
history: 3,
history: cfg.nats.history,
..Default::default()
};
if bucket_name == "nodes" || bucket_name == "clusters" {
+1 -1
View File
@@ -339,7 +339,7 @@ impl Sql {
if meta.limit == 0 {
meta.offset = req_query.from as usize;
meta.limit = req_query.size as usize;
if meta.limit == 0 && sql_mode.eq(&SqlMode::Full) {
if meta.limit == 0 && sql_mode.eq(&SqlMode::Full) && !track_total_hits {
// sql mode context, allow limit 0, used to no hits, but return aggs
// sql mode full, disallow without limit, default limit 1000
meta.limit = cfg.limit.query_full_mode_limit;