diff --git a/Cargo.lock b/Cargo.lock
index be3454628c..4dd50297b7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2265,6 +2265,7 @@ dependencies = [
"actix-web-prometheus",
"ahash 0.8.11",
"anyhow",
+ "arc-swap",
"arrow",
"arrow-json",
"arrow-schema",
@@ -5328,7 +5329,7 @@ dependencies = [
"rayon",
"regex",
"regex-syntax 0.8.3",
- "reportserver",
+ "report_server",
"reqwest 0.12.4",
"rust-embed-for-web",
"segment",
@@ -6356,7 +6357,7 @@ dependencies = [
]
[[package]]
-name = "reportserver"
+name = "report_server"
version = "0.1.0"
dependencies = [
"actix-web",
diff --git a/Cargo.toml b/Cargo.toml
index b869fe12ed..b86677ec0e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -159,7 +159,7 @@ zstd.workspace = true
config.workspace = true
infra.workspace = true
ingester.workspace = true
-reportserver.workspace = true
+report_server.workspace = true
chromiumoxide.workspace = true
lettre.workspace = true
@@ -173,7 +173,7 @@ float-cmp = "0.9"
async-walkdir = "1.0.0"
[workspace]
-members = ["src/config", "src/infra", "src/ingester", "src/wal", "src/proto", "src/reportserver"]
+members = ["src/config", "src/infra", "src/ingester", "src/wal", "src/proto", "src/report_server"]
resolver = "2"
[workspace.package]
@@ -187,12 +187,13 @@ infra = { path = "src/infra" }
ingester = { path = "src/ingester" }
wal = { path = "src/wal" }
proto = { path = "src/proto" }
-reportserver = { path = "src/reportserver" }
+report_server = { path = "src/report_server" }
ahash = { version = "0.8", features = ["serde"] }
actix-web = "4.5"
actix-web-prometheus = { version = "0.1", features = ["process"] }
anyhow = "1.0"
+arc-swap = "1.7.1"
argon2 = { version = "0.5", features = ["alloc", "password-hash"] }
async-trait = "0.1"
async-recursion = "1.0"
diff --git a/build.rs b/build.rs
index 03d6e93940..4a0196ac0f 100644
--- a/build.rs
+++ b/build.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/cli/basic/cli.rs b/src/cli/basic/cli.rs
index 499e032edc..625417bd2d 100644
--- a/src/cli/basic/cli.rs
+++ b/src/cli/basic/cli.rs
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{utils::file::set_permission, CONFIG};
+use config::utils::file::set_permission;
use infra::file_list as infra_file_list;
use crate::{
@@ -115,7 +115,7 @@ pub async fn cli() -> Result {
if app.subcommand().is_none() {
return Ok(false);
}
-
+ let cfg = config::get_config();
#[cfg(not(feature = "tokio-console"))]
env_logger::init_from_env(env_logger::Env::new().default_filter_or("INFO"));
@@ -142,13 +142,13 @@ pub async fn cli() -> Result {
"root" => {
let _ = users::update_user(
meta::organization::DEFAULT_ORG,
- CONFIG.auth.root_user_email.as_str(),
+ cfg.auth.root_user_email.as_str(),
false,
- CONFIG.auth.root_user_email.as_str(),
+ cfg.auth.root_user_email.as_str(),
meta::user::UpdateUser {
change_password: true,
old_password: None,
- new_password: Some(CONFIG.auth.root_user_password.clone()),
+ new_password: Some(cfg.auth.root_user_password.clone()),
role: Some(meta::user::UserRole::Root),
first_name: Some("root".to_owned()),
last_name: Some("".to_owned()),
diff --git a/src/cli/basic/mod.rs b/src/cli/basic/mod.rs
index 2131ab39fd..c0ae357c90 100644
--- a/src/cli/basic/mod.rs
+++ b/src/cli/basic/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/cli/data/cli.rs b/src/cli/data/cli.rs
index 547e6c7e1d..53cfdbffca 100644
--- a/src/cli/data/cli.rs
+++ b/src/cli/data/cli.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/cli/data/export.rs b/src/cli/data/export.rs
index a83951c5d6..2422149d56 100644
--- a/src/cli/data/export.rs
+++ b/src/cli/data/export.rs
@@ -37,6 +37,7 @@ impl Context for Export {
Err(_) => return Ok(false),
};
+ let cfg = config::get_config();
let table = c.stream_name;
let query = search::Query {
sql: format!("select * from {}", table),
@@ -47,7 +48,7 @@ impl Context for Export {
query_type: "".to_owned(),
start_time: c.start_time,
end_time: c.end_time,
- sort_by: Some(format!("{} ASC", config::CONFIG.common.column_timestamp)),
+ sort_by: Some(format!("{} ASC", cfg.common.column_timestamp)),
track_total_hits: false,
query_context: None,
uses_zo_fn: false,
diff --git a/src/cli/mod.rs b/src/cli/mod.rs
index 6101b53b9c..40e03640e6 100644
--- a/src/cli/mod.rs
+++ b/src/cli/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/infra/cluster/etcd.rs b/src/common/infra/cluster/etcd.rs
index 131a519786..33704d1720 100644
--- a/src/common/infra/cluster/etcd.rs
+++ b/src/common/infra/cluster/etcd.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,9 +15,9 @@
use config::{
cluster::*,
+ get_config,
meta::cluster::{Node, NodeStatus, Role},
utils::json,
- CONFIG,
};
use etcd_client::PutOptions;
use infra::{
@@ -49,9 +49,12 @@ pub(crate) async fn register_and_keepalive() -> Result<()> {
}
let lease_id = unsafe { LOCAL_NODE_KEY_LEASE_ID };
- let ret =
- etcd::keepalive_lease_id(lease_id, CONFIG.limit.node_heartbeat_ttl, is_offline)
- .await;
+ let ret = etcd::keepalive_lease_id(
+ lease_id,
+ get_config().limit.node_heartbeat_ttl,
+ is_offline,
+ )
+ .await;
if ret.is_ok() {
break;
}
@@ -77,8 +80,9 @@ pub(crate) async fn register_and_keepalive() -> Result<()> {
/// Register to cluster
async fn register() -> Result<()> {
+ let cfg = get_config();
// 1. create a cluster lock for node register
- let locker = dist_lock::lock("/nodes/register", CONFIG.limit.node_heartbeat_ttl as u64).await?;
+ let locker = dist_lock::lock("/nodes/register", cfg.limit.node_heartbeat_ttl as u64).await?;
// 2. watch node list
tokio::task::spawn(async move { super::watch_node_list().await });
@@ -126,15 +130,15 @@ async fn register() -> Result<()> {
}
// 4. join the cluster
- let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
+ let key = format!("{}nodes/{}", &cfg.etcd.prefix, *LOCAL_NODE_UUID);
let node = Node {
id: new_node_id,
uuid: LOCAL_NODE_UUID.clone(),
- name: CONFIG.common.instance_name.clone(),
- http_addr: format!("http://{}:{}", get_local_http_ip(), CONFIG.http.port),
- grpc_addr: format!("http://{}:{}", get_local_grpc_ip(), CONFIG.grpc.port),
+ name: cfg.common.instance_name.clone(),
+ http_addr: format!("http://{}:{}", get_local_http_ip(), cfg.http.port),
+ grpc_addr: format!("http://{}:{}", get_local_grpc_ip(), cfg.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
- cpu_num: CONFIG.limit.cpu_num as u64,
+ cpu_num: cfg.limit.cpu_num as u64,
status: NodeStatus::Prepare,
scheduled: true,
broadcasted: false,
@@ -155,10 +159,7 @@ async fn register() -> Result<()> {
// register node to cluster
let mut client = etcd::get_etcd_client().await.clone();
- let resp = match client
- .lease_grant(CONFIG.limit.node_heartbeat_ttl, None)
- .await
- {
+ let resp = match client.lease_grant(cfg.limit.node_heartbeat_ttl, None).await {
Ok(v) => v,
Err(e) => {
dist_lock::unlock(&locker).await?;
@@ -193,6 +194,7 @@ pub(crate) async fn set_offline(new_lease_id: bool) -> Result<()> {
/// set online to cluster
pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result<()> {
+ let cfg = get_config();
// set node status to online
let node = match super::NODES.read().await.get(LOCAL_NODE_UUID.as_str()) {
Some(node) => {
@@ -203,11 +205,11 @@ pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result
None => Node {
id: unsafe { LOCAL_NODE_ID },
uuid: LOCAL_NODE_UUID.clone(),
- name: CONFIG.common.instance_name.clone(),
- http_addr: format!("http://{}:{}", get_local_node_ip(), CONFIG.http.port),
- grpc_addr: format!("http://{}:{}", get_local_node_ip(), CONFIG.grpc.port),
+ name: cfg.common.instance_name.clone(),
+ http_addr: format!("http://{}:{}", get_local_node_ip(), cfg.http.port),
+ grpc_addr: format!("http://{}:{}", get_local_node_ip(), cfg.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
- cpu_num: CONFIG.limit.cpu_num as u64,
+ cpu_num: cfg.limit.cpu_num as u64,
status: status.clone(),
scheduled: true,
broadcasted: false,
@@ -223,7 +225,7 @@ pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result
// get new lease id
let mut client = etcd::get_etcd_client().await.clone();
let resp = client
- .lease_grant(CONFIG.limit.node_heartbeat_ttl, None)
+ .lease_grant(cfg.limit.node_heartbeat_ttl, None)
.await?;
let lease_id = resp.id();
// update local node key lease id
@@ -232,7 +234,7 @@ pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result
}
}
- let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
+ let key = format!("{}nodes/{}", &cfg.etcd.prefix, *LOCAL_NODE_UUID);
let opt = PutOptions::new().with_lease(unsafe { LOCAL_NODE_KEY_LEASE_ID });
let mut client = etcd::get_etcd_client().await.clone();
if let Err(e) = client.put(key, val, Some(opt)).await {
@@ -244,7 +246,7 @@ pub(crate) async fn set_status(status: NodeStatus, new_lease_id: bool) -> Result
/// Leave cluster
pub(crate) async fn leave() -> Result<()> {
- let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
+ let key = format!("{}nodes/{}", get_config().etcd.prefix, *LOCAL_NODE_UUID);
let mut client = etcd::get_etcd_client().await.clone();
if let Err(e) = client.delete(key, None).await {
return Err(Error::Message(format!("leave node error: {}", e)));
@@ -254,7 +256,7 @@ pub(crate) async fn leave() -> Result<()> {
}
pub(crate) async fn update_local_node(node: &Node) -> Result<()> {
- let key = format!("{}nodes/{}", &CONFIG.etcd.prefix, *LOCAL_NODE_UUID);
+ let key = format!("{}nodes/{}", get_config().etcd.prefix, *LOCAL_NODE_UUID);
let opt = PutOptions::new().with_lease(unsafe { LOCAL_NODE_KEY_LEASE_ID });
let val = json::to_string(&node).unwrap();
let mut client = etcd::get_etcd_client().await.clone();
diff --git a/src/common/infra/cluster/mod.rs b/src/common/infra/cluster/mod.rs
index 3dd5e732b8..d3759b1656 100644
--- a/src/common/infra/cluster/mod.rs
+++ b/src/common/infra/cluster/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,12 +17,13 @@ use std::{cmp::min, ops::Bound, sync::Arc, time::Duration};
use config::{
cluster::*,
+ get_config, get_instance_id,
meta::{
cluster::{Node, NodeStatus, Role},
meta_store::MetaStore,
},
utils::{hash::Sum64, json},
- RwAHashMap, RwBTreeMap, CONFIG, INSTANCE_ID,
+ RwAHashMap, RwBTreeMap,
};
use infra::{
db::{get_coordinator, Event},
@@ -51,7 +52,7 @@ pub async fn add_node_to_consistent_hash(node: &Node, role: &Role) {
_ => return,
};
let mut h = config::utils::hash::gxhash::new();
- for i in 0..CONFIG.limit.consistent_hash_vnodes {
+ for i in 0..get_config().limit.consistent_hash_vnodes {
let key = format!("{}{}", node.uuid, i);
let hash = h.sum64(&key);
nodes.insert(hash, node.uuid.clone());
@@ -65,7 +66,7 @@ pub async fn remove_node_from_consistent_hash(node: &Node, role: &Role) {
_ => return,
};
let mut h = config::utils::hash::gxhash::new();
- for i in 0..CONFIG.limit.consistent_hash_vnodes {
+ for i in 0..get_config().limit.consistent_hash_vnodes {
let key = format!("{}{}", node.uuid, i);
let hash = h.sum64(&key);
nodes.remove(&hash);
@@ -94,16 +95,18 @@ pub async fn get_node_from_consistent_hash(key: &str, role: &Role) -> Option String {
- if CONFIG.grpc.internal_grpc_token.is_empty() {
- INSTANCE_ID.get("instance_id").unwrap().to_string()
+ let cfg = get_config();
+ if cfg.grpc.internal_grpc_token.is_empty() {
+ get_instance_id()
} else {
- CONFIG.grpc.internal_grpc_token.clone()
+ cfg.grpc.internal_grpc_token.clone()
}
}
/// Register and keepalive the node to cluster
pub async fn register_and_keepalive() -> Result<()> {
- if CONFIG.common.local_mode {
+ let cfg = get_config();
+ if cfg.common.local_mode {
let roles = load_local_node_role();
if !is_single_node(&roles) {
panic!("Local mode only support NODE_ROLE=all");
@@ -116,14 +119,14 @@ pub async fn register_and_keepalive() -> Result<()> {
return Ok(());
}
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => nats::register_and_keepalive().await?,
_ => etcd::register_and_keepalive().await?,
};
// check node heatbeat
tokio::task::spawn(async move {
- let ttl_keep_alive = min(10, (CONFIG.limit.node_heartbeat_ttl / 2) as u64);
+ let ttl_keep_alive = min(10, (cfg.limit.node_heartbeat_ttl / 2) as u64);
let client = reqwest::Client::new();
loop {
time::sleep(time::Duration::from_secs(ttl_keep_alive)).await;
@@ -137,33 +140,36 @@ pub async fn register_and_keepalive() -> Result<()> {
}
pub async fn set_online(new_lease_id: bool) -> Result<()> {
- if CONFIG.common.local_mode {
+ let cfg = get_config();
+ if cfg.common.local_mode {
return Ok(());
}
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => nats::set_online().await,
_ => etcd::set_online(new_lease_id).await,
}
}
pub async fn set_offline(new_lease_id: bool) -> Result<()> {
- if CONFIG.common.local_mode {
+ let cfg = get_config();
+ if cfg.common.local_mode {
return Ok(());
}
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => nats::set_offline().await,
_ => etcd::set_offline(new_lease_id).await,
}
}
pub async fn update_local_node(node: &Node) -> Result<()> {
- if CONFIG.common.local_mode {
+ let cfg = get_config();
+ if cfg.common.local_mode {
return Ok(());
}
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => nats::update_local_node(node).await,
_ => etcd::update_local_node(node).await,
}
@@ -173,12 +179,12 @@ pub async fn leave() -> Result<()> {
unsafe {
LOCAL_NODE_STATUS = NodeStatus::Offline;
}
-
- if CONFIG.common.local_mode {
+ let cfg = get_config();
+ if cfg.common.local_mode {
return Ok(());
}
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => nats::leave().await,
_ => etcd::leave().await,
}
@@ -207,7 +213,9 @@ async fn watch_node_list() -> Result<()> {
let mut events = client.watch(key).await?;
let events = Arc::get_mut(&mut events).unwrap();
log::info!("Start watching node_list");
+
loop {
+ let cfg = get_config();
let ev = match events.recv().await {
Some(ev) => ev,
None => {
@@ -238,7 +246,7 @@ async fn watch_node_list() -> Result<()> {
continue;
}
log::info!("[CLUSTER] join {:?}", item_value);
- if !CONFIG.common.meta_store_external && !broadcasted {
+ if !cfg.common.meta_store_external && !broadcasted {
// The ingester need broadcast local file list to the new node
if is_ingester(&LOCAL_NODE_ROLE)
&& (item_value.status.eq(&NodeStatus::Prepare)
@@ -297,12 +305,13 @@ async fn watch_node_list() -> Result<()> {
}
async fn check_nodes_status(client: &reqwest::Client) -> Result<()> {
+ let cfg = get_config();
let nodes = get_cached_online_nodes().await.unwrap_or_default();
for node in nodes {
if node.uuid.eq(LOCAL_NODE_UUID.as_str()) {
continue;
}
- let url = format!("{}{}/healthz", node.http_addr, CONFIG.common.base_uri);
+ let url = format!("{}{}/healthz", node.http_addr, cfg.common.base_uri);
let resp = client.get(url).timeout(HEALTH_CHECK_TIMEOUT).send().await;
if resp.is_err() || !resp.unwrap().status().is_success() {
log::error!("[CLUSTER] node {} health check failed", node.name);
@@ -350,14 +359,15 @@ pub async fn get_cached_nodes(cond: fn(&Node) -> bool) -> Option> {
#[inline(always)]
pub fn load_local_mode_node() -> Node {
+ let cfg = get_config();
Node {
id: 1,
uuid: LOCAL_NODE_UUID.clone(),
- name: CONFIG.common.instance_name.clone(),
- http_addr: format!("http://127.0.0.1:{}", CONFIG.http.port),
- grpc_addr: format!("http://127.0.0.1:{}", CONFIG.grpc.port),
+ name: cfg.common.instance_name.clone(),
+ http_addr: format!("http://127.0.0.1:{}", cfg.http.port),
+ grpc_addr: format!("http://127.0.0.1:{}", cfg.grpc.port),
role: [Role::All].to_vec(),
- cpu_num: CONFIG.limit.cpu_num as u64,
+ cpu_num: cfg.limit.cpu_num as u64,
status: NodeStatus::Online,
scheduled: true,
broadcasted: false,
diff --git a/src/common/infra/cluster/nats.rs b/src/common/infra/cluster/nats.rs
index d271cac50d..7ac3a3aa52 100644
--- a/src/common/infra/cluster/nats.rs
+++ b/src/common/infra/cluster/nats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,9 +17,9 @@ use core::cmp::min;
use config::{
cluster::*,
+ get_config,
meta::cluster::{Node, NodeStatus, Role},
utils::json,
- CONFIG,
};
use infra::{
db::{get_coordinator, NEED_WATCH},
@@ -46,7 +46,7 @@ pub(crate) async fn register_and_keepalive() -> Result<()> {
}
}
// after the node is online, keepalive
- let ttl_keep_alive = min(10, (CONFIG.limit.node_heartbeat_ttl / 2) as u64);
+ let ttl_keep_alive = min(10, (get_config().limit.node_heartbeat_ttl / 2) as u64);
loop {
time::sleep(time::Duration::from_secs(ttl_keep_alive)).await;
loop {
@@ -72,8 +72,9 @@ pub(crate) async fn register_and_keepalive() -> Result<()> {
/// Register to cluster
async fn register() -> Result<()> {
+ let cfg = get_config();
// 1. create a cluster lock for node register
- let locker = dist_lock::lock("/nodes/register", CONFIG.limit.node_heartbeat_ttl as u64).await?;
+ let locker = dist_lock::lock("/nodes/register", cfg.limit.node_heartbeat_ttl as u64).await?;
// 2. watch node list
task::spawn(async move { super::watch_node_list().await });
@@ -125,11 +126,11 @@ async fn register() -> Result<()> {
let node = Node {
id: new_node_id,
uuid: LOCAL_NODE_UUID.clone(),
- name: CONFIG.common.instance_name.clone(),
- http_addr: format!("http://{}:{}", get_local_http_ip(), CONFIG.http.port),
- grpc_addr: format!("http://{}:{}", get_local_grpc_ip(), CONFIG.grpc.port),
+ name: cfg.common.instance_name.clone(),
+ http_addr: format!("http://{}:{}", get_local_http_ip(), cfg.http.port),
+ grpc_addr: format!("http://{}:{}", get_local_grpc_ip(), cfg.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
- cpu_num: CONFIG.limit.cpu_num as u64,
+ cpu_num: cfg.limit.cpu_num as u64,
status: NodeStatus::Prepare,
scheduled: true,
broadcasted: false,
@@ -172,6 +173,7 @@ pub(crate) async fn set_offline() -> Result<()> {
/// set online to cluster
pub(crate) async fn set_status(status: NodeStatus) -> Result<()> {
+ let cfg = get_config();
// set node status to online
let node = match super::NODES.read().await.get(LOCAL_NODE_UUID.as_str()) {
Some(node) => {
@@ -182,11 +184,11 @@ pub(crate) async fn set_status(status: NodeStatus) -> Result<()> {
None => Node {
id: unsafe { LOCAL_NODE_ID },
uuid: LOCAL_NODE_UUID.clone(),
- name: CONFIG.common.instance_name.clone(),
- http_addr: format!("http://{}:{}", get_local_node_ip(), CONFIG.http.port),
- grpc_addr: format!("http://{}:{}", get_local_node_ip(), CONFIG.grpc.port),
+ name: cfg.common.instance_name.clone(),
+ http_addr: format!("http://{}:{}", get_local_node_ip(), cfg.http.port),
+ grpc_addr: format!("http://{}:{}", get_local_node_ip(), cfg.grpc.port),
role: LOCAL_NODE_ROLE.clone(),
- cpu_num: CONFIG.limit.cpu_num as u64,
+ cpu_num: cfg.limit.cpu_num as u64,
status: status.clone(),
scheduled: true,
broadcasted: false,
diff --git a/src/common/infra/mod.rs b/src/common/infra/mod.rs
index d0c05f6a3f..c1b23cd020 100644
--- a/src/common/infra/mod.rs
+++ b/src/common/infra/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use ::config::{ider, INSTANCE_ID};
+use ::config::{cache_instance_id, ider};
use crate::service::db::instance;
@@ -32,7 +32,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
id
}
};
- INSTANCE_ID.insert("instance_id".to_owned(), instance_id);
+ cache_instance_id(&instance_id);
wal::init().await?;
// because of asynchronous, we need to wait for a while
diff --git a/src/common/infra/wal.rs b/src/common/infra/wal.rs
index a4006b32af..db714e1971 100644
--- a/src/common/infra/wal.rs
+++ b/src/common/infra/wal.rs
@@ -19,11 +19,11 @@ use arrow::{ipc::writer::StreamWriter, record_batch::RecordBatch};
use arrow_schema::Schema;
use chrono::{DateTime, Datelike, TimeZone, Utc};
use config::{
- ider,
+ get_config, ider,
meta::stream::{PartitionTimeLevel, StreamType},
metrics,
utils::asynchronism::file::get_file_contents,
- CONFIG, FILE_EXT_ARROW, FILE_EXT_JSON,
+ FILE_EXT_ARROW, FILE_EXT_JSON,
};
use hashbrown::HashMap;
use once_cell::sync::Lazy;
@@ -147,7 +147,7 @@ impl Default for Manager {
impl Manager {
pub fn new() -> Manager {
- let size = CONFIG.limit.cpu_num;
+ let size = get_config().limit.cpu_num;
let mut data = Vec::with_capacity(size);
for _ in 0..size {
data.push(RwLock::new(HashMap::>::default()));
@@ -178,7 +178,7 @@ impl Manager {
drop(manager);
// check size & ttl
- if file.size().await >= (CONFIG.limit.max_file_size_on_disk as i64)
+ if file.size().await >= (get_config().limit.max_file_size_on_disk as i64)
|| file.expired() <= Utc::now().timestamp()
{
let mut manager = locker.write().await;
@@ -258,16 +258,16 @@ impl RwFile {
schema: Option,
) -> RwFile {
let use_arrow = schema.is_some();
-
+ let cfg = get_config();
let mut dir_path = if use_arrow {
format!(
"{}files/{}/{}/{}/",
- &CONFIG.common.data_idx_dir, stream.org_id, stream.stream_type, stream.stream_name
+ &cfg.common.data_idx_dir, stream.org_id, stream.stream_type, stream.stream_name
)
} else {
format!(
"{}files/{}/{}/{}/",
- &CONFIG.common.data_wal_dir, stream.org_id, stream.stream_type, stream.stream_name
+ &cfg.common.data_wal_dir, stream.org_id, stream.stream_type, stream.stream_name
)
};
// Hack for file_list
@@ -308,7 +308,7 @@ impl RwFile {
let time_now: DateTime = Utc::now();
let level_duration = partition_time_level.unwrap_or_default().duration();
- let ttl = if !CONFIG.limit.ignore_file_retention_by_stream && level_duration > 0 {
+ let ttl = if !cfg.limit.ignore_file_retention_by_stream && level_duration > 0 {
let time_end_day = Utc
.with_ymd_and_hms(
time_now.year(),
@@ -324,12 +324,12 @@ impl RwFile {
if expired > time_end_day {
// if the file expired time is tomorrow, it should be deleted at 23:59:59 +
// 10min
- time_end_day + CONFIG.limit.max_file_retention_time as i64
+ time_end_day + cfg.limit.max_file_retention_time as i64
} else {
expired
}
} else {
- time_now.timestamp() + CONFIG.limit.max_file_retention_time as i64
+ time_now.timestamp() + cfg.limit.max_file_retention_time as i64
};
RwFile {
diff --git a/src/common/meta/alerts/destinations.rs b/src/common/meta/alerts/destinations.rs
index 1b29414fa0..86be738c2f 100644
--- a/src/common/meta/alerts/destinations.rs
+++ b/src/common/meta/alerts/destinations.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/alerts/mod.rs b/src/common/meta/alerts/mod.rs
index 66e4869c88..2dd421ea0a 100644
--- a/src/common/meta/alerts/mod.rs
+++ b/src/common/meta/alerts/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/alerts/templates.rs b/src/common/meta/alerts/templates.rs
index 799edc9fbd..51f32dc500 100644
--- a/src/common/meta/alerts/templates.rs
+++ b/src/common/meta/alerts/templates.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/authz.rs b/src/common/meta/authz.rs
index 59d31bf85d..1dec9a8dc2 100644
--- a/src/common/meta/authz.rs
+++ b/src/common/meta/authz.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/dashboards/mod.rs b/src/common/meta/dashboards/mod.rs
index d22e022563..0633904282 100644
--- a/src/common/meta/dashboards/mod.rs
+++ b/src/common/meta/dashboards/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/dashboards/reports.rs b/src/common/meta/dashboards/reports.rs
index 4146de1e99..d95298b886 100644
--- a/src/common/meta/dashboards/reports.rs
+++ b/src/common/meta/dashboards/reports.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/dashboards/v1/mod.rs b/src/common/meta/dashboards/v1/mod.rs
index 78728f2f39..3ff0f0fd62 100644
--- a/src/common/meta/dashboards/v1/mod.rs
+++ b/src/common/meta/dashboards/v1/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/dashboards/v2/mod.rs b/src/common/meta/dashboards/v2/mod.rs
index 526f2ddadb..04ba7cb769 100644
--- a/src/common/meta/dashboards/v2/mod.rs
+++ b/src/common/meta/dashboards/v2/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/dashboards/v3/mod.rs b/src/common/meta/dashboards/v3/mod.rs
index 8518d4d8b0..e9ca542669 100644
--- a/src/common/meta/dashboards/v3/mod.rs
+++ b/src/common/meta/dashboards/v3/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/functions.rs b/src/common/meta/functions.rs
index 54fb87d6db..c56ec69211 100644
--- a/src/common/meta/functions.rs
+++ b/src/common/meta/functions.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/ingestion.rs b/src/common/meta/ingestion.rs
index 934a9d196f..8b54f392cb 100644
--- a/src/common/meta/ingestion.rs
+++ b/src/common/meta/ingestion.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/maxmind/client.rs b/src/common/meta/maxmind/client.rs
index 8eb7f31360..eab46b5f06 100644
--- a/src/common/meta/maxmind/client.rs
+++ b/src/common/meta/maxmind/client.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/maxmind/mod.rs b/src/common/meta/maxmind/mod.rs
index 61b29f0388..90d8a2d26e 100644
--- a/src/common/meta/maxmind/mod.rs
+++ b/src/common/meta/maxmind/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/middleware_data.rs b/src/common/meta/middleware_data.rs
index d9a69d443f..3d3bba211b 100644
--- a/src/common/meta/middleware_data.rs
+++ b/src/common/meta/middleware_data.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/organization.rs b/src/common/meta/organization.rs
index ee40d7fa51..656157f8c5 100644
--- a/src/common/meta/organization.rs
+++ b/src/common/meta/organization.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
@@ -99,7 +98,7 @@ pub struct RumIngestionResponse {
}
fn default_scrape_interval() -> u32 {
- CONFIG.common.default_scrape_interval
+ config::get_config().common.default_scrape_interval
}
#[derive(Serialize, ToSchema, Deserialize, Debug, Clone)]
diff --git a/src/common/meta/prom.rs b/src/common/meta/prom.rs
index 56f880f69f..81456e7474 100644
--- a/src/common/meta/prom.rs
+++ b/src/common/meta/prom.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/saved_view.rs b/src/common/meta/saved_view.rs
index 5ae58db692..5dd31aece3 100644
--- a/src/common/meta/saved_view.rs
+++ b/src/common/meta/saved_view.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/service.rs b/src/common/meta/service.rs
index 428a55806d..61f8a0ce89 100644
--- a/src/common/meta/service.rs
+++ b/src/common/meta/service.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/syslog.rs b/src/common/meta/syslog.rs
index 103eb52055..b32d7933e8 100644
--- a/src/common/meta/syslog.rs
+++ b/src/common/meta/syslog.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/meta/telemetry.rs b/src/common/meta/telemetry.rs
index 4382c50b3c..3108725f1c 100644
--- a/src/common/meta/telemetry.rs
+++ b/src/common/meta/telemetry.rs
@@ -17,8 +17,9 @@ use std::collections::HashMap;
use config::{
cluster::{is_single_node, load_local_node_role},
+ get_config, get_instance_id,
utils::json,
- CONFIG, INSTANCE_ID, SIZE_IN_MB, TELEMETRY_CLIENT,
+ SIZE_IN_MB, TELEMETRY_CLIENT,
};
use hashbrown::HashSet;
use infra::{cache::stats, db as infra_db, schema::STREAM_SCHEMAS_LATEST};
@@ -53,7 +54,7 @@ impl Telemetry {
data: Option>,
send_zo_data: bool,
) {
- if !CONFIG.common.telemetry_enabled {
+ if !get_config().common.telemetry_enabled {
return;
}
log::info!("sending event {}", event);
@@ -69,7 +70,7 @@ impl Telemetry {
self.add_event(Track {
user: segment::message::User::UserId {
user_id: segment::message::User::AnonymousId {
- anonymous_id: INSTANCE_ID.get("instance_id").unwrap().to_string(),
+ anonymous_id: get_instance_id(),
}
.to_string(),
},
@@ -80,10 +81,7 @@ impl Telemetry {
});
let res = TELEMETRY_CLIENT
- .send(
- INSTANCE_ID.get("instance_id").unwrap().to_string(),
- Message::from(self.event.clone()),
- )
+ .send(get_instance_id(), Message::from(self.event.clone()))
.await;
if res.is_err() {
@@ -109,7 +107,7 @@ pub fn get_base_info(data: &mut HashMap) -> HashMap) -> HashMap) -> HashMap.
use config::{
+ get_config,
meta::stream::{FileKey, PartitionTimeLevel, StreamType},
utils::{file::get_file_meta, time::BASE_TIME},
- CONFIG,
};
use infra::file_list as infra_file_list;
@@ -26,7 +26,7 @@ use crate::{
};
pub async fn run(prefix: &str, from: &str, to: &str) -> Result<(), anyhow::Error> {
- if get_file_meta(&CONFIG.common.data_wal_dir).is_err() {
+ if get_file_meta(&get_config().common.data_wal_dir).is_err() {
// there is no local wal files, no need upgrade
return Ok(());
}
diff --git a/src/common/mod.rs b/src/common/mod.rs
index aa2b22e63e..bd21a2e3f0 100644
--- a/src/common/mod.rs
+++ b/src/common/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/utils/auth.rs b/src/common/utils/auth.rs
index 44712dacf6..28593c9c47 100644
--- a/src/common/utils/auth.rs
+++ b/src/common/utils/auth.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,8 +16,6 @@
use actix_web::{dev::Payload, Error, FromRequest, HttpRequest};
use argon2::{password_hash::SaltString, Algorithm, Argon2, Params, PasswordHasher, Version};
use config::utils::json;
-#[cfg(feature = "enterprise")]
-use config::CONFIG;
use futures::future::{ready, Ready};
#[cfg(feature = "enterprise")]
@@ -177,11 +175,12 @@ impl FromRequest for AuthExtractor {
let mut method = req.method().to_string();
let local_path = req.path().to_string();
- let path =
- match local_path.strip_prefix(format!("{}/api/", CONFIG.common.base_uri).as_str()) {
- Some(path) => path,
- None => &local_path,
- };
+ let path = match local_path
+ .strip_prefix(format!("{}/api/", config::get_config().common.base_uri).as_str())
+ {
+ Some(path) => path,
+ None => &local_path,
+ };
let path_columns = path.split('/').collect::>();
let url_len = path_columns.len();
diff --git a/src/common/utils/functions.rs b/src/common/utils/functions.rs
index 84c8eedeb6..33fe5353ef 100644
--- a/src/common/utils/functions.rs
+++ b/src/common/utils/functions.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/utils/mod.rs b/src/common/utils/mod.rs
index fa7a8b7b65..f1e4b653d8 100644
--- a/src/common/utils/mod.rs
+++ b/src/common/utils/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/utils/record_batch_parquet_ext.rs b/src/common/utils/record_batch_parquet_ext.rs
index 3958f7ece7..74ea1011ac 100644
--- a/src/common/utils/record_batch_parquet_ext.rs
+++ b/src/common/utils/record_batch_parquet_ext.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/common/utils/stream.rs b/src/common/utils/stream.rs
index 9808fc15c9..45850e83fe 100644
--- a/src/common/utils/stream.rs
+++ b/src/common/utils/stream.rs
@@ -20,9 +20,10 @@ use std::{
use actix_web::HttpResponse;
use config::{
+ get_config,
meta::stream::{FileMeta, StreamType},
utils::{arrow::record_batches_to_json_rows, json},
- CONFIG, FILE_EXT_JSON,
+ FILE_EXT_JSON,
};
use datafusion::{
arrow::{datatypes::Schema, record_batch::RecordBatch},
@@ -60,7 +61,7 @@ pub fn get_file_name_v1(org_id: &str, stream_name: &str, suffix: u32) -> String
// "./data/openobserve/olympics/olympics#2022#09#13#13_1.json"
format!(
"{}{}/{}/{}/{}_{}{}",
- &CONFIG.common.data_wal_dir,
+ get_config().common.data_wal_dir,
org_id,
StreamType::Logs,
stream_name,
@@ -84,7 +85,7 @@ pub async fn populate_file_meta(
let sql = format!(
"SELECT min({0}) as min, max({0}) as max, count({0}) as num_records FROM temp;",
- CONFIG.common.column_timestamp
+ get_config().common.column_timestamp
);
let df = ctx.sql(sql.as_str()).await?;
let batches = df.collect().await?;
diff --git a/src/common/utils/zo_logger.rs b/src/common/utils/zo_logger.rs
index 9aa70de47b..f35d6d085b 100644
--- a/src/common/utils/zo_logger.rs
+++ b/src/common/utils/zo_logger.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
use std::sync::Arc;
use cloudevents::{Event, EventBuilder, EventBuilderV10};
-use config::{ider, utils::json, CONFIG};
+use config::{get_config, ider, utils::json};
use log::{Metadata, Record};
use once_cell::sync::Lazy;
use reqwest::Client;
@@ -35,7 +35,7 @@ pub struct ZoLogger {
impl log::Log for ZoLogger {
fn enabled(&self, _metadata: &Metadata) -> bool {
- CONFIG.log.events_enabled
+ get_config().log.events_enabled
}
fn log(self: &ZoLogger, record: &Record) {
@@ -59,7 +59,7 @@ impl log::Log for ZoLogger {
let event = EventBuilderV10::new()
.id(ider::generate())
.ty("debug_log")
- .source(CONFIG.common.instance_name.clone())
+ .source(get_config().common.instance_name.clone())
.data("application/json", payload)
.build()
.unwrap();
@@ -78,10 +78,11 @@ pub async fn send_logs() {
let mut logs = LOGS.write().await;
logs.push(val);
- if logs.len() >= CONFIG.log.events_batch_size {
+ let cfg = get_config();
+ if logs.len() >= cfg.log.events_batch_size {
let old_logs = std::mem::take(&mut *logs);
- let url = url::Url::parse(&CONFIG.log.events_url).unwrap();
- let auth = format!("Basic {}", &CONFIG.log.events_auth);
+ let url = url::Url::parse(&cfg.log.events_url).unwrap();
+ let auth = format!("Basic {}", &cfg.log.events_auth);
let cl = Arc::clone(&cl);
tokio::task::spawn(async move {
let _ = cl
diff --git a/src/config/Cargo.toml b/src/config/Cargo.toml
index 4263c88472..ec6a9d2859 100644
--- a/src/config/Cargo.toml
+++ b/src/config/Cargo.toml
@@ -8,6 +8,7 @@ license.workspace = true
actix-web-prometheus.workspace = true
ahash.workspace = true
anyhow.workspace = true
+arc-swap.workspace = true
arrow.workspace = true
arrow-json.workspace = true
arrow-schema.workspace = true
diff --git a/src/config/src/cluster.rs b/src/config/src/cluster.rs
index df616357e5..28b1361068 100644
--- a/src/config/src/cluster.rs
+++ b/src/config/src/cluster.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -18,9 +18,8 @@ use std::net::IpAddr;
use once_cell::sync::Lazy;
use crate::{
- ider,
+ get_config, ider,
meta::cluster::{NodeStatus, Role},
- CONFIG,
};
pub static mut LOCAL_NODE_ID: i32 = 0;
@@ -36,8 +35,9 @@ pub fn load_local_node_uuid() -> String {
#[inline(always)]
pub fn get_local_http_ip() -> String {
- if !CONFIG.http.addr.is_empty() {
- CONFIG.http.addr.clone()
+ let cfg = get_config();
+ if !cfg.http.addr.is_empty() {
+ cfg.http.addr.clone()
} else {
get_local_node_ip()
}
@@ -45,8 +45,9 @@ pub fn get_local_http_ip() -> String {
#[inline(always)]
pub fn get_local_grpc_ip() -> String {
- if !CONFIG.grpc.addr.is_empty() {
- CONFIG.grpc.addr.clone()
+ let cfg = get_config();
+ if !cfg.grpc.addr.is_empty() {
+ cfg.grpc.addr.clone()
} else {
get_local_node_ip()
}
@@ -64,7 +65,7 @@ pub fn get_local_node_ip() -> String {
#[inline(always)]
pub fn load_local_node_role() -> Vec {
- CONFIG
+ get_config()
.common
.node_role
.clone()
diff --git a/src/config/src/config.rs b/src/config/src/config.rs
index f89b87ec4e..1de2b30898 100644
--- a/src/config/src/config.rs
+++ b/src/config/src/config.rs
@@ -13,8 +13,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use std::{cmp::max, collections::BTreeMap, path::Path, time::Duration};
+use std::{cmp::max, collections::BTreeMap, path::Path, sync::Arc, time::Duration};
+use arc_swap::ArcSwap;
use chromiumoxide::{browser::BrowserConfig, handler::viewport::Viewport};
use dotenv_config::EnvConfig;
use dotenvy::dotenv;
@@ -69,7 +70,7 @@ pub static SQL_FULL_TEXT_SEARCH_FIELDS: Lazy> = Lazy::new(|| {
_DEFAULT_SQL_FULL_TEXT_SEARCH_FIELDS
.iter()
.map(|s| s.to_string()),
- CONFIG
+ get_config()
.common
.feature_fulltext_extra_fields
.split(',')
@@ -89,7 +90,7 @@ pub static SQL_FULL_TEXT_SEARCH_FIELDS: Lazy> = Lazy::new(|| {
});
pub static QUICK_MODEL_FIELDS: Lazy> = Lazy::new(|| {
- let mut fields = CONFIG
+ let mut fields = get_config()
.common
.feature_quick_mode_fields
.split(',')
@@ -111,7 +112,7 @@ const _DEFAULT_DISTINCT_FIELDS: [&str; 2] = ["service_name", "operation_name"];
pub static DISTINCT_FIELDS: Lazy> = Lazy::new(|| {
let mut fields = chain(
_DEFAULT_DISTINCT_FIELDS.iter().map(|s| s.to_string()),
- CONFIG
+ get_config()
.common
.feature_distinct_extra_fields
.split(',')
@@ -134,7 +135,7 @@ const _DEFAULT_BLOOM_FILTER_FIELDS: [&str; 1] = ["trace_id"];
pub static BLOOM_FILTER_DEFAULT_FIELDS: Lazy> = Lazy::new(|| {
let mut fields = chain(
_DEFAULT_BLOOM_FILTER_FIELDS.iter().map(|s| s.to_string()),
- CONFIG
+ get_config()
.common
.bloom_filter_default_fields
.split(',')
@@ -153,8 +154,8 @@ pub static BLOOM_FILTER_DEFAULT_FIELDS: Lazy> = Lazy::new(|| {
fields
});
-pub static CONFIG: Lazy = Lazy::new(init);
-pub static INSTANCE_ID: Lazy> = Lazy::new(Default::default);
+static CONFIG: Lazy> = Lazy::new(|| ArcSwap::from(Arc::new(init())));
+static INSTANCE_ID: Lazy> = Lazy::new(Default::default);
pub static TELEMETRY_CLIENT: Lazy = Lazy::new(|| {
segment::HttpClient::new(
@@ -162,10 +163,27 @@ pub static TELEMETRY_CLIENT: Lazy = Lazy::new(|| {
.connect_timeout(Duration::new(10, 0))
.build()
.unwrap(),
- CONFIG.common.telemetry_url.clone(),
+ CONFIG.load().common.telemetry_url.clone(),
)
});
+pub fn get_config() -> Arc {
+ CONFIG.load().clone()
+}
+
+pub fn refresh_config() -> Result<(), anyhow::Error> {
+ CONFIG.store(Arc::new(init()));
+ Ok(())
+}
+
+pub fn cache_instance_id(instance_id: &str) {
+ INSTANCE_ID.insert("instance_id".to_owned(), instance_id.to_owned());
+}
+
+pub fn get_instance_id() -> String {
+ INSTANCE_ID.get("instance_id").unwrap().clone()
+}
+
static CHROME_LAUNCHER_OPTIONS: tokio::sync::OnceCell> =
tokio::sync::OnceCell::const_new();
@@ -176,31 +194,32 @@ pub async fn get_chrome_launch_options() -> &'static Option {
}
async fn init_chrome_launch_options() -> Option {
- if !CONFIG.chrome.chrome_enabled || !CONFIG.common.report_server_url.is_empty() {
+ let cfg = get_config();
+ if !cfg.chrome.chrome_enabled || !cfg.common.report_server_url.is_empty() {
None
} else {
let mut browser_config = BrowserConfig::builder()
.window_size(
- CONFIG.chrome.chrome_window_width,
- CONFIG.chrome.chrome_window_height,
+ cfg.chrome.chrome_window_width,
+ cfg.chrome.chrome_window_height,
)
.viewport(Viewport {
- width: CONFIG.chrome.chrome_window_width,
- height: CONFIG.chrome.chrome_window_height,
+ width: cfg.chrome.chrome_window_width,
+ height: cfg.chrome.chrome_window_height,
device_scale_factor: Some(1.0),
..Viewport::default()
});
- if CONFIG.chrome.chrome_with_head {
+ if cfg.chrome.chrome_with_head {
browser_config = browser_config.with_head();
}
- if CONFIG.chrome.chrome_no_sandbox {
+ if cfg.chrome.chrome_no_sandbox {
browser_config = browser_config.no_sandbox();
}
- if !CONFIG.chrome.chrome_path.is_empty() {
- browser_config = browser_config.chrome_executable(CONFIG.chrome.chrome_path.as_str());
+ if !cfg.chrome.chrome_path.is_empty() {
+ browser_config = browser_config.chrome_executable(cfg.chrome.chrome_path.as_str());
} else {
panic!("Chrome path must be specified");
}
@@ -209,15 +228,16 @@ async fn init_chrome_launch_options() -> Option {
}
pub static SMTP_CLIENT: Lazy>> = Lazy::new(|| {
- if !CONFIG.smtp.smtp_enabled {
+ let cfg = get_config();
+ if !cfg.smtp.smtp_enabled {
None
} else {
- let tls_parameters = TlsParameters::new(CONFIG.smtp.smtp_host.clone()).unwrap();
+ let tls_parameters = TlsParameters::new(cfg.smtp.smtp_host.clone()).unwrap();
let mut transport_builder =
- AsyncSmtpTransport::::builder_dangerous(&CONFIG.smtp.smtp_host)
- .port(CONFIG.smtp.smtp_port);
+ AsyncSmtpTransport::::builder_dangerous(&cfg.smtp.smtp_host)
+ .port(cfg.smtp.smtp_port);
- let option = &CONFIG.smtp.smtp_encryption;
+ let option = &cfg.smtp.smtp_encryption;
transport_builder = if option == "starttls" {
transport_builder.tls(Tls::Required(tls_parameters))
} else if option == "ssltls" {
@@ -226,18 +246,25 @@ pub static SMTP_CLIENT: Lazy>> = Lazy:
transport_builder
};
- if !CONFIG.smtp.smtp_username.is_empty() && !CONFIG.smtp.smtp_password.is_empty() {
+ if !cfg.smtp.smtp_username.is_empty() && !cfg.smtp.smtp_password.is_empty() {
transport_builder = transport_builder.credentials(Credentials::new(
- CONFIG.smtp.smtp_username.clone(),
- CONFIG.smtp.smtp_password.clone(),
+ cfg.smtp.smtp_username.clone(),
+ cfg.smtp.smtp_password.clone(),
));
}
Some(transport_builder.build())
}
});
-pub static BLOCKED_STREAMS: Lazy> =
- Lazy::new(|| CONFIG.common.blocked_streams.split(',').collect());
+pub static BLOCKED_STREAMS: Lazy> = Lazy::new(|| {
+ let blocked_streams = get_config()
+ .common
+ .blocked_streams
+ .split(',')
+ .map(|x| x.to_string())
+ .collect();
+ blocked_streams
+});
#[derive(EnvConfig)]
pub struct Config {
@@ -1448,13 +1475,15 @@ fn check_s3_config(cfg: &mut Config) -> Result<(), anyhow::Error> {
#[inline]
pub fn is_local_disk_storage() -> bool {
- CONFIG.common.local_mode && CONFIG.common.local_mode_storage.eq("disk")
+ let cfg = get_config();
+ cfg.common.local_mode && cfg.common.local_mode_storage.eq("disk")
}
#[inline]
pub fn get_cluster_name() -> String {
- if !CONFIG.common.cluster_name.is_empty() {
- CONFIG.common.cluster_name.to_string()
+ let cfg = get_config();
+ if !cfg.common.cluster_name.is_empty() {
+ cfg.common.cluster_name.to_string()
} else {
INSTANCE_ID.get("instance_id").unwrap().to_string()
}
diff --git a/src/config/src/ider.rs b/src/config/src/ider.rs
index ad97f1c6fc..7d795230cc 100644
--- a/src/config/src/ider.rs
+++ b/src/config/src/ider.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/lib.rs b/src/config/src/lib.rs
index ec169126ba..c7347ef759 100644
--- a/src/config/src/lib.rs
+++ b/src/config/src/lib.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/meta/cluster.rs b/src/config/src/meta/cluster.rs
index d7992c6834..ff42ae6ea1 100644
--- a/src/config/src/meta/cluster.rs
+++ b/src/config/src/meta/cluster.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/meta/logger.rs b/src/config/src/meta/logger.rs
index 7859dfc463..ca491af0ac 100644
--- a/src/config/src/meta/logger.rs
+++ b/src/config/src/meta/logger.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -27,16 +27,15 @@ use tracing_subscriber::{
registry::LookupSpan,
};
-use crate::CONFIG;
-
pub struct CustomTimeFormat;
impl FormatTime for CustomTimeFormat {
fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result {
- if CONFIG.log.local_time_format.is_empty() {
+ let cfg = crate::get_config();
+ if cfg.log.local_time_format.is_empty() {
write!(w, "{}", Utc::now().to_rfc3339())
} else {
- write!(w, "{}", Local::now().format(&CONFIG.log.local_time_format))
+ write!(w, "{}", Local::now().format(&cfg.log.local_time_format))
}
}
}
diff --git a/src/config/src/meta/meta_store.rs b/src/config/src/meta/meta_store.rs
index 566020bc10..a7c9dd7efd 100644
--- a/src/config/src/meta/meta_store.rs
+++ b/src/config/src/meta/meta_store.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/meta/sql.rs b/src/config/src/meta/sql.rs
index 83efc873f3..1e071fbf74 100644
--- a/src/config/src/meta/sql.rs
+++ b/src/config/src/meta/sql.rs
@@ -25,7 +25,7 @@ use sqlparser::{
parser::Parser,
};
-use crate::CONFIG;
+use crate::get_config;
const MAX_LIMIT: usize = 100000;
const MAX_OFFSET: usize = 100000;
@@ -313,7 +313,7 @@ impl<'a> TryFrom> for Option<(i64, i64)> {
Some(expr) => parse_expr_for_field(
expr,
&SqlOperator::And,
- &CONFIG.common.column_timestamp,
+ &get_config().common.column_timestamp,
&mut fields,
)?,
None => {}
@@ -544,7 +544,7 @@ fn parse_expr_check_field_name(s: &str, field: &str) -> bool {
if s == field {
return true;
}
- if field == "*" && s != "_all" && s != CONFIG.common.column_timestamp.clone() {
+ if field == "*" && s != "_all" && s != get_config().common.column_timestamp.clone() {
return true;
}
diff --git a/src/config/src/meta/stream.rs b/src/config/src/meta/stream.rs
index 58aa315441..9d7b8f89a1 100644
--- a/src/config/src/meta/stream.rs
+++ b/src/config/src/meta/stream.rs
@@ -24,12 +24,11 @@ use utoipa::ToSchema;
use super::usage::Stats;
use crate::{
+ get_config,
utils::{
hash::{gxhash, Sum64},
- json,
- json::{Map, Value},
+ json::{self, Map, Value},
},
- CONFIG,
};
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize, ToSchema, Hash)]
@@ -223,7 +222,7 @@ impl StreamStats {
fn time_range(&self) -> (i64, i64) {
assert!(self.doc_time_min <= self.doc_time_max);
- let file_push_interval = Duration::try_seconds(CONFIG.limit.file_push_interval as _)
+ let file_push_interval = Duration::try_seconds(get_config().limit.file_push_interval as _)
.unwrap()
.num_microseconds()
.unwrap();
diff --git a/src/config/src/meta/usage.rs b/src/config/src/meta/usage.rs
index 7b7aea4a2a..c2ebc6fcdb 100644
--- a/src/config/src/meta/usage.rs
+++ b/src/config/src/meta/usage.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/metrics.rs b/src/config/src/metrics.rs
index c5b26f64cd..0a45f36203 100644
--- a/src/config/src/metrics.rs
+++ b/src/config/src/metrics.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -21,8 +21,6 @@ use prometheus::{
CounterVec, HistogramOpts, HistogramVec, IntCounterVec, IntGaugeVec, Opts, Registry,
};
-use crate::CONFIG;
-
pub const NAMESPACE: &str = "zo";
const HELP_SUFFIX: &str =
"Please include 'organization, 'stream type', and 'stream' labels for this metric.";
@@ -682,10 +680,11 @@ fn register_metrics(registry: &Registry) {
}
fn create_const_labels() -> HashMap {
+ let cfg = crate::config::get_config();
let mut labels = HashMap::new();
- labels.insert("cluster".to_string(), CONFIG.common.cluster_name.clone());
- labels.insert("instance".to_string(), CONFIG.common.instance_name.clone());
- labels.insert("role".to_string(), CONFIG.common.node_role.clone());
+ labels.insert("cluster".to_string(), cfg.common.cluster_name.clone());
+ labels.insert("instance".to_string(), cfg.common.instance_name.clone());
+ labels.insert("role".to_string(), cfg.common.node_role.clone());
labels
}
@@ -694,7 +693,7 @@ pub fn create_prometheus_handler() -> PrometheusMetrics {
register_metrics(®istry);
PrometheusMetricsBuilder::new(NAMESPACE)
- .endpoint(format!("{}/metrics", CONFIG.common.base_uri).as_str())
+ .endpoint(format!("{}/metrics", crate::config::get_config().common.base_uri).as_str())
.const_labels(create_const_labels())
.registry(registry)
.build()
diff --git a/src/config/src/utils/asynchronism/mod.rs b/src/config/src/utils/asynchronism/mod.rs
index 517385c0e9..a0f934d208 100644
--- a/src/config/src/utils/asynchronism/mod.rs
+++ b/src/config/src/utils/asynchronism/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/cgroup.rs b/src/config/src/utils/cgroup.rs
index d08bd26dce..a60e773825 100644
--- a/src/config/src/utils/cgroup.rs
+++ b/src/config/src/utils/cgroup.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/flatten.rs b/src/config/src/utils/flatten.rs
index b1cd712b31..a434246449 100644
--- a/src/config/src/utils/flatten.rs
+++ b/src/config/src/utils/flatten.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/hash/cityhash.rs b/src/config/src/utils/hash/cityhash.rs
index f41ca9ffdc..a7f831ba70 100644
--- a/src/config/src/utils/hash/cityhash.rs
+++ b/src/config/src/utils/hash/cityhash.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/hash/fnv.rs b/src/config/src/utils/hash/fnv.rs
index 6e8ee31dd2..d1ee05a179 100644
--- a/src/config/src/utils/hash/fnv.rs
+++ b/src/config/src/utils/hash/fnv.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/hash/gxhash.rs b/src/config/src/utils/hash/gxhash.rs
index ab7be4d718..e504028b0e 100644
--- a/src/config/src/utils/hash/gxhash.rs
+++ b/src/config/src/utils/hash/gxhash.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/hash/mod.rs b/src/config/src/utils/hash/mod.rs
index 134b708f16..c176be829f 100644
--- a/src/config/src/utils/hash/mod.rs
+++ b/src/config/src/utils/hash/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/hash/murmur3.rs b/src/config/src/utils/hash/murmur3.rs
index 00f8f529e2..0cbf564dd3 100644
--- a/src/config/src/utils/hash/murmur3.rs
+++ b/src/config/src/utils/hash/murmur3.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/parquet.rs b/src/config/src/utils/parquet.rs
index 04fbe3da67..b7d634549c 100644
--- a/src/config/src/utils/parquet.rs
+++ b/src/config/src/utils/parquet.rs
@@ -33,8 +33,9 @@ pub fn new_parquet_writer<'a>(
full_text_search_fields: &'a [String],
metadata: &'a FileMeta,
) -> AsyncArrowWriter<&'a mut Vec> {
- let row_group_size = if CONFIG.limit.parquet_max_row_group_size > 0 {
- CONFIG.limit.parquet_max_row_group_size
+ let cfg = get_config();
+ let row_group_size = if cfg.limit.parquet_max_row_group_size > 0 {
+ cfg.limit.parquet_max_row_group_size
} else {
PARQUET_MAX_ROW_GROUP_SIZE
};
@@ -46,11 +47,11 @@ pub fn new_parquet_writer<'a>(
.set_dictionary_enabled(true)
.set_encoding(Encoding::PLAIN)
.set_column_dictionary_enabled(
- CONFIG.common.column_timestamp.as_str().into(),
+ cfg.common.column_timestamp.as_str().into(),
false,
)
.set_column_encoding(
- CONFIG.common.column_timestamp.as_str().into(),
+ cfg.common.column_timestamp.as_str().into(),
Encoding::DELTA_BINARY_PACKED,
)
.set_key_value_metadata(Some(vec![
@@ -70,9 +71,9 @@ pub fn new_parquet_writer<'a>(
}
// Bloom filter stored by row_group, so if the num_rows can limit to row_group_size
let num_rows = min(metadata.records as u64, row_group_size as u64);
- if CONFIG.common.bloom_filter_enabled {
+ if cfg.common.bloom_filter_enabled {
// if bloom_filter_on_all_fields is true, then use all string fields
- let fields = if CONFIG.common.bloom_filter_on_all_fields {
+ let fields = if cfg.common.bloom_filter_on_all_fields {
schema
.fields()
.iter()
diff --git a/src/config/src/utils/rand.rs b/src/config/src/utils/rand.rs
index 3a1ab3e1d4..f428fc5435 100644
--- a/src/config/src/utils/rand.rs
+++ b/src/config/src/utils/rand.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/schema.rs b/src/config/src/utils/schema.rs
index 92c49f6b1b..d1f106393e 100644
--- a/src/config/src/utils/schema.rs
+++ b/src/config/src/utils/schema.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/schema_ext.rs b/src/config/src/utils/schema_ext.rs
index 26f1c04618..3040cdd997 100644
--- a/src/config/src/utils/schema_ext.rs
+++ b/src/config/src/utils/schema_ext.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/str.rs b/src/config/src/utils/str.rs
index a8d78706c1..d247422309 100644
--- a/src/config/src/utils/str.rs
+++ b/src/config/src/utils/str.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/config/src/utils/time.rs b/src/config/src/utils/time.rs
index 51c860dc8d..d9e02678ab 100644
--- a/src/config/src/utils/time.rs
+++ b/src/config/src/utils/time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/grpc/auth/mod.rs b/src/handler/grpc/auth/mod.rs
index 2cb29350b6..bba8b5ff5e 100644
--- a/src/handler/grpc/auth/mod.rs
+++ b/src/handler/grpc/auth/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
use http_auth_basic::Credentials;
use tonic::{metadata::MetadataValue, Request, Status};
@@ -26,10 +25,9 @@ use crate::common::{
};
pub fn check_auth(req: Request<()>) -> Result, Status> {
+ let cfg = config::get_config();
let metadata = req.metadata();
- if !metadata.contains_key(&CONFIG.grpc.org_header_key)
- && !metadata.contains_key("authorization")
- {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) && !metadata.contains_key("authorization") {
return Err(Status::unauthenticated("No valid auth token"));
}
@@ -43,11 +41,11 @@ pub fn check_auth(req: Request<()>) -> Result, Status> {
if token.eq(get_internal_grpc_token().as_str()) {
Ok(req)
} else {
- let org_id = metadata.get(&CONFIG.grpc.org_header_key);
+ let org_id = metadata.get(&cfg.grpc.org_header_key);
if org_id.is_none() {
return Err(Status::invalid_argument(format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
)));
}
@@ -92,14 +90,14 @@ pub fn check_auth(req: Request<()>) -> Result, Status> {
#[cfg(test)]
mod tests {
- use config::INSTANCE_ID;
+ use config::cache_instance_id;
use super::*;
use crate::common::meta::user::User;
#[tokio::test]
async fn test_check_no_auth() {
- INSTANCE_ID.insert("instance_id".to_owned(), "instance".to_string());
+ cache_instance_id("instance");
ROOT_USER.insert(
"root".to_string(),
User {
@@ -128,7 +126,7 @@ mod tests {
#[tokio::test]
async fn test_check_auth() {
- INSTANCE_ID.insert("instance_id".to_owned(), "instance".to_string());
+ cache_instance_id("instance");
ROOT_USER.insert(
"root".to_string(),
User {
@@ -155,7 +153,7 @@ mod tests {
#[tokio::test]
async fn test_check_err_auth() {
- INSTANCE_ID.insert("instance_id".to_owned(), "instance".to_string());
+ cache_instance_id("instance");
ROOT_USER.insert(
"root".to_string(),
User {
diff --git a/src/handler/grpc/request/event.rs b/src/handler/grpc/request/event.rs
index 3f2b8eafe5..5af904fa5c 100644
--- a/src/handler/grpc/request/event.rs
+++ b/src/handler/grpc/request/event.rs
@@ -16,10 +16,10 @@
use chrono::{Duration, Utc};
use config::{
cluster::{is_compactor, is_querier, LOCAL_NODE_ROLE, LOCAL_NODE_UUID},
+ get_config,
meta::{cluster::Role, stream::FileKey},
metrics,
utils::parquet::read_recordbatch_from_bytes,
- CONFIG,
};
use hashbrown::HashSet;
use infra::{file_list as infra_file_list, schema::STREAM_SCHEMAS_FIELDS};
@@ -57,11 +57,11 @@ impl Event for Eventer {
.filter(|v| v.deleted)
.map(|v| v.key.clone())
.collect::>();
-
+ let cfg = get_config();
// Warning: external meta store should not accept any file list
// querier and compactor can accept add new files
// ingester only accept remove old files
- if !CONFIG.common.meta_store_external {
+ if !cfg.common.meta_store_external {
if is_querier(&LOCAL_NODE_ROLE) || is_compactor(&LOCAL_NODE_ROLE) {
if let Err(e) = infra_file_list::batch_add(&put_items).await {
// metrics
@@ -89,7 +89,7 @@ impl Event for Eventer {
}
// cache latest files for querier
- if CONFIG.memory_cache.cache_latest_files && is_querier(&LOCAL_NODE_ROLE) {
+ if cfg.memory_cache.cache_latest_files && is_querier(&LOCAL_NODE_ROLE) {
let mut cached_field_stream = HashSet::new();
for item in put_items.iter() {
let Some(node) = get_node_from_consistent_hash(&item.key, &Role::Querier).await
@@ -102,7 +102,7 @@ impl Event for Eventer {
if infra::cache::file_data::download("download", &item.key)
.await
.is_ok()
- && CONFIG.limit.quick_mode_file_list_enabled
+ && cfg.limit.quick_mode_file_list_enabled
{
let columns = item.key.split('/').collect::>();
if columns[2] != "logs" {
@@ -138,7 +138,7 @@ async fn cache_latest_fields(stream: &str, file: &str) -> Result<(), anyhow::Err
drop(fr);
if field_cache_time
- + Duration::try_seconds(CONFIG.limit.quick_mode_file_list_interval)
+ + Duration::try_seconds(get_config().limit.quick_mode_file_list_interval)
.unwrap()
.num_microseconds()
.unwrap()
diff --git a/src/handler/grpc/request/file_list.rs b/src/handler/grpc/request/file_list.rs
index 5d71f4fef6..6de00d7ace 100644
--- a/src/handler/grpc/request/file_list.rs
+++ b/src/handler/grpc/request/file_list.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/grpc/request/logs.rs b/src/handler/grpc/request/logs.rs
index b0ef913b16..7c00413340 100644
--- a/src/handler/grpc/request/logs.rs
+++ b/src/handler/grpc/request/logs.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::logs::v1::{
logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse,
};
@@ -29,21 +28,22 @@ impl LogsService for LogsServer {
&self,
request: tonic::Request,
) -> Result, tonic::Status> {
+ let cfg = config::get_config();
let metadata = request.metadata().clone();
let msg = format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
);
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(msg));
}
let in_req = request.into_inner();
- let org_id = metadata.get(&CONFIG.grpc.org_header_key);
+ let org_id = metadata.get(&cfg.grpc.org_header_key);
if org_id.is_none() {
return Err(Status::invalid_argument(msg));
}
- let stream_name = metadata.get(&CONFIG.grpc.stream_header_key);
+ let stream_name = metadata.get(&cfg.grpc.stream_header_key);
let mut in_stream_name: Option<&str> = None;
if let Some(stream_name) = stream_name {
in_stream_name = Some(stream_name.to_str().unwrap());
diff --git a/src/handler/grpc/request/metrics/ingester.rs b/src/handler/grpc/request/metrics/ingester.rs
index 4a5d8f8477..a9181e3947 100644
--- a/src/handler/grpc/request/metrics/ingester.rs
+++ b/src/handler/grpc/request/metrics/ingester.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::metrics::v1::{
metrics_service_server::MetricsService, ExportMetricsServiceRequest,
ExportMetricsServiceResponse,
@@ -30,17 +29,18 @@ impl MetricsService for Ingester {
&self,
request: tonic::Request,
) -> Result, tonic::Status> {
+ let cfg = config::get_config();
let metadata = request.metadata().clone();
let msg = format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
);
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(msg));
}
let in_req = request.into_inner();
- let org_id = metadata.get(&CONFIG.grpc.org_header_key);
+ let org_id = metadata.get(&cfg.grpc.org_header_key);
if org_id.is_none() {
return Err(Status::invalid_argument(msg));
}
diff --git a/src/handler/grpc/request/metrics/mod.rs b/src/handler/grpc/request/metrics/mod.rs
index b8f1b86327..f2821e34ca 100644
--- a/src/handler/grpc/request/metrics/mod.rs
+++ b/src/handler/grpc/request/metrics/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/grpc/request/metrics/querier.rs b/src/handler/grpc/request/metrics/querier.rs
index 0fc2a96d2f..d8a5cede31 100644
--- a/src/handler/grpc/request/metrics/querier.rs
+++ b/src/handler/grpc/request/metrics/querier.rs
@@ -23,7 +23,6 @@ use config::{
parquet::{parse_time_range_from_filename, read_metadata_from_bytes},
schema_ext::SchemaExt,
},
- CONFIG,
};
use infra::errors;
use opentelemetry::global;
@@ -150,20 +149,21 @@ impl Metrics for Querier {
}
// get parquet files
- let wal_dir = match std::path::Path::new(&CONFIG.common.data_wal_dir).canonicalize() {
- Ok(path) => {
- let mut path = path.to_str().unwrap().to_string();
- // Hack for windows
- if path.starts_with("\\\\?\\") {
- path = path[4..].to_string();
- path = path.replace('\\', "/");
+ let wal_dir =
+ match std::path::Path::new(&config::get_config().common.data_wal_dir).canonicalize() {
+ Ok(path) => {
+ let mut path = path.to_str().unwrap().to_string();
+ // Hack for windows
+ if path.starts_with("\\\\?\\") {
+ path = path[4..].to_string();
+ path = path.replace('\\', "/");
+ }
+ path
}
- path
- }
- Err(_) => {
- return Ok(Response::new(resp));
- }
- };
+ Err(_) => {
+ return Ok(Response::new(resp));
+ }
+ };
let pattern = format!("{wal_dir}/files/{org_id}/metrics/{stream_name}/");
let files = scan_files(&pattern, "parquet", None).unwrap_or_default();
diff --git a/src/handler/grpc/request/mod.rs b/src/handler/grpc/request/mod.rs
index e7e2d3809c..918a4e09a4 100644
--- a/src/handler/grpc/request/mod.rs
+++ b/src/handler/grpc/request/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/grpc/request/traces.rs b/src/handler/grpc/request/traces.rs
index e5a35873cb..7957cff0ed 100644
--- a/src/handler/grpc/request/traces.rs
+++ b/src/handler/grpc/request/traces.rs
@@ -13,7 +13,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::trace::v1::{
trace_service_server::TraceService, ExportTraceServiceRequest, ExportTraceServiceResponse,
};
@@ -30,22 +29,23 @@ impl TraceService for TraceServer {
&self,
request: tonic::Request,
) -> Result, tonic::Status> {
+ let cfg = config::get_config();
let metadata = request.metadata().clone();
let msg = format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
);
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(msg));
}
let in_req = request.into_inner();
- let org_id = metadata.get(&CONFIG.grpc.org_header_key);
+ let org_id = metadata.get(&cfg.grpc.org_header_key);
if org_id.is_none() {
return Err(Status::invalid_argument(msg));
}
- let stream_name = metadata.get(&CONFIG.grpc.stream_header_key);
+ let stream_name = metadata.get(&cfg.grpc.stream_header_key);
let mut in_stream_name: Option<&str> = None;
if let Some(stream_name) = stream_name {
in_stream_name = Some(stream_name.to_str().unwrap());
diff --git a/src/handler/grpc/request/usage.rs b/src/handler/grpc/request/usage.rs
index 0a6a388971..bfe720e357 100644
--- a/src/handler/grpc/request/usage.rs
+++ b/src/handler/grpc/request/usage.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use proto::cluster_rpc::{usage_server::Usage, UsageRequest, UsageResponse};
use tonic::{Request, Response, Status};
@@ -30,7 +29,7 @@ impl Usage for UsageServerImpl {
let metadata = request.metadata().clone();
let req = request.into_inner();
let report_to_stream = req.stream_name;
- let report_to_org_id = metadata.get(&CONFIG.grpc.org_header_key);
+ let report_to_org_id = metadata.get(&config::get_config().grpc.org_header_key);
let in_data = req.data.unwrap_or_default();
let resp = crate::service::logs::otlp_grpc::usage_ingest(
report_to_org_id.unwrap().to_str().unwrap(),
diff --git a/src/handler/http/auth/jwt.rs b/src/handler/http/auth/jwt.rs
index bced55778f..7722473ee0 100644
--- a/src/handler/http/auth/jwt.rs
+++ b/src/handler/http/auth/jwt.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
use std::str::FromStr;
#[cfg(feature = "enterprise")]
-use config::CONFIG;
+use config::get_config;
#[cfg(feature = "enterprise")]
use jsonwebtoken::TokenData;
#[cfg(feature = "enterprise")]
@@ -220,7 +220,7 @@ pub async fn process_token(
&org.name,
&user_email,
org.role.clone(),
- &CONFIG.auth.root_user_email,
+ &get_config().auth.root_user_email,
)
.await
{
@@ -239,8 +239,12 @@ pub async fn process_token(
}
for org in orgs_removed {
- match users::remove_user_from_org(&org.name, &user_email, &CONFIG.auth.root_user_email)
- .await
+ match users::remove_user_from_org(
+ &org.name,
+ &user_email,
+ &get_config().auth.root_user_email,
+ )
+ .await
{
Ok(_) => {
log::info!("User removed from the organization {}", org.name);
@@ -264,7 +268,7 @@ pub async fn process_token(
&org.name,
&user_email,
false,
- &CONFIG.auth.root_user_email,
+ &get_config().auth.root_user_email,
crate::common::meta::user::UpdateUser {
role: Some(org.role.clone()),
..Default::default()
diff --git a/src/handler/http/auth/mod.rs b/src/handler/http/auth/mod.rs
index fdd5ca3edd..7a692e52d7 100644
--- a/src/handler/http/auth/mod.rs
+++ b/src/handler/http/auth/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/auth/token.rs b/src/handler/http/auth/token.rs
index e448002f1d..f063225170 100644
--- a/src/handler/http/auth/token.rs
+++ b/src/handler/http/auth/token.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -20,8 +20,6 @@ use actix_web::{
http::{header, Method},
};
#[cfg(feature = "enterprise")]
-use config::CONFIG;
-#[cfg(feature = "enterprise")]
use o2_enterprise::enterprise::{common::infra::config::O2_CONFIG, dex::service::auth::get_jwks};
use crate::common::utils::auth::AuthExtractor;
@@ -44,7 +42,7 @@ pub async fn token_validator(
let path = match req
.request()
.path()
- .strip_prefix(format!("{}/api/", CONFIG.common.base_uri).as_str())
+ .strip_prefix(format!("{}/api/", config::get_config().common.base_uri).as_str())
{
Some(path) => path,
None => req.request().path(),
diff --git a/src/handler/http/auth/validator.rs b/src/handler/http/auth/validator.rs
index 07e0241ced..e95e4b14d8 100644
--- a/src/handler/http/auth/validator.rs
+++ b/src/handler/http/auth/validator.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -20,7 +20,7 @@ use actix_web::{
web, Error,
};
use actix_web_httpauth::extractors::basic::BasicAuth;
-use config::{utils::base64, CONFIG};
+use config::{get_config, utils::base64};
use crate::{
common::{
@@ -45,10 +45,11 @@ pub async fn validator(
auth_info: AuthExtractor,
path_prefix: &str,
) -> Result {
+ let cfg = get_config();
let path = match req
.request()
.path()
- .strip_prefix(format!("{}{}", CONFIG.common.base_uri, path_prefix).as_str())
+ .strip_prefix(format!("{}{}", cfg.common.base_uri, path_prefix).as_str())
{
Some(path) => path,
None => req.request().path(),
@@ -244,10 +245,11 @@ pub async fn validator_aws(
req: ServiceRequest,
_credentials: Option,
) -> Result {
+ let cfg = get_config();
let path = req
.request()
.path()
- .strip_prefix(format!("{}/aws/", CONFIG.common.base_uri).as_str())
+ .strip_prefix(format!("{}/aws/", cfg.common.base_uri).as_str())
.unwrap_or(req.request().path());
match req.headers().get("X-Amz-Firehose-Access-Key") {
@@ -288,10 +290,11 @@ pub async fn validator_gcp(
req: ServiceRequest,
_credentials: Option,
) -> Result {
+ let cfg = get_config();
let path = req
.request()
.path()
- .strip_prefix(format!("{}/gcp/", CONFIG.common.base_uri).as_str())
+ .strip_prefix(format!("{}/gcp/", cfg.common.base_uri).as_str())
.unwrap_or(req.request().path());
let query =
@@ -332,7 +335,7 @@ pub async fn validator_rum(
let path = req
.request()
.path()
- .strip_prefix(format!("{}/rum/v1/", CONFIG.common.base_uri).as_str())
+ .strip_prefix(format!("{}/rum/v1/", get_config().common.base_uri).as_str())
.unwrap_or(req.request().path());
// After this previous path clean we should get only the
diff --git a/src/handler/http/mod.rs b/src/handler/http/mod.rs
index 0f719e9acd..169c50d469 100644
--- a/src/handler/http/mod.rs
+++ b/src/handler/http/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/authz/fga.rs b/src/handler/http/request/authz/fga.rs
index 1c7da1ce38..1af9f0d63e 100644
--- a/src/handler/http/request/authz/fga.rs
+++ b/src/handler/http/request/authz/fga.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/authz/mod.rs b/src/handler/http/request/authz/mod.rs
index 56306869ee..70b14e0c9b 100644
--- a/src/handler/http/request/authz/mod.rs
+++ b/src/handler/http/request/authz/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/enrichment_table/mod.rs b/src/handler/http/request/enrichment_table/mod.rs
index 01b7ac44cd..7df00c10af 100644
--- a/src/handler/http/request/enrichment_table/mod.rs
+++ b/src/handler/http/request/enrichment_table/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ use std::io::Error;
use actix_multipart::Multipart;
use actix_web::{post, web, HttpRequest, HttpResponse};
-use config::{CONFIG, SIZE_IN_MB};
+use config::SIZE_IN_MB;
use hashbrown::HashMap;
use crate::{
@@ -62,10 +62,11 @@ pub async fn save_enrichment_table(
/ SIZE_IN_MB
}
};
- if content_length > CONFIG.limit.enrichment_table_limit as f64 {
+ let cfg = config::get_config();
+ if content_length > cfg.limit.enrichment_table_limit as f64 {
return Ok(MetaHttpResponse::bad_request(format!(
"exceeds allowed limit of {} mb",
- CONFIG.limit.enrichment_table_limit
+ cfg.limit.enrichment_table_limit
)));
}
match content_type {
diff --git a/src/handler/http/request/functions/mod.rs b/src/handler/http/request/functions/mod.rs
index 7b38abe53d..a84fcb1819 100644
--- a/src/handler/http/request/functions/mod.rs
+++ b/src/handler/http/request/functions/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/kv/mod.rs b/src/handler/http/request/kv/mod.rs
index 394ce14e07..4f702f99d5 100644
--- a/src/handler/http/request/kv/mod.rs
+++ b/src/handler/http/request/kv/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/logs/ingest.rs b/src/handler/http/request/logs/ingest.rs
index 83f757424a..ef69de8048 100644
--- a/src/handler/http/request/logs/ingest.rs
+++ b/src/handler/http/request/logs/ingest.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,6 @@
use std::io::Error;
use actix_web::{http, post, web, HttpRequest, HttpResponse};
-use config::CONFIG;
use crate::{
common::meta::{
@@ -288,7 +287,7 @@ pub async fn otlp_logs_write(
let user_email = req.headers().get("user_id").unwrap().to_str().unwrap();
let in_stream_name = req
.headers()
- .get(&CONFIG.grpc.stream_header_key)
+ .get(&config::get_config().grpc.stream_header_key)
.map(|header| header.to_str().unwrap());
if content_type.eq(CONTENT_TYPE_PROTO) {
// log::info!("otlp::logs_proto_handler");
diff --git a/src/handler/http/request/logs/mod.rs b/src/handler/http/request/logs/mod.rs
index ad13676aa2..9558bd7dd9 100644
--- a/src/handler/http/request/logs/mod.rs
+++ b/src/handler/http/request/logs/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/metrics/ingest.rs b/src/handler/http/request/metrics/ingest.rs
index 534f94ca2f..92f168e18b 100644
--- a/src/handler/http/request/metrics/ingest.rs
+++ b/src/handler/http/request/metrics/ingest.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/metrics/mod.rs b/src/handler/http/request/metrics/mod.rs
index ad13676aa2..9558bd7dd9 100644
--- a/src/handler/http/request/metrics/mod.rs
+++ b/src/handler/http/request/metrics/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/mod.rs b/src/handler/http/request/mod.rs
index a6d90a009a..7dbe9029a0 100644
--- a/src/handler/http/request/mod.rs
+++ b/src/handler/http/request/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/organization/es.rs b/src/handler/http/request/organization/es.rs
index 209ce15c31..b0df5bd770 100644
--- a/src/handler/http/request/organization/es.rs
+++ b/src/handler/http/request/organization/es.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/organization/mod.rs b/src/handler/http/request/organization/mod.rs
index 11b6935ae2..774581604a 100644
--- a/src/handler/http/request/organization/mod.rs
+++ b/src/handler/http/request/organization/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/organization/settings.rs b/src/handler/http/request/organization/settings.rs
index 24585e31a1..339b724029 100644
--- a/src/handler/http/request/organization/settings.rs
+++ b/src/handler/http/request/organization/settings.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/prom/mod.rs b/src/handler/http/request/prom/mod.rs
index ae9897e59f..ea8648f7be 100644
--- a/src/handler/http/request/prom/mod.rs
+++ b/src/handler/http/request/prom/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/rum/ingest.rs b/src/handler/http/request/rum/ingest.rs
index 9c48eec291..f86bcd0414 100644
--- a/src/handler/http/request/rum/ingest.rs
+++ b/src/handler/http/request/rum/ingest.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/rum/mod.rs b/src/handler/http/request/rum/mod.rs
index ad13676aa2..9558bd7dd9 100644
--- a/src/handler/http/request/rum/mod.rs
+++ b/src/handler/http/request/rum/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/search/mod.rs b/src/handler/http/request/search/mod.rs
index 3b660591fd..9cf619dfce 100644
--- a/src/handler/http/request/search/mod.rs
+++ b/src/handler/http/request/search/mod.rs
@@ -18,14 +18,14 @@ use std::{collections::HashMap, io::Error};
use actix_web::{get, http::StatusCode, post, web, HttpRequest, HttpResponse};
use chrono::Duration;
use config::{
- ider,
+ get_config, ider,
meta::{
stream::StreamType,
usage::{RequestStats, UsageType},
},
metrics,
utils::{base64, json},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use infra::{errors, schema::STREAM_SCHEMAS_LATEST};
use opentelemetry::{global, trace::TraceContextExt};
@@ -123,13 +123,14 @@ pub async fn search(
let start = std::time::Instant::now();
let org_id = org_id.into_inner();
+ let cfg = get_config();
let mut http_span = None;
- let trace_id = if CONFIG.common.tracing_enabled {
+ let trace_id = if cfg.common.tracing_enabled {
let ctx = global::get_text_map_propagator(|propagator| {
propagator.extract(&RequestHeaderExtractor::new(in_req.headers()))
});
ctx.span().span_context().trace_id().to_string()
- } else if CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_search_enabled {
let span = tracing::info_span!("/api/{org_id}/_search", org_id = org_id.clone());
let trace_id = span.context().span().span_context().trace_id().to_string();
http_span = Some(span);
@@ -219,13 +220,14 @@ pub async fn search(
}
}
+ let cfg = get_config();
// get a local search queue lock
#[cfg(not(feature = "enterprise"))]
let locker = SearchService::QUEUE_LOCKER.clone();
#[cfg(not(feature = "enterprise"))]
let locker = locker.lock().await;
#[cfg(not(feature = "enterprise"))]
- if !CONFIG.common.feature_query_queue_enabled {
+ if !cfg.common.feature_query_queue_enabled {
drop(locker);
}
#[cfg(not(feature = "enterprise"))]
@@ -241,7 +243,7 @@ pub async fn search(
Some(user_id.to_str().unwrap().to_string()),
&req,
);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.unwrap()).await
} else {
search_fut.await
@@ -388,14 +390,14 @@ pub async fn around(
) -> Result {
let start = std::time::Instant::now();
let (org_id, stream_name) = path.into_inner();
-
+ let cfg = get_config();
let mut http_span = None;
- let trace_id = if CONFIG.common.tracing_enabled {
+ let trace_id = if cfg.common.tracing_enabled {
let ctx = global::get_text_map_propagator(|propagator| {
propagator.extract(&RequestHeaderExtractor::new(in_req.headers()))
});
ctx.span().span_context().trace_id().to_string()
- } else if CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_search_enabled {
let span = tracing::info_span!(
"/api/{org_id}/{stream_name}/_around",
org_id = org_id.clone(),
@@ -468,7 +470,7 @@ pub async fn around(
#[cfg(not(feature = "enterprise"))]
let locker = locker.lock().await;
#[cfg(not(feature = "enterprise"))]
- if !CONFIG.common.feature_query_queue_enabled {
+ if !cfg.common.feature_query_queue_enabled {
drop(locker);
}
#[cfg(not(feature = "enterprise"))]
@@ -501,7 +503,7 @@ pub async fn around(
size: around_size / 2,
start_time: around_start_time,
end_time: around_key,
- sort_by: Some(format!("{} DESC", CONFIG.common.column_timestamp)),
+ sort_by: Some(format!("{} DESC", cfg.common.column_timestamp)),
sql_mode: "".to_string(),
quick_mode: false,
query_type: "".to_string(),
@@ -525,7 +527,7 @@ pub async fn around(
.ok()
.map(|v| v.to_string());
let search_fut = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.clone().unwrap()).await
} else {
search_fut.await
@@ -580,7 +582,7 @@ pub async fn around(
size: around_size / 2,
start_time: around_key,
end_time: around_end_time,
- sort_by: Some(format!("{} ASC", CONFIG.common.column_timestamp)),
+ sort_by: Some(format!("{} ASC", cfg.common.column_timestamp)),
sql_mode: "".to_string(),
quick_mode: false,
query_type: "".to_string(),
@@ -597,7 +599,7 @@ pub async fn around(
timeout,
};
let search_fut = SearchService::search(&trace_id, &org_id, stream_type, user_id, &req);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.unwrap()).await
} else {
search_fut.await
@@ -769,13 +771,15 @@ pub async fn values(
Some(v) => v.to_str().unwrap(),
None => "",
};
+
+ let cfg = get_config();
let mut http_span = None;
- let trace_id = if CONFIG.common.tracing_enabled {
+ let trace_id = if cfg.common.tracing_enabled {
let ctx = global::get_text_map_propagator(|propagator| {
propagator.extract(&RequestHeaderExtractor::new(in_req.headers()))
});
ctx.span().span_context().trace_id().to_string()
- } else if CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_search_enabled {
let span = tracing::info_span!(
"/api/{org_id}/{stream_name}/_values",
org_id = org_id.clone(),
@@ -879,9 +883,10 @@ async fn values_v1(
}
}
+ let cfg = get_config();
let default_sql = format!(
"SELECT {} FROM \"{stream_name}\"",
- CONFIG.common.column_timestamp
+ cfg.common.column_timestamp
);
let mut query_sql = match query.get("filter") {
None => default_sql,
@@ -919,7 +924,7 @@ async fn values_v1(
query_sql = RE_ONLY_SELECT
.replace(
&query_sql,
- format!("SELECT {} ", CONFIG.common.column_timestamp).as_str(),
+ format!("SELECT {} ", cfg.common.column_timestamp).as_str(),
)
.to_string()
};
@@ -965,7 +970,7 @@ async fn values_v1(
#[cfg(not(feature = "enterprise"))]
let locker = locker.lock().await;
#[cfg(not(feature = "enterprise"))]
- if !CONFIG.common.feature_query_queue_enabled {
+ if !cfg.common.feature_query_queue_enabled {
drop(locker);
}
#[cfg(not(feature = "enterprise"))]
@@ -1028,7 +1033,7 @@ async fn values_v1(
Some(user_id.to_string()),
&req,
);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.unwrap()).await
} else {
search_fut.await
@@ -1203,13 +1208,14 @@ async fn values_v2(
.get("timeout")
.map_or(0, |v| v.parse::().unwrap_or(0));
+ let cfg = get_config();
// get a local search queue lock
#[cfg(not(feature = "enterprise"))]
let locker = SearchService::QUEUE_LOCKER.clone();
#[cfg(not(feature = "enterprise"))]
let locker = locker.lock().await;
#[cfg(not(feature = "enterprise"))]
- if !CONFIG.common.feature_query_queue_enabled {
+ if !cfg.common.feature_query_queue_enabled {
drop(locker);
}
#[cfg(not(feature = "enterprise"))]
@@ -1249,7 +1255,7 @@ async fn values_v2(
Some(user_id.to_string()),
&req,
);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.unwrap()).await
} else {
search_fut.await
@@ -1393,14 +1399,14 @@ pub async fn search_partition(
body: web::Bytes,
) -> Result {
let start = std::time::Instant::now();
-
+ let cfg = get_config();
let mut http_span = None;
- let trace_id = if CONFIG.common.tracing_enabled {
+ let trace_id = if cfg.common.tracing_enabled {
let ctx = global::get_text_map_propagator(|propagator| {
propagator.extract(&RequestHeaderExtractor::new(in_req.headers()))
});
ctx.span().span_context().trace_id().to_string()
- } else if CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_search_enabled {
let span = tracing::info_span!("/api/{org_id}/_search_partition", org_id = org_id.clone());
let trace_id = span.context().span().span_context().trace_id().to_string();
http_span = Some(span);
@@ -1425,7 +1431,7 @@ pub async fn search_partition(
}
let search_fut = SearchService::search_partition(&trace_id, &org_id, stream_type, &req);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.unwrap()).await
} else {
search_fut.await
diff --git a/src/handler/http/request/search/saved_view.rs b/src/handler/http/request/search/saved_view.rs
index b10785438b..d32111be48 100644
--- a/src/handler/http/request/search/saved_view.rs
+++ b/src/handler/http/request/search/saved_view.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/status/mod.rs b/src/handler/http/request/status/mod.rs
index b12a6c2374..0d15c17215 100644
--- a/src/handler/http/request/status/mod.rs
+++ b/src/handler/http/request/status/mod.rs
@@ -25,9 +25,10 @@ use actix_web::{
use arrow_schema::Schema;
use config::{
cluster::{is_ingester, LOCAL_NODE_ROLE, LOCAL_NODE_UUID},
+ get_config, get_instance_id,
meta::cluster::NodeStatus,
utils::{json, schema_ext::SchemaExt},
- CONFIG, INSTANCE_ID, QUICK_MODEL_FIELDS, SQL_FULL_TEXT_SEARCH_FIELDS,
+ QUICK_MODEL_FIELDS, SQL_FULL_TEXT_SEARCH_FIELDS,
};
use hashbrown::HashMap;
use infra::{
@@ -216,60 +217,62 @@ pub async fn zo_config() -> Result {
#[cfg(not(feature = "enterprise"))]
let build_type = "opensource";
+ let cfg = get_config();
Ok(HttpResponse::Ok().json(ConfigResponse {
version: VERSION.to_string(),
- instance: INSTANCE_ID.get("instance_id").unwrap().to_string(),
+ instance: get_instance_id(),
commit_hash: COMMIT_HASH.to_string(),
build_date: BUILD_DATE.to_string(),
build_type: build_type.to_string(),
- telemetry_enabled: CONFIG.common.telemetry_enabled,
+ telemetry_enabled: cfg.common.telemetry_enabled,
default_fts_keys: SQL_FULL_TEXT_SEARCH_FIELDS
.iter()
.map(|s| s.to_string())
.collect(),
default_quick_mode_fields: QUICK_MODEL_FIELDS.to_vec(),
default_functions: DEFAULT_FUNCTIONS.to_vec(),
- sql_base64_enabled: CONFIG.common.ui_sql_base64_enabled,
- timestamp_column: CONFIG.common.column_timestamp.clone(),
+ sql_base64_enabled: cfg.common.ui_sql_base64_enabled,
+ timestamp_column: cfg.common.column_timestamp.clone(),
syslog_enabled: *SYSLOG_ENABLED.read(),
- data_retention_days: CONFIG.compact.data_retention_days,
- restricted_routes_on_empty_data: CONFIG.common.restricted_routes_on_empty_data,
+ data_retention_days: cfg.compact.data_retention_days,
+ restricted_routes_on_empty_data: cfg.common.restricted_routes_on_empty_data,
sso_enabled,
native_login_enabled,
rbac_enabled,
super_cluster_enabled,
- query_on_stream_selection: CONFIG.common.query_on_stream_selection,
- show_stream_stats_doc_num: CONFIG.common.show_stream_dates_doc_num,
+ query_on_stream_selection: cfg.common.query_on_stream_selection,
+ show_stream_stats_doc_num: cfg.common.show_stream_dates_doc_num,
custom_logo_text,
custom_slack_url: custom_slack_url.to_string(),
custom_docs_url: custom_docs_url.to_string(),
custom_logo_img: logo,
custom_hide_menus: custom_hide_menus.to_string(),
rum: Rum {
- enabled: CONFIG.rum.enabled,
- client_token: CONFIG.rum.client_token.to_string(),
- application_id: CONFIG.rum.application_id.to_string(),
- site: CONFIG.rum.site.to_string(),
- service: CONFIG.rum.service.to_string(),
- env: CONFIG.rum.env.to_string(),
- version: CONFIG.rum.version.to_string(),
- organization_identifier: CONFIG.rum.organization_identifier.to_string(),
- api_version: CONFIG.rum.api_version.to_string(),
- insecure_http: CONFIG.rum.insecure_http,
+ enabled: cfg.rum.enabled,
+ client_token: cfg.rum.client_token.to_string(),
+ application_id: cfg.rum.application_id.to_string(),
+ site: cfg.rum.site.to_string(),
+ service: cfg.rum.service.to_string(),
+ env: cfg.rum.env.to_string(),
+ version: cfg.rum.version.to_string(),
+ organization_identifier: cfg.rum.organization_identifier.to_string(),
+ api_version: cfg.rum.api_version.to_string(),
+ insecure_http: cfg.rum.insecure_http,
},
- meta_org: CONFIG.common.usage_org.to_string(),
- quick_mode_enabled: CONFIG.limit.quick_mode_enabled,
- user_defined_schemas_enabled: CONFIG.common.allow_user_defined_schemas,
- all_fields_name: CONFIG.common.all_fields_name.to_string(),
+ meta_org: cfg.common.usage_org.to_string(),
+ quick_mode_enabled: cfg.limit.quick_mode_enabled,
+ user_defined_schemas_enabled: cfg.common.allow_user_defined_schemas,
+ all_fields_name: cfg.common.all_fields_name.to_string(),
}))
}
#[get("/status")]
pub async fn cache_status() -> Result {
+ let cfg = get_config();
let mut stats: HashMap<&str, json::Value> = HashMap::default();
stats.insert("LOCAL_NODE_UUID", json::json!(LOCAL_NODE_UUID.clone()));
- stats.insert("LOCAL_NODE_NAME", json::json!(&CONFIG.common.instance_name));
- stats.insert("LOCAL_NODE_ROLE", json::json!(&CONFIG.common.node_role));
+ stats.insert("LOCAL_NODE_NAME", json::json!(&cfg.common.instance_name));
+ stats.insert("LOCAL_NODE_ROLE", json::json!(&cfg.common.node_role));
let nodes = cluster::get_cached_online_nodes().await;
stats.insert("NODE_LIST", json::json!(nodes));
@@ -318,6 +321,17 @@ pub async fn cache_status() -> Result {
Ok(HttpResponse::Ok().json(stats))
}
+#[get("")]
+pub async fn config_reload() -> Result {
+ if let Err(e) = config::refresh_config() {
+ return Ok(
+ HttpResponse::InternalServerError().json(serde_json::json!({"status": e.to_string()}))
+ );
+ }
+ let status = "succcessfully reloaded config";
+ Ok(HttpResponse::Ok().json(serde_json::json!({"status": status})))
+}
+
async fn get_stream_schema_status() -> (usize, usize, usize) {
let mut stream_num = 0;
let mut stream_schema_num = 0;
@@ -424,15 +438,16 @@ pub async fn redirect(req: HttpRequest) -> Result {
})
.unwrap();
+ let cfg = get_config();
let mut auth_cookie = Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(SameSite::Lax);
} else {
auth_cookie.set_same_site(SameSite::None);
@@ -462,6 +477,7 @@ pub async fn dex_login() -> Result {
#[cfg(feature = "enterprise")]
#[get("/dex_refresh")]
async fn refresh_token_with_dex(req: actix_web::HttpRequest) -> HttpResponse {
+ let cfg = get_config();
let token = if let Some(cookie) = req.cookie("auth_tokens") {
let auth_tokens: AuthTokens = json::from_str(cookie.value()).unwrap_or_default();
@@ -498,12 +514,12 @@ async fn refresh_token_with_dex(req: actix_web::HttpRequest) -> HttpResponse {
let mut auth_cookie = Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(SameSite::Lax);
} else {
auth_cookie.set_same_site(SameSite::None);
@@ -516,12 +532,12 @@ async fn refresh_token_with_dex(req: actix_web::HttpRequest) -> HttpResponse {
let mut auth_cookie = Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(SameSite::Lax);
} else {
auth_cookie.set_same_site(SameSite::None);
@@ -548,16 +564,17 @@ async fn logout(req: actix_web::HttpRequest) -> HttpResponse {
}
};
+ let cfg = get_config();
let tokens = json::to_string(&AuthTokens::default()).unwrap();
let mut auth_cookie = Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(SameSite::Lax);
} else {
auth_cookie.set_same_site(SameSite::None);
diff --git a/src/handler/http/request/syslog/mod.rs b/src/handler/http/request/syslog/mod.rs
index 9fd5351c40..b16d24d00f 100644
--- a/src/handler/http/request/syslog/mod.rs
+++ b/src/handler/http/request/syslog/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/http/request/traces/mod.rs b/src/handler/http/request/traces/mod.rs
index c35fed9d7b..0bb0a44ee6 100644
--- a/src/handler/http/request/traces/mod.rs
+++ b/src/handler/http/request/traces/mod.rs
@@ -16,7 +16,7 @@
use std::{collections::HashMap, io::Error};
use actix_web::{get, http, post, web, HttpRequest, HttpResponse};
-use config::{ider, meta::stream::StreamType, metrics, utils::json, CONFIG};
+use config::{get_config, ider, meta::stream::StreamType, metrics, utils::json};
use infra::errors;
use opentelemetry::{global, trace::TraceContextExt};
use serde::Serialize;
@@ -76,7 +76,7 @@ async fn handle_req(
let content_type = req.headers().get("Content-Type").unwrap().to_str().unwrap();
let in_stream_name = req
.headers()
- .get(&CONFIG.grpc.stream_header_key)
+ .get(&get_config().grpc.stream_header_key)
.map(|header| header.to_str().unwrap());
if content_type.eq(CONTENT_TYPE_PROTO) {
otlp_http::traces_proto(&org_id, **thread_id, body, in_stream_name).await
@@ -136,14 +136,14 @@ pub async fn get_latest_traces(
) -> Result {
let start = std::time::Instant::now();
let (org_id, stream_name) = path.into_inner();
-
+ let cfg = get_config();
let mut http_span = None;
- let trace_id = if CONFIG.common.tracing_enabled {
+ let trace_id = if cfg.common.tracing_enabled {
let ctx = global::get_text_map_propagator(|propagator| {
propagator.extract(&RequestHeaderExtractor::new(in_req.headers()))
});
ctx.span().span_context().trace_id().to_string()
- } else if CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_search_enabled {
let span = tracing::info_span!(
"/api/{org_id}/{stream_name}/traces/latest",
org_id = org_id.clone(),
@@ -221,13 +221,14 @@ pub async fn get_latest_traces(
.get("timeout")
.map_or(0, |v| v.parse::().unwrap_or(0));
+ let cfg = get_config();
// get a local search queue lock
#[cfg(not(feature = "enterprise"))]
let locker = SearchService::QUEUE_LOCKER.clone();
#[cfg(not(feature = "enterprise"))]
let locker = locker.lock().await;
#[cfg(not(feature = "enterprise"))]
- if !CONFIG.common.feature_query_queue_enabled {
+ if !cfg.common.feature_query_queue_enabled {
drop(locker);
}
#[cfg(not(feature = "enterprise"))]
@@ -239,7 +240,7 @@ pub async fn get_latest_traces(
// search
let query_sql = format!(
"SELECT trace_id, min({}) as zo_sql_timestamp, min(start_time) as trace_start_time, max(end_time) as trace_end_time FROM {stream_name}",
- CONFIG.common.column_timestamp
+ cfg.common.column_timestamp
);
let query_sql = if filter.is_empty() {
format!("{query_sql} GROUP BY trace_id ORDER BY zo_sql_timestamp DESC")
@@ -279,7 +280,7 @@ pub async fn get_latest_traces(
.map(|v| v.to_string());
let search_fut = SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.clone().unwrap()).await
} else {
search_fut.await
@@ -360,7 +361,7 @@ pub async fn get_latest_traces(
.join("','");
let query_sql = format!(
"SELECT {}, trace_id, start_time, end_time, duration, service_name, operation_name, span_status FROM {stream_name} WHERE trace_id IN ('{}') ORDER BY {} ASC",
- CONFIG.common.column_timestamp, trace_ids, CONFIG.common.column_timestamp,
+ cfg.common.column_timestamp, trace_ids, cfg.common.column_timestamp,
);
req.query.from = 0;
req.query.size = 9999;
@@ -372,7 +373,7 @@ pub async fn get_latest_traces(
loop {
let search_fut =
SearchService::search(&trace_id, &org_id, stream_type, user_id.clone(), &req);
- let search_res = if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ let search_res = if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
search_fut.instrument(http_span.clone().unwrap()).await
} else {
search_fut.await
diff --git a/src/handler/http/request/users/mod.rs b/src/handler/http/request/users/mod.rs
index 50a86603c1..740867ac76 100644
--- a/src/handler/http/request/users/mod.rs
+++ b/src/handler/http/request/users/mod.rs
@@ -21,8 +21,8 @@ use actix_web::{
post, put, web, HttpRequest, HttpResponse,
};
use config::{
+ get_config,
utils::{base64, json},
- CONFIG,
};
use strum::IntoEnumIterator;
@@ -275,6 +275,7 @@ pub async fn authentication(
}
};
if resp.status {
+ let cfg = get_config();
let access_token = format!(
"Basic {}",
base64::encode(&format!("{}:{}", auth.name, auth.password))
@@ -287,12 +288,12 @@ pub async fn authentication(
let mut auth_cookie = cookie::Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(cookie::SameSite::Lax);
} else {
auth_cookie.set_same_site(cookie::SameSite::None);
@@ -311,6 +312,7 @@ pub async fn get_auth(_req: HttpRequest) -> Result {
use crate::handler::http::auth::validator::ID_TOKEN_HEADER;
+ let cfg = get_config();
let mut resp = SignInResponse::default();
if let Some(auth_header) = _req.headers().get("Authorization") {
if let Ok(auth_header) = auth_header.to_str() {
@@ -351,21 +353,21 @@ pub async fn get_auth(_req: HttpRequest) -> Result {
let mut auth_cookie = cookie::Cookie::new("auth_tokens", tokens);
auth_cookie.set_expires(
cookie::time::OffsetDateTime::now_utc()
- + cookie::time::Duration::seconds(CONFIG.auth.cookie_max_age),
+ + cookie::time::Duration::seconds(cfg.auth.cookie_max_age),
);
auth_cookie.set_http_only(true);
- auth_cookie.set_secure(CONFIG.auth.cookie_secure_only);
+ auth_cookie.set_secure(cfg.auth.cookie_secure_only);
auth_cookie.set_path("/");
- if CONFIG.auth.cookie_same_site_lax {
+ if cfg.auth.cookie_same_site_lax {
auth_cookie.set_same_site(cookie::SameSite::Lax);
} else {
auth_cookie.set_same_site(cookie::SameSite::None);
}
let url = format!(
"{}{}/web/cb#id_token={}.{}",
- CONFIG.common.web_url,
- CONFIG.common.base_uri,
+ cfg.common.web_url,
+ cfg.common.base_uri,
ID_TOKEN_HEADER,
base64::encode(&id_token.to_string())
);
diff --git a/src/handler/http/router/mod.rs b/src/handler/http/router/mod.rs
index 2a19e62908..4a207285c9 100644
--- a/src/handler/http/router/mod.rs
+++ b/src/handler/http/router/mod.rs
@@ -24,7 +24,7 @@ use actix_web::{
};
use actix_web_httpauth::middleware::HttpAuthentication;
use actix_web_lab::middleware::{from_fn, Next};
-use config::CONFIG;
+use config::get_config;
use futures::FutureExt;
use utoipa::OpenApi;
use utoipa_swagger_ui::SwaggerUi;
@@ -84,7 +84,7 @@ async fn audit_middleware(
next: Next,
) -> Result, actix_web::Error> {
let method = req.method().to_string();
- let prefix = format!("{}/api/", CONFIG.common.base_uri);
+ let prefix = format!("{}/api/", get_config().common.base_uri);
let path = req.path().strip_prefix(&prefix).unwrap().to_string();
let path_columns = path.split('/').collect::>();
let path_len = path_columns.len();
@@ -219,7 +219,7 @@ pub fn get_basic_routes(cfg: &mut web::ServiceConfig) {
cfg.service(
SwaggerUi::new("/swagger/{_:.*}")
.url(
- format!("{}/api-doc/openapi.json", CONFIG.common.base_uri),
+ format!("{}/api-doc/openapi.json", get_config().common.base_uri),
openapi::ApiDoc::openapi(),
)
.url("/api-doc/openapi.json", openapi::ApiDoc::openapi()),
@@ -227,13 +227,14 @@ pub fn get_basic_routes(cfg: &mut web::ServiceConfig) {
cfg.service(web::redirect("/swagger", "/swagger/"));
cfg.service(web::redirect("/docs", "/swagger/"));
- if CONFIG.common.ui_enabled {
+ if get_config().common.ui_enabled {
cfg.service(web::redirect("/", "./web/"));
cfg.service(web::redirect("/web", "./web/"));
cfg.service(
web::scope("/web")
.wrap_fn(|req, srv| {
- let prefix = format!("{}/web/", CONFIG.common.base_uri);
+ let cfg = get_config();
+ let prefix = format!("{}/web/", cfg.common.base_uri);
let path = req.path().strip_prefix(&prefix).unwrap().to_string();
srv.call(req).map(move |res| {
@@ -272,9 +273,10 @@ pub fn get_config_routes(cfg: &mut web::ServiceConfig) {
let cors = get_cors();
cfg.service(
web::scope("/config")
- .wrap(cors)
+ .wrap(cors.clone())
.service(status::zo_config)
- .service(status::logout),
+ .service(status::logout)
+ .service(web::scope("/reload").service(status::config_reload)),
);
}
@@ -288,13 +290,13 @@ pub fn get_config_routes(cfg: &mut web::ServiceConfig) {
.service(status::redirect)
.service(status::dex_login)
.service(status::refresh_token_with_dex)
- .service(status::logout),
+ .service(status::logout)
+ .service(web::scope("/reload").service(status::config_reload)),
);
}
pub fn get_service_routes(cfg: &mut web::ServiceConfig) {
let cors = get_cors();
-
// set server header
#[cfg(feature = "enterprise")]
let server = format!(
@@ -302,10 +304,10 @@ pub fn get_service_routes(cfg: &mut web::ServiceConfig) {
o2_enterprise::enterprise::common::infra::config::O2_CONFIG
.super_cluster
.region,
- CONFIG.common.instance_name_short
+ get_config().common.instance_name_short
);
#[cfg(not(feature = "enterprise"))]
- let server = CONFIG.common.instance_name_short.to_string();
+ let server = get_config().common.instance_name_short.to_string();
cfg.service(
web::scope("/api")
diff --git a/src/handler/http/router/openapi.rs b/src/handler/http/router/openapi.rs
index 5a1b2bfb5d..da824ec73e 100644
--- a/src/handler/http/router/openapi.rs
+++ b/src/handler/http/router/openapi.rs
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{meta::stream::StreamType, CONFIG};
+use config::{get_config, meta::stream::StreamType};
use utoipa::{openapi::security::SecurityScheme, Modify, OpenApi};
use crate::{common::meta, handler::http::request};
@@ -246,8 +246,9 @@ pub struct SecurityAddon;
impl Modify for SecurityAddon {
fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
- if !CONFIG.common.base_uri.is_empty() {
- openapi.servers = Some(vec![utoipa::openapi::Server::new(&CONFIG.common.base_uri)]);
+ let cfg = get_config();
+ if !cfg.common.base_uri.is_empty() {
+ openapi.servers = Some(vec![utoipa::openapi::Server::new(&cfg.common.base_uri)]);
}
let components = openapi.components.as_mut().unwrap();
components.add_security_scheme(
diff --git a/src/handler/http/router/ui.rs b/src/handler/http/router/ui.rs
index bb9f50e979..98f8ca6119 100644
--- a/src/handler/http/router/ui.rs
+++ b/src/handler/http/router/ui.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/mod.rs b/src/handler/mod.rs
index dd4f8f53d2..2672bc5ad8 100644
--- a/src/handler/mod.rs
+++ b/src/handler/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/handler/tcp_udp/mod.rs b/src/handler/tcp_udp/mod.rs
index 071b90e1ac..c36e121a5a 100644
--- a/src/handler/tcp_udp/mod.rs
+++ b/src/handler/tcp_udp/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/cache/file_data/disk.rs b/src/infra/src/cache/file_data/disk.rs
index e111e0d684..6f1313b297 100644
--- a/src/infra/src/cache/file_data/disk.rs
+++ b/src/infra/src/cache/file_data/disk.rs
@@ -22,12 +22,11 @@ use std::{
use async_recursion::async_recursion;
use bytes::Bytes;
use config::{
- is_local_disk_storage, metrics,
+ get_config, is_local_disk_storage, metrics,
utils::{
asynchronism::file::*,
hash::{gxhash, Sum32},
},
- CONFIG,
};
use once_cell::sync::Lazy;
use tokio::{fs, sync::RwLock};
@@ -36,8 +35,9 @@ use super::CacheStrategy;
use crate::storage;
static FILES: Lazy>> = Lazy::new(|| {
- let mut files = Vec::with_capacity(CONFIG.disk_cache.bucket_num);
- for _ in 0..CONFIG.disk_cache.bucket_num {
+ let cfg = get_config();
+ let mut files = Vec::with_capacity(cfg.disk_cache.bucket_num);
+ for _ in 0..cfg.disk_cache.bucket_num {
files.push(RwLock::new(FileData::new()));
}
files
@@ -59,22 +59,24 @@ impl Default for FileData {
impl FileData {
pub fn new() -> FileData {
+ let cfg = get_config();
FileData::with_capacity_and_cache_strategy(
- CONFIG.disk_cache.max_size,
- &CONFIG.disk_cache.cache_strategy,
+ cfg.disk_cache.max_size,
+ &cfg.disk_cache.cache_strategy,
)
}
pub fn with_capacity_and_cache_strategy(max_size: usize, strategy: &str) -> FileData {
+ let cfg = get_config();
FileData {
max_size,
cur_size: 0,
root_dir: format!(
"{}{}",
- CONFIG.common.data_cache_dir,
+ cfg.common.data_cache_dir,
storage::format_key("", true)
),
- multi_dir: CONFIG
+ multi_dir: cfg
.disk_cache
.multi_dir
.split(',')
@@ -114,7 +116,7 @@ impl FileData {
// cache is full, need release some space
let need_release_size = min(
self.max_size,
- max(CONFIG.disk_cache.release_size, data_size * 100),
+ max(get_config().disk_cache.release_size, data_size * 100),
);
self.gc(trace_id, need_release_size).await?;
}
@@ -210,6 +212,7 @@ impl FileData {
}
pub async fn init() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
for file in FILES.iter() {
let root_dir = file.read().await.root_dir.clone();
std::fs::create_dir_all(&root_dir).expect("create cache dir success");
@@ -226,12 +229,11 @@ pub async fn init() -> Result<(), anyhow::Error> {
});
tokio::task::spawn(async move {
- if CONFIG.disk_cache.gc_interval == 0 {
+ if cfg.disk_cache.gc_interval == 0 {
return;
}
- let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
- config::CONFIG.disk_cache.gc_interval,
- ));
+ let mut interval =
+ tokio::time::interval(tokio::time::Duration::from_secs(cfg.disk_cache.gc_interval));
interval.tick().await; // the first tick is immediate
loop {
if let Err(e) = gc().await {
@@ -245,7 +247,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
#[inline]
pub async fn get(file: &str, range: Option>) -> Option {
- if !CONFIG.disk_cache.enabled {
+ if !get_config().disk_cache.enabled {
return None;
}
let idx = get_bucket_idx(file);
@@ -255,7 +257,7 @@ pub async fn get(file: &str, range: Option>) -> Option {
#[inline]
pub async fn exist(file: &str) -> bool {
- if !CONFIG.disk_cache.enabled {
+ if !get_config().disk_cache.enabled {
return false;
}
let idx = get_bucket_idx(file);
@@ -265,7 +267,7 @@ pub async fn exist(file: &str) -> bool {
#[inline]
pub async fn set(trace_id: &str, file: &str, data: Bytes) -> Result<(), anyhow::Error> {
- if !CONFIG.disk_cache.enabled || is_local_disk_storage() {
+ if !get_config().disk_cache.enabled || is_local_disk_storage() {
return Ok(());
}
let idx = get_bucket_idx(file);
@@ -318,7 +320,7 @@ async fn load(root_dir: &PathBuf, scan_dir: &PathBuf) -> Result<(), anyhow::Erro
.unwrap()
.replace('\\', "/");
- if !CONFIG.disk_cache.multi_dir.is_empty() {
+ if !get_config().disk_cache.multi_dir.is_empty() {
file_key = file_key.split('/').skip(1).collect::>().join("/");
}
// check file already exists
@@ -352,18 +354,19 @@ async fn load(root_dir: &PathBuf, scan_dir: &PathBuf) -> Result<(), anyhow::Erro
}
async fn gc() -> Result<(), anyhow::Error> {
- if !CONFIG.disk_cache.enabled || is_local_disk_storage() {
+ let cfg = get_config();
+ if !cfg.disk_cache.enabled || is_local_disk_storage() {
return Ok(());
}
for file in FILES.iter() {
let r = file.read().await;
- if r.cur_size + CONFIG.disk_cache.release_size < r.max_size {
+ if r.cur_size + cfg.disk_cache.release_size < r.max_size {
drop(r);
continue;
}
drop(r);
let mut w = file.write().await;
- w.gc("global", CONFIG.disk_cache.gc_size).await?;
+ w.gc("global", cfg.disk_cache.gc_size).await?;
drop(w);
}
Ok(())
@@ -419,11 +422,12 @@ pub async fn download(trace_id: &str, file: &str) -> Result<(), anyhow::Error> {
}
fn get_bucket_idx(file: &str) -> usize {
- if CONFIG.disk_cache.bucket_num <= 1 {
+ let cfg = get_config();
+ if cfg.disk_cache.bucket_num <= 1 {
0
} else {
let h = gxhash::new().sum32(file);
- (h as usize) % CONFIG.disk_cache.bucket_num
+ (h as usize) % cfg.disk_cache.bucket_num
}
}
@@ -450,7 +454,7 @@ mod tests {
async fn test_lru_cache_get_file() {
let trace_id = "session_123";
let mut file_data =
- FileData::with_capacity_and_cache_strategy(CONFIG.disk_cache.max_size, "lru");
+ FileData::with_capacity_and_cache_strategy(get_config().disk_cache.max_size, "lru");
let file_key = "files/default/logs/olympics/2022/10/03/10/6982652937134804993_2_1.parquet";
let content = Bytes::from("Some text");
@@ -510,7 +514,7 @@ mod tests {
async fn test_fifo_cache_get_file() {
let trace_id = "session_123";
let mut file_data =
- FileData::with_capacity_and_cache_strategy(CONFIG.disk_cache.max_size, "fifo");
+ FileData::with_capacity_and_cache_strategy(get_config().disk_cache.max_size, "fifo");
let file_key = "files/default/logs/olympics/2022/10/03/10/6982652937134804993_2_1.parquet";
let content = Bytes::from("Some text");
@@ -561,7 +565,7 @@ mod tests {
let trace_id = "session_123";
let mut file_data =
- FileData::with_capacity_and_cache_strategy(CONFIG.disk_cache.max_size, "lru");
+ FileData::with_capacity_and_cache_strategy(get_config().disk_cache.max_size, "lru");
file_data.multi_dir = multi_dir;
let file_key = "files/default/logs/olympics/2022/10/03/10/6982652937134804993_2_1.parquet";
let content = Bytes::from("Some text");
diff --git a/src/infra/src/cache/file_data/memory.rs b/src/infra/src/cache/file_data/memory.rs
index 86224bd5bb..1627b863bf 100644
--- a/src/infra/src/cache/file_data/memory.rs
+++ b/src/infra/src/cache/file_data/memory.rs
@@ -20,9 +20,9 @@ use std::{
use bytes::Bytes;
use config::{
- metrics,
+ get_config, metrics,
utils::hash::{gxhash, Sum32},
- RwHashMap, CONFIG,
+ RwHashMap,
};
use once_cell::sync::Lazy;
use tokio::sync::RwLock;
@@ -31,15 +31,17 @@ use super::CacheStrategy;
use crate::storage;
static FILES: Lazy>> = Lazy::new(|| {
- let mut files = Vec::with_capacity(CONFIG.memory_cache.bucket_num);
- for _ in 0..CONFIG.memory_cache.bucket_num {
+ let cfg = get_config();
+ let mut files = Vec::with_capacity(cfg.memory_cache.bucket_num);
+ for _ in 0..cfg.memory_cache.bucket_num {
files.push(RwLock::new(FileData::new()));
}
files
});
static DATA: Lazy>> = Lazy::new(|| {
- let mut datas = Vec::with_capacity(CONFIG.memory_cache.bucket_num);
- for _ in 0..CONFIG.memory_cache.bucket_num {
+ let cfg = get_config();
+ let mut datas = Vec::with_capacity(cfg.memory_cache.bucket_num);
+ for _ in 0..cfg.memory_cache.bucket_num {
datas.push(Default::default());
}
datas
@@ -59,9 +61,10 @@ impl Default for FileData {
impl FileData {
pub fn new() -> FileData {
+ let cfg = get_config();
FileData::with_capacity_and_cache_strategy(
- CONFIG.memory_cache.max_size,
- &CONFIG.memory_cache.cache_strategy,
+ cfg.memory_cache.max_size,
+ &cfg.memory_cache.cache_strategy,
)
}
@@ -97,7 +100,7 @@ impl FileData {
// cache is full, need release some space
let need_release_size = min(
self.max_size,
- max(CONFIG.memory_cache.release_size, data_size * 100),
+ max(get_config().memory_cache.release_size, data_size * 100),
);
self.gc(trace_id, need_release_size).await?;
}
@@ -180,16 +183,18 @@ impl FileData {
}
pub async fn init() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
+
for file in FILES.iter() {
_ = file.read().await.get("", None).await;
}
tokio::task::spawn(async move {
- if CONFIG.memory_cache.gc_interval == 0 {
+ if cfg.memory_cache.gc_interval == 0 {
return;
}
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
- config::CONFIG.memory_cache.gc_interval,
+ cfg.memory_cache.gc_interval,
));
interval.tick().await; // the first tick is immediate
loop {
@@ -204,7 +209,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
#[inline]
pub async fn get(file: &str, range: Option>) -> Option {
- if !CONFIG.memory_cache.enabled {
+ if !get_config().memory_cache.enabled {
return None;
}
let idx = get_bucket_idx(file);
@@ -214,7 +219,7 @@ pub async fn get(file: &str, range: Option>) -> Option {
#[inline]
pub async fn exist(file: &str) -> bool {
- if !CONFIG.memory_cache.enabled {
+ if !get_config().memory_cache.enabled {
return false;
}
let idx = get_bucket_idx(file);
@@ -224,7 +229,7 @@ pub async fn exist(file: &str) -> bool {
#[inline]
pub async fn set(trace_id: &str, file: &str, data: Bytes) -> Result<(), anyhow::Error> {
- if !CONFIG.memory_cache.enabled {
+ if !get_config().memory_cache.enabled {
return Ok(());
}
let idx = get_bucket_idx(file);
@@ -236,19 +241,20 @@ pub async fn set(trace_id: &str, file: &str, data: Bytes) -> Result<(), anyhow::
}
async fn gc() -> Result<(), anyhow::Error> {
- if !CONFIG.memory_cache.enabled {
+ let cfg = get_config();
+ if !cfg.memory_cache.enabled {
return Ok(());
}
for file in FILES.iter() {
let r = file.read().await;
- if r.cur_size + CONFIG.memory_cache.release_size < r.max_size {
+ if r.cur_size + cfg.memory_cache.release_size < r.max_size {
drop(r);
continue;
}
drop(r);
let mut w = file.write().await;
- w.gc("global", CONFIG.memory_cache.gc_size).await?;
+ w.gc("global", cfg.memory_cache.gc_size).await?;
drop(w);
}
@@ -305,11 +311,12 @@ pub async fn download(trace_id: &str, file: &str) -> Result<(), anyhow::Error> {
}
fn get_bucket_idx(file: &str) -> usize {
- if CONFIG.memory_cache.bucket_num <= 1 {
+ let cfg = get_config();
+ if cfg.memory_cache.bucket_num <= 1 {
0
} else {
let h = gxhash::new().sum32(file);
- (h as usize) % CONFIG.memory_cache.bucket_num
+ (h as usize) % cfg.memory_cache.bucket_num
}
}
@@ -336,7 +343,7 @@ mod tests {
async fn test_lru_cache_get_file() {
let trace_id = "session_123";
let mut file_data =
- FileData::with_capacity_and_cache_strategy(CONFIG.memory_cache.max_size, "lru");
+ FileData::with_capacity_and_cache_strategy(get_config().memory_cache.max_size, "lru");
let file_key = "files/default/logs/olympics/2022/10/03/10/6982652937134804993_2_1.parquet";
let content = Bytes::from("Some text");
@@ -397,7 +404,7 @@ mod tests {
async fn test_fifo_cache_get_file() {
let trace_id = "session_123";
let mut file_data =
- FileData::with_capacity_and_cache_strategy(CONFIG.memory_cache.max_size, "fifo");
+ FileData::with_capacity_and_cache_strategy(get_config().memory_cache.max_size, "fifo");
let file_key = "files/default/logs/olympics/2022/10/03/10/6982652937134804993_2_1.parquet";
let content = Bytes::from("Some text");
diff --git a/src/infra/src/cache/file_data/mod.rs b/src/infra/src/cache/file_data/mod.rs
index 770a8e4da5..fe83f3a50e 100644
--- a/src/infra/src/cache/file_data/mod.rs
+++ b/src/infra/src/cache/file_data/mod.rs
@@ -102,9 +102,10 @@ pub async fn init() -> Result<(), anyhow::Error> {
}
pub async fn download(trace_id: &str, file: &str) -> Result<(), anyhow::Error> {
- if config::CONFIG.memory_cache.enabled {
+ let cfg = config::get_config();
+ if cfg.memory_cache.enabled {
memory::download(trace_id, file).await
- } else if config::CONFIG.disk_cache.enabled {
+ } else if cfg.disk_cache.enabled {
disk::download(trace_id, file).await
} else {
Ok(())
diff --git a/src/infra/src/cache/stats.rs b/src/infra/src/cache/stats.rs
index 45310d6317..2a93836edb 100644
--- a/src/infra/src/cache/stats.rs
+++ b/src/infra/src/cache/stats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/cache/tmpfs.rs b/src/infra/src/cache/tmpfs.rs
index 4ad7cc3887..c5baba7c1b 100644
--- a/src/infra/src/cache/tmpfs.rs
+++ b/src/infra/src/cache/tmpfs.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/db/etcd.rs b/src/infra/src/db/etcd.rs
index 03a2d42d1b..0ec6fac8ea 100644
--- a/src/infra/src/db/etcd.rs
+++ b/src/infra/src/db/etcd.rs
@@ -23,7 +23,7 @@ use std::{
use async_trait::async_trait;
use bytes::Bytes;
-use config::{cluster, CONFIG};
+use config::{cluster, get_config};
use etcd_client::{
Certificate, DeleteOptions, EventType, GetOptions, Identity, SortOrder, SortTarget, TlsOptions,
};
@@ -47,7 +47,8 @@ pub async fn get_etcd_client() -> &'static etcd_client::Client {
}
pub async fn init() {
- if CONFIG.common.local_mode || CONFIG.common.cluster_coordinator.to_lowercase() == "nats" {
+ let cfg = get_config();
+ if cfg.common.local_mode || cfg.common.cluster_coordinator.to_lowercase() == "nats" {
return;
}
// enable keep alive for auth token
@@ -91,7 +92,7 @@ impl Etcd {
impl Default for Etcd {
fn default() -> Self {
- Self::new(&CONFIG.etcd.prefix)
+ Self::new(&get_config().etcd.prefix)
}
}
@@ -228,20 +229,21 @@ impl super::Db for Etcd {
}
async fn list(&self, prefix: &str) -> Result> {
+ let cfg = get_config();
let mut result = HashMap::default();
let key = format!("{}{}", self.prefix, prefix);
let mut client = get_etcd_client().await.clone();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
- .with_limit(CONFIG.etcd.load_page_size);
+ .with_limit(cfg.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
- if kvs_num < CONFIG.etcd.load_page_size {
+ if kvs_num < cfg.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
@@ -272,20 +274,21 @@ impl super::Db for Etcd {
}
async fn list_keys(&self, prefix: &str) -> Result> {
+ let cfg = get_config();
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let mut client = get_etcd_client().await.clone();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
- .with_limit(CONFIG.etcd.load_page_size);
+ .with_limit(cfg.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
- if kvs_num < CONFIG.etcd.load_page_size {
+ if kvs_num < cfg.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
@@ -316,20 +319,21 @@ impl super::Db for Etcd {
}
async fn list_values(&self, prefix: &str) -> Result> {
+ let cfg = get_config();
let mut result = Vec::new();
let key = format!("{}{}", self.prefix, prefix);
let mut client = get_etcd_client().await.clone();
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
- .with_limit(CONFIG.etcd.load_page_size);
+ .with_limit(cfg.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
- if kvs_num < CONFIG.etcd.load_page_size {
+ if kvs_num < cfg.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
@@ -363,6 +367,7 @@ impl super::Db for Etcd {
prefix: &str,
start_dt: Option<(i64, i64)>,
) -> Result> {
+ let cfg = get_config();
if start_dt.is_none() || start_dt == Some((0, 0)) {
let vals = self.list_values(prefix).await?;
return Ok(vals.into_iter().map(|v| (0, v)).collect());
@@ -375,14 +380,14 @@ impl super::Db for Etcd {
let mut opt = GetOptions::new()
.with_prefix()
.with_sort(SortTarget::Key, SortOrder::Ascend)
- .with_limit(CONFIG.etcd.load_page_size);
+ .with_limit(cfg.etcd.load_page_size);
let mut resp = client.get(key.clone(), Some(opt.clone())).await?;
let mut first_call = true;
let mut have_next = true;
let mut last_key = String::new();
loop {
let kvs_num = resp.kvs().len() as i64;
- if kvs_num < CONFIG.etcd.load_page_size {
+ if kvs_num < cfg.etcd.load_page_size {
have_next = false;
}
for kv in resp.kvs() {
@@ -500,29 +505,30 @@ pub async fn create_table() -> Result<()> {
}
pub async fn connect() -> etcd_client::Client {
- if CONFIG.common.print_key_config {
- log::info!("Etcd init config: {:?}", CONFIG.etcd);
+ let cfg = get_config();
+ if cfg.common.print_key_config {
+ log::info!("Etcd init cfg: {:?}", cfg.etcd);
}
let mut opts = etcd_client::ConnectOptions::new()
- .with_timeout(core::time::Duration::from_secs(CONFIG.etcd.command_timeout))
- .with_connect_timeout(core::time::Duration::from_secs(CONFIG.etcd.connect_timeout));
- if !&CONFIG.etcd.user.is_empty() {
- opts = opts.with_user(&CONFIG.etcd.user, &CONFIG.etcd.password);
+ .with_timeout(core::time::Duration::from_secs(cfg.etcd.command_timeout))
+ .with_connect_timeout(core::time::Duration::from_secs(cfg.etcd.connect_timeout));
+ if !&cfg.etcd.user.is_empty() {
+ opts = opts.with_user(&cfg.etcd.user, &cfg.etcd.password);
}
- if CONFIG.etcd.cert_auth {
- let server_root_ca_cert = tokio::fs::read(&CONFIG.etcd.ca_file).await.unwrap();
+ if cfg.etcd.cert_auth {
+ let server_root_ca_cert = tokio::fs::read(&cfg.etcd.ca_file).await.unwrap();
let server_root_ca_cert = Certificate::from_pem(server_root_ca_cert);
- let client_cert = tokio::fs::read(&CONFIG.etcd.cert_file).await.unwrap();
- let client_key = tokio::fs::read(&CONFIG.etcd.key_file).await.unwrap();
+ let client_cert = tokio::fs::read(&cfg.etcd.cert_file).await.unwrap();
+ let client_key = tokio::fs::read(&cfg.etcd.key_file).await.unwrap();
let client_identity = Identity::from_pem(client_cert, client_key);
let tls = TlsOptions::new()
- .domain_name(&CONFIG.etcd.domain_name)
+ .domain_name(&cfg.etcd.domain_name)
.ca_certificate(server_root_ca_cert)
.identity(client_identity);
opts = opts.with_tls(tls);
}
- let addrs = CONFIG.etcd.addr.split(',').collect::>();
+ let addrs = cfg.etcd.addr.split(',').collect::>();
etcd_client::Client::connect(addrs, Some(opts))
.await
.expect("Etcd connect failed")
@@ -534,7 +540,7 @@ pub async fn keepalive_connection() -> Result<()> {
break;
}
let mut client = get_etcd_client().await.clone();
- let key = format!("{}healthz", &CONFIG.etcd.prefix);
+ let key = format!("{}healthz", get_config().etcd.prefix);
let key = key.as_str();
client.put(key, "OK", None).await?;
let mut interval = time::interval(time::Duration::from_secs(60));
@@ -616,7 +622,7 @@ pub(crate) struct Locker {
impl Locker {
pub(crate) fn new(key: &str) -> Self {
Self {
- key: format!("{}locker{}", &CONFIG.etcd.prefix, key),
+ key: format!("{}locker{}", get_config().etcd.prefix, key),
lock_id: "".to_string(),
state: Arc::new(AtomicU8::new(0)),
}
@@ -624,14 +630,15 @@ impl Locker {
/// lock with timeout, 0 means use default timeout, unit: second
pub(crate) async fn lock(&mut self, timeout: u64) -> Result<()> {
+ let cfg = get_config();
let mut client = get_etcd_client().await.clone();
let mut last_err = None;
let timeout = if timeout == 0 {
- CONFIG.etcd.lock_wait_timeout
+ cfg.etcd.lock_wait_timeout
} else {
timeout
};
- let mut n = timeout / CONFIG.etcd.command_timeout;
+ let mut n = timeout / cfg.etcd.command_timeout;
if n < 1 {
n = 1;
}
@@ -686,7 +693,7 @@ mod tests {
#[tokio::test]
async fn test_etcd_count() {
- if CONFIG.common.local_mode {
+ if get_config().common.local_mode {
return;
}
let client = Etcd::default();
diff --git a/src/infra/src/db/mod.rs b/src/infra/src/db/mod.rs
index 5baffbb5e0..5033c61869 100644
--- a/src/infra/src/db/mod.rs
+++ b/src/infra/src/db/mod.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use bytes::Bytes;
-use config::{meta::meta_store::MetaStore, CONFIG};
+use config::{get_config, meta::meta_store::MetaStore};
use hashbrown::HashMap;
use tokio::sync::{mpsc, OnceCell};
@@ -57,13 +57,14 @@ pub async fn init() -> Result<()> {
}
async fn default() -> Box {
- if !CONFIG.common.local_mode
- && (CONFIG.common.meta_store == "sled" || CONFIG.common.meta_store == "sqlite")
+ let cfg = get_config();
+ if !cfg.common.local_mode
+ && (cfg.common.meta_store == "sled" || cfg.common.meta_store == "sqlite")
{
panic!("cluster mode is not supported for ZO_META_STORE=sqlite");
}
- match CONFIG.common.meta_store.as_str().into() {
+ match cfg.common.meta_store.as_str().into() {
MetaStore::Sqlite => Box::::default(),
MetaStore::Etcd => Box::::default(),
MetaStore::Nats => Box::::default(),
@@ -73,13 +74,14 @@ async fn default() -> Box {
}
async fn init_cluster_coordinator() -> Box {
- if CONFIG.common.local_mode {
- match CONFIG.common.meta_store.as_str().into() {
+ let cfg = get_config();
+ if cfg.common.local_mode {
+ match cfg.common.meta_store.as_str().into() {
MetaStore::Sqlite => Box::::default(),
_ => Box::::default(),
}
} else {
- match CONFIG.common.cluster_coordinator.as_str().into() {
+ match cfg.common.cluster_coordinator.as_str().into() {
MetaStore::Nats => Box::::default(),
_ => Box::::default(),
}
@@ -87,7 +89,7 @@ async fn init_cluster_coordinator() -> Box {
}
async fn init_super_cluster() -> Box {
- if CONFIG.common.local_mode {
+ if get_config().common.local_mode {
panic!("super cluster is not supported in local mode");
}
Box::new(nats::NatsDb::super_cluster())
@@ -95,7 +97,7 @@ async fn init_super_cluster() -> Box {
pub async fn create_table() -> Result<()> {
// check db dir
- std::fs::create_dir_all(&CONFIG.common.data_db_dir)?;
+ std::fs::create_dir_all(&get_config().common.data_db_dir)?;
// create for meta store
let cluster_coordinator = get_coordinator().await;
cluster_coordinator.create_table().await?;
diff --git a/src/infra/src/db/mysql.rs b/src/infra/src/db/mysql.rs
index 296d3da60c..b9d9d44159 100644
--- a/src/infra/src/db/mysql.rs
+++ b/src/infra/src/db/mysql.rs
@@ -17,7 +17,6 @@ use std::{str::FromStr, sync::Arc};
use async_trait::async_trait;
use bytes::Bytes;
-use config::CONFIG;
use hashbrown::HashMap;
use once_cell::sync::Lazy;
use sqlx::{
@@ -31,13 +30,14 @@ use crate::errors::*;
pub static CLIENT: Lazy> = Lazy::new(connect);
fn connect() -> Pool {
- let db_opts = MySqlConnectOptions::from_str(&CONFIG.common.meta_mysql_dsn)
+ let cfg = config::get_config();
+ let db_opts = MySqlConnectOptions::from_str(&cfg.common.meta_mysql_dsn)
.expect("mysql connect options create failed")
.disable_statement_logging();
MySqlPoolOptions::new()
- .min_connections(CONFIG.limit.sql_min_db_connections)
- .max_connections(CONFIG.limit.sql_max_db_connections)
+ .min_connections(cfg.limit.sql_min_db_connections)
+ .max_connections(cfg.limit.sql_max_db_connections)
.connect_lazy_with(db_opts)
}
@@ -164,7 +164,8 @@ impl super::Db for MysqlDb {
let lock_key = format!("get_for_update_{}", key);
let lock_sql = format!(
"SELECT GET_LOCK('{}', {})",
- lock_key, CONFIG.limit.meta_transaction_lock_timeout
+ lock_key,
+ config::get_config().limit.meta_transaction_lock_timeout
);
let unlock_sql = format!("SELECT RELEASE_LOCK('{}')", lock_key);
let mut lock_tx = lock_pool.begin().await?;
diff --git a/src/infra/src/db/nats.rs b/src/infra/src/db/nats.rs
index 7a083f3483..1d007f60a7 100644
--- a/src/infra/src/db/nats.rs
+++ b/src/infra/src/db/nats.rs
@@ -24,7 +24,7 @@ use std::{
use async_nats::{jetstream, Client, ServerAddr};
use async_trait::async_trait;
use bytes::Bytes;
-use config::{cluster, ider, utils::base64, CONFIG};
+use config::{cluster, get_config, ider, utils::base64};
use futures::{StreamExt, TryStreamExt};
use hashbrown::HashMap;
use tokio::{
@@ -50,20 +50,21 @@ async fn get_bucket_by_key<'a>(
prefix: &'a str,
key: &'a str,
) -> Result<(jetstream::kv::Store, &'a str)> {
+ let cfg = get_config();
let client = get_nats_client().await.clone();
let jetstream = jetstream::new(client);
let key = key.trim_start_matches('/');
let bucket_name = key.split('/').next().unwrap();
let mut bucket = jetstream::kv::Config {
bucket: format!("{}{}", prefix, bucket_name),
- num_replicas: CONFIG.nats.replicas,
+ num_replicas: cfg.nats.replicas,
history: 3,
..Default::default()
};
if bucket_name == "nodes" || bucket_name == "clusters" {
// if changed ttl need recreate the bucket
// CMD: nats kv del -f o2_nodes
- bucket.max_age = Duration::from_secs(CONFIG.limit.node_heartbeat_ttl as u64);
+ bucket.max_age = Duration::from_secs(cfg.limit.node_heartbeat_ttl as u64);
}
let kv = jetstream.create_key_value(bucket).await?;
Ok((kv, key.trim_start_matches(bucket_name)))
@@ -127,7 +128,7 @@ impl NatsDb {
impl Default for NatsDb {
fn default() -> Self {
- Self::new(&CONFIG.nats.prefix)
+ Self::new(&get_config().nats.prefix)
}
}
@@ -321,13 +322,14 @@ impl super::Db for NatsDb {
if keys_len == 0 {
return Ok(HashMap::new());
}
+
let values = futures::stream::iter(keys)
.map(|key| async move {
let encoded_key = key_encode(&key);
let value = bucket.get(&encoded_key).await?;
Ok::<(String, Option), Error>((key, value))
})
- .buffer_unordered(CONFIG.limit.cpu_num)
+ .buffer_unordered(get_config().limit.cpu_num)
.try_collect::)>>()
.await
.map_err(|e| Error::Message(e.to_string()))?;
@@ -385,7 +387,7 @@ impl super::Db for NatsDb {
let value = bucket.get(&encoded_key).await?;
Ok::, Error>(value)
})
- .buffer_unordered(CONFIG.limit.cpu_num)
+ .buffer_unordered(get_config().limit.cpu_num)
.try_collect::>>()
.await
.map_err(|e| Error::Message(e.to_string()))?;
@@ -441,7 +443,7 @@ impl super::Db for NatsDb {
let value = bucket.get(&encoded_key).await?;
Ok::, Error>(value.map(|value| (start_dt, value)))
})
- .buffer_unordered(CONFIG.limit.cpu_num)
+ .buffer_unordered(get_config().limit.cpu_num)
.try_collect::>>()
.await
.map_err(|e| Error::Message(e.to_string()))?;
@@ -530,19 +532,17 @@ pub async fn create_table() -> Result<()> {
}
pub async fn connect() -> async_nats::Client {
- if CONFIG.common.print_key_config {
- log::info!("Nats init config: {:?}", CONFIG.nats);
+ let cfg = get_config();
+ if cfg.common.print_key_config {
+ log::info!("Nats init get_config(): {:?}", cfg.nats);
}
let mut opts = async_nats::ConnectOptions::new()
- .connection_timeout(Duration::from_secs(CONFIG.nats.connect_timeout));
- if !CONFIG.nats.user.is_empty() {
- opts = opts.user_and_password(
- CONFIG.nats.user.to_string(),
- CONFIG.nats.password.to_string(),
- );
+ .connection_timeout(Duration::from_secs(cfg.nats.connect_timeout));
+ if !cfg.nats.user.is_empty() {
+ opts = opts.user_and_password(cfg.nats.user.to_string(), cfg.nats.password.to_string());
}
- let addrs = CONFIG
+ let addrs = cfg
.nats
.addr
.split(',')
@@ -578,9 +578,10 @@ impl Locker {
/// lock with timeout, 0 means use default timeout, unit: second
pub(crate) async fn lock(&mut self, timeout: u64) -> Result<()> {
- let (bucket, new_key) = get_bucket_by_key(&CONFIG.nats.prefix, &self.key).await?;
+ let cfg = get_config();
+ let (bucket, new_key) = get_bucket_by_key(&cfg.nats.prefix, &self.key).await?;
let timeout = if timeout == 0 {
- CONFIG.nats.lock_wait_timeout
+ cfg.nats.lock_wait_timeout
} else {
timeout
};
@@ -636,7 +637,9 @@ impl Locker {
if self.state.load(Ordering::SeqCst) != 1 {
return Ok(());
}
- let (bucket, new_key) = get_bucket_by_key(&CONFIG.nats.prefix, &self.key).await?;
+
+ let cfg = get_config();
+ let (bucket, new_key) = get_bucket_by_key(&cfg.nats.prefix, &self.key).await?;
let key = key_encode(new_key);
let ret = bucket.get(&key).await?;
let Some(ret) = ret else {
diff --git a/src/infra/src/db/postgres.rs b/src/infra/src/db/postgres.rs
index 641a807cf1..4eddfdff7a 100644
--- a/src/infra/src/db/postgres.rs
+++ b/src/infra/src/db/postgres.rs
@@ -17,7 +17,7 @@ use std::{str::FromStr, sync::Arc};
use async_trait::async_trait;
use bytes::Bytes;
-use config::{utils::hash::Sum32, CONFIG};
+use config::utils::hash::Sum32;
use hashbrown::HashMap;
use once_cell::sync::Lazy;
use sqlx::{
@@ -31,13 +31,14 @@ use crate::errors::*;
pub static CLIENT: Lazy> = Lazy::new(connect);
fn connect() -> Pool {
- let db_opts = PgConnectOptions::from_str(&CONFIG.common.meta_postgres_dsn)
+ let cfg = config::get_config();
+ let db_opts = PgConnectOptions::from_str(&cfg.common.meta_postgres_dsn)
.expect("postgres connect options create failed")
.disable_statement_logging();
PgPoolOptions::new()
- .min_connections(CONFIG.limit.sql_min_db_connections)
- .max_connections(CONFIG.limit.sql_max_db_connections)
+ .min_connections(cfg.limit.sql_min_db_connections)
+ .max_connections(cfg.limit.sql_max_db_connections)
.connect_lazy_with(db_opts)
}
diff --git a/src/infra/src/db/sqlite.rs b/src/infra/src/db/sqlite.rs
index 2cbcb6847a..21fa1de9d3 100644
--- a/src/infra/src/db/sqlite.rs
+++ b/src/infra/src/db/sqlite.rs
@@ -17,7 +17,7 @@ use std::{str::FromStr, sync::Arc, time::Duration};
use async_trait::async_trait;
use bytes::Bytes;
-use config::{cluster, FxIndexMap, CONFIG};
+use config::{cluster, FxIndexMap};
use hashbrown::HashMap;
use once_cell::sync::Lazy;
use sqlx::{
@@ -45,8 +45,9 @@ static WATCHERS: Lazy>> =
type EventChannel = Arc>;
fn connect_rw() -> Pool {
- let url = format!("{}{}", CONFIG.common.data_db_dir, "metadata.sqlite");
- if !CONFIG.common.local_mode && std::path::Path::new(&url).exists() {
+ let cfg = config::get_config();
+ let url = format!("{}{}", cfg.common.data_db_dir, "metadata.sqlite");
+ if !cfg.common.local_mode && std::path::Path::new(&url).exists() {
std::fs::remove_file(&url).expect("remove file sqlite failed");
_ = std::fs::remove_file(format!("{url}-shm"));
_ = std::fs::remove_file(format!("{url}-wal"));
@@ -68,7 +69,9 @@ fn connect_rw() -> Pool {
}
fn connect_ro() -> Pool {
- let url = format!("{}{}", CONFIG.common.data_db_dir, "metadata.sqlite");
+ let cfg = config::get_config();
+
+ let url = format!("{}{}", cfg.common.data_db_dir, "metadata.sqlite");
let db_opts = SqliteConnectOptions::from_str(&url)
.expect("sqlite connect options create failed")
.journal_mode(SqliteJournalMode::Wal)
@@ -78,8 +81,8 @@ fn connect_ro() -> Pool {
// .disable_statement_logging()
.read_only(true);
SqlitePoolOptions::new()
- .min_connections(CONFIG.limit.sql_min_db_connections)
- .max_connections(CONFIG.limit.sql_max_db_connections)
+ .min_connections(cfg.limit.sql_min_db_connections)
+ .max_connections(cfg.limit.sql_max_db_connections)
.acquire_timeout(Duration::from_secs(30))
.connect_lazy_with(db_opts)
}
@@ -109,7 +112,7 @@ impl SqliteDbChannel {
break;
}
};
- if CONFIG.common.print_key_event {
+ if config::get_config().common.print_key_event {
log::info!("[SQLITE] watch event: {:?}", event);
}
for (prefix, tx) in WATCHERS.read().await.iter() {
diff --git a/src/infra/src/dist_lock.rs b/src/infra/src/dist_lock.rs
index 3ebbe5d533..d50df48df2 100644
--- a/src/infra/src/dist_lock.rs
+++ b/src/infra/src/dist_lock.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,8 +13,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
-
use crate::{
db::{etcd, nats},
errors::Result,
@@ -30,10 +28,11 @@ enum LockerStore {
/// lock key in etcd, wait_ttl is 0 means wait forever
#[inline(always)]
pub async fn lock(key: &str, wait_ttl: u64) -> Result> {
- if CONFIG.common.local_mode {
+ let cfg = config::get_config();
+ if cfg.common.local_mode {
return Ok(None);
}
- match CONFIG.common.cluster_coordinator.as_str() {
+ match cfg.common.cluster_coordinator.as_str() {
"nats" => {
let mut lock = nats::Locker::new(key);
lock.lock(wait_ttl).await?;
diff --git a/src/infra/src/errors/grpc.rs b/src/infra/src/errors/grpc.rs
index 94d640a6ed..d886a4018b 100644
--- a/src/infra/src/errors/grpc.rs
+++ b/src/infra/src/errors/grpc.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/errors/mod.rs b/src/infra/src/errors/mod.rs
index 8fb27e2f46..538e8bc1bb 100644
--- a/src/infra/src/errors/mod.rs
+++ b/src/infra/src/errors/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/file_list/mod.rs b/src/infra/src/file_list/mod.rs
index c90923a60b..0c131e8946 100644
--- a/src/infra/src/file_list/mod.rs
+++ b/src/infra/src/file_list/mod.rs
@@ -14,12 +14,9 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::{
- meta::{
- meta_store::MetaStore,
- stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType},
- },
- CONFIG,
+use config::meta::{
+ meta_store::MetaStore,
+ stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType},
};
use once_cell::sync::Lazy;
@@ -32,7 +29,7 @@ pub mod sqlite;
static CLIENT: Lazy> = Lazy::new(connect);
pub fn connect() -> Box {
- match CONFIG.common.meta_store.as_str().into() {
+ match config::get_config().common.meta_store.as_str().into() {
MetaStore::Sqlite => Box::::default(),
MetaStore::Etcd => Box::::default(),
MetaStore::Nats => Box::::default(),
diff --git a/src/infra/src/queue/mod.rs b/src/infra/src/queue/mod.rs
index 0e75b6eeeb..cd45797c3b 100644
--- a/src/infra/src/queue/mod.rs
+++ b/src/infra/src/queue/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use bytes::Bytes;
-use config::{meta::meta_store::MetaStore, CONFIG};
+use config::meta::meta_store::MetaStore;
use tokio::sync::{mpsc, OnceCell};
use crate::errors::{Error, Result};
@@ -41,14 +41,14 @@ pub async fn init() -> Result<()> {
}
async fn default() -> Box {
- match CONFIG.common.queue_store.as_str().into() {
+ match config::get_config().common.queue_store.as_str().into() {
MetaStore::Nats => Box::::default(),
_ => Box::::default(),
}
}
async fn init_super_cluster() -> Box {
- if CONFIG.common.local_mode {
+ if config::get_config().common.local_mode {
panic!("super cluster is not supported in local mode");
}
Box::new(nats::NatsQueue::super_cluster())
diff --git a/src/infra/src/queue/nats.rs b/src/infra/src/queue/nats.rs
index 1e200ade1b..1b1d9fc9cb 100644
--- a/src/infra/src/queue/nats.rs
+++ b/src/infra/src/queue/nats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@ use std::{cmp::max, sync::Arc, time::Duration};
use async_nats::jetstream;
use async_trait::async_trait;
use bytes::Bytes;
-use config::{get_cluster_name, CONFIG};
+use config::get_cluster_name;
use futures::TryStreamExt;
use tokio::{sync::mpsc, task::JoinHandle};
@@ -47,13 +47,14 @@ impl NatsQueue {
impl Default for NatsQueue {
fn default() -> Self {
- Self::new(&CONFIG.nats.prefix)
+ Self::new(&config::get_config().nats.prefix)
}
}
#[async_trait]
impl super::Queue for NatsQueue {
async fn create(&self, topic: &str) -> Result<()> {
+ let cfg = config::get_config();
let client = get_nats_client().await.clone();
let jetstream = jetstream::new(client);
let topic_name = format!("{}{}", self.prefix, topic);
@@ -61,8 +62,8 @@ impl super::Queue for NatsQueue {
name: topic_name.to_string(),
subjects: vec![topic_name.to_string(), format!("{}.*", topic_name)],
retention: jetstream::stream::RetentionPolicy::Limits,
- max_age: Duration::from_secs(60 * 60 * 24 * max(1, CONFIG.nats.queue_max_age)),
- num_replicas: CONFIG.nats.replicas,
+ max_age: Duration::from_secs(60 * 60 * 24 * max(1, cfg.nats.queue_max_age)),
+ num_replicas: cfg.nats.replicas,
..Default::default()
};
_ = jetstream.get_or_create_stream(config).await?;
diff --git a/src/infra/src/queue/nop.rs b/src/infra/src/queue/nop.rs
index 306ee5d5e5..60f938db01 100644
--- a/src/infra/src/queue/nop.rs
+++ b/src/infra/src/queue/nop.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/infra/src/scheduler/mod.rs b/src/infra/src/scheduler/mod.rs
index 8ddd7c0e4f..6427cddae0 100644
--- a/src/infra/src/scheduler/mod.rs
+++ b/src/infra/src/scheduler/mod.rs
@@ -14,7 +14,7 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::{meta::meta_store::MetaStore, CONFIG};
+use config::meta::meta_store::MetaStore;
use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
@@ -28,7 +28,7 @@ static CLIENT: Lazy> = Lazy::new(connect);
pub const TRIGGERS_KEY: &str = "/triggers/";
pub fn connect() -> Box {
- match CONFIG.common.meta_store.as_str().into() {
+ match config::get_config().common.meta_store.as_str().into() {
MetaStore::MySQL => Box::::default(),
MetaStore::PostgreSQL => Box::::default(),
_ => Box::::default(),
diff --git a/src/infra/src/scheduler/mysql.rs b/src/infra/src/scheduler/mysql.rs
index 9b78de13e2..2e7ffc4abd 100644
--- a/src/infra/src/scheduler/mysql.rs
+++ b/src/infra/src/scheduler/mysql.rs
@@ -16,7 +16,6 @@
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Duration;
-use config::CONFIG;
use sqlx::Row;
use super::{Trigger, TriggerId, TriggerModule, TriggerStatus, TRIGGERS_KEY};
@@ -284,7 +283,7 @@ FOR UPDATE;
)
.bind(TriggerStatus::Waiting)
.bind(now)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.bind(true)
.bind(false)
.bind(concurrency)
@@ -397,7 +396,7 @@ WHERE id IN (?);
let pool = CLIENT.clone();
sqlx::query(r#"DELETE FROM scheduled_jobs WHERE status = ? OR retries >= ?;"#)
.bind(TriggerStatus::Completed)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.execute(&pool)
.await?;
Ok(())
diff --git a/src/infra/src/scheduler/postgres.rs b/src/infra/src/scheduler/postgres.rs
index 2229071abb..f9611cd8e8 100644
--- a/src/infra/src/scheduler/postgres.rs
+++ b/src/infra/src/scheduler/postgres.rs
@@ -16,7 +16,6 @@
use async_trait::async_trait;
use bytes::Bytes;
use chrono::Duration;
-use config::CONFIG;
use sqlx::Row;
use super::{Trigger, TriggerModule, TriggerStatus, TRIGGERS_KEY};
@@ -294,7 +293,7 @@ RETURNING *;"#;
.bind(report_max_time)
.bind(TriggerStatus::Waiting)
.bind(now)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.bind(true)
.bind(false)
.bind(concurrency)
@@ -361,7 +360,7 @@ WHERE org = $1 AND module = $2 AND module_key = $3;"#;
let pool = CLIENT.clone();
sqlx::query(r#"DELETE FROM scheduled_jobs WHERE status = $1 OR retries >= $2;"#)
.bind(TriggerStatus::Completed)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.execute(&pool)
.await?;
Ok(())
diff --git a/src/infra/src/scheduler/sqlite.rs b/src/infra/src/scheduler/sqlite.rs
index 78d519b70d..20e02983ba 100644
--- a/src/infra/src/scheduler/sqlite.rs
+++ b/src/infra/src/scheduler/sqlite.rs
@@ -15,7 +15,7 @@
use async_trait::async_trait;
use chrono::Duration;
-use config::{utils::json, CONFIG};
+use config::utils::json;
use sqlx::Row;
use super::{Trigger, TriggerModule, TriggerStatus, TRIGGERS_KEY};
@@ -311,7 +311,7 @@ RETURNING *;"#;
.bind(report_max_time)
.bind(TriggerStatus::Waiting)
.bind(now)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.bind(true)
.bind(false)
.bind(concurrency)
@@ -367,7 +367,7 @@ WHERE org = $1 AND module = $2 AND module_key = $3;"#;
let client = client.lock().await;
sqlx::query(r#"DELETE FROM scheduled_jobs WHERE status = $1 OR retries >= $2;"#)
.bind(TriggerStatus::Completed)
- .bind(CONFIG.limit.scheduler_max_retries)
+ .bind(config::get_config().limit.scheduler_max_retries)
.execute(&*client)
.await?;
Ok(())
diff --git a/src/infra/src/schema/history/mod.rs b/src/infra/src/schema/history/mod.rs
index f17077b973..8997e29995 100644
--- a/src/infra/src/schema/history/mod.rs
+++ b/src/infra/src/schema/history/mod.rs
@@ -14,10 +14,7 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::{
- meta::{meta_store::MetaStore, stream::StreamType},
- CONFIG,
-};
+use config::meta::{meta_store::MetaStore, stream::StreamType};
use datafusion::arrow::datatypes::Schema;
use once_cell::sync::Lazy;
@@ -30,7 +27,7 @@ pub mod sqlite;
static CLIENT: Lazy> = Lazy::new(connect);
pub fn connect() -> Box {
- match CONFIG.common.meta_store.as_str().into() {
+ match config::get_config().common.meta_store.as_str().into() {
MetaStore::Sqlite => Box::::default(),
MetaStore::Etcd => Box::::default(),
MetaStore::Nats => Box::::default(),
diff --git a/src/infra/src/schema/mod.rs b/src/infra/src/schema/mod.rs
index 1e6360126c..9105afd6c6 100644
--- a/src/infra/src/schema/mod.rs
+++ b/src/infra/src/schema/mod.rs
@@ -17,9 +17,10 @@ use std::collections::HashMap;
use chrono::Utc;
use config::{
+ get_config,
meta::stream::{PartitionTimeLevel, StreamSettings, StreamType},
utils::{json, schema_ext::SchemaExt},
- RwAHashMap, CONFIG,
+ RwAHashMap,
};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use futures::{StreamExt, TryStreamExt};
@@ -121,8 +122,9 @@ pub async fn get_versions(
let key = mk_key(org_id, stream_type, stream_name);
let cache_key = key.strip_prefix("/schema/").unwrap();
+ let cfg = get_config();
let (min_ts, max_ts) = time_range.unwrap_or((0, 0));
- if CONFIG.common.schema_cache_compress_enabled {
+ if cfg.common.schema_cache_compress_enabled {
let mut last_schema_index = None;
let r = STREAM_SCHEMAS_COMPRESSED.read().await;
if let Some(versions) = r.get(cache_key) {
@@ -154,7 +156,7 @@ pub async fn get_versions(
let mut schemas: Vec = json::from_slice(&de_bytes)?;
Ok::, Error>(schemas.pop())
})
- .buffer_unordered(CONFIG.limit.cpu_num)
+ .buffer_unordered(cfg.limit.cpu_num)
.try_collect::>>()
.await
.map_err(|e| Error::Message(e.to_string()))?;
@@ -250,13 +252,14 @@ pub fn unwrap_partition_time_level(
if level != PartitionTimeLevel::Unset {
level
} else {
+ let cfg = get_config();
match stream_type {
- StreamType::Logs => PartitionTimeLevel::from(CONFIG.limit.logs_file_retention.as_str()),
+ StreamType::Logs => PartitionTimeLevel::from(cfg.limit.logs_file_retention.as_str()),
StreamType::Metrics => {
- PartitionTimeLevel::from(CONFIG.limit.metrics_file_retention.as_str())
+ PartitionTimeLevel::from(cfg.limit.metrics_file_retention.as_str())
}
StreamType::Traces => {
- PartitionTimeLevel::from(CONFIG.limit.traces_file_retention.as_str())
+ PartitionTimeLevel::from(cfg.limit.traces_file_retention.as_str())
}
_ => PartitionTimeLevel::default(),
}
@@ -608,7 +611,7 @@ pub fn get_merge_schema_changes(
Some(idx) => {
let existing_field = &merged_fields[*idx];
if existing_field.data_type() != item_data_type {
- if !CONFIG.common.widening_schema_evolution {
+ if !get_config().common.widening_schema_evolution {
field_datatype_delta.push(existing_field.as_ref().clone());
} else if is_widening_conversion(existing_field.data_type(), item_data_type) {
is_schema_changed = true;
diff --git a/src/infra/src/storage/local.rs b/src/infra/src/storage/local.rs
index 5656142409..0f2b91be6f 100644
--- a/src/infra/src/storage/local.rs
+++ b/src/infra/src/storage/local.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ use std::ops::Range;
use async_trait::async_trait;
use bytes::Bytes;
-use config::{metrics, CONFIG};
+use config::metrics;
use futures::stream::BoxStream;
use object_store::{
limit::LimitStore, local::LocalFileSystem, path::Path, Error, GetOptions, GetResult,
@@ -43,7 +43,7 @@ impl Local {
impl Default for Local {
fn default() -> Self {
- Local::new(&CONFIG.common.data_stream_dir, true)
+ Local::new(&config::get_config().common.data_stream_dir, true)
}
}
diff --git a/src/infra/src/storage/mod.rs b/src/infra/src/storage/mod.rs
index 5ee5e4b77f..f9d236a215 100644
--- a/src/infra/src/storage/mod.rs
+++ b/src/infra/src/storage/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{is_local_disk_storage, metrics, CONFIG};
+use config::{get_config, is_local_disk_storage, metrics};
use futures::{StreamExt, TryStreamExt};
use object_store::ObjectStore;
use once_cell::sync::Lazy;
@@ -40,7 +40,7 @@ pub static LOCAL_WAL: Lazy> = Lazy::new(local_wal);
/// ```
fn default() -> Box {
if is_local_disk_storage() {
- std::fs::create_dir_all(&CONFIG.common.data_stream_dir)
+ std::fs::create_dir_all(&get_config().common.data_stream_dir)
.expect("create stream data dir success");
Box::::default()
} else {
@@ -49,13 +49,15 @@ fn default() -> Box {
}
fn local_cache() -> Box {
- std::fs::create_dir_all(&CONFIG.common.data_cache_dir).expect("create cache dir success");
- Box::new(local::Local::new(&CONFIG.common.data_cache_dir, true))
+ let cfg = get_config();
+ std::fs::create_dir_all(&cfg.common.data_cache_dir).expect("create cache dir success");
+ Box::new(local::Local::new(&cfg.common.data_cache_dir, true))
}
fn local_wal() -> Box {
- std::fs::create_dir_all(&CONFIG.common.data_wal_dir).expect("create wal dir success");
- Box::new(local::Local::new(&CONFIG.common.data_wal_dir, false))
+ let cfg = get_config();
+ std::fs::create_dir_all(&cfg.common.data_wal_dir).expect("create wal dir success");
+ Box::new(local::Local::new(&cfg.common.data_wal_dir, false))
}
pub async fn list(prefix: &str) -> Result, anyhow::Error> {
@@ -92,7 +94,7 @@ pub async fn del(files: &[&str]) -> Result<(), anyhow::Error> {
.collect::>();
let files_stream = futures::stream::iter(files);
files_stream
- .for_each_concurrent(CONFIG.limit.cpu_num, |file| async move {
+ .for_each_concurrent(get_config().limit.cpu_num, |file| async move {
match DEFAULT.delete(&(file.as_str().into())).await {
Ok(_) => {
log::debug!("Deleted object: {}", file);
@@ -115,12 +117,13 @@ pub async fn del(files: &[&str]) -> Result<(), anyhow::Error> {
}
pub fn format_key(key: &str, with_prefix: bool) -> String {
+ let cfg = get_config();
if !is_local_disk_storage()
&& with_prefix
- && !CONFIG.s3.bucket_prefix.is_empty()
- && !key.starts_with(&CONFIG.s3.bucket_prefix)
+ && !cfg.s3.bucket_prefix.is_empty()
+ && !key.starts_with(&cfg.s3.bucket_prefix)
{
- format!("{}{}", CONFIG.s3.bucket_prefix, key)
+ format!("{}{}", cfg.s3.bucket_prefix, key)
} else {
key.to_string()
}
diff --git a/src/infra/src/storage/remote.rs b/src/infra/src/storage/remote.rs
index 933a58abf5..62cf6070c9 100644
--- a/src/infra/src/storage/remote.rs
+++ b/src/infra/src/storage/remote.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ use std::ops::Range;
use async_trait::async_trait;
use bytes::Bytes;
-use config::{metrics, CONFIG};
+use config::{get_config, metrics};
use futures::stream::BoxStream;
use object_store::{
limit::LimitStore, path::Path, Error, GetOptions, GetResult, ListResult, MultipartId,
@@ -227,77 +227,80 @@ impl ObjectStore for Remote {
}
fn init_aws_config() -> object_store::Result {
+ let cfg = get_config();
let mut opts = object_store::ClientOptions::default()
- .with_connect_timeout(std::time::Duration::from_secs(CONFIG.s3.connect_timeout))
- .with_timeout(std::time::Duration::from_secs(CONFIG.s3.request_timeout))
- .with_allow_invalid_certificates(CONFIG.s3.allow_invalid_certificates)
+ .with_connect_timeout(std::time::Duration::from_secs(cfg.s3.connect_timeout))
+ .with_timeout(std::time::Duration::from_secs(cfg.s3.request_timeout))
+ .with_allow_invalid_certificates(cfg.s3.allow_invalid_certificates)
.with_allow_http(true);
- if CONFIG.s3.feature_http1_only {
+ if cfg.s3.feature_http1_only {
opts = opts.with_http1_only();
}
- if CONFIG.s3.feature_http2_only {
+ if cfg.s3.feature_http2_only {
opts = opts.with_http2_only();
}
- let force_hosted_style =
- CONFIG.s3.feature_force_hosted_style || CONFIG.s3.feature_force_path_style;
+ let force_hosted_style = cfg.s3.feature_force_hosted_style || cfg.s3.feature_force_path_style;
let mut builder = object_store::aws::AmazonS3Builder::from_env()
.with_client_options(opts)
- .with_bucket_name(&CONFIG.s3.bucket_name)
+ .with_bucket_name(&cfg.s3.bucket_name)
.with_virtual_hosted_style_request(force_hosted_style);
- if !CONFIG.s3.server_url.is_empty() {
- builder = builder.with_endpoint(&CONFIG.s3.server_url);
+ if !cfg.s3.server_url.is_empty() {
+ builder = builder.with_endpoint(&cfg.s3.server_url);
}
- if !CONFIG.s3.region_name.is_empty() {
- builder = builder.with_region(&CONFIG.s3.region_name);
+ if !cfg.s3.region_name.is_empty() {
+ builder = builder.with_region(&cfg.s3.region_name);
}
- if !CONFIG.s3.access_key.is_empty() {
- builder = builder.with_access_key_id(&CONFIG.s3.access_key);
+ if !cfg.s3.access_key.is_empty() {
+ builder = builder.with_access_key_id(&cfg.s3.access_key);
}
- if !CONFIG.s3.secret_key.is_empty() {
- builder = builder.with_secret_access_key(&CONFIG.s3.secret_key);
+ if !cfg.s3.secret_key.is_empty() {
+ builder = builder.with_secret_access_key(&cfg.s3.secret_key);
}
builder.build()
}
fn init_azure_config() -> object_store::Result {
+ let cfg = get_config();
let mut builder = object_store::azure::MicrosoftAzureBuilder::from_env()
.with_client_options(
object_store::ClientOptions::default()
- .with_connect_timeout(std::time::Duration::from_secs(CONFIG.s3.connect_timeout))
- .with_timeout(std::time::Duration::from_secs(CONFIG.s3.request_timeout))
- .with_allow_invalid_certificates(CONFIG.s3.allow_invalid_certificates),
+ .with_connect_timeout(std::time::Duration::from_secs(cfg.s3.connect_timeout))
+ .with_timeout(std::time::Duration::from_secs(cfg.s3.request_timeout))
+ .with_allow_invalid_certificates(cfg.s3.allow_invalid_certificates),
)
- .with_container_name(&CONFIG.s3.bucket_name);
- if !CONFIG.s3.access_key.is_empty() {
- builder = builder.with_account(&CONFIG.s3.access_key);
+ .with_container_name(&cfg.s3.bucket_name);
+ if !cfg.s3.access_key.is_empty() {
+ builder = builder.with_account(&cfg.s3.access_key);
}
- if !CONFIG.s3.secret_key.is_empty() {
- builder = builder.with_access_key(&CONFIG.s3.secret_key);
+ if !cfg.s3.secret_key.is_empty() {
+ builder = builder.with_access_key(&cfg.s3.secret_key);
}
builder.build()
}
fn init_gcp_config() -> object_store::Result {
+ let cfg = get_config();
let mut builder = object_store::gcp::GoogleCloudStorageBuilder::from_env()
.with_client_options(
object_store::ClientOptions::default()
- .with_connect_timeout(std::time::Duration::from_secs(CONFIG.s3.connect_timeout))
- .with_timeout(std::time::Duration::from_secs(CONFIG.s3.request_timeout))
- .with_allow_invalid_certificates(CONFIG.s3.allow_invalid_certificates),
+ .with_connect_timeout(std::time::Duration::from_secs(cfg.s3.connect_timeout))
+ .with_timeout(std::time::Duration::from_secs(cfg.s3.request_timeout))
+ .with_allow_invalid_certificates(cfg.s3.allow_invalid_certificates),
)
- .with_bucket_name(&CONFIG.s3.bucket_name);
- if !CONFIG.s3.access_key.is_empty() {
- builder = builder.with_service_account_path(&CONFIG.s3.access_key);
+ .with_bucket_name(&cfg.s3.bucket_name);
+ if !cfg.s3.access_key.is_empty() {
+ builder = builder.with_service_account_path(&cfg.s3.access_key);
}
builder.build()
}
fn init_client() -> Box {
- if CONFIG.common.print_key_config {
- log::info!("s3 init config: {:?}", CONFIG.s3);
+ let cfg = get_config();
+ if cfg.common.print_key_config {
+ log::info!("s3 init config: {:?}", cfg.s3);
}
- match CONFIG.s3.provider.as_str() {
+ match cfg.s3.provider.as_str() {
"aws" | "s3" => match init_aws_config() {
Ok(client) => Box::new(client),
Err(e) => {
@@ -319,7 +322,7 @@ fn init_client() -> Box {
_ => match init_aws_config() {
Ok(client) => Box::new(client),
Err(e) => {
- panic!("{} init config error: {:?}", CONFIG.s3.provider, e);
+ panic!("{} init config error: {:?}", cfg.s3.provider, e);
}
},
}
diff --git a/src/ingester/bin/wal-reader.rs b/src/ingester/bin/wal-reader.rs
index 22e353be4b..498ae07e2a 100644
--- a/src/ingester/bin/wal-reader.rs
+++ b/src/ingester/bin/wal-reader.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/ingester/src/entry.rs b/src/ingester/src/entry.rs
index d4f3ace5f7..109ab3025b 100644
--- a/src/ingester/src/entry.rs
+++ b/src/ingester/src/entry.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -21,10 +21,7 @@ use std::{
use arrow::{array::Int64Array, record_batch::RecordBatch};
use arrow_schema::Schema;
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
-use config::{
- utils::record_batch_ext::{convert_json_to_record_batch, RecordBatchExt},
- CONFIG,
-};
+use config::utils::record_batch_ext::{convert_json_to_record_batch, RecordBatchExt};
use snafu::ResultExt;
use crate::errors::*;
@@ -132,23 +129,24 @@ impl RecordBatchEntry {
data_json_size: usize,
data_arrow_size: usize,
) -> Arc {
- let (min_ts, max_ts) = match data.column_by_name(&CONFIG.common.column_timestamp) {
- None => (0, 0),
- Some(v) => {
- let v = v.as_any().downcast_ref::().unwrap();
- let mut min_v = v.value(0);
- let mut max_v = v.value(0);
- for v in v.values() {
- if &min_v > v {
- min_v = *v;
- }
- if &max_v < v {
- max_v = *v;
+ let (min_ts, max_ts) =
+ match data.column_by_name(&config::get_config().common.column_timestamp) {
+ None => (0, 0),
+ Some(v) => {
+ let v = v.as_any().downcast_ref::().unwrap();
+ let mut min_v = v.value(0);
+ let mut max_v = v.value(0);
+ for v in v.values() {
+ if &min_v > v {
+ min_v = *v;
+ }
+ if &max_v < v {
+ max_v = *v;
+ }
}
+ (min_v, max_v)
}
- (min_v, max_v)
- }
- };
+ };
Arc::new(Self {
data,
data_json_size,
diff --git a/src/ingester/src/errors.rs b/src/ingester/src/errors.rs
index bb5f37465c..566ea4bd16 100644
--- a/src/ingester/src/errors.rs
+++ b/src/ingester/src/errors.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/ingester/src/immutable.rs b/src/ingester/src/immutable.rs
index 2a909a4a96..89102736ad 100644
--- a/src/ingester/src/immutable.rs
+++ b/src/ingester/src/immutable.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
use std::{path::PathBuf, sync::Arc};
use arrow_schema::Schema;
-use config::{metrics, CONFIG};
+use config::metrics;
use futures::future::join_all;
use once_cell::sync::Lazy;
use snafu::ResultExt;
@@ -116,7 +116,9 @@ pub(crate) async fn persist() -> Result<()> {
drop(r);
let mut tasks = Vec::with_capacity(paths.len());
- let semaphore = Arc::new(Semaphore::new(CONFIG.limit.file_move_thread_num));
+ let semaphore = Arc::new(Semaphore::new(
+ config::get_config().limit.file_move_thread_num,
+ ));
for path in paths {
let permit = semaphore.clone().acquire_owned().await.unwrap();
let task: task::JoinHandle>> =
diff --git a/src/ingester/src/lib.rs b/src/ingester/src/lib.rs
index d6b3d524be..f98628cbf2 100644
--- a/src/ingester/src/lib.rs
+++ b/src/ingester/src/lib.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -43,7 +43,7 @@ pub async fn init() -> errors::Result<()> {
tokio::task::spawn(async move {
// immutable persist every 10 (default) seconds
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
- config::CONFIG.limit.mem_persist_interval,
+ config::get_config().limit.mem_persist_interval,
));
interval.tick().await; // the first tick is immediate
loop {
@@ -60,7 +60,7 @@ pub async fn init() -> errors::Result<()> {
// start a job to flush memtable to immutable
tokio::task::spawn(async move {
let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
- config::CONFIG.limit.max_file_retention_time,
+ config::get_config().limit.max_file_retention_time,
));
interval.tick().await; // the first tick is immediate
loop {
diff --git a/src/ingester/src/memtable.rs b/src/ingester/src/memtable.rs
index 3e97cc3f63..8c7b1db691 100644
--- a/src/ingester/src/memtable.rs
+++ b/src/ingester/src/memtable.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/ingester/src/partition.rs b/src/ingester/src/partition.rs
index ebc93b0815..95a3ef93d4 100644
--- a/src/ingester/src/partition.rs
+++ b/src/ingester/src/partition.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -23,7 +23,6 @@ use config::{
parquet::{generate_filename_with_time_range, new_parquet_writer},
schema_ext::SchemaExt,
},
- CONFIG,
};
use snafu::ResultExt;
use tokio::{fs::OpenOptions, io::AsyncWriteExt};
@@ -78,7 +77,8 @@ impl Partition {
stream_type: &str,
stream_name: &str,
) -> Result<(usize, Vec<(PathBuf, PersistStat)>)> {
- let base_path = PathBuf::from(&CONFIG.common.data_wal_dir);
+ let cfg = config::get_config();
+ let base_path = PathBuf::from(&cfg.common.data_wal_dir);
let mut path = base_path.clone();
path.push("files");
path.push(org_id);
@@ -110,7 +110,7 @@ impl Partition {
};
// write into parquet buf
let (bloom_filter_fields, full_text_search_fields) =
- if self.schema.fields().len() > CONFIG.limit.file_move_fields_limit {
+ if self.schema.fields().len() > cfg.limit.file_move_fields_limit {
let bloom_filter_fields =
infra::schema::get_stream_setting_bloom_filter_fields(self.schema.as_ref())
.unwrap();
diff --git a/src/ingester/src/rwmap.rs b/src/ingester/src/rwmap.rs
index c6f847faf3..996ffb6297 100644
--- a/src/ingester/src/rwmap.rs
+++ b/src/ingester/src/rwmap.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/ingester/src/stream.rs b/src/ingester/src/stream.rs
index a691cbb821..284d939d15 100644
--- a/src/ingester/src/stream.rs
+++ b/src/ingester/src/stream.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/ingester/src/wal.rs b/src/ingester/src/wal.rs
index 179440ccfb..bc66234028 100644
--- a/src/ingester/src/wal.rs
+++ b/src/ingester/src/wal.rs
@@ -21,10 +21,7 @@ use std::{
};
use async_walkdir::WalkDir;
-use config::{
- utils::{schema::infer_json_schema_from_values, schema_ext::SchemaExt},
- CONFIG,
-};
+use config::utils::{schema::infer_json_schema_from_values, schema_ext::SchemaExt};
use futures::StreamExt;
use snafu::ResultExt;
@@ -48,8 +45,9 @@ use crate::{errors::*, immutable, memtable, writer::WriterKey};
// 4. the process is killed before step 5, so there are some .parquet files and have lock file, the
// files actually wrote to disk completely, need to continue step 5
pub(crate) async fn check_uncompleted_parquet_files() -> Result<()> {
+ let cfg = config::get_config();
// 1. get all .lock files
- let wal_dir = PathBuf::from(&CONFIG.common.data_wal_dir).join("logs");
+ let wal_dir = PathBuf::from(&cfg.common.data_wal_dir).join("logs");
// create wal dir if not exists
create_dir_all(&wal_dir).context(OpenDirSnafu {
path: wal_dir.clone(),
@@ -90,7 +88,7 @@ pub(crate) async fn check_uncompleted_parquet_files() -> Result<()> {
}
// 4. delete all the .par files
- let parquet_dir = PathBuf::from(&CONFIG.common.data_wal_dir).join("files");
+ let parquet_dir = PathBuf::from(&cfg.common.data_wal_dir).join("files");
// create wal dir if not exists
create_dir_all(&parquet_dir).context(OpenDirSnafu {
path: parquet_dir.clone(),
@@ -105,7 +103,7 @@ pub(crate) async fn check_uncompleted_parquet_files() -> Result<()> {
// replay wal files to create immutable
pub(crate) async fn replay_wal_files() -> Result<()> {
- let wal_dir = PathBuf::from(&CONFIG.common.data_wal_dir).join("logs");
+ let wal_dir = PathBuf::from(&config::get_config().common.data_wal_dir).join("logs");
create_dir_all(&wal_dir).context(OpenDirSnafu {
path: wal_dir.clone(),
})?;
diff --git a/src/ingester/src/writer.rs b/src/ingester/src/writer.rs
index 821e224ce3..7c5a42221a 100644
--- a/src/ingester/src/writer.rs
+++ b/src/ingester/src/writer.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -23,7 +23,7 @@ use std::{
use arrow_schema::Schema;
use chrono::{Duration, Utc};
-use config::{metrics, CONFIG};
+use config::{get_config, metrics};
use once_cell::sync::Lazy;
use snafu::ResultExt;
use tokio::sync::{Mutex, RwLock};
@@ -39,8 +39,9 @@ use crate::{
};
static WRITERS: Lazy>>> = Lazy::new(|| {
- let writer_num = if CONFIG.common.feature_per_thread_lock {
- CONFIG.limit.http_worker_num
+ let cfg = get_config();
+ let writer_num = if cfg.common.feature_per_thread_lock {
+ cfg.limit.http_worker_num
} else {
1
};
@@ -65,7 +66,7 @@ pub fn check_memtable_size() -> Result<()> {
let total_mem_size = metrics::INGEST_MEMTABLE_ARROW_BYTES
.with_label_values(&[])
.get();
- if total_mem_size >= CONFIG.limit.mem_table_max_size as i64 {
+ if total_mem_size >= get_config().limit.mem_table_max_size as i64 {
Err(Error::MemoryTableOverflowError {})
} else {
Ok(())
@@ -88,9 +89,10 @@ pub async fn read_from_memtable(
stream_name: &str,
time_range: Option<(i64, i64)>,
) -> Result, Vec>)>> {
+ let cfg = get_config();
let key = WriterKey::new(org_id, stream_type);
- let writer_num = if CONFIG.common.feature_per_thread_lock {
- CONFIG.limit.http_worker_num
+ let writer_num = if cfg.common.feature_per_thread_lock {
+ cfg.limit.http_worker_num
} else {
1
};
@@ -137,9 +139,10 @@ pub async fn flush_all() -> Result<()> {
impl Writer {
pub(crate) fn new(thread_id: usize, key: WriterKey) -> Self {
let now = Utc::now().timestamp_micros();
+ let cfg = get_config();
let next_seq = AtomicU64::new(now as u64);
let wal_id = next_seq.fetch_add(1, Ordering::SeqCst);
- let wal_dir = PathBuf::from(&CONFIG.common.data_wal_dir)
+ let wal_dir = PathBuf::from(&cfg.common.data_wal_dir)
.join("logs")
.join(thread_id.to_string());
log::info!(
@@ -158,7 +161,7 @@ impl Writer {
&key.org_id,
&key.stream_type,
wal_id,
- CONFIG.limit.max_file_size_on_disk as u64,
+ cfg.limit.max_file_size_on_disk as u64,
)
.expect("wal file create error"),
)),
@@ -187,11 +190,12 @@ impl Writer {
if self.check_wal_threshold(wal.size(), entry_bytes.len())
|| self.check_mem_threshold(self.memtable.read().await.size(), entry.data_size)
{
+ let cfg = get_config();
// sync wal before rotation
wal.sync().context(WalSnafu)?;
// rotation wal
let wal_id = self.next_seq.fetch_add(1, Ordering::SeqCst);
- let wal_dir = PathBuf::from(&CONFIG.common.data_wal_dir)
+ let wal_dir = PathBuf::from(&cfg.common.data_wal_dir)
.join("logs")
.join(self.thread_id.to_string());
log::info!(
@@ -206,7 +210,7 @@ impl Writer {
&self.key.org_id,
&self.key.stream_type,
wal_id,
- CONFIG.limit.max_file_size_on_disk as u64,
+ cfg.limit.max_file_size_on_disk as u64,
)
.context(WalSnafu)?;
let old_wal = std::mem::replace(&mut *wal, new_wal);
@@ -285,11 +289,12 @@ impl Writer {
/// Check if the wal file size is over the threshold or the file is too old
fn check_wal_threshold(&self, written_size: (usize, usize), data_size: usize) -> bool {
+ let cfg = get_config();
let (compressed_size, _uncompressed_size) = written_size;
compressed_size > 0
- && (compressed_size + data_size > CONFIG.limit.max_file_size_on_disk
+ && (compressed_size + data_size > cfg.limit.max_file_size_on_disk
|| self.created_at.load(Ordering::Relaxed)
- + Duration::try_seconds(CONFIG.limit.max_file_retention_time as i64)
+ + Duration::try_seconds(cfg.limit.max_file_retention_time as i64)
.unwrap()
.num_microseconds()
.unwrap()
@@ -298,10 +303,11 @@ impl Writer {
/// Check if the memtable size is over the threshold
fn check_mem_threshold(&self, written_size: (usize, usize), data_size: usize) -> bool {
+ let cfg = get_config();
let (json_size, arrow_size) = written_size;
json_size > 0
- && (json_size + data_size > CONFIG.limit.max_file_size_in_memory
- || arrow_size + data_size > CONFIG.limit.max_file_size_in_memory)
+ && (json_size + data_size > cfg.limit.max_file_size_in_memory
+ || arrow_size + data_size > cfg.limit.max_file_size_in_memory)
}
}
diff --git a/src/job/alert_manager.rs b/src/job/alert_manager.rs
index e8d78e1843..97d7602300 100644
--- a/src/job/alert_manager.rs
+++ b/src/job/alert_manager.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{cluster, config::CONFIG};
+use config::{cluster, get_config};
#[cfg(feature = "enterprise")]
use o2_enterprise::enterprise::common::infra::config::O2_CONFIG;
use tokio::time;
@@ -25,13 +25,14 @@ pub async fn run() -> Result<(), anyhow::Error> {
return Ok(());
}
+ let cfg = get_config();
log::info!(
"Spawning embedded report server {}",
- CONFIG.report_server.enable_report_server
+ cfg.report_server.enable_report_server
);
- if CONFIG.report_server.enable_report_server {
+ if cfg.report_server.enable_report_server {
tokio::task::spawn(async move {
- if let Err(e) = reportserver::spawn_server().await {
+ if let Err(e) = report_server::spawn_server().await {
log::error!("report server failed to spawn {}", e);
}
});
@@ -77,7 +78,7 @@ async fn run_schedule_jobs() -> Result<(), anyhow::Error> {
async fn clean_complete_jobs() -> Result<(), anyhow::Error> {
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.limit.scheduler_clean_interval,
+ get_config().limit.scheduler_clean_interval,
));
interval.tick().await; // trigger the first run
loop {
@@ -90,7 +91,7 @@ async fn clean_complete_jobs() -> Result<(), anyhow::Error> {
async fn watch_timeout_jobs() -> Result<(), anyhow::Error> {
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.limit.scheduler_watch_interval,
+ get_config().limit.scheduler_watch_interval,
));
interval.tick().await; // trigger the first run
loop {
diff --git a/src/job/compact.rs b/src/job/compact.rs
index bdf23aa8d1..d25cbf2cc2 100644
--- a/src/job/compact.rs
+++ b/src/job/compact.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,8 +17,8 @@ use std::sync::Arc;
use config::{
cluster::{is_compactor, LOCAL_NODE_ROLE},
+ get_config,
meta::stream::FileKey,
- CONFIG,
};
use tokio::{
sync::{mpsc, Mutex},
@@ -35,14 +35,15 @@ pub async fn run() -> Result<(), anyhow::Error> {
return Ok(());
}
- if !CONFIG.compact.enabled {
+ let cfg = get_config();
+ if !cfg.compact.enabled {
return Ok(());
}
- let (tx, rx) = mpsc::channel::<(MergeSender, MergeBatch)>(CONFIG.limit.file_move_thread_num);
+ let (tx, rx) = mpsc::channel::<(MergeSender, MergeBatch)>(cfg.limit.file_move_thread_num);
let rx = Arc::new(Mutex::new(rx));
// start merge workers
- for thread_id in 0..CONFIG.limit.file_move_thread_num {
+ for thread_id in 0..cfg.limit.file_move_thread_num {
let rx = rx.clone();
tokio::spawn(async move {
loop {
@@ -100,7 +101,7 @@ pub async fn run() -> Result<(), anyhow::Error> {
/// Merge small files
async fn run_merge(tx: mpsc::Sender<(MergeSender, MergeBatch)>) -> Result<(), anyhow::Error> {
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.compact.interval));
+ let mut interval = time::interval(time::Duration::from_secs(get_config().compact.interval));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
@@ -117,7 +118,7 @@ async fn run_merge(tx: mpsc::Sender<(MergeSender, MergeBatch)>) -> Result<(), an
/// Deletion for data retention
async fn run_retention() -> Result<(), anyhow::Error> {
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.compact.interval + 1));
+ let mut interval = time::interval(time::Duration::from_secs(get_config().compact.interval + 1));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
@@ -134,7 +135,7 @@ async fn run_retention() -> Result<(), anyhow::Error> {
/// Delete files based on the file_file_deleted in the database
async fn run_delay_deletion() -> Result<(), anyhow::Error> {
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.compact.interval + 2));
+ let mut interval = time::interval(time::Duration::from_secs(get_config().compact.interval + 2));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
@@ -151,7 +152,7 @@ async fn run_delay_deletion() -> Result<(), anyhow::Error> {
async fn run_sync_to_db() -> Result<(), anyhow::Error> {
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.compact.sync_to_db_interval,
+ get_config().compact.sync_to_db_interval,
));
interval.tick().await; // trigger the first run
loop {
diff --git a/src/job/file_list.rs b/src/job/file_list.rs
index abf47983cf..de977c705e 100644
--- a/src/job/file_list.rs
+++ b/src/job/file_list.rs
@@ -17,9 +17,9 @@ use std::{fs, path::Path};
use config::{
cluster::{is_compactor, is_ingester, is_querier, LOCAL_NODE_ROLE},
+ get_config,
meta::stream::StreamType,
utils::file::scan_files,
- CONFIG,
};
use infra::storage;
use tokio::time;
@@ -30,7 +30,8 @@ use crate::{
};
pub async fn run() -> Result<(), anyhow::Error> {
- if CONFIG.common.local_mode || CONFIG.common.meta_store_external {
+ let cfg = get_config();
+ if cfg.common.local_mode || cfg.common.meta_store_external {
return Ok(());
}
@@ -45,7 +46,9 @@ pub async fn run_move_file_to_s3() -> Result<(), anyhow::Error> {
return Ok(()); // not an ingester, no need to init job
}
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.limit.file_push_interval));
+ let mut interval = time::interval(time::Duration::from_secs(
+ get_config().limit.file_push_interval,
+ ));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
@@ -57,11 +60,10 @@ pub async fn run_move_file_to_s3() -> Result<(), anyhow::Error> {
// upload compressed file_list to storage & delete moved files from local
pub async fn move_file_list_to_storage(check_in_use: bool) -> Result<(), anyhow::Error> {
- let data_dir = Path::new(&CONFIG.common.data_wal_dir)
- .canonicalize()
- .unwrap();
+ let cfg = get_config();
+ let data_dir = Path::new(&cfg.common.data_wal_dir).canonicalize().unwrap();
- let pattern = format!("{}file_list/", &CONFIG.common.data_wal_dir);
+ let pattern = format!("{}file_list/", &cfg.common.data_wal_dir);
let files = scan_files(&pattern, "json", None).unwrap_or_default();
for file in files {
@@ -147,7 +149,9 @@ async fn _run_sync_s3_to_cache() -> Result<(), anyhow::Error> {
return Ok(()); // only querier or compactor need to sync
}
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.s3.sync_to_cache_interval));
+ let mut interval = time::interval(time::Duration::from_secs(
+ get_config().s3.sync_to_cache_interval,
+ ));
interval.tick().await; // trigger the first run
loop {
interval.tick().await;
diff --git a/src/job/files/mod.rs b/src/job/files/mod.rs
index f890cabbb3..4524d3e43b 100644
--- a/src/job/files/mod.rs
+++ b/src/job/files/mod.rs
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{cluster, ider, meta::stream::StreamType, CONFIG, FILE_EXT_PARQUET};
+use config::{cluster, ider, meta::stream::StreamType, FILE_EXT_PARQUET};
use tokio::time;
pub mod broadcast;
@@ -41,7 +41,7 @@ async fn clean_empty_dirs() -> Result<(), anyhow::Error> {
interval.tick().await;
let last_updated = std::time::SystemTime::now() - std::time::Duration::from_secs(3600);
if let Err(e) = config::utils::asynchronism::file::clean_empty_dirs(
- &CONFIG.common.data_wal_dir,
+ &config::get_config().common.data_wal_dir,
Some(last_updated),
)
.await
diff --git a/src/job/files/parquet.rs b/src/job/files/parquet.rs
index e0756ea9f5..134ae253e8 100644
--- a/src/job/files/parquet.rs
+++ b/src/job/files/parquet.rs
@@ -23,7 +23,7 @@ use arrow_schema::{DataType, Schema, SchemaRef};
use bytes::Bytes;
use chrono::{Duration, Utc};
use config::{
- cluster,
+ cluster, get_config,
meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamType},
metrics,
utils::{
@@ -37,7 +37,7 @@ use config::{
},
schema_ext::SchemaExt,
},
- FxIndexMap, CONFIG, DEFAULT_INDEX_TRIM_CHARS, INDEX_MIN_CHAR_LEN,
+ FxIndexMap, DEFAULT_INDEX_TRIM_CHARS, INDEX_MIN_CHAR_LEN,
};
use datafusion::{datasource::MemTable, prelude::*};
use hashbrown::HashSet;
@@ -70,11 +70,12 @@ use crate::{
static PROCESSING_FILES: Lazy>> = Lazy::new(|| RwLock::new(HashSet::new()));
pub async fn run() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
let (tx, rx) =
- tokio::sync::mpsc::channel::<(String, Vec)>(CONFIG.limit.file_move_thread_num);
+ tokio::sync::mpsc::channel::<(String, Vec)>(cfg.limit.file_move_thread_num);
let rx = Arc::new(Mutex::new(rx));
// move files
- for thread_id in 0..CONFIG.limit.file_move_thread_num {
+ for thread_id in 0..cfg.limit.file_move_thread_num {
let rx = rx.clone();
tokio::spawn(async move {
loop {
@@ -98,7 +99,7 @@ pub async fn run() -> Result<(), anyhow::Error> {
}
// prepare files
- let mut interval = time::interval(time::Duration::from_secs(CONFIG.limit.file_push_interval));
+ let mut interval = time::interval(time::Duration::from_secs(cfg.limit.file_push_interval));
interval.tick().await; // trigger the first run
loop {
if cluster::is_offline() {
@@ -123,14 +124,13 @@ pub async fn run() -> Result<(), anyhow::Error> {
}
async fn prepare_files() -> Result>, anyhow::Error> {
+ let cfg = get_config();
let start = std::time::Instant::now();
- let wal_dir = Path::new(&CONFIG.common.data_wal_dir)
- .canonicalize()
- .unwrap();
+ let wal_dir = Path::new(&cfg.common.data_wal_dir).canonicalize().unwrap();
let pattern = wal_dir.join("files/");
let files =
- scan_files(&pattern, "parquet", Some(CONFIG.limit.file_push_limit)).unwrap_or_default();
+ scan_files(&pattern, "parquet", Some(cfg.limit.file_push_limit)).unwrap_or_default();
if files.is_empty() {
return Ok(FxIndexMap::default());
}
@@ -220,9 +220,8 @@ async fn move_files(
// log::debug!("[INGESTER:JOB:{thread_id}] check deletion for partition: {}", prefix);
- let wal_dir = Path::new(&CONFIG.common.data_wal_dir)
- .canonicalize()
- .unwrap();
+ let cfg = get_config();
+ let wal_dir = Path::new(&cfg.common.data_wal_dir).canonicalize().unwrap();
// check if we are allowed to ingest or just delete the file
if db::compact::retention::is_deleting_stream(&org_id, stream_type, &stream_name, None) {
@@ -289,16 +288,16 @@ async fn move_files(
.sum::();
if total_original_size
< std::cmp::min(
- CONFIG.limit.max_file_size_on_disk as i64,
- CONFIG.compact.max_file_size as i64,
+ cfg.limit.max_file_size_on_disk as i64,
+ cfg.compact.max_file_size as i64,
)
- && (CONFIG.limit.file_move_fields_limit == 0
- || group_schema_field_num < CONFIG.limit.file_move_fields_limit)
+ && (cfg.limit.file_move_fields_limit == 0
+ || group_schema_field_num < cfg.limit.file_move_fields_limit)
{
let mut has_expired_files = false;
// not enough files to upload, check if some files are too old
let min_ts = Utc::now().timestamp_micros()
- - Duration::try_seconds(CONFIG.limit.max_file_retention_time as i64)
+ - Duration::try_seconds(cfg.limit.max_file_retention_time as i64)
.unwrap()
.num_microseconds()
.unwrap();
@@ -445,6 +444,7 @@ async fn merge_files(
return Ok((String::from(""), FileMeta::default(), Vec::new()));
}
+ let cfg = get_config();
let mut new_file_size: i64 = 0;
let mut new_file_list = Vec::new();
let mut deleted_files = Vec::new();
@@ -454,9 +454,9 @@ async fn merge_files(
for file in files_with_size.iter() {
if new_file_size > 0
- && ((new_file_size + file.meta.original_size > CONFIG.compact.max_file_size as i64)
- || (CONFIG.limit.file_move_fields_limit > 0
- && group_schema_field_num > CONFIG.limit.file_move_fields_limit))
+ && ((new_file_size + file.meta.original_size > cfg.compact.max_file_size as i64)
+ || (cfg.limit.file_move_fields_limit > 0
+ && group_schema_field_num > cfg.limit.file_move_fields_limit))
{
break;
}
@@ -617,7 +617,7 @@ async fn merge_files(
let buf = Bytes::from(buf);
match storage::put(&new_file_key, buf).await {
Ok(_) => {
- if CONFIG.common.inverted_index_enabled && stream_type != StreamType::Index {
+ if cfg.common.inverted_index_enabled && stream_type != StreamType::Index {
generate_index_on_ingester(
inverted_idx_batches,
new_file_key.clone(),
@@ -680,7 +680,7 @@ pub(crate) async fn generate_index_on_ingester(
let recs: Vec = json_rows.into_iter().map(json::Value::Object).collect();
for record_val in recs {
let timestamp: i64 = record_val
- .get(&CONFIG.common.column_timestamp)
+ .get(&get_config().common.column_timestamp)
.unwrap()
.as_i64()
.unwrap();
@@ -782,6 +782,7 @@ async fn prepare_index_record_batches(
return Ok(vec![]);
}
+ let cfg = get_config();
// filter null columns
let schema = batches.first().unwrap().schema();
let batches = batches.iter().collect::>();
@@ -796,7 +797,7 @@ async fn prepare_index_record_batches(
}
let schema = new_batch.schema();
if schema.fields().is_empty()
- || (schema.fields().len() == 1 && schema.field(0).name() == &CONFIG.common.column_timestamp)
+ || (schema.fields().len() == 1 && schema.field(0).name() == &cfg.common.column_timestamp)
{
return Ok(vec![]);
}
@@ -816,7 +817,7 @@ async fn prepare_index_record_batches(
let index_df = ctx.table("_tbl_raw_data").await?;
let column_name = column.name();
- let split_chars = &CONFIG.common.inverted_index_split_chars;
+ let split_chars = &cfg.common.inverted_index_split_chars;
let remove_chars_btrim = DEFAULT_INDEX_TRIM_CHARS;
let lower_case_expr = lower(concat(vec![col(column_name), lit("")]));
let split_arr = STRING_TO_ARRAY_V2_UDF.call(vec![lower_case_expr, lit(split_chars)]);
diff --git a/src/job/metrics.rs b/src/job/metrics.rs
index b9ceef7648..3f0fb834b2 100644
--- a/src/job/metrics.rs
+++ b/src/job/metrics.rs
@@ -16,11 +16,10 @@
use std::path::Path;
use config::{
- cluster,
+ cluster, get_config,
meta::{cluster::Role, stream::StreamType},
metrics,
utils::file::scan_files,
- CONFIG,
};
use hashbrown::HashMap;
use infra::{cache, db::get_db};
@@ -54,21 +53,23 @@ pub async fn run() -> Result<(), anyhow::Error> {
}
async fn load_query_cache_limit_bytes() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
metrics::QUERY_MEMORY_CACHE_LIMIT_BYTES
.with_label_values(&[])
- .set(CONFIG.memory_cache.max_size as i64);
+ .set(cfg.memory_cache.max_size as i64);
metrics::QUERY_DISK_CACHE_LIMIT_BYTES
.with_label_values(&[])
- .set(CONFIG.disk_cache.max_size as i64);
+ .set(cfg.disk_cache.max_size as i64);
Ok(())
}
async fn load_ingest_wal_used_bytes() -> Result<(), anyhow::Error> {
- let data_dir = match Path::new(&CONFIG.common.data_wal_dir).canonicalize() {
+ let cfg = get_config();
+ let data_dir = match Path::new(&cfg.common.data_wal_dir).canonicalize() {
Ok(path) => path,
Err(_) => return Ok(()),
};
- let pattern = format!("{}files/", &CONFIG.common.data_wal_dir);
+ let pattern = format!("{}files/", &cfg.common.data_wal_dir);
let files = scan_files(pattern, "parquet", None).unwrap_or_default();
let mut sizes = HashMap::new();
for file in files {
@@ -110,7 +111,7 @@ async fn update_metadata_metrics() -> Result<(), anyhow::Error> {
.with_label_values(&[])
.set(stats.keys_count);
- if CONFIG.common.local_mode {
+ if get_config().common.local_mode {
metrics::META_NUM_NODES.with_label_values(&["all"]).set(1);
} else {
metrics::META_NUM_NODES.reset();
diff --git a/src/job/mmdb_downloader.rs b/src/job/mmdb_downloader.rs
index cd56ccbbbb..2ad1e5ca60 100644
--- a/src/job/mmdb_downloader.rs
+++ b/src/job/mmdb_downloader.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
use std::{cmp::min, path::Path};
-use config::{CONFIG, MMDB_ASN_FILE_NAME, MMDB_CITY_FILE_NAME};
+use config::{MMDB_ASN_FILE_NAME, MMDB_CITY_FILE_NAME};
use futures::stream::StreamExt;
use once_cell::sync::Lazy;
use reqwest::Client;
@@ -33,11 +33,10 @@ use crate::{
static CLIENT_INITIALIZED: Lazy = Lazy::new(|| true);
pub async fn run() -> Result<(), anyhow::Error> {
- std::fs::create_dir_all(&CONFIG.common.mmdb_data_dir)?;
+ let cfg = config::get_config();
+ std::fs::create_dir_all(&cfg.common.mmdb_data_dir)?;
// should run it every 24 hours
- let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.common.mmdb_update_duration,
- ));
+ let mut interval = time::interval(time::Duration::from_secs(cfg.common.mmdb_update_duration));
loop {
interval.tick().await;
@@ -46,13 +45,15 @@ pub async fn run() -> Result<(), anyhow::Error> {
}
async fn run_download_files() {
+ let cfg = config::get_config();
+
// send request and await response
let client = reqwest::Client::new();
- let city_fname = format!("{}{}", &CONFIG.common.mmdb_data_dir, MMDB_CITY_FILE_NAME);
- let asn_fname = format!("{}{}", &CONFIG.common.mmdb_data_dir, MMDB_ASN_FILE_NAME);
+ let city_fname = format!("{}{}", &cfg.common.mmdb_data_dir, MMDB_CITY_FILE_NAME);
+ let asn_fname = format!("{}{}", &cfg.common.mmdb_data_dir, MMDB_ASN_FILE_NAME);
let download_city_files =
- is_digest_different(&city_fname, &CONFIG.common.mmdb_geolite_citydb_sha256_url)
+ is_digest_different(&city_fname, &cfg.common.mmdb_geolite_citydb_sha256_url)
.await
.unwrap_or_else(|e| {
log::error!("Error checking digest difference: {e}");
@@ -60,7 +61,7 @@ async fn run_download_files() {
});
let download_asn_files =
- is_digest_different(&asn_fname, &CONFIG.common.mmdb_geolite_asndb_sha256_url)
+ is_digest_different(&asn_fname, &cfg.common.mmdb_geolite_asndb_sha256_url)
.await
.unwrap_or_else(|e| {
log::error!("Error checking digest difference: {e}");
@@ -68,14 +69,14 @@ async fn run_download_files() {
});
if download_city_files {
- match download_file(&client, &CONFIG.common.mmdb_geolite_citydb_url, &city_fname).await {
+ match download_file(&client, &cfg.common.mmdb_geolite_citydb_url, &city_fname).await {
Ok(()) => {}
Err(e) => log::error!("failed to download the files {}", e),
}
}
if download_asn_files {
- match download_file(&client, &CONFIG.common.mmdb_geolite_asndb_url, &asn_fname).await {
+ match download_file(&client, &cfg.common.mmdb_geolite_asndb_url, &asn_fname).await {
Ok(()) => {}
Err(e) => log::error!("failed to download the files {}", e),
}
diff --git a/src/job/mod.rs b/src/job/mod.rs
index 3c19520d2d..0398752996 100644
--- a/src/job/mod.rs
+++ b/src/job/mod.rs
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{cluster, CONFIG};
+use config::cluster;
use infra::file_list as infra_file_list;
#[cfg(feature = "enterprise")]
use o2_enterprise::enterprise::common::infra::config::O2_CONFIG;
@@ -44,11 +44,12 @@ pub async fn init() -> Result<(), anyhow::Error> {
)
.expect("Email regex is valid");
+ let cfg = config::get_config();
// init root user
if !db::user::root_user_exists().await {
- if CONFIG.auth.root_user_email.is_empty()
- || !email_regex.is_match(&CONFIG.auth.root_user_email)
- || CONFIG.auth.root_user_password.is_empty()
+ if cfg.auth.root_user_email.is_empty()
+ || !email_regex.is_match(&cfg.auth.root_user_email)
+ || cfg.auth.root_user_password.is_empty()
{
panic!(
"Please set root user email-id & password using ZO_ROOT_USER_EMAIL & ZO_ROOT_USER_PASSWORD environment variables. This can also indicate an invalid email ID. Email ID must comply with ([a-z0-9_+]([a-z0-9_+.-]*[a-z0-9_+])?)@([a-z0-9]+([\\-\\.]{{1}}[a-z0-9]+)*\\.[a-z]{{2,6}})"
@@ -57,8 +58,8 @@ pub async fn init() -> Result<(), anyhow::Error> {
let _ = users::create_root_user(
DEFAULT_ORG,
UserRequest {
- email: CONFIG.auth.root_user_email.clone(),
- password: CONFIG.auth.root_user_password.clone(),
+ email: cfg.auth.root_user_email.clone(),
+ password: cfg.auth.root_user_password.clone(),
role: crate::common::meta::user::UserRole::Root,
first_name: "root".to_owned(),
last_name: "".to_owned(),
@@ -68,7 +69,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
.await;
}
- if !CONFIG.common.mmdb_disable_download {
+ if !cfg.common.mmdb_disable_download {
// Try to download the mmdb files, if its not disabled.
tokio::task::spawn(async move { mmdb_downloader::run().await });
}
@@ -89,7 +90,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
}
// telemetry run
- if CONFIG.common.telemetry_enabled && cluster::is_querier(&cluster::LOCAL_NODE_ROLE) {
+ if cfg.common.telemetry_enabled && cluster::is_querier(&cluster::LOCAL_NODE_ROLE) {
tokio::task::spawn(async move { telemetry::run().await });
}
@@ -154,7 +155,7 @@ pub async fn init() -> Result<(), anyhow::Error> {
}
// cache file list
- if !CONFIG.common.meta_store_external {
+ if !cfg.common.meta_store_external {
if cluster::is_ingester(&cluster::LOCAL_NODE_ROLE) {
// load the wal file_list into memory
db::file_list::local::load_wal_in_cache()
@@ -194,10 +195,10 @@ pub async fn init() -> Result<(), anyhow::Error> {
// check wal directory
if cluster::is_ingester(&cluster::LOCAL_NODE_ROLE) {
// create wal dir
- if let Err(e) = std::fs::create_dir_all(&CONFIG.common.data_wal_dir) {
+ if let Err(e) = std::fs::create_dir_all(&cfg.common.data_wal_dir) {
log::error!("Failed to create wal dir: {}", e);
}
- if let Err(e) = std::fs::create_dir_all(&CONFIG.common.data_idx_dir) {
+ if let Err(e) = std::fs::create_dir_all(&cfg.common.data_idx_dir) {
log::error!("Failed to create wal dir: {}", e);
}
}
diff --git a/src/job/prom.rs b/src/job/prom.rs
index 629887c0d9..d7faaecae0 100644
--- a/src/job/prom.rs
+++ b/src/job/prom.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{cluster, CONFIG};
+use config::cluster;
use hashbrown::HashMap;
use tokio::time::{self, Duration};
@@ -27,13 +27,12 @@ pub async fn run() -> Result<(), anyhow::Error> {
return Ok(()); // not an ingester, no need to init job
}
- if !CONFIG.common.metrics_dedup_enabled {
+ let cfg = config::get_config();
+ if !cfg.common.metrics_dedup_enabled {
return Ok(());
}
- let mut interval = time::interval(Duration::from_secs(
- CONFIG.limit.metrics_leader_push_interval,
- ));
+ let mut interval = time::interval(Duration::from_secs(cfg.limit.metrics_leader_push_interval));
interval.tick().await; // trigger the first run
let mut last_leaders: HashMap = HashMap::new(); // maintain the last state
diff --git a/src/job/stats.rs b/src/job/stats.rs
index e1a489dc01..687c84e6fc 100644
--- a/src/job/stats.rs
+++ b/src/job/stats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
use config::{
cluster::{is_compactor, is_querier},
- CONFIG,
+ get_config,
};
use tokio::time;
@@ -29,13 +29,14 @@ pub async fn run() -> Result<(), anyhow::Error> {
}
async fn _usage_report_stats() -> Result<(), anyhow::Error> {
- if !is_compactor(&super::cluster::LOCAL_NODE_ROLE) || !CONFIG.common.usage_enabled {
+ let cfg = get_config();
+ if !is_compactor(&super::cluster::LOCAL_NODE_ROLE) || !cfg.common.usage_enabled {
return Ok(());
}
// should run it every 10 minutes
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.limit.calculate_stats_interval,
+ cfg.limit.calculate_stats_interval,
));
interval.tick().await; // trigger the first run
loop {
@@ -48,14 +49,15 @@ async fn _usage_report_stats() -> Result<(), anyhow::Error> {
// get stats from file_list to update stream_stats
async fn file_list_update_stats() -> Result<(), anyhow::Error> {
- if (CONFIG.common.meta_store_external || !is_querier(&super::cluster::LOCAL_NODE_ROLE))
+ let cfg = get_config();
+ if (cfg.common.meta_store_external || !is_querier(&super::cluster::LOCAL_NODE_ROLE))
&& !is_compactor(&super::cluster::LOCAL_NODE_ROLE)
{
return Ok(());
}
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.limit.calculate_stats_interval,
+ cfg.limit.calculate_stats_interval,
));
interval.tick().await; // trigger the first run
loop {
@@ -84,7 +86,7 @@ async fn cache_stream_stats() -> Result<(), anyhow::Error> {
// should run it every minute
let mut interval = time::interval(time::Duration::from_secs(std::cmp::min(
- CONFIG.limit.calculate_stats_interval,
+ get_config().limit.calculate_stats_interval,
60,
)));
interval.tick().await; // trigger the first run
diff --git a/src/job/syslog_server.rs b/src/job/syslog_server.rs
index feeb491944..dcb6721446 100644
--- a/src/job/syslog_server.rs
+++ b/src/job/syslog_server.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,6 @@ use std::{
net::{SocketAddr, TcpStream},
};
-use config::CONFIG;
use once_cell::sync::Lazy;
use tokio::{
net::{TcpListener, UdpSocket},
@@ -38,10 +37,11 @@ pub static BROADCASTER: Lazy>> = Lazy::new(|| {
});
pub async fn run(start_srv: bool, is_init: bool) -> Result<(), anyhow::Error> {
+ let cfg = config::get_config();
let server_running = *SYSLOG_ENABLED.read();
let bind_addr = "0.0.0.0";
- let tcp_addr: SocketAddr = format!("{bind_addr}:{}", CONFIG.tcp.tcp_port).parse()?;
- let udp_addr: SocketAddr = format!("{bind_addr}:{}", CONFIG.tcp.udp_port).parse()?;
+ let tcp_addr: SocketAddr = format!("{bind_addr}:{}", cfg.tcp.tcp_port).parse()?;
+ let udp_addr: SocketAddr = format!("{bind_addr}:{}", cfg.tcp.udp_port).parse()?;
if (!server_running || is_init) && start_srv {
log::info!("Starting TCP UDP server");
let tcp_listener: TcpListener = TcpListener::bind(tcp_addr).await?;
diff --git a/src/job/telemetry.rs b/src/job/telemetry.rs
index f1ffc091aa..dc211ac1e8 100644
--- a/src/job/telemetry.rs
+++ b/src/job/telemetry.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,18 +13,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
use tokio::time;
use crate::common::meta::telemetry::Telemetry;
pub async fn run() -> Result<(), anyhow::Error> {
- if !CONFIG.common.telemetry_enabled {
+ let cfg = config::get_config();
+ if !cfg.common.telemetry_enabled {
return Ok(());
}
let mut interval = time::interval(time::Duration::from_secs(
- (CONFIG.common.telemetry_heartbeat).try_into().unwrap(),
+ (cfg.common.telemetry_heartbeat).try_into().unwrap(),
));
interval.tick().await;
loop {
diff --git a/src/main.rs b/src/main.rs
index 386a317b70..3073d9be4e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -29,7 +29,7 @@ use actix_web::{dev::ServerHandle, http::KeepAlive, middleware, web, App, HttpSe
use actix_web_opentelemetry::RequestTracing;
use config::{
cluster::{is_router, LOCAL_NODE_ROLE},
- CONFIG,
+ get_config,
};
use log::LevelFilter;
use openobserve::{
@@ -91,16 +91,17 @@ use tracing_subscriber::{
#[tokio::main]
async fn main() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
#[cfg(feature = "tokio-console")]
console_subscriber::ConsoleLayer::builder()
.retention(Duration::from_secs(
- CONFIG.tokio_console.tokio_console_retention,
+ cfg.tokio_console.tokio_console_retention,
))
.server_addr(
format!(
"{}:{}",
- CONFIG.tokio_console.tokio_console_server_addr,
- CONFIG.tokio_console.tokio_console_server_port
+ cfg.tokio_console.tokio_console_server_addr,
+ cfg.tokio_console.tokio_console_server_port
)
.as_str()
.parse::()?,
@@ -118,22 +119,21 @@ async fn main() -> Result<(), anyhow::Error> {
// setup profiling
#[cfg(feature = "profiling")]
- let agent = if !CONFIG.profiling.enabled {
+ let agent = if !cfg.profiling.enabled {
None
} else {
- let agent =
- PyroscopeAgent::builder(&CONFIG.profiling.server_url, &CONFIG.profiling.project_name)
- .tags(
- [
- ("role", CONFIG.common.node_role.as_str()),
- ("instance", CONFIG.common.instance_name.as_str()),
- ("version", VERSION),
- ]
- .to_vec(),
- )
- .backend(pprof_backend(PprofConfig::new().sample_rate(100)))
- .build()
- .expect("Failed to setup pyroscope agent");
+ let agent = PyroscopeAgent::builder(&cfg.profiling.server_url, &cfg.profiling.project_name)
+ .tags(
+ [
+ ("role", cfg.common.node_role.as_str()),
+ ("instance", cfg.common.instance_name.as_str()),
+ ("version", VERSION),
+ ]
+ .to_vec(),
+ )
+ .backend(pprof_backend(PprofConfig::new().sample_rate(100)))
+ .build()
+ .expect("Failed to setup pyroscope agent");
#[cfg(feature = "profiling")]
let agent_running = agent.start().expect("Failed to start pyroscope agent");
Some(agent_running)
@@ -151,17 +151,15 @@ async fn main() -> Result<(), anyhow::Error> {
let enable_tokio_console = false;
let _guard: Option = if enable_tokio_console {
None
- } else if CONFIG.log.events_enabled {
+ } else if cfg.log.events_enabled {
let logger = zo_logger::ZoLogger {
sender: zo_logger::EVENT_SENDER.clone(),
};
log::set_boxed_logger(Box::new(logger)).map(|()| {
- log::set_max_level(
- LevelFilter::from_str(&CONFIG.log.level).unwrap_or(LevelFilter::Info),
- )
+ log::set_max_level(LevelFilter::from_str(&cfg.log.level).unwrap_or(LevelFilter::Info))
})?;
None
- } else if CONFIG.common.tracing_enabled || CONFIG.common.tracing_search_enabled {
+ } else if cfg.common.tracing_enabled || cfg.common.tracing_search_enabled {
enable_tracing()?;
None
} else {
@@ -171,10 +169,10 @@ async fn main() -> Result<(), anyhow::Error> {
log::info!("Starting OpenObserve {}", VERSION);
log::info!(
"System info: CPU cores {}, MEM total {} MB, Disk total {} GB, free {} GB",
- CONFIG.limit.cpu_num,
- CONFIG.limit.mem_total / 1024 / 1024,
- CONFIG.limit.disk_total / 1024 / 1024 / 1024,
- CONFIG.limit.disk_free / 1024 / 1024 / 1024,
+ cfg.limit.cpu_num,
+ cfg.limit.mem_total / 1024 / 1024,
+ cfg.limit.disk_total / 1024 / 1024 / 1024,
+ cfg.limit.disk_free / 1024 / 1024 / 1024,
);
// it must be initialized before the server starts
@@ -225,10 +223,10 @@ async fn main() -> Result<(), anyhow::Error> {
.await
.expect("EnrichmentTables cache failed");
- if CONFIG.log.events_enabled {
+ if cfg.log.events_enabled {
tokio::task::spawn(async move { zo_logger::send_logs().await });
}
- if CONFIG.common.telemetry_enabled {
+ if cfg.common.telemetry_enabled {
tokio::task::spawn(async move {
meta::telemetry::Telemetry::new()
.event("OpenObserve - Starting server", None, false)
@@ -237,7 +235,7 @@ async fn main() -> Result<(), anyhow::Error> {
}
// init http server
- if !CONFIG.common.tracing_enabled && CONFIG.common.tracing_search_enabled {
+ if !cfg.common.tracing_enabled && cfg.common.tracing_search_enabled {
if let Err(e) = init_http_server_without_tracing().await {
log::error!("HTTP server runs failed: {}", e);
}
@@ -269,7 +267,7 @@ async fn main() -> Result<(), anyhow::Error> {
_ = db.close().await;
// stop telemetry
- if CONFIG.common.telemetry_enabled {
+ if cfg.common.telemetry_enabled {
meta::telemetry::Telemetry::new()
.event("OpenObserve - Server stopped", None, false)
.await;
@@ -290,12 +288,13 @@ fn init_common_grpc_server(
shutdown_rx: oneshot::Receiver<()>,
stopped_tx: oneshot::Sender<()>,
) -> Result<(), anyhow::Error> {
- let ip = if !CONFIG.grpc.addr.is_empty() {
- CONFIG.grpc.addr.clone()
+ let cfg = get_config();
+ let ip = if !cfg.grpc.addr.is_empty() {
+ cfg.grpc.addr.clone()
} else {
"0.0.0.0".to_string()
};
- let gaddr: SocketAddr = format!("{}:{}", ip, CONFIG.grpc.port).parse()?;
+ let gaddr: SocketAddr = format!("{}:{}", ip, cfg.grpc.port).parse()?;
let event_svc = EventServer::new(Eventer)
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip);
@@ -349,12 +348,13 @@ fn init_router_grpc_server(
shutdown_rx: oneshot::Receiver<()>,
stopped_tx: oneshot::Sender<()>,
) -> Result<(), anyhow::Error> {
- let gaddr: SocketAddr = format!("0.0.0.0:{}", CONFIG.grpc.port).parse()?;
+ let cfg = get_config();
+ let gaddr: SocketAddr = format!("0.0.0.0:{}", cfg.grpc.port).parse()?;
let logs_svc = LogsServiceServer::new(router::grpc::ingest::logs::LogsServer)
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
let metrics_svc = MetricsServiceServer::new(router::grpc::ingest::metrics::MetricsServer)
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip);
@@ -381,24 +381,26 @@ fn init_router_grpc_server(
}
async fn init_http_server() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
// metrics
let prometheus = config::metrics::create_prometheus_handler();
let thread_id = Arc::new(AtomicU16::new(0));
- let haddr: SocketAddr = if CONFIG.http.ipv6_enabled {
- format!("[::]:{}", CONFIG.http.port).parse()?
+ let haddr: SocketAddr = if cfg.http.ipv6_enabled {
+ format!("[::]:{}", cfg.http.port).parse()?
} else {
- let ip = if !CONFIG.http.addr.is_empty() {
- CONFIG.http.addr.clone()
+ let ip = if !cfg.http.addr.is_empty() {
+ cfg.http.addr.clone()
} else {
"0.0.0.0".to_string()
};
- format!("{}:{}", ip, CONFIG.http.port).parse()?
+ format!("{}:{}", ip, cfg.http.port).parse()?
};
let server = HttpServer::new(move || {
+ let cfg = get_config();
let local_id = thread_id.load(Ordering::SeqCst) as usize;
- if CONFIG.common.feature_per_thread_lock {
+ if cfg.common.feature_per_thread_lock {
thread_id.fetch_add(1, Ordering::SeqCst);
}
log::info!(
@@ -409,14 +411,14 @@ async fn init_http_server() -> Result<(), anyhow::Error> {
let mut app = App::new().wrap(prometheus.clone());
if is_router(&LOCAL_NODE_ROLE) {
let client = awc::Client::builder()
- .connector(awc::Connector::new().limit(CONFIG.route.max_connections))
- .timeout(Duration::from_secs(CONFIG.route.timeout))
+ .connector(awc::Connector::new().limit(cfg.route.max_connections))
+ .timeout(Duration::from_secs(cfg.route.timeout))
.disable_redirects()
.finish();
app = app
.service(
- // if `CONFIG.common.base_uri` is empty, scope("") still works as expected.
- web::scope(&CONFIG.common.base_uri)
+ // if `cfg.common.base_uri` is empty, scope("") still works as expected.
+ web::scope(&cfg.common.base_uri)
.service(router::http::config)
.service(router::http::config_paths)
.service(router::http::api)
@@ -429,7 +431,7 @@ async fn init_http_server() -> Result<(), anyhow::Error> {
.app_data(web::Data::new(client))
} else {
app = app.service(
- web::scope(&CONFIG.common.base_uri)
+ web::scope(&cfg.common.base_uri)
.configure(get_config_routes)
.configure(get_service_routes)
.configure(get_other_service_routes)
@@ -437,8 +439,8 @@ async fn init_http_server() -> Result<(), anyhow::Error> {
.configure(get_proxy_routes),
)
}
- app.app_data(web::JsonConfig::default().limit(CONFIG.limit.req_json_limit))
- .app_data(web::PayloadConfig::new(CONFIG.limit.req_payload_limit)) // size is in bytes
+ app.app_data(web::JsonConfig::default().limit(cfg.limit.req_json_limit))
+ .app_data(web::PayloadConfig::new(cfg.limit.req_payload_limit)) // size is in bytes
.app_data(web::Data::new(local_id%10))
.wrap(middleware::Compress::default())
.wrap(middleware::Logger::new(
@@ -448,17 +450,15 @@ async fn init_http_server() -> Result<(), anyhow::Error> {
})
.keep_alive(KeepAlive::Timeout(Duration::from_secs(max(
15,
- CONFIG.limit.keep_alive,
+ cfg.limit.keep_alive,
))))
- .client_request_timeout(Duration::from_secs(max(5, CONFIG.limit.request_timeout)))
- .shutdown_timeout(max(1, CONFIG.limit.shutdown_timeout))
+ .client_request_timeout(Duration::from_secs(max(5, cfg.limit.request_timeout)))
+ .shutdown_timeout(max(1, cfg.limit.shutdown_timeout))
.bind(haddr)?;
let server = server
- .workers(CONFIG.limit.http_worker_num)
- .worker_max_blocking_threads(
- CONFIG.limit.http_worker_num * CONFIG.limit.http_worker_max_blocking,
- )
+ .workers(cfg.limit.http_worker_num)
+ .worker_max_blocking_threads(cfg.limit.http_worker_num * cfg.limit.http_worker_max_blocking)
.disable_signals()
.run();
let handle = server.handle();
@@ -470,24 +470,26 @@ async fn init_http_server() -> Result<(), anyhow::Error> {
}
async fn init_http_server_without_tracing() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
// metrics
let prometheus = config::metrics::create_prometheus_handler();
let thread_id = Arc::new(AtomicU16::new(0));
- let haddr: SocketAddr = if CONFIG.http.ipv6_enabled {
- format!("[::]:{}", CONFIG.http.port).parse()?
+ let haddr: SocketAddr = if cfg.http.ipv6_enabled {
+ format!("[::]:{}", cfg.http.port).parse()?
} else {
- let ip = if !CONFIG.http.addr.is_empty() {
- CONFIG.http.addr.clone()
+ let ip = if !cfg.http.addr.is_empty() {
+ cfg.http.addr.clone()
} else {
"0.0.0.0".to_string()
};
- format!("{}:{}", ip, CONFIG.http.port).parse()?
+ format!("{}:{}", ip, cfg.http.port).parse()?
};
let server = HttpServer::new(move || {
+ let cfg = get_config();
let local_id = thread_id.load(Ordering::SeqCst) as usize;
- if CONFIG.common.feature_per_thread_lock {
+ if cfg.common.feature_per_thread_lock {
thread_id.fetch_add(1, Ordering::SeqCst);
}
log::info!(
@@ -498,14 +500,14 @@ async fn init_http_server_without_tracing() -> Result<(), anyhow::Error> {
let mut app = App::new().wrap(prometheus.clone());
if is_router(&LOCAL_NODE_ROLE) {
let client = awc::Client::builder()
- .connector(awc::Connector::new().limit(CONFIG.route.max_connections))
- .timeout(Duration::from_secs(CONFIG.route.timeout))
+ .connector(awc::Connector::new().limit(cfg.route.max_connections))
+ .timeout(Duration::from_secs(cfg.route.timeout))
.disable_redirects()
.finish();
app = app
.service(
- // if `CONFIG.common.base_uri` is empty, scope("") still works as expected.
- web::scope(&CONFIG.common.base_uri)
+ // if `cfg.common.base_uri` is empty, scope("") still works as expected.
+ web::scope(&cfg.common.base_uri)
.service(router::http::config)
.service(router::http::config_paths)
.service(router::http::api)
@@ -518,7 +520,7 @@ async fn init_http_server_without_tracing() -> Result<(), anyhow::Error> {
.app_data(web::Data::new(client))
} else {
app = app.service(
- web::scope(&CONFIG.common.base_uri)
+ web::scope(&cfg.common.base_uri)
.configure(get_config_routes)
.configure(get_service_routes)
.configure(get_other_service_routes)
@@ -526,8 +528,8 @@ async fn init_http_server_without_tracing() -> Result<(), anyhow::Error> {
.configure(get_proxy_routes),
)
}
- app.app_data(web::JsonConfig::default().limit(CONFIG.limit.req_json_limit))
- .app_data(web::PayloadConfig::new(CONFIG.limit.req_payload_limit)) // size is in bytes
+ app.app_data(web::JsonConfig::default().limit(cfg.limit.req_json_limit))
+ .app_data(web::PayloadConfig::new(cfg.limit.req_payload_limit)) // size is in bytes
.app_data(web::Data::new(local_id%10))
.wrap(middleware::Compress::default())
.wrap(middleware::Logger::new(
@@ -536,17 +538,15 @@ async fn init_http_server_without_tracing() -> Result<(), anyhow::Error> {
})
.keep_alive(KeepAlive::Timeout(Duration::from_secs(max(
15,
- CONFIG.limit.keep_alive,
+ cfg.limit.keep_alive,
))))
- .client_request_timeout(Duration::from_secs(max(5, CONFIG.limit.request_timeout)))
- .shutdown_timeout(max(1, CONFIG.limit.shutdown_timeout))
+ .client_request_timeout(Duration::from_secs(max(5, cfg.limit.request_timeout)))
+ .shutdown_timeout(max(1, cfg.limit.shutdown_timeout))
.bind(haddr)?;
let server = server
- .workers(CONFIG.limit.http_worker_num)
- .worker_max_blocking_threads(
- CONFIG.limit.http_worker_num * CONFIG.limit.http_worker_max_blocking,
- )
+ .workers(cfg.limit.http_worker_num)
+ .worker_max_blocking_threads(cfg.limit.http_worker_num * cfg.limit.http_worker_max_blocking)
.disable_signals()
.run();
let handle = server.handle();
@@ -605,21 +605,21 @@ async fn graceful_shutdown(handle: ServerHandle) {
pub(crate) fn setup_logs() -> tracing_appender::non_blocking::WorkerGuard {
use tracing_subscriber::fmt::writer::BoxMakeWriter;
- let (writer, guard) = if CONFIG.log.file_dir.is_empty() {
+ let cfg = get_config();
+ let (writer, guard) = if cfg.log.file_dir.is_empty() {
let (non_blocking, _guard) = tracing_appender::non_blocking(std::io::stdout());
(BoxMakeWriter::new(non_blocking), _guard)
} else {
- let file_name_prefix = if CONFIG.log.file_name_prefix.is_empty() {
- format!("o2.{}.log", CONFIG.common.instance_name.as_str())
+ let file_name_prefix = if cfg.log.file_name_prefix.is_empty() {
+ format!("o2.{}.log", cfg.common.instance_name.as_str())
} else {
- CONFIG.log.file_name_prefix.to_string()
+ cfg.log.file_name_prefix.to_string()
};
- let file_appender =
- tracing_appender::rolling::daily(&CONFIG.log.file_dir, file_name_prefix);
+ let file_appender = tracing_appender::rolling::daily(&cfg.log.file_dir, file_name_prefix);
let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);
(BoxMakeWriter::new(non_blocking), _guard)
};
- let layer = if CONFIG.log.json_format {
+ let layer = if cfg.log.json_format {
Layer::default()
.with_writer(writer)
.with_timer(config::meta::logger::CustomTimeFormat)
@@ -649,28 +649,29 @@ pub(crate) fn setup_logs() -> tracing_appender::non_blocking::WorkerGuard {
}
fn enable_tracing() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
opentelemetry::global::set_text_map_propagator(TraceContextPropagator::new());
let mut headers = HashMap::new();
headers.insert(
- CONFIG.common.tracing_header_key.clone(),
- CONFIG.common.tracing_header_value.clone(),
+ cfg.common.tracing_header_key.clone(),
+ cfg.common.tracing_header_value.clone(),
);
let tracer = opentelemetry_otlp::new_pipeline()
.tracing()
.with_exporter(
opentelemetry_otlp::new_exporter()
.http()
- .with_endpoint(&CONFIG.common.otel_otlp_url)
+ .with_endpoint(&cfg.common.otel_otlp_url)
.with_headers(headers),
)
.with_trace_config(sdktrace::config().with_resource(Resource::new(vec![
- KeyValue::new("service.name", CONFIG.common.node_role.as_str()),
- KeyValue::new("service.instance", CONFIG.common.instance_name.as_str()),
+ KeyValue::new("service.name", cfg.common.node_role.to_string()),
+ KeyValue::new("service.instance", cfg.common.instance_name.to_string()),
KeyValue::new("service.version", VERSION),
])))
.install_batch(opentelemetry_sdk::runtime::Tokio)?;
- let layer = if CONFIG.log.json_format {
+ let layer = if cfg.log.json_format {
tracing_subscriber::fmt::layer()
.with_ansi(false)
.json()
@@ -680,7 +681,7 @@ fn enable_tracing() -> Result<(), anyhow::Error> {
};
Registry::default()
- .with(tracing_subscriber::EnvFilter::new(&CONFIG.log.level))
+ .with(tracing_subscriber::EnvFilter::new(&cfg.log.level))
.with(layer)
.with(tracing_opentelemetry::layer().with_tracer(tracer))
.init();
diff --git a/src/proto/build.rs b/src/proto/build.rs
index 0a01ffa5a2..8e51828d79 100644
--- a/src/proto/build.rs
+++ b/src/proto/build.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/reportserver/Cargo.toml b/src/report_server/Cargo.toml
similarity index 94%
rename from src/reportserver/Cargo.toml
rename to src/report_server/Cargo.toml
index b2709f63ed..3727e81520 100644
--- a/src/reportserver/Cargo.toml
+++ b/src/report_server/Cargo.toml
@@ -1,5 +1,5 @@
[package]
-name = "reportserver"
+name = "report_server"
version.workspace = true
edition.workspace = true
license.workspace = true
diff --git a/src/reportserver/src/lib.rs b/src/report_server/src/lib.rs
similarity index 100%
rename from src/reportserver/src/lib.rs
rename to src/report_server/src/lib.rs
diff --git a/src/reportserver/src/models.rs b/src/report_server/src/models.rs
similarity index 100%
rename from src/reportserver/src/models.rs
rename to src/report_server/src/models.rs
diff --git a/src/reportserver/src/report.rs b/src/report_server/src/report.rs
similarity index 92%
rename from src/reportserver/src/report.rs
rename to src/report_server/src/report.rs
index 1a080b9efc..386ad57e2e 100644
--- a/src/reportserver/src/report.rs
+++ b/src/report_server/src/report.rs
@@ -6,7 +6,7 @@ use chromiumoxide::{
handler::viewport::Viewport,
Page,
};
-use config::CONFIG;
+use config::get_config;
use futures::StreamExt;
use lettre::{
message::{header::ContentType, MultiPart, SinglePart},
@@ -31,32 +31,33 @@ pub async fn get_chrome_launch_options() -> &'static BrowserConfig {
}
async fn init_chrome_launch_options() -> BrowserConfig {
+ let cfg = get_config();
let mut browser_config = BrowserConfig::builder()
.window_size(
- CONFIG.chrome.chrome_window_width,
- CONFIG.chrome.chrome_window_height,
+ cfg.chrome.chrome_window_width,
+ cfg.chrome.chrome_window_height,
)
.viewport(Viewport {
- width: CONFIG.chrome.chrome_window_width,
- height: CONFIG.chrome.chrome_window_height,
+ width: cfg.chrome.chrome_window_width,
+ height: cfg.chrome.chrome_window_height,
device_scale_factor: Some(1.0),
..Viewport::default()
});
- if CONFIG.chrome.chrome_with_head {
+ if cfg.chrome.chrome_with_head {
browser_config = browser_config.with_head();
}
- if CONFIG.chrome.chrome_no_sandbox {
+ if cfg.chrome.chrome_no_sandbox {
browser_config = browser_config.no_sandbox();
}
- if !CONFIG.chrome.chrome_path.is_empty() {
- browser_config = browser_config.chrome_executable(CONFIG.chrome.chrome_path.as_str());
+ if !cfg.chrome.chrome_path.is_empty() {
+ browser_config = browser_config.chrome_executable(cfg.chrome.chrome_path.as_str());
} else {
let mut should_download = false;
- if !CONFIG.chrome.chrome_check_default {
+ if !cfg.chrome.chrome_check_default {
should_download = true;
} else {
// Check if chrome is available on default paths
@@ -70,14 +71,14 @@ async fn init_chrome_launch_options() -> BrowserConfig {
should_download = true;
}
}
- if should_download && !CONFIG.chrome.chrome_auto_download {
+ if should_download && !cfg.chrome.chrome_auto_download {
should_download = false;
log::error!("Chrome binary could not be detected");
}
if should_download {
// Download known good chrome version
- let download_path = &CONFIG.chrome.chrome_download_path;
+ let download_path = &cfg.chrome.chrome_download_path;
log::info!("fetching chrome at: {download_path}");
tokio::fs::create_dir_all(download_path).await.unwrap();
let fetcher = BrowserFetcher::new(
@@ -111,12 +112,13 @@ async fn init_chrome_launch_options() -> BrowserConfig {
}
pub static SMTP_CLIENT: Lazy> = Lazy::new(|| {
- let tls_parameters = TlsParameters::new(CONFIG.smtp.smtp_host.clone()).unwrap();
+ let cfg = get_config();
+ let tls_parameters = TlsParameters::new(cfg.smtp.smtp_host.clone()).unwrap();
let mut transport_builder =
- AsyncSmtpTransport::::builder_dangerous(&CONFIG.smtp.smtp_host)
- .port(CONFIG.smtp.smtp_port);
+ AsyncSmtpTransport::::builder_dangerous(&cfg.smtp.smtp_host)
+ .port(cfg.smtp.smtp_port);
- let option = &CONFIG.smtp.smtp_encryption;
+ let option = &cfg.smtp.smtp_encryption;
transport_builder = if option == "starttls" {
transport_builder.tls(Tls::Required(tls_parameters))
} else if option == "ssltls" {
@@ -125,10 +127,10 @@ pub static SMTP_CLIENT: Lazy> = Lazy::new(||
transport_builder
};
- if !CONFIG.smtp.smtp_username.is_empty() && !CONFIG.smtp.smtp_password.is_empty() {
+ if !cfg.smtp.smtp_username.is_empty() && !cfg.smtp.smtp_password.is_empty() {
transport_builder = transport_builder.credentials(Credentials::new(
- CONFIG.smtp.smtp_username.clone(),
- CONFIG.smtp.smtp_password.clone(),
+ cfg.smtp.smtp_username.clone(),
+ cfg.smtp.smtp_password.clone(),
));
}
transport_builder.build()
@@ -378,7 +380,7 @@ pub async fn send_email(
pub async fn wait_for_panel_data_load(page: &Page) -> Result<(), anyhow::Error> {
let start = std::time::Instant::now();
- let timeout = std::time::Duration::from_secs(CONFIG.chrome.chrome_sleep_secs.into());
+ let timeout = std::time::Duration::from_secs(get_config().chrome.chrome_sleep_secs.into());
loop {
if page
.find_element("span#dashboardVariablesAndPanelsDataLoaded")
diff --git a/src/reportserver/src/router.rs b/src/report_server/src/router.rs
similarity index 92%
rename from src/reportserver/src/router.rs
rename to src/report_server/src/router.rs
index 7a66b4d840..878f918e31 100644
--- a/src/reportserver/src/router.rs
+++ b/src/report_server/src/router.rs
@@ -1,7 +1,6 @@
use std::{collections::HashMap, io::Error};
use actix_web::{get, http::StatusCode, put, web, HttpRequest, HttpResponse as ActixHttpResponse};
-use config::CONFIG;
use serde::{Deserialize, Serialize};
use crate::{
@@ -65,11 +64,13 @@ pub async fn send_report(
Some(v) => v,
None => "Europe/London",
};
+
+ let cfg = config::get_config();
let (pdf_data, email_dashboard_url) = match generate_report(
&report.dashboards[0],
&org_id,
- &CONFIG.report_server.user_email,
- &CONFIG.report_server.user_password,
+ &cfg.report_server.user_email,
+ &cfg.report_server.user_password,
&report.email_details.dashb_url,
timezone,
)
@@ -90,8 +91,8 @@ pub async fn send_report(
..report.email_details
},
models::SmtpConfig {
- from_email: CONFIG.smtp.smtp_from_email.to_string(),
- reply_to: CONFIG.smtp.smtp_reply_to.to_string(),
+ from_email: cfg.smtp.smtp_from_email.to_string(),
+ reply_to: cfg.smtp.smtp_reply_to.to_string(),
client: &SMTP_CLIENT,
},
)
diff --git a/src/reportserver/src/server.rs b/src/report_server/src/server.rs
similarity index 85%
rename from src/reportserver/src/server.rs
rename to src/report_server/src/server.rs
index 55033a4a6c..17c5b0d1d6 100644
--- a/src/reportserver/src/server.rs
+++ b/src/report_server/src/server.rs
@@ -1,7 +1,6 @@
use std::net::SocketAddr;
use actix_web::{dev::ServerHandle, middleware, web, App, HttpServer};
-use config::CONFIG;
use crate::router::{healthz, send_report};
@@ -11,22 +10,23 @@ pub async fn spawn_server() -> Result<(), anyhow::Error> {
log::info!("starting o2 chrome server");
- if CONFIG.report_server.user_email.is_empty() || CONFIG.report_server.user_password.is_empty() {
+ let cfg = config::get_config();
+ if cfg.report_server.user_email.is_empty() || cfg.report_server.user_password.is_empty() {
log::error!("Missing ZO_REPORT_USER_EMAIL or ZO_REPORT_USER_PASSWORD env vars");
return Err(anyhow::anyhow!(
"Please set ZO_REPORT_USER_EMAIL and ZO_REPORT_USER_PASSWORD to use report server"
));
}
- let haddr: SocketAddr = if CONFIG.report_server.ipv6_enabled {
- format!("[::]:{}", CONFIG.report_server.port).parse()?
+ let haddr: SocketAddr = if cfg.report_server.ipv6_enabled {
+ format!("[::]:{}", cfg.report_server.port).parse()?
} else {
- let ip = if !CONFIG.report_server.addr.is_empty() {
- CONFIG.report_server.addr.clone()
+ let ip = if !cfg.report_server.addr.is_empty() {
+ cfg.report_server.addr.clone()
} else {
"0.0.0.0".to_string()
};
- format!("{}:{}", ip, CONFIG.report_server.port).parse()?
+ format!("{}:{}", ip, cfg.report_server.port).parse()?
};
log::info!("starting Report server at: {}", haddr);
let server = HttpServer::new(move || {
diff --git a/src/router/grpc/ingest/logs.rs b/src/router/grpc/ingest/logs.rs
index e67ef56b19..40ec36bf6c 100644
--- a/src/router/grpc/ingest/logs.rs
+++ b/src/router/grpc/ingest/logs.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::logs::v1::{
logs_service_client::LogsServiceClient, logs_service_server::LogsService,
ExportLogsServiceRequest, ExportLogsServiceResponse,
@@ -35,11 +34,12 @@ impl LogsService for LogsServer {
) -> Result, Status> {
let (metadata, extensions, message) = request.into_parts();
+ let cfg = config::get_config();
// basic validation
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
)));
}
diff --git a/src/router/grpc/ingest/metrics.rs b/src/router/grpc/ingest/metrics.rs
index 42ee6ee01c..984f693b98 100644
--- a/src/router/grpc/ingest/metrics.rs
+++ b/src/router/grpc/ingest/metrics.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::metrics::v1::{
metrics_service_client::MetricsServiceClient, metrics_service_server::MetricsService,
ExportMetricsServiceRequest, ExportMetricsServiceResponse,
@@ -35,11 +34,12 @@ impl MetricsService for MetricsServer {
) -> Result, Status> {
let (metadata, extensions, message) = request.into_parts();
+ let cfg = config::get_config();
// basic validation
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
)));
}
diff --git a/src/router/grpc/ingest/mod.rs b/src/router/grpc/ingest/mod.rs
index 442f4e7e7a..85140df7e4 100644
--- a/src/router/grpc/ingest/mod.rs
+++ b/src/router/grpc/ingest/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{utils::rand::get_rand_element, RwAHashMap, CONFIG};
+use config::{utils::rand::get_rand_element, RwAHashMap};
use once_cell::sync::Lazy;
use tonic::{transport::Channel, Status};
@@ -37,7 +37,9 @@ pub(crate) async fn get_ingester_channel() -> Result {
// cache miss, connect to ingester
let channel = Channel::from_shared(grpc_addr.clone())
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(
+ config::get_config().grpc.connect_timeout,
+ ))
.connect()
.await
.map_err(|err| {
@@ -56,12 +58,13 @@ pub(crate) async fn get_ingester_channel() -> Result {
}
async fn get_rand_ingester_addr() -> Result {
+ let cfg = config::get_config();
let nodes = cluster::get_cached_online_ingester_nodes().await;
if nodes.is_none() || nodes.as_ref().unwrap().is_empty() {
- if !CONFIG.route.ingester_srv_url.is_empty() {
+ if !cfg.route.ingester_srv_url.is_empty() {
Ok(format!(
"http://{}:{}",
- CONFIG.route.ingester_srv_url, CONFIG.grpc.port
+ cfg.route.ingester_srv_url, cfg.grpc.port
))
} else {
Err(tonic::Status::internal(
diff --git a/src/router/grpc/ingest/traces.rs b/src/router/grpc/ingest/traces.rs
index 6d5de555d9..3878d7f849 100644
--- a/src/router/grpc/ingest/traces.rs
+++ b/src/router/grpc/ingest/traces.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use async_trait::async_trait;
-use config::CONFIG;
use opentelemetry_proto::tonic::collector::trace::v1::{
trace_service_client::TraceServiceClient, trace_service_server::TraceService,
ExportTraceServiceRequest, ExportTraceServiceResponse,
@@ -35,11 +34,12 @@ impl TraceService for TraceServer {
) -> Result, Status> {
let (metadata, extensions, message) = request.into_parts();
+ let cfg = config::get_config();
// basic validation
- if !metadata.contains_key(&CONFIG.grpc.org_header_key) {
+ if !metadata.contains_key(&cfg.grpc.org_header_key) {
return Err(Status::invalid_argument(format!(
"Please specify organization id with header key '{}' ",
- &CONFIG.grpc.org_header_key
+ &cfg.grpc.org_header_key
)));
}
diff --git a/src/router/grpc/mod.rs b/src/router/grpc/mod.rs
index ad13676aa2..9558bd7dd9 100644
--- a/src/router/grpc/mod.rs
+++ b/src/router/grpc/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/router/http/mod.rs b/src/router/http/mod.rs
index 9e82bde06b..6cc89aefa8 100644
--- a/src/router/http/mod.rs
+++ b/src/router/http/mod.rs
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use ::config::{meta::cluster::Role, utils::rand::get_rand_element, CONFIG};
+use ::config::{get_config, meta::cluster::Role, utils::rand::get_rand_element};
use actix_web::{http::Error, route, web, HttpRequest, HttpResponse};
use crate::common::infra::cluster;
@@ -173,7 +173,11 @@ async fn dispatch(
}
// set body
- let body = match resp.body().limit(CONFIG.limit.req_payload_limit).await {
+ let body = match resp
+ .body()
+ .limit(get_config().limit.req_payload_limit)
+ .await
+ {
Ok(b) => b,
Err(e) => {
log::error!("{}: {}", new_url.value, e);
@@ -200,12 +204,13 @@ async fn get_url(path: &str) -> URLDetails {
cluster::get_cached_online_ingester_nodes().await
};
if nodes.is_none() || nodes.as_ref().unwrap().is_empty() {
- if node_type == Role::Ingester && !CONFIG.route.ingester_srv_url.is_empty() {
+ let cfg = get_config();
+ if node_type == Role::Ingester && !cfg.route.ingester_srv_url.is_empty() {
return URLDetails {
is_error: false,
value: format!(
"http://{}:{}{}",
- CONFIG.route.ingester_srv_url, CONFIG.http.port, path
+ cfg.route.ingester_srv_url, cfg.http.port, path
),
};
}
diff --git a/src/router/mod.rs b/src/router/mod.rs
index b003afa861..0cfb601e64 100644
--- a/src/router/mod.rs
+++ b/src/router/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/alerts/alert_manager.rs b/src/service/alerts/alert_manager.rs
index 5324d1d0c6..3197988807 100644
--- a/src/service/alerts/alert_manager.rs
+++ b/src/service/alerts/alert_manager.rs
@@ -17,11 +17,11 @@ use std::str::FromStr;
use chrono::{Duration, FixedOffset, Utc};
use config::{
+ get_config,
meta::{
stream::StreamType,
usage::{TriggerData, TriggerDataStatus, TriggerDataType},
},
- CONFIG,
};
use cron::Schedule;
@@ -32,14 +32,15 @@ use crate::{
pub async fn run() -> Result<(), anyhow::Error> {
log::debug!("Pulling jobs from scheduler");
+ let cfg = get_config();
// Scheduler pulls only those triggers that match the conditions-
// - trigger.next_run_at <= now
// - !(trigger.is_realtime && !trigger.is_silenced)
// - trigger.status == "Waiting"
let triggers = db::scheduler::pull(
- CONFIG.limit.alert_schedule_concurrency,
- CONFIG.limit.alert_schedule_timeout,
- CONFIG.limit.report_schedule_timeout,
+ cfg.limit.alert_schedule_concurrency,
+ cfg.limit.alert_schedule_timeout,
+ cfg.limit.report_schedule_timeout,
)
.await?;
@@ -174,7 +175,7 @@ async fn handle_alert_triggers(trigger: db::scheduler::Trigger) -> Result<(), an
&new_trigger.org,
&new_trigger.module_key
);
- if trigger.retries + 1 >= CONFIG.limit.scheduler_max_retries {
+ if trigger.retries + 1 >= get_config().limit.scheduler_max_retries {
// It has been tried the maximum time, just update the
// next_run_at to the next expected trigger time
log::debug!(
@@ -324,7 +325,7 @@ async fn handle_report_triggers(trigger: db::scheduler::Trigger) -> Result<(), a
}
Err(e) => {
log::error!("Error sending report to subscribers: {e}");
- if trigger.retries + 1 >= CONFIG.limit.scheduler_max_retries && !run_once {
+ if trigger.retries + 1 >= get_config().limit.scheduler_max_retries && !run_once {
// It has been tried the maximum time, just update the
// next_run_at to the next expected trigger time
log::debug!(
diff --git a/src/service/alerts/destinations.rs b/src/service/alerts/destinations.rs
index 3841b4924b..9c0225ad0a 100644
--- a/src/service/alerts/destinations.rs
+++ b/src/service/alerts/destinations.rs
@@ -14,7 +14,6 @@
// along with this program. If not, see .
use actix_web::http;
-use config::CONFIG;
use crate::{
common::{
@@ -51,7 +50,7 @@ pub async fn save(
anyhow::anyhow!("Atleast one alert destination email is required"),
));
}
- if !CONFIG.smtp.smtp_enabled {
+ if !config::get_config().smtp.smtp_enabled {
return Err((
http::StatusCode::INTERNAL_SERVER_ERROR,
anyhow::anyhow!("SMTP not configured"),
diff --git a/src/service/alerts/mod.rs b/src/service/alerts/mod.rs
index e6e4db1a13..8deb71fc57 100644
--- a/src/service/alerts/mod.rs
+++ b/src/service/alerts/mod.rs
@@ -22,13 +22,13 @@ use actix_web::http;
use arrow_schema::DataType;
use chrono::{Duration, Local, TimeZone, Utc};
use config::{
- ider,
+ get_config, ider,
meta::stream::StreamType,
utils::{
base64,
json::{Map, Value},
},
- CONFIG, SMTP_CLIENT,
+ SMTP_CLIENT,
};
use cron::Schedule;
use lettre::{message::SinglePart, AsyncTransport, Message};
@@ -90,7 +90,8 @@ pub async fn save(
Schedule::from_str(&alert.trigger_condition.cron)?;
} else if alert.trigger_condition.frequency == 0 {
// default frequency is 60 seconds
- alert.trigger_condition.frequency = std::cmp::max(10, CONFIG.limit.alert_schedule_interval);
+ alert.trigger_condition.frequency =
+ std::cmp::max(10, get_config().limit.alert_schedule_interval);
}
if alert.name.is_empty() || alert.stream_name.is_empty() {
@@ -613,14 +614,15 @@ async fn build_sql(alert: &Alert, conditions: &[Condition]) -> Result format!("approx_percentile_cont(\"{}\", 0.99)", agg.having.column),
};
+ let cfg = get_config();
if let Some(group) = agg.group_by.as_ref() {
if !group.is_empty() {
sql = format!(
"SELECT {}, {} AS alert_agg_value, MIN({}) as zo_sql_min_time, MAX({}) AS zo_sql_max_time FROM \"{}\" {} GROUP BY {} HAVING {}",
group.join(", "),
func_expr,
- CONFIG.common.column_timestamp,
- CONFIG.common.column_timestamp,
+ cfg.common.column_timestamp,
+ cfg.common.column_timestamp,
alert.stream_name,
where_sql,
group.join(", "),
@@ -632,8 +634,8 @@ async fn build_sql(alert: &Alert, conditions: &[Condition]) -> Result Result<(), anyhow::Error> {
- if !CONFIG.smtp.smtp_enabled {
+ let cfg = get_config();
+ if !cfg.smtp.smtp_enabled {
return Err(anyhow::anyhow!("SMTP configuration not enabled"));
}
@@ -875,15 +878,15 @@ pub async fn send_email_notification(
}
let mut email = Message::builder()
- .from(CONFIG.smtp.smtp_from_email.parse()?)
+ .from(cfg.smtp.smtp_from_email.parse()?)
.subject(format!("Openobserve Alert - {}", alert_name));
for recepient in recepients {
email = email.to(recepient.parse()?);
}
- if !CONFIG.smtp.smtp_reply_to.is_empty() {
- email = email.reply_to(CONFIG.smtp.smtp_reply_to.parse()?);
+ if !cfg.smtp.smtp_reply_to.is_empty() {
+ email = email.reply_to(cfg.smtp.smtp_reply_to.parse()?);
}
let email = email.singlepart(SinglePart::html(msg)).unwrap();
@@ -918,7 +921,7 @@ fn process_row_template(tpl: &String, alert: &Alert, rows: &[Map]
process_variable_replace(&mut resp, key, &VarValue::Str(&value));
// calculate start and end time
- if key == &CONFIG.common.column_timestamp {
+ if key == &get_config().common.column_timestamp {
let val = value.parse::().unwrap_or_default();
if alert_start_time == 0 || val < alert_start_time {
alert_start_time = val;
@@ -1003,6 +1006,7 @@ async fn process_dest_template(
rows: &[Map],
rows_tpl_val: &[String],
) -> String {
+ let cfg = get_config();
// format values
let alert_count = rows.len();
let mut vars = HashMap::with_capacity(rows.len());
@@ -1023,7 +1027,7 @@ async fn process_dest_template(
// calculate start and end time
let mut alert_start_time = 0;
let mut alert_end_time = 0;
- if let Some(values) = vars.get(&CONFIG.common.column_timestamp) {
+ if let Some(values) = vars.get(&cfg.common.column_timestamp) {
for val in values {
let val = val.parse::().unwrap_or_default();
if alert_start_time == 0 || val < alert_start_time {
@@ -1122,8 +1126,8 @@ async fn process_dest_template(
// http://localhost:5080/web/metrics?stream=zo_http_response_time_bucket&from=1705248000000000&to=1705334340000000&query=em9faHR0cF9yZXNwb25zZV90aW1lX2J1Y2tldHt9&org_identifier=default
format!(
"{}{}/web/metrics?stream_type={}&stream={}&stream_value={}&from={}&to={}&query={}&org_identifier={}",
- CONFIG.common.web_url,
- CONFIG.common.base_uri,
+ cfg.common.web_url,
+ cfg.common.base_uri,
alert.stream_type,
alert.stream_name,
alert.stream_name,
@@ -1151,8 +1155,8 @@ async fn process_dest_template(
// http://localhost:5080/web/logs?stream_type=logs&stream=test&from=1708416534519324&to=1708416597898186&sql_mode=true&query=U0VMRUNUICogRlJPTSAidGVzdCIgd2hlcmUgbGV2ZWwgPSAnaW5mbyc=&org_identifier=default
format!(
"{}{}/web/logs?stream_type={}&stream={}&stream_value={}&from={}&to={}&sql_mode=true&query={}&org_identifier={}",
- CONFIG.common.web_url,
- CONFIG.common.base_uri,
+ cfg.common.web_url,
+ cfg.common.base_uri,
alert.stream_type,
alert.stream_name,
alert.stream_name,
diff --git a/src/service/compact/file_list.rs b/src/service/compact/file_list.rs
index 97d7485b33..7ac1c77dc7 100644
--- a/src/service/compact/file_list.rs
+++ b/src/service/compact/file_list.rs
@@ -20,7 +20,7 @@ use std::{
use bytes::Buf;
use chrono::{DateTime, Datelike, Duration, TimeZone, Timelike, Utc};
-use config::{cluster::LOCAL_NODE_UUID, ider, meta::stream::FileKey, utils::json, CONFIG};
+use config::{cluster::LOCAL_NODE_UUID, ider, meta::stream::FileKey, utils::json};
use hashbrown::HashMap;
use infra::{dist_lock, schema::STREAM_SCHEMAS_LATEST, storage};
use tokio::sync::{RwLock, Semaphore};
@@ -223,7 +223,9 @@ async fn merge_file_list(offset: i64) -> Result<(), anyhow::Error> {
// filter deleted file keys
let filter_file_keys: Arc>> =
Arc::new(RwLock::new(HashMap::new()));
- let semaphore = std::sync::Arc::new(Semaphore::new(CONFIG.limit.file_move_thread_num));
+ let semaphore = std::sync::Arc::new(Semaphore::new(
+ config::get_config().limit.file_move_thread_num,
+ ));
let mut tasks = Vec::new();
for file in file_list.clone() {
let filter_file_keys = filter_file_keys.clone();
diff --git a/src/service/compact/file_list_deleted.rs b/src/service/compact/file_list_deleted.rs
index 9807983b7f..47a4ccc54e 100644
--- a/src/service/compact/file_list_deleted.rs
+++ b/src/service/compact/file_list_deleted.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,6 @@ use std::io::{BufRead, BufReader};
use bytes::Buf;
use chrono::{DateTime, Duration, TimeZone, Utc};
-use config::CONFIG;
use futures::future::try_join_all;
use hashbrown::HashMap;
use infra::{file_list as infra_file_list, storage};
@@ -84,7 +83,7 @@ async fn query_deleted(
time_max: i64,
limit: i64,
) -> Result>, anyhow::Error> {
- if CONFIG.common.meta_store_external {
+ if config::get_config().common.meta_store_external {
query_deleted_from_table(org_id, time_min, time_max, limit).await
} else {
query_deleted_from_s3(org_id, time_min, time_max, limit).await
@@ -160,8 +159,9 @@ pub async fn load_prefix_from_s3(
}
log::info!("Load file_list_deleted gets {} files", files_num);
- let mut tasks = Vec::with_capacity(CONFIG.limit.query_thread_num + 1);
- let chunk_size = std::cmp::max(1, files_num / CONFIG.limit.query_thread_num);
+ let cfg = config::get_config();
+ let mut tasks = Vec::with_capacity(cfg.limit.query_thread_num + 1);
+ let chunk_size = std::cmp::max(1, files_num / cfg.limit.query_thread_num);
for chunk in files.chunks(chunk_size) {
let chunk = chunk.to_vec();
let task: tokio::task::JoinHandle>, anyhow::Error>> =
diff --git a/src/service/compact/merge.rs b/src/service/compact/merge.rs
index ba07fc2dd7..dd77e3ff73 100644
--- a/src/service/compact/merge.rs
+++ b/src/service/compact/merge.rs
@@ -21,7 +21,7 @@ use bytes::Bytes;
use chrono::{DateTime, Datelike, Duration, TimeZone, Timelike, Utc};
use config::{
cluster::LOCAL_NODE_UUID,
- ider,
+ get_config, ider,
meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType},
metrics,
utils::{
@@ -31,7 +31,7 @@ use config::{
},
record_batch_ext::format_recordbatch_by_schema,
},
- CONFIG, FILE_EXT_PARQUET,
+ FILE_EXT_PARQUET,
};
use hashbrown::HashSet;
use infra::{
@@ -167,6 +167,7 @@ pub async fn merge_by_stream(
.unwrap()
.timestamp_micros();
+ let cfg = get_config();
// check offset
let time_now: DateTime = Utc::now();
let time_now_hour = Utc
@@ -186,16 +187,16 @@ pub async fn merge_by_stream(
// -- second period, the last hour file list upload to storage
// -- third period, we can do the merge, so, at least 3 times of
// max_file_retention_time
- if (CONFIG.compact.step_secs < 3600
+ if (cfg.compact.step_secs < 3600
&& time_now.timestamp_micros() - offset
- <= Duration::try_seconds(CONFIG.limit.max_file_retention_time as i64)
+ <= Duration::try_seconds(cfg.limit.max_file_retention_time as i64)
.unwrap()
.num_microseconds()
.unwrap())
- || (CONFIG.compact.step_secs >= 3600
+ || (cfg.compact.step_secs >= 3600
&& (offset >= time_now_hour
|| time_now.timestamp_micros() - offset
- <= Duration::try_seconds(CONFIG.limit.max_file_retention_time as i64)
+ <= Duration::try_seconds(cfg.limit.max_file_retention_time as i64)
.unwrap()
.num_microseconds()
.unwrap()
@@ -230,8 +231,8 @@ pub async fn merge_by_stream(
.map_err(|e| anyhow::anyhow!("query file list failed: {}", e))?;
// check lookback files
- if CONFIG.compact.lookback_hours > 0 {
- let lookback_offset = Duration::try_hours(CONFIG.compact.lookback_hours)
+ if cfg.compact.lookback_hours > 0 {
+ let lookback_offset = Duration::try_hours(cfg.compact.lookback_hours)
.unwrap()
.num_microseconds()
.unwrap();
@@ -268,11 +269,11 @@ pub async fn merge_by_stream(
if files.is_empty() {
// this hour is no data, and check if pass allowed_upto, then just write new
// offset if offset > 0 && offset_time_hour +
- // Duration::try_hours(CONFIG.limit.allowed_upto).unwrap().num_microseconds().unwrap() <
+ // Duration::try_hours(cfg.limit.allowed_upto).unwrap().num_microseconds().unwrap() <
// time_now_hour { -- no check it
// }
let offset = offset
- + Duration::try_seconds(CONFIG.compact.step_secs)
+ + Duration::try_seconds(cfg.compact.step_secs)
.unwrap()
.num_microseconds()
.unwrap();
@@ -300,7 +301,7 @@ pub async fn merge_by_stream(
let mut stream_stats = StreamStats::default();
// use mutiple threads to merge
- let semaphore = std::sync::Arc::new(Semaphore::new(CONFIG.limit.file_move_thread_num));
+ let semaphore = std::sync::Arc::new(Semaphore::new(cfg.limit.file_move_thread_num));
let mut tasks = Vec::with_capacity(partition_files_with_size.len());
for (prefix, files_with_size) in partition_files_with_size.into_iter() {
let org_id = org_id.to_string();
@@ -308,6 +309,7 @@ pub async fn merge_by_stream(
let permit = semaphore.clone().acquire_owned().await.unwrap();
let worker_tx = worker_tx.clone();
let task: JoinHandle> = tokio::task::spawn(async move {
+ let cfg = get_config();
// sort by file size
let mut files_with_size = files_with_size.to_owned();
files_with_size.sort_by(|a, b| a.meta.original_size.cmp(&b.meta.original_size));
@@ -323,7 +325,7 @@ pub async fn merge_by_stream(
let mut new_file_list = Vec::new();
let mut new_file_size = 0;
for file in files_with_size.iter() {
- if new_file_size + file.meta.original_size > CONFIG.compact.max_file_size as i64 {
+ if new_file_size + file.meta.original_size > cfg.compact.max_file_size as i64 {
if new_file_list.len() <= 1 {
break; // no files need to merge
}
@@ -445,7 +447,7 @@ pub async fn merge_by_stream(
// write new offset
let offset = offset
- + Duration::try_seconds(CONFIG.compact.step_secs)
+ + Duration::try_seconds(cfg.compact.step_secs)
.unwrap()
.num_microseconds()
.unwrap();
@@ -503,8 +505,9 @@ pub async fn merge_files(
let mut total_records = 0;
let mut new_file_list = Vec::new();
let mut deleted_files = Vec::new();
+ let cfg = get_config();
for file in files_with_size.iter() {
- if new_file_size + file.meta.original_size > CONFIG.compact.max_file_size as i64 {
+ if new_file_size + file.meta.original_size > cfg.compact.max_file_size as i64 {
break;
}
new_file_size += file.meta.original_size;
@@ -585,7 +588,7 @@ pub async fn merge_files(
let schema_latest_id = schema_versions.len() - 1;
let bloom_filter_fields = get_stream_setting_bloom_filter_fields(&schema_latest).unwrap();
let full_text_search_fields = get_stream_setting_fts_fields(&schema_latest).unwrap();
- if CONFIG.common.widening_schema_evolution && schema_versions.len() > 1 {
+ if cfg.common.widening_schema_evolution && schema_versions.len() > 1 {
for file in &new_file_list {
// get the schema version of the file
let schema_ver_id = match db::schema::filter_schema_version_id(
@@ -735,7 +738,7 @@ pub async fn merge_files(
// upload file
match storage::put(&new_file_key, buf.clone()).await {
Ok(_) => {
- if CONFIG.common.inverted_index_enabled && stream_type == StreamType::Logs {
+ if cfg.common.inverted_index_enabled && stream_type == StreamType::Logs {
let (index_file_name, filemeta) = generate_index_on_compactor(
&retain_file_list,
inverted_idx_batches,
@@ -788,7 +791,7 @@ async fn write_file_list(org_id: &str, events: &[FileKey]) -> Result<(), anyhow:
if events.is_empty() {
return Ok(());
}
- if CONFIG.common.meta_store_external {
+ if get_config().common.meta_store_external {
write_file_list_db_only(org_id, events).await
} else {
write_file_list_s3(org_id, events).await
@@ -833,7 +836,7 @@ async fn write_file_list_db_only(org_id: &str, events: &[FileKey]) -> Result<(),
continue;
}
// send broadcast to other nodes
- if CONFIG.memory_cache.cache_latest_files {
+ if get_config().memory_cache.cache_latest_files {
if let Err(e) = db::file_list::broadcast::send(events, None).await {
log::error!("[COMPACT] send broadcast for file_list failed: {}", e);
}
@@ -925,10 +928,8 @@ pub fn generate_inverted_idx_recordbatch(
stream_type: StreamType,
full_text_search_fields: &[String],
) -> Vec {
- if !CONFIG.common.inverted_index_enabled
- || batches.is_empty()
- || stream_type != StreamType::Logs
- {
+ let cfg = get_config();
+ if !cfg.common.inverted_index_enabled || batches.is_empty() || stream_type != StreamType::Logs {
return Vec::new();
}
@@ -948,8 +949,8 @@ pub fn generate_inverted_idx_recordbatch(
return Vec::new();
}
// add _timestamp column to columns_to_index
- if !inverted_idx_columns.contains(&CONFIG.common.column_timestamp) {
- inverted_idx_columns.push(CONFIG.common.column_timestamp.to_string());
+ if !inverted_idx_columns.contains(&cfg.common.column_timestamp) {
+ inverted_idx_columns.push(cfg.common.column_timestamp.to_string());
}
let mut inverted_idx_batches = Vec::with_capacity(batches.len());
@@ -1029,7 +1030,7 @@ pub async fn merge_parquet_files(
// 3. sort concatenated record batch by timestamp col in desc order
let sort_indices = arrow::compute::sort_to_indices(
concatenated_record_batch
- .column_by_name(&CONFIG.common.column_timestamp)
+ .column_by_name(&get_config().common.column_timestamp)
.ok_or_else(|| {
log::error!(
"[INGESTER:JOB:{thread_id}] merge small files failed to find _timestamp column from merged record batch.",
diff --git a/src/service/compact/mod.rs b/src/service/compact/mod.rs
index b21a4d5f2b..f89bf3a202 100644
--- a/src/service/compact/mod.rs
+++ b/src/service/compact/mod.rs
@@ -18,8 +18,8 @@ use std::sync::Arc;
use chrono::{Datelike, Duration, TimeZone, Timelike, Utc};
use config::{
cluster::LOCAL_NODE_UUID,
+ get_config,
meta::{cluster::Role, stream::StreamType},
- CONFIG,
};
use infra::dist_lock;
use once_cell::sync::Lazy;
@@ -41,10 +41,11 @@ pub(crate) static QUEUE_LOCKER: Lazy>> =
/// compactor retention run steps:
pub async fn run_retention() -> Result<(), anyhow::Error> {
+ let cfg = get_config();
// check data retention
- if CONFIG.compact.data_retention_days > 0 {
+ if cfg.compact.data_retention_days > 0 {
let now = Utc::now();
- let date = now - Duration::try_days(CONFIG.compact.data_retention_days).unwrap();
+ let date = now - Duration::try_days(cfg.compact.data_retention_days).unwrap();
let data_lifecycle_end = date.format("%Y-%m-%d").to_string();
let orgs = db::schema::list_organizations_from_cache().await;
@@ -181,7 +182,8 @@ pub async fn run_retention() -> Result<(), anyhow::Error> {
pub async fn run_merge(
worker_tx: mpsc::Sender<(merge::MergeSender, merge::MergeBatch)>,
) -> Result<(), anyhow::Error> {
- let semaphore = std::sync::Arc::new(Semaphore::new(CONFIG.limit.file_move_thread_num));
+ let cfg = get_config();
+ let semaphore = std::sync::Arc::new(Semaphore::new(cfg.limit.file_move_thread_num));
let orgs = db::schema::list_organizations_from_cache().await;
let stream_types = [
StreamType::Logs,
@@ -193,8 +195,7 @@ pub async fn run_merge(
];
for org_id in orgs {
// check backlist
- if !db::file_list::BLOCKED_ORGS.is_empty()
- && db::file_list::BLOCKED_ORGS.contains(&org_id.as_str())
+ if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id)
{
continue;
}
@@ -271,7 +272,7 @@ pub async fn run_merge(
}
// after compact, compact file list from storage
- if !CONFIG.common.meta_store_external {
+ if !cfg.common.meta_store_external {
let last_file_list_offset = db::compact::file_list::get_offset().await?;
if let Err(e) = file_list::run(last_file_list_offset).await {
log::error!("[COMPACTOR] merge file list error: {}", e);
@@ -286,7 +287,8 @@ pub async fn run_merge(
/// 2. delete files from storage
pub async fn run_delay_deletion() -> Result<(), anyhow::Error> {
let now = Utc::now();
- let time_max = now - Duration::try_hours(CONFIG.compact.delete_files_delay_hours).unwrap();
+ let time_max =
+ now - Duration::try_hours(get_config().compact.delete_files_delay_hours).unwrap();
let time_max = Utc
.with_ymd_and_hms(
time_max.year(),
diff --git a/src/service/compact/retention.rs b/src/service/compact/retention.rs
index f66198c8bf..a59f83992f 100644
--- a/src/service/compact/retention.rs
+++ b/src/service/compact/retention.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -21,10 +21,9 @@ use std::{
use chrono::{DateTime, Duration, TimeZone, Utc};
use config::{
cluster::LOCAL_NODE_UUID,
- ider, is_local_disk_storage,
+ get_config, ider, is_local_disk_storage,
meta::stream::{FileKey, FileMeta, PartitionTimeLevel, StreamStats, StreamType},
utils::{json, time::BASE_TIME},
- CONFIG,
};
use infra::{cache, dist_lock, file_list as infra_file_list, storage};
@@ -106,10 +105,11 @@ pub async fn delete_all(
let start_time = BASE_TIME.timestamp_micros();
let end_time = Utc::now().timestamp_micros();
+ let cfg = get_config();
if is_local_disk_storage() {
let data_dir = format!(
"{}files/{org_id}/{stream_type}/{stream_name}",
- CONFIG.common.data_stream_dir
+ cfg.common.data_stream_dir
);
let path = std::path::Path::new(&data_dir);
if path.exists() {
@@ -129,7 +129,7 @@ pub async fn delete_all(
true,
)
.await?;
- if CONFIG.compact.data_retention_history {
+ if cfg.compact.data_retention_history {
// only store the file_list into history, don't delete files
if let Err(e) = infra_file_list::batch_add_history(&files).await {
log::error!("[COMPACT] file_list batch_add_history failed: {}", e);
@@ -208,11 +208,12 @@ pub async fn delete_by_date(
DateTime::parse_from_rfc3339(&format!("{}T00:00:00Z", date_range.1))?.with_timezone(&Utc);
let time_range = { (date_start.timestamp_micros(), date_end.timestamp_micros()) };
+ let cfg = get_config();
if is_local_disk_storage() {
while date_start <= date_end {
let data_dir = format!(
"{}files/{org_id}/{stream_type}/{stream_name}/{}",
- CONFIG.common.data_stream_dir,
+ cfg.common.data_stream_dir,
date_start.format("%Y/%m/%d")
);
let path = std::path::Path::new(&data_dir);
@@ -234,7 +235,7 @@ pub async fn delete_by_date(
true,
)
.await?;
- if CONFIG.compact.data_retention_history {
+ if cfg.compact.data_retention_history {
// only store the file_list into history, don't delete files
if let Err(e) = infra_file_list::batch_add_history(&files).await {
log::error!("[COMPACT] file_list batch_add_history failed: {}", e);
@@ -358,7 +359,7 @@ async fn write_file_list(
file_list_days: HashSet,
hours_files: HashMap>,
) -> Result<(), anyhow::Error> {
- if CONFIG.common.meta_store_external {
+ if get_config().common.meta_store_external {
write_file_list_db_only(org_id, hours_files).await
} else {
write_file_list_s3(org_id, file_list_days, hours_files).await
diff --git a/src/service/compact/stats.rs b/src/service/compact/stats.rs
index 05976e77b9..5f8c8ca399 100644
--- a/src/service/compact/stats.rs
+++ b/src/service/compact/stats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{cluster::LOCAL_NODE_UUID, CONFIG};
+use config::cluster::LOCAL_NODE_UUID;
use infra::{dist_lock, file_list as infra_file_list};
use tokio::time;
@@ -27,7 +27,9 @@ pub async fn update_stats_from_file_list() -> Result, anyhow:
}
// before starting, set current node to lock the job
- if CONFIG.common.meta_store_external && (node.is_empty() || LOCAL_NODE_UUID.ne(&node)) {
+ if config::get_config().common.meta_store_external
+ && (node.is_empty() || LOCAL_NODE_UUID.ne(&node))
+ {
offset = match update_stats_lock_node().await {
Ok(Some(offset)) => offset,
Ok(None) => return Ok(None),
diff --git a/src/service/dashboards/folders.rs b/src/service/dashboards/folders.rs
index 35c70c15c7..a6e7ce7faf 100644
--- a/src/service/dashboards/folders.rs
+++ b/src/service/dashboards/folders.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/dashboards/mod.rs b/src/service/dashboards/mod.rs
index 4367ff5088..cacddb1028 100644
--- a/src/service/dashboards/mod.rs
+++ b/src/service/dashboards/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/dashboards/reports.rs b/src/service/dashboards/reports.rs
index 38efdd83e8..df893adfb8 100644
--- a/src/service/dashboards/reports.rs
+++ b/src/service/dashboards/reports.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@ use std::{str::FromStr, time::Duration};
use actix_web::http;
use chromiumoxide::{browser::Browser, cdp::browser_protocol::page::PrintToPdfParams, Page};
-use config::{get_chrome_launch_options, CONFIG, SMTP_CLIENT};
+use config::{get_chrome_launch_options, get_config, SMTP_CLIENT};
use cron::Schedule;
use futures::{future::try_join_all, StreamExt};
use lettre::{
@@ -46,20 +46,19 @@ pub async fn save(
mut report: Report,
create: bool,
) -> Result<(), anyhow::Error> {
- if CONFIG.common.report_server_url.is_empty() {
+ let cfg = get_config();
+ if cfg.common.report_server_url.is_empty() {
// Check if SMTP is enabled, otherwise don't save the report
- if !CONFIG.smtp.smtp_enabled {
+ if !cfg.smtp.smtp_enabled {
return Err(anyhow::anyhow!("SMTP configuration not enabled"));
}
// Check if Chrome is enabled, otherwise don't save the report
- if !CONFIG.chrome.chrome_enabled || CONFIG.chrome.chrome_path.is_empty() {
+ if !cfg.chrome.chrome_enabled || cfg.chrome.chrome_path.is_empty() {
return Err(anyhow::anyhow!("Chrome not enabled"));
}
- if CONFIG.common.report_user_name.is_empty()
- || CONFIG.common.report_user_password.is_empty()
- {
+ if cfg.common.report_user_name.is_empty() || cfg.common.report_user_password.is_empty() {
return Err(anyhow::anyhow!("Report username and password ENVs not set"));
}
}
@@ -240,7 +239,8 @@ impl Report {
return Err(anyhow::anyhow!("Atleast one dashboard is required"));
}
- if !CONFIG.common.report_server_url.is_empty() {
+ let cfg = get_config();
+ if !cfg.common.report_server_url.is_empty() {
let mut recepients = vec![];
for recepient in &self.destinations {
match recepient {
@@ -255,17 +255,17 @@ impl Report {
recepients,
name: self.name.clone(),
message: self.message.clone(),
- dashb_url: format!("{}{}/web", CONFIG.common.web_url, CONFIG.common.base_uri),
+ dashb_url: format!("{}{}/web", cfg.common.web_url, cfg.common.base_uri),
},
};
let url = url::Url::parse(&format!(
"{}/{}/reports/{}/send",
- &CONFIG.common.report_server_url, &self.org_id, &self.name
+ &cfg.common.report_server_url, &self.org_id, &self.name
))
.unwrap();
match Client::builder()
- .danger_accept_invalid_certs(CONFIG.common.report_server_skip_tls_verify)
+ .danger_accept_invalid_certs(cfg.common.report_server_skip_tls_verify)
.build()
.unwrap()
.put(url)
@@ -295,8 +295,8 @@ impl Report {
let report = generate_report(
dashboard,
&self.org_id,
- &CONFIG.common.report_user_name,
- &CONFIG.common.report_user_password,
+ &cfg.common.report_user_name,
+ &cfg.common.report_user_password,
&self.timezone,
)
.await?;
@@ -306,7 +306,8 @@ impl Report {
/// Sends emails to the [`Report`] recepients. Currently only one pdf data is supported.
async fn send_email(&self, pdf_data: &[u8], dashb_url: String) -> Result<(), anyhow::Error> {
- if !CONFIG.smtp.smtp_enabled {
+ let cfg = get_config();
+ if !cfg.smtp.smtp_enabled {
return Err(anyhow::anyhow!("SMTP configuration not enabled"));
}
@@ -318,15 +319,15 @@ impl Report {
}
let mut email = Message::builder()
- .from(CONFIG.smtp.smtp_from_email.parse()?)
+ .from(cfg.smtp.smtp_from_email.parse()?)
.subject(format!("Openobserve Report - {}", &self.title));
for recepient in recepients {
email = email.to(recepient.parse()?);
}
- if !CONFIG.smtp.smtp_reply_to.is_empty() {
- email = email.reply_to(CONFIG.smtp.smtp_reply_to.parse()?);
+ if !cfg.smtp.smtp_reply_to.is_empty() {
+ email = email.reply_to(cfg.smtp.smtp_reply_to.parse()?);
}
let email = email
@@ -364,8 +365,9 @@ async fn generate_report(
user_pass: &str,
timezone: &str,
) -> Result<(Vec, String), anyhow::Error> {
+ let cfg = get_config();
// Check if Chrome is enabled, otherwise don't save the report
- if !CONFIG.chrome.chrome_enabled {
+ if !cfg.chrome.chrome_enabled {
return Err(anyhow::anyhow!("Chrome not enabled"));
}
@@ -396,7 +398,7 @@ async fn generate_report(
}
});
- let web_url = format!("{}{}/web", CONFIG.common.web_url, CONFIG.common.base_uri);
+ let web_url = format!("{}{}/web", cfg.common.web_url, cfg.common.base_uri);
log::debug!("Navigating to web url: {}", &web_url);
let page = browser
.new_page(&format!("{web_url}/login?login_as_internal_user=true"))
@@ -548,7 +550,7 @@ async fn generate_report(
async fn wait_for_panel_data_load(page: &Page) -> Result<(), anyhow::Error> {
let start = std::time::Instant::now();
- let timeout = Duration::from_secs(CONFIG.chrome.chrome_sleep_secs.into());
+ let timeout = Duration::from_secs(get_config().chrome.chrome_sleep_secs.into());
log::info!("waiting for headless data to load");
loop {
if page
diff --git a/src/service/db/alerts/destinations.rs b/src/service/db/alerts/destinations.rs
index 10ae3ec8ab..1d666ecac0 100644
--- a/src/service/db/alerts/destinations.rs
+++ b/src/service/db/alerts/destinations.rs
@@ -91,7 +91,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value: Destination = if config::CONFIG.common.meta_store_external {
+ let item_value: Destination = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/alerts/mod.rs b/src/service/db/alerts/mod.rs
index 1b68db80c6..d84fdf5b14 100644
--- a/src/service/db/alerts/mod.rs
+++ b/src/service/db/alerts/mod.rs
@@ -155,7 +155,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
let stream_key = item_key[0..item_key.rfind('/').unwrap()].to_string();
- let item_value: Alert = if config::CONFIG.common.meta_store_external {
+ let item_value: Alert = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/alerts/templates.rs b/src/service/db/alerts/templates.rs
index 0509d29f9a..fb800fb6ea 100644
--- a/src/service/db/alerts/templates.rs
+++ b/src/service/db/alerts/templates.rs
@@ -103,7 +103,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value: Template = if config::CONFIG.common.meta_store_external {
+ let item_value: Template = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/compact/file_list.rs b/src/service/db/compact/file_list.rs
index 68fea90a93..79bd4151ad 100644
--- a/src/service/db/compact/file_list.rs
+++ b/src/service/db/compact/file_list.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/compact/files.rs b/src/service/db/compact/files.rs
index 3afcfbf9b1..b495005705 100644
--- a/src/service/db/compact/files.rs
+++ b/src/service/db/compact/files.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/compact/mod.rs b/src/service/db/compact/mod.rs
index 99939ce2fd..9d475a5cea 100644
--- a/src/service/db/compact/mod.rs
+++ b/src/service/db/compact/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/compact/organization.rs b/src/service/db/compact/organization.rs
index 160897dea3..d80d3fe772 100644
--- a/src/service/db/compact/organization.rs
+++ b/src/service/db/compact/organization.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/compact/retention.rs b/src/service/db/compact/retention.rs
index 0b290e596d..4da403d1b0 100644
--- a/src/service/db/compact/retention.rs
+++ b/src/service/db/compact/retention.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/compact/stats.rs b/src/service/db/compact/stats.rs
index f9ecfcd192..5b5c56d481 100644
--- a/src/service/db/compact/stats.rs
+++ b/src/service/db/compact/stats.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,14 +15,12 @@
use std::sync::atomic::{AtomicI64, Ordering};
-use config::CONFIG;
-
use crate::service::db;
static LOCAL_OFFSET: AtomicI64 = AtomicI64::new(0);
pub async fn get_offset() -> (i64, String) {
- if !CONFIG.common.meta_store_external {
+ if !config::get_config().common.meta_store_external {
let offset = LOCAL_OFFSET.load(Ordering::Relaxed);
return (offset, String::from(""));
}
@@ -43,7 +41,7 @@ pub async fn get_offset() -> (i64, String) {
}
pub async fn set_offset(offset: i64, node: Option<&str>) -> Result<(), anyhow::Error> {
- if !CONFIG.common.meta_store_external {
+ if !config::get_config().common.meta_store_external {
LOCAL_OFFSET.store(offset, Ordering::Release);
return Ok(());
}
diff --git a/src/service/db/dashboards/folders.rs b/src/service/db/dashboards/folders.rs
index 41fa96e8f0..8ab91f9490 100644
--- a/src/service/db/dashboards/folders.rs
+++ b/src/service/db/dashboards/folders.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/dashboards/mod.rs b/src/service/db/dashboards/mod.rs
index 9b87a3ecf7..719b27e070 100644
--- a/src/service/db/dashboards/mod.rs
+++ b/src/service/db/dashboards/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/dashboards/reports.rs b/src/service/db/dashboards/reports.rs
index eaef4b79ff..d335bed84e 100644
--- a/src/service/db/dashboards/reports.rs
+++ b/src/service/db/dashboards/reports.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -145,7 +145,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value: Report = if config::CONFIG.common.meta_store_external {
+ let item_value: Report = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/file_list/broadcast.rs b/src/service/db/file_list/broadcast.rs
index 01d56ccfe7..cf74063cbc 100644
--- a/src/service/db/file_list/broadcast.rs
+++ b/src/service/db/file_list/broadcast.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,11 +17,11 @@ use std::sync::Arc;
use config::{
cluster::{is_compactor, is_ingester, is_querier, LOCAL_NODE_UUID},
+ get_config,
meta::{
cluster::{Node, NodeStatus},
stream::FileKey,
},
- CONFIG,
};
use hashbrown::HashMap;
use once_cell::sync::Lazy;
@@ -38,7 +38,8 @@ type EventChannel = Arc>>;
/// send an event to broadcast, will create a new channel for each nodes
pub async fn send(items: &[FileKey], node_uuid: Option) -> Result<(), anyhow::Error> {
- if CONFIG.common.local_mode || items.is_empty() {
+ let cfg = get_config();
+ if cfg.common.local_mode || items.is_empty() {
return Ok(());
}
let nodes = if let Some(node_uuid) = node_uuid {
@@ -62,7 +63,7 @@ pub async fn send(items: &[FileKey], node_uuid: Option) -> Result<(), an
continue;
}
// if meta_store_external is true, only send to querier
- if CONFIG.common.meta_store_external && !is_querier(&node.role) {
+ if cfg.common.meta_store_external && !is_querier(&node.role) {
continue;
}
if !is_querier(&node.role) && !is_compactor(&node.role) && !is_ingester(&node.role) {
@@ -70,7 +71,7 @@ pub async fn send(items: &[FileKey], node_uuid: Option) -> Result<(), an
}
let node_uuid = node.uuid.clone();
let node_addr = node.grpc_addr.clone();
- if CONFIG.common.print_key_event {
+ if cfg.common.print_key_event {
items.iter().for_each(|item| {
log::info!(
"[broadcast] send event to node[{}]: file: {}, deleted: {}",
@@ -148,13 +149,14 @@ async fn send_to_node(
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
tokio::task::yield_now().await;
}
+ let cfg = get_config();
// connect to the node
let token: MetadataValue<_> = cluster::get_internal_grpc_token()
.parse()
.expect("parse internal grpc token faile");
let channel = match Channel::from_shared(node.grpc_addr.clone())
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(cfg.grpc.connect_timeout))
.connect()
.await
{
@@ -177,11 +179,12 @@ async fn send_to_node(
Ok(req)
},
);
+
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
loop {
let items = match rx.recv().await {
Some(v) => v,
diff --git a/src/service/db/file_list/local.rs b/src/service/db/file_list/local.rs
index b5f097dff0..54b397d531 100644
--- a/src/service/db/file_list/local.rs
+++ b/src/service/db/file_list/local.rs
@@ -21,7 +21,6 @@ use config::{
asynchronism::file::get_file_contents, file::scan_files, json,
parquet::parse_file_key_columns,
},
- CONFIG,
};
use infra::file_list as infra_file_list;
use once_cell::sync::Lazy;
@@ -48,8 +47,9 @@ pub async fn set(key: &str, meta: Option, deleted: bool) -> Result<(),
}
}
+ let cfg = config::get_config();
// write into local cache for s3
- if !CONFIG.common.meta_store_external {
+ if !cfg.common.meta_store_external {
let mut write_buf = json::to_vec(&file_data)?;
write_buf.push(b'\n');
let file = wal::get_or_create(
@@ -63,7 +63,7 @@ pub async fn set(key: &str, meta: Option, deleted: bool) -> Result<(),
}
// notify other nodes
- if !CONFIG.common.meta_store_external || CONFIG.memory_cache.cache_latest_files {
+ if !cfg.common.meta_store_external || cfg.memory_cache.cache_latest_files {
let mut q = BROADCAST_QUEUE.write().await;
q.push(file_data);
}
@@ -74,7 +74,7 @@ pub async fn set(key: &str, meta: Option, deleted: bool) -> Result<(),
/// need to load wal file_list when the ingester
async fn get_in_wal() -> Result, anyhow::Error> {
let mut result = Vec::with_capacity(1024);
- let pattern = format!("{}file_list/", &CONFIG.common.data_wal_dir);
+ let pattern = format!("{}file_list/", &config::get_config().common.data_wal_dir);
let files = scan_files(&pattern, "json", None).unwrap_or_default();
let mut line_no = 0;
for file in files {
diff --git a/src/service/db/file_list/mod.rs b/src/service/db/file_list/mod.rs
index ef407dce0b..49987d4019 100644
--- a/src/service/db/file_list/mod.rs
+++ b/src/service/db/file_list/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -13,7 +13,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::{meta::stream::FileMeta, RwHashMap, RwHashSet, CONFIG};
+use config::{meta::stream::FileMeta, RwHashMap, RwHashSet};
use dashmap::{DashMap, DashSet};
use infra::{cache, file_list};
use once_cell::sync::Lazy;
@@ -28,8 +28,14 @@ pub static DEPULICATE_FILES: Lazy> =
pub static DELETED_FILES: Lazy> =
Lazy::new(|| DashMap::with_capacity_and_hasher(64, Default::default()));
-pub static BLOCKED_ORGS: Lazy> =
- Lazy::new(|| CONFIG.compact.blocked_orgs.split(',').collect());
+pub static BLOCKED_ORGS: Lazy> = Lazy::new(|| {
+ config::get_config()
+ .compact
+ .blocked_orgs
+ .split(',')
+ .map(|x| x.to_string())
+ .collect()
+});
pub async fn progress(
key: &str,
@@ -53,7 +59,7 @@ pub async fn progress(
);
}
// update stream stats realtime
- if CONFIG.common.local_mode {
+ if config::get_config().common.local_mode {
if let Err(e) = cache::stats::incr_stream_stats(key, data.unwrap()) {
log::error!(
"service:db:file_list: add {}, incr_stream_stats error: {}",
diff --git a/src/service/db/file_list/remote.rs b/src/service/db/file_list/remote.rs
index c3a62841cf..3c056245c5 100644
--- a/src/service/db/file_list/remote.rs
+++ b/src/service/db/file_list/remote.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -21,7 +21,7 @@ use std::{
use bytes::Buf;
use chrono::{DateTime, Duration, TimeZone, Utc};
-use config::{meta::stream::FileKey, utils::json, CONFIG};
+use config::{meta::stream::FileKey, utils::json};
use futures::future::try_join_all;
use infra::{cache::stats, file_list as infra_file_list, storage};
use once_cell::sync::Lazy;
@@ -53,8 +53,9 @@ pub async fn cache(prefix: &str, force: bool) -> Result<(), anyhow::Error> {
return Ok(());
}
- let mut tasks = Vec::with_capacity(CONFIG.limit.query_thread_num + 1);
- let chunk_size = std::cmp::max(1, files_num / CONFIG.limit.query_thread_num);
+ let cfg = config::get_config();
+ let mut tasks = Vec::with_capacity(cfg.limit.query_thread_num + 1);
+ let chunk_size = std::cmp::max(1, files_num / cfg.limit.query_thread_num);
for chunk in files.chunks(chunk_size) {
let chunk = chunk.to_vec();
let task: tokio::task::JoinHandle> =
@@ -181,7 +182,7 @@ async fn process_file(file: &str) -> Result, anyhow::Error> {
if !super::BLOCKED_ORGS.is_empty() {
let columns = item.key.split('/').collect::>();
let org_id = columns.get(1).unwrap_or(&"");
- if super::BLOCKED_ORGS.contains(org_id) {
+ if super::BLOCKED_ORGS.contains(&org_id.to_string()) {
// log::error!("Load file_list skip blacklist org: {}", org_id);
continue;
}
@@ -195,7 +196,7 @@ async fn process_file(file: &str) -> Result, anyhow::Error> {
continue;
}
// check duplicate files
- if CONFIG.common.feature_filelist_dedup_enabled {
+ if config::get_config().common.feature_filelist_dedup_enabled {
if super::DEPULICATE_FILES.contains(&item.key) {
continue;
}
diff --git a/src/service/db/functions.rs b/src/service/db/functions.rs
index 095dfc49a4..138f83974e 100644
--- a/src/service/db/functions.rs
+++ b/src/service/db/functions.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
use std::sync::Arc;
-use config::{utils::json, CONFIG};
+use config::utils::json;
use crate::{
common::{
@@ -88,7 +88,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
let org_id = &item_key[0..item_key.find('/').unwrap()];
- let item_value: Transform = if CONFIG.common.meta_store_external {
+ let item_value: Transform = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/instance.rs b/src/service/db/instance.rs
index 97b41b3208..82f4ebe571 100644
--- a/src/service/db/instance.rs
+++ b/src/service/db/instance.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/kv.rs b/src/service/db/kv.rs
index ec5ae587df..efcac0e876 100644
--- a/src/service/db/kv.rs
+++ b/src/service/db/kv.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,8 +15,6 @@
use std::sync::Arc;
-use config::CONFIG;
-
use crate::{common::infra::config::KVS, service::db};
fn mk_keys(org_id: &str, key: &str) -> (String, String) {
@@ -79,7 +77,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value = if CONFIG.common.meta_store_external {
+ let item_value = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => val,
Err(e) => {
diff --git a/src/service/db/metrics.rs b/src/service/db/metrics.rs
index 401901534c..50c26e7f79 100644
--- a/src/service/db/metrics.rs
+++ b/src/service/db/metrics.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -73,7 +73,7 @@ pub async fn watch_prom_cluster_leader() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value: ClusterLeader = if config::CONFIG.common.meta_store_external {
+ let item_value: ClusterLeader = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/ofga.rs b/src/service/db/ofga.rs
index 59a5491fc0..79f68f0b08 100644
--- a/src/service/db/ofga.rs
+++ b/src/service/db/ofga.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -110,7 +110,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
};
match ev {
db::Event::Put(ev) => {
- let item_value: OFGAModel = if config::CONFIG.common.meta_store_external {
+ let item_value: OFGAModel = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/organization.rs b/src/service/db/organization.rs
index edc036745f..d40a8ebc7f 100644
--- a/src/service/db/organization.rs
+++ b/src/service/db/organization.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -93,7 +93,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
if let db::Event::Put(ev) = ev {
let item_key = ev.key;
let item_value = ev.value.unwrap();
- let json_val: OrganizationSetting = if config::CONFIG.common.meta_store_external {
+ let json_val: OrganizationSetting = if config::get_config().common.meta_store_external {
match db::get(&item_key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/pipelines.rs b/src/service/db/pipelines.rs
index 49122a1709..8747de6cfd 100644
--- a/src/service/db/pipelines.rs
+++ b/src/service/db/pipelines.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
use std::sync::Arc;
-use config::{meta::stream::StreamType, utils::json, CONFIG};
+use config::{meta::stream::StreamType, utils::json};
use crate::{
common::{infra::config::STREAM_PIPELINES, meta::pipelines::PipeLine},
@@ -101,7 +101,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
let org_id = &item_key[0..item_key.find('/').unwrap()];
- let item_value: PipeLine = if CONFIG.common.meta_store_external {
+ let item_value: PipeLine = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/saved_view.rs b/src/service/db/saved_view.rs
index 763521825f..b2d7fa45c5 100644
--- a/src/service/db/saved_view.rs
+++ b/src/service/db/saved_view.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/db/schema.rs b/src/service/db/schema.rs
index d7a78430b1..eaf059bc24 100644
--- a/src/service/db/schema.rs
+++ b/src/service/db/schema.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
use arrow_schema::{Field, Schema};
use bytes::Bytes;
-use config::{is_local_disk_storage, meta::stream::StreamType, utils::json, CONFIG};
+use config::{get_config, is_local_disk_storage, meta::stream::StreamType, utils::json};
use hashbrown::{HashMap, HashSet};
use infra::{
cache,
@@ -331,7 +331,8 @@ pub async fn watch() -> Result<(), anyhow::Error> {
SchemaCache::new(latest_schema.clone()),
);
drop(w);
- if CONFIG.common.schema_cache_compress_enabled {
+ let cfg = get_config();
+ if cfg.common.schema_cache_compress_enabled {
let schema_versions = schema_versions
.into_iter()
.map(|(start_dt, data)| {
@@ -436,7 +437,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
if stream_type.eq(&StreamType::EnrichmentTables) && is_local_disk_storage() {
let data_dir = format!(
"{}files/{org_id}/{stream_type}/{stream_name}",
- CONFIG.common.data_wal_dir
+ get_config().common.data_wal_dir
);
let path = std::path::Path::new(&data_dir);
if path.exists() {
@@ -492,7 +493,7 @@ pub async fn cache() -> Result<(), anyhow::Error> {
SchemaCache::new(latest_schema.clone()),
);
drop(w);
- if CONFIG.common.schema_cache_compress_enabled {
+ if get_config().common.schema_cache_compress_enabled {
let schema_versions = schema_versions
.into_iter()
.map(|(start_dt, data)| {
@@ -555,7 +556,7 @@ pub async fn cache_enrichment_tables() -> Result<(), anyhow::Error> {
}
// waiting for querier to be ready
- let expect_querier_num = CONFIG.limit.starting_expect_querier_num;
+ let expect_querier_num = get_config().limit.starting_expect_querier_num;
loop {
let nodes = get_cached_online_querier_nodes().await.unwrap_or_default();
if nodes.len() >= expect_querier_num {
diff --git a/src/service/db/session.rs b/src/service/db/session.rs
index e9cd9f5e9e..193c80549c 100644
--- a/src/service/db/session.rs
+++ b/src/service/db/session.rs
@@ -15,7 +15,7 @@
use std::sync::Arc;
-use config::{utils::json, CONFIG};
+use config::utils::json;
use crate::{
common::infra::config::USER_SESSIONS,
@@ -74,7 +74,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
match ev {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value = if CONFIG.common.meta_store_external {
+ let item_value = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/syslog.rs b/src/service/db/syslog.rs
index d83566a41c..6d3374174d 100644
--- a/src/service/db/syslog.rs
+++ b/src/service/db/syslog.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,7 @@
use std::sync::Arc;
-use config::{utils::json, CONFIG};
+use config::utils::json;
use crate::{
common::{
@@ -83,7 +83,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
};
match ev {
db::Event::Put(ev) => {
- let item_value: SyslogRoute = if CONFIG.common.meta_store_external {
+ let item_value: SyslogRoute = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/user.rs b/src/service/db/user.rs
index 6db26fb124..c913d4fa6b 100644
--- a/src/service/db/user.rs
+++ b/src/service/db/user.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@
use std::sync::Arc;
use anyhow::bail;
-use config::{utils::json, CONFIG};
+use config::utils::json;
use crate::{
common::{
@@ -157,7 +157,7 @@ pub async fn watch() -> Result<(), anyhow::Error> {
db::Event::Put(ev) => {
let item_key = ev.key.strip_prefix(key).unwrap();
- let item_value: DBUser = if CONFIG.common.meta_store_external {
+ let item_value: DBUser = if config::get_config().common.meta_store_external {
match db::get(&ev.key).await {
Ok(val) => match json::from_slice(&val) {
Ok(val) => val,
diff --git a/src/service/db/version.rs b/src/service/db/version.rs
index 3bfb1b2ea7..717469db92 100644
--- a/src/service/db/version.rs
+++ b/src/service/db/version.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/enrichment_table/geoip.rs b/src/service/enrichment_table/geoip.rs
index 26f59b71b7..fca542f48e 100644
--- a/src/service/enrichment_table/geoip.rs
+++ b/src/service/enrichment_table/geoip.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -17,7 +17,7 @@
use std::{collections::BTreeMap, fs, net::IpAddr, sync::Arc, time::SystemTime};
-use config::{CONFIG, MMDB_CITY_FILE_NAME};
+use config::{get_config, MMDB_CITY_FILE_NAME};
use maxminddb::{
geoip2::{City, ConnectionType, Isp},
MaxMindDBError, Reader,
@@ -92,7 +92,11 @@ fn default_locale() -> String {
impl Default for GeoipConfig {
fn default() -> self::GeoipConfig {
GeoipConfig {
- path: format!("{}{}", &CONFIG.common.mmdb_data_dir, MMDB_CITY_FILE_NAME),
+ path: format!(
+ "{}{}",
+ &get_config().common.mmdb_data_dir,
+ MMDB_CITY_FILE_NAME
+ ),
locale: default_locale(),
}
}
@@ -101,7 +105,7 @@ impl Default for GeoipConfig {
impl GeoipConfig {
pub fn new(name: &str) -> self::GeoipConfig {
GeoipConfig {
- path: format!("{}{}", &CONFIG.common.mmdb_data_dir, name),
+ path: format!("{}{}", &get_config().common.mmdb_data_dir, name),
locale: default_locale(),
}
}
diff --git a/src/service/enrichment_table/mod.rs b/src/service/enrichment_table/mod.rs
index f0b21b5178..e2e09ae22d 100644
--- a/src/service/enrichment_table/mod.rs
+++ b/src/service/enrichment_table/mod.rs
@@ -23,13 +23,12 @@ use actix_web::{
use bytes::Bytes;
use chrono::Utc;
use config::{
- cluster,
+ cluster, get_config,
meta::{
stream::{PartitionTimeLevel, StreamType},
usage::UsageType,
},
utils::{flatten::format_key, json, schema_ext::SchemaExt},
- CONFIG,
};
use futures::{StreamExt, TryStreamExt};
use infra::{
@@ -152,7 +151,7 @@ pub async fn save_enrichment_data(
json_record.insert(header.into(), json::Value::String(field.into()));
}
json_record.insert(
- CONFIG.common.column_timestamp.clone(),
+ get_config().common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
diff --git a/src/service/file_list.rs b/src/service/file_list.rs
index 2e72e2d0de..1977080616 100644
--- a/src/service/file_list.rs
+++ b/src/service/file_list.rs
@@ -17,14 +17,13 @@ use std::io::Write;
use config::{
cluster::LOCAL_NODE_UUID,
- ider,
+ get_config, ider,
meta::{
cluster::Node,
search::ScanStats,
stream::{FileKey, FileMeta, PartitionTimeLevel, StreamType},
},
utils::{file::get_file_meta as util_get_file_meta, json},
- CONFIG,
};
use futures::future::try_join_all;
use infra::{
@@ -32,7 +31,12 @@ use infra::{
file_list, storage,
};
use proto::cluster_rpc;
-use tonic::{codec::CompressionEncoding, metadata::MetadataValue, transport::Channel, Request};
+use tonic::{
+ codec::CompressionEncoding,
+ metadata::{MetadataKey, MetadataValue},
+ transport::Channel,
+ Request,
+};
use tracing_opentelemetry::OpenTelemetrySpanExt;
use crate::{
@@ -49,7 +53,8 @@ pub async fn query(
time_max: i64,
is_local: bool,
) -> Result, anyhow::Error> {
- if is_local || CONFIG.common.local_mode {
+ let cfg = get_config();
+ if is_local || cfg.common.local_mode {
return query_inner(
org_id,
stream_name,
@@ -72,6 +77,7 @@ pub async fn query(
let mut tasks = Vec::with_capacity(3);
// get first three nodes to check file list max id
for node in nodes.into_iter().take(3) {
+ let cfg = cfg.clone();
let org_id = org_id.to_string();
let task = tokio::task::spawn(async move {
let req = cluster_rpc::EmptyRequest::default();
@@ -87,12 +93,17 @@ pub async fn query(
)
});
+ let org_header_key: MetadataKey<_> = cfg
+ .grpc
+ .org_header_key
+ .parse()
+ .map_err(|_| Error::Message("invalid org_header_key".to_string()))?;
let token: MetadataValue<_> = cluster::get_internal_grpc_token()
.parse()
.map_err(|_| Error::Message("invalid token".to_string()))?;
let channel = Channel::from_shared(node.grpc_addr.clone())
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(cfg.grpc.connect_timeout))
.connect()
.await
.map_err(|err| {
@@ -110,15 +121,15 @@ pub async fn query(
move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
- .insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
+ .insert(org_header_key.clone(), org_id.clone());
Ok(req)
},
);
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
let response: cluster_rpc::MaxIdResponse = match client.max_id(request).await {
Ok(res) => res.into_inner(),
Err(err) => {
@@ -194,12 +205,17 @@ pub async fn query(
)
});
+ let org_header_key: MetadataKey<_> = cfg
+ .grpc
+ .org_header_key
+ .parse()
+ .map_err(|_| Error::Message("invalid org_header_key".to_string()))?;
let token: MetadataValue<_> = cluster::get_internal_grpc_token()
.parse()
.map_err(|_| Error::Message("invalid token".to_string()))?;
let channel = Channel::from_shared(node.grpc_addr.clone())
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(cfg.grpc.connect_timeout))
.connect()
.await
.map_err(|err| {
@@ -218,15 +234,16 @@ pub async fn query(
move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
- .insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
+ .insert(org_header_key.clone(), org_id.clone());
Ok(req)
},
);
+
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
let response: cluster_rpc::FileList = match client.query(request).await {
Ok(res) => res.into_inner(),
Err(err) => {
@@ -312,7 +329,7 @@ pub fn calculate_local_files_size(files: &[String]) -> Result Result<(), anyhow::Error> {
- if CONFIG.common.meta_store_external {
+ if get_config().common.meta_store_external {
delete_parquet_file_db_only(key, file_list_only).await
} else {
delete_parquet_file_s3(key, file_list_only).await
diff --git a/src/service/ingestion/grpc.rs b/src/service/ingestion/grpc.rs
index 50e57a312a..db0a744aae 100644
--- a/src/service/ingestion/grpc.rs
+++ b/src/service/ingestion/grpc.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/ingestion/mod.rs b/src/service/ingestion/mod.rs
index 70a2d1cac3..63e0c46d07 100644
--- a/src/service/ingestion/mod.rs
+++ b/src/service/ingestion/mod.rs
@@ -21,13 +21,13 @@ use std::{
use anyhow::{anyhow, Result};
use chrono::{Duration, TimeZone, Utc};
use config::{
- cluster,
+ cluster, get_config,
meta::{
stream::{PartitionTimeLevel, PartitioningDetails, Routing, StreamPartition, StreamType},
usage::{RequestStats, TriggerData, TriggerDataStatus, TriggerDataType},
},
utils::{flatten, json::*},
- CONFIG, SIZE_IN_MB,
+ SIZE_IN_MB,
};
use vector_enrichment::TableRegistry;
use vrl::{
@@ -368,7 +368,7 @@ pub fn apply_stream_functions(
value = apply_vrl_fn(runtime, vrl_runtime, &value, org_id, stream_name);
}
}
- flatten::flatten_with_level(value, CONFIG.limit.ingest_flatten_level)
+ flatten::flatten_with_level(value, get_config().limit.ingest_flatten_level)
}
pub fn init_functions_runtime() -> Runtime {
@@ -413,7 +413,9 @@ pub fn check_ingestion_allowed(org_id: &str, stream_name: Option<&str>) -> Resul
if !cluster::is_ingester(&cluster::LOCAL_NODE_ROLE) {
return Err(anyhow!("not an ingester"));
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Err(anyhow!("Quota exceeded for this organization [{}]", org_id));
}
@@ -535,6 +537,7 @@ pub async fn get_user_defined_schema(
streams: &[StreamParams],
user_defined_schema_map: &mut HashMap>,
) {
+ let cfg = get_config();
for stream in streams {
let stream_settings =
infra::schema::get_settings(&stream.org_id, &stream.stream_name, stream.stream_type)
@@ -543,8 +546,8 @@ pub async fn get_user_defined_schema(
if let Some(fields) = stream_settings.defined_schema_fields {
if !fields.is_empty() {
let mut fields: HashSet<_> = fields.iter().cloned().collect();
- if !fields.contains(&CONFIG.common.column_timestamp) {
- fields.insert(CONFIG.common.column_timestamp.to_string());
+ if !fields.contains(&cfg.common.column_timestamp) {
+ fields.insert(cfg.common.column_timestamp.to_string());
}
user_defined_schema_map.insert(stream.stream_name.to_string(), fields);
}
diff --git a/src/service/logs/bulk.rs b/src/service/logs/bulk.rs
index 94174fd615..860aaa1fe1 100644
--- a/src/service/logs/bulk.rs
+++ b/src/service/logs/bulk.rs
@@ -23,14 +23,14 @@ use actix_web::web;
use anyhow::{Error, Result};
use chrono::{Duration, Utc};
use config::{
- cluster,
+ cluster, get_config,
meta::{
stream::{PartitioningDetails, Routing, StreamType},
usage::UsageType,
},
metrics,
utils::{flatten, json, schema_ext::SchemaExt, time::parse_timestamp_micro_from_value},
- BLOCKED_STREAMS, CONFIG, DISTINCT_FIELDS,
+ BLOCKED_STREAMS, DISTINCT_FIELDS,
};
use infra::schema::{unwrap_partition_time_level, SchemaCache};
@@ -69,7 +69,9 @@ pub async fn ingest(
return Err(anyhow::anyhow!("not an ingester"));
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Err(anyhow::anyhow!(
"Quota exceeded for this organization [{}]",
org_id
@@ -86,7 +88,8 @@ pub async fn ingest(
items: vec![],
};
- let min_ts = (Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap())
+ let cfg = get_config();
+ let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap())
.timestamp_micros();
let mut runtime = crate::service::ingestion::init_functions_runtime();
@@ -130,13 +133,13 @@ pub async fn ingest(
}
(action, stream_name, doc_id) = ret.unwrap();
- if !CONFIG.common.skip_formatting_bulk_stream_name {
+ if !cfg.common.skip_formatting_bulk_stream_name {
stream_name = format_stream_name(&stream_name);
}
// skip blocked streams
let key = format!("{org_id}/{}/{stream_name}", StreamType::Logs);
- if BLOCKED_STREAMS.contains(&key.as_str()) {
+ if BLOCKED_STREAMS.contains(&key) {
// print warning only once
blocked_stream_warnings.entry(key).or_insert_with(|| {
log::warn!("stream [{stream_name}] is blocked from ingestion");
@@ -225,7 +228,7 @@ pub async fn ingest(
next_line_is_data = false;
// JSON Flattening
- let mut value = flatten::flatten_with_level(value, CONFIG.limit.ingest_flatten_level)?;
+ let mut value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level)?;
if let Some(routing) = stream_routing_map.get(&stream_name) {
if !routing.is_empty() {
@@ -305,7 +308,7 @@ pub async fn ingest(
}
// handle timestamp
- let timestamp = match local_val.get(&CONFIG.common.column_timestamp) {
+ let timestamp = match local_val.get(&cfg.common.column_timestamp) {
Some(v) => match parse_timestamp_micro_from_value(v) {
Ok(t) => t,
Err(_e) => {
@@ -340,7 +343,7 @@ pub async fn ingest(
continue;
}
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
let (partition_keys, partition_time_level) =
@@ -375,7 +378,7 @@ pub async fn ingest(
// this is for schema inference at stream level , which avoids locks in case schema
// changes are frequent within request
- if CONFIG.common.infer_schema_per_request {
+ if cfg.common.infer_schema_per_request {
if let Err(e) = add_record(
&StreamMeta {
org_id: org_id.to_string(),
@@ -478,7 +481,7 @@ pub async fn ingest(
}
// new flow for schema inference at stream level
- stream_data.data = if CONFIG.common.infer_schema_per_request {
+ stream_data.data = if cfg.common.infer_schema_per_request {
process_record(
&mut stream_data,
&StreamParams {
@@ -563,6 +566,7 @@ async fn process_record(
) -> Result, Error> {
let mut new_stream_buf = HashMap::new();
let mut trigger: TriggerAlertData = Vec::new();
+ let cfg = get_config();
for schema_records in stream_data.data.values_mut() {
// check schema
let mut timestamp = 0;
@@ -573,7 +577,7 @@ async fn process_record(
.map(|record| {
let rec = record.as_ref();
let rec_ts = rec
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap();
@@ -614,7 +618,7 @@ async fn process_record(
match cast_to_schema_v1(&mut local_rec, &schema_latest_map) {
Ok(_) => {
let timestamp: i64 = local_rec
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap();
@@ -744,7 +748,7 @@ fn add_record_status(
action,
BulkResponseItem::new(stream_name.clone(), doc_id, value, stream_name),
);
- if !CONFIG.common.bulk_api_response_errors_only {
+ if !get_config().common.bulk_api_response_errors_only {
bulk_res.items.push(item);
}
}
diff --git a/src/service/logs/ingest.rs b/src/service/logs/ingest.rs
index 3df9b25d81..3d6568825d 100644
--- a/src/service/logs/ingest.rs
+++ b/src/service/logs/ingest.rs
@@ -22,10 +22,11 @@ use actix_web::http;
use anyhow::Result;
use chrono::{Duration, Utc};
use config::{
+ get_config,
meta::{stream::StreamType, usage::UsageType},
metrics,
utils::{flatten, json, time::parse_timestamp_micro_from_value},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use flate2::read::GzDecoder;
use infra::schema::SchemaCache;
@@ -74,7 +75,8 @@ pub async fn ingest(
});
}
- let min_ts = (Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap())
+ let cfg = get_config();
+ let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap())
.timestamp_micros();
// Start Register Transforms for stream
@@ -299,7 +301,8 @@ pub fn apply_functions<'a>(
stream_name: &'a str,
runtime: &mut Runtime,
) -> Result {
- let mut value = flatten::flatten_with_level(item, CONFIG.limit.ingest_flatten_level)?;
+ let cfg = get_config();
+ let mut value = flatten::flatten_with_level(item, cfg.limit.ingest_flatten_level)?;
if !local_trans.is_empty() {
value = crate::service::ingestion::apply_stream_functions(
@@ -323,8 +326,9 @@ pub fn handle_timestamp(
local_val: &mut json::Map,
min_ts: i64,
) -> Result<(), anyhow::Error> {
+ let cfg = get_config();
// handle timestamp
- let timestamp = match local_val.get(&CONFIG.common.column_timestamp) {
+ let timestamp = match local_val.get(&cfg.common.column_timestamp) {
Some(v) => match parse_timestamp_micro_from_value(v) {
Ok(t) => t,
Err(_) => return Err(anyhow::Error::msg("Can't parse timestamp")),
@@ -336,7 +340,7 @@ pub fn handle_timestamp(
return Err(get_upto_discard_error());
}
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
Ok(())
@@ -502,7 +506,7 @@ fn deserialize_aws_record_from_str(data: &str, request_id: &str) -> Result Result,
need_trigger: bool,
) -> Result> {
+ let cfg = get_config();
let mut trigger: TriggerAlertData = Vec::new();
let timestamp: i64 = record_val
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap();
@@ -324,21 +325,20 @@ async fn add_valid_record(
let valid_record = match schema_evolution.types_delta {
None => true,
Some(delta) => {
- let ret_val = if !CONFIG.common.widening_schema_evolution
- || !schema_evolution.is_schema_changed
- {
- cast_to_type(&mut record_val, delta)
- } else {
- let local_delta = delta
- .into_iter()
- .filter(|x| x.metadata().contains_key("zo_cast"))
- .collect::>();
- if !local_delta.is_empty() {
- cast_to_type(&mut record_val, local_delta)
+ let ret_val =
+ if !cfg.common.widening_schema_evolution || !schema_evolution.is_schema_changed {
+ cast_to_type(&mut record_val, delta)
} else {
- Ok(())
- }
- };
+ let local_delta = delta
+ .into_iter()
+ .filter(|x| x.metadata().contains_key("zo_cast"))
+ .collect::>();
+ if !local_delta.is_empty() {
+ cast_to_type(&mut record_val, local_delta)
+ } else {
+ Ok(())
+ }
+ };
match ret_val {
Ok(_) => true,
Err(e) => {
@@ -407,8 +407,9 @@ async fn add_record(
write_buf: &mut HashMap,
record_val: Map,
) -> Result<()> {
+ let cfg = get_config();
let timestamp: i64 = record_val
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap();
@@ -475,7 +476,7 @@ pub fn refactor_map(
if has_elements {
new_map.insert(
- CONFIG.common.all_fields_name.to_string(),
+ get_config().common.all_fields_name.to_string(),
Value::String(String::from_utf8(non_schema_map).unwrap()),
);
}
diff --git a/src/service/logs/multi.rs b/src/service/logs/multi.rs
index f7473399e8..62e7f1e76e 100644
--- a/src/service/logs/multi.rs
+++ b/src/service/logs/multi.rs
@@ -25,7 +25,7 @@ use config::{
meta::{stream::StreamType, usage::UsageType},
metrics,
utils::{flatten, json, time::parse_timestamp_micro_from_value},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use infra::schema::SchemaCache;
@@ -71,6 +71,7 @@ async fn ingest_inner(
thread_id: usize,
) -> Result {
let start = std::time::Instant::now();
+ let cfg = config::get_config();
let mut stream_schema_map: HashMap = HashMap::new();
let mut distinct_values = Vec::with_capacity(16);
@@ -80,7 +81,7 @@ async fn ingest_inner(
check_ingestion_allowed(org_id, Some(stream_name))?;
let mut runtime = crate::service::ingestion::init_functions_runtime();
- let min_ts = (Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap())
+ let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap())
.timestamp_micros();
let mut stream_alerts_map: HashMap> = HashMap::new();
@@ -131,7 +132,7 @@ async fn ingest_inner(
}
// JSON Flattening
- value = flatten::flatten_with_level(value, CONFIG.limit.ingest_flatten_level)?;
+ value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level)?;
// Start row based transform
if !local_trans.is_empty() {
@@ -158,7 +159,7 @@ async fn ingest_inner(
};
// handle timestamp
- let timestamp = match local_val.get(&CONFIG.common.column_timestamp) {
+ let timestamp = match local_val.get(&cfg.common.column_timestamp) {
Some(v) => match parse_timestamp_micro_from_value(v) {
Ok(t) => t,
Err(e) => {
@@ -176,7 +177,7 @@ async fn ingest_inner(
continue;
}
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
diff --git a/src/service/logs/otlp_grpc.rs b/src/service/logs/otlp_grpc.rs
index cfd83b231c..899091f1ea 100644
--- a/src/service/logs/otlp_grpc.rs
+++ b/src/service/logs/otlp_grpc.rs
@@ -24,7 +24,7 @@ use config::{
meta::{stream::StreamType, usage::UsageType},
metrics,
utils::{flatten, json, time::parse_timestamp_micro_from_value},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use infra::schema::SchemaCache;
use opentelemetry::trace::{SpanId, TraceId};
@@ -72,7 +72,9 @@ pub async fn usage_ingest(
return Err(anyhow::anyhow!("not an ingester"));
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Err(anyhow::anyhow!(
"Quota exceeded for this organization [{}]",
org_id
@@ -84,7 +86,8 @@ pub async fn usage_ingest(
return Err(anyhow::anyhow!("stream [{stream_name}] is being deleted"));
}
- let min_ts = (Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap())
+ let cfg = config::get_config();
+ let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap())
.timestamp_micros();
let mut stream_alerts_map: HashMap> = HashMap::new();
@@ -117,7 +120,7 @@ pub async fn usage_ingest(
let reader: Vec = json::from_slice(&body)?;
for item in reader.into_iter() {
// JSON Flattening
- let mut value = flatten::flatten_with_level(item, CONFIG.limit.ingest_flatten_level)?;
+ let mut value = flatten::flatten_with_level(item, cfg.limit.ingest_flatten_level)?;
// get json object
let mut local_val = match value.take() {
@@ -129,7 +132,7 @@ pub async fn usage_ingest(
};
// handle timestamp
- let timestamp = match local_val.get(&CONFIG.common.column_timestamp) {
+ let timestamp = match local_val.get(&cfg.common.column_timestamp) {
Some(v) => match parse_timestamp_micro_from_value(v) {
Ok(t) => t,
Err(e) => {
@@ -147,7 +150,7 @@ pub async fn usage_ingest(
continue;
}
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
@@ -259,7 +262,9 @@ pub async fn handle_grpc_request(
);
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Ok(HttpResponse::Forbidden().json(MetaHttpResponse::error(
http::StatusCode::FORBIDDEN.into(),
format!("Quota exceeded for this organization [{}]", org_id),
@@ -342,6 +347,7 @@ pub async fn handle_grpc_request(
let mut data_buf: HashMap = HashMap::new();
+ let cfg = config::get_config();
for resource_log in &request.resource_logs {
for instrumentation_logs in &resource_log.scope_logs {
for log_record in &instrumentation_logs.log_records {
@@ -374,7 +380,7 @@ pub async fn handle_grpc_request(
// check ingestion time
let earlest_time =
- Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap();
+ Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap();
let ts = if log_record.time_unix_nano != 0 {
log_record.time_unix_nano / 1000
@@ -388,7 +394,7 @@ pub async fn handle_grpc_request(
continue;
}
- rec[CONFIG.common.column_timestamp.clone()] = ts.into();
+ rec[cfg.common.column_timestamp.clone()] = ts.into();
rec["severity"] = log_record.severity_text.to_owned().into();
// rec["name"] = log_record.name.to_owned().into();
rec["body"] = get_val(&log_record.body.as_ref());
@@ -425,7 +431,7 @@ pub async fn handle_grpc_request(
};
// flattening
- rec = flatten::flatten_with_level(rec, CONFIG.limit.ingest_flatten_level)?;
+ rec = flatten::flatten_with_level(rec, cfg.limit.ingest_flatten_level)?;
if !local_trans.is_empty() {
rec = crate::service::ingestion::apply_stream_functions(
diff --git a/src/service/logs/otlp_http.rs b/src/service/logs/otlp_http.rs
index cb1d55210f..94886d6f97 100644
--- a/src/service/logs/otlp_http.rs
+++ b/src/service/logs/otlp_http.rs
@@ -23,7 +23,7 @@ use config::{
meta::{stream::StreamType, usage::UsageType},
metrics,
utils::{flatten, json},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use infra::schema::SchemaCache;
use opentelemetry::trace::{SpanId, TraceId};
@@ -102,7 +102,9 @@ pub async fn logs_json_handler(
);
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Ok(HttpResponse::Forbidden().json(MetaHttpResponse::error(
http::StatusCode::FORBIDDEN.into(),
format!("Quota exceeded for this organization [{}]", org_id),
@@ -144,7 +146,8 @@ pub async fn logs_json_handler(
let mut stream_status = StreamStatus::new(stream_name);
let mut trigger: Option = None;
- let min_ts = (Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap())
+ let cfg = config::get_config();
+ let min_ts = (Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap())
.timestamp_micros();
let partition_det = crate::service::ingestion::get_stream_partition_keys(
@@ -350,9 +353,8 @@ pub async fn logs_json_handler(
stream_status.status.error = get_upto_discard_error().to_string();
continue;
}
-
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
@@ -361,8 +363,7 @@ pub async fn logs_json_handler(
value = json::to_value(local_val)?;
// JSON Flattening
- value =
- flatten::flatten_with_level(value, CONFIG.limit.ingest_flatten_level).unwrap();
+ value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap();
if !local_trans.is_empty() {
value = crate::service::ingestion::apply_stream_functions(
diff --git a/src/service/logs/syslog.rs b/src/service/logs/syslog.rs
index 00154960cd..09bc95d090 100644
--- a/src/service/logs/syslog.rs
+++ b/src/service/logs/syslog.rs
@@ -23,7 +23,7 @@ use config::{
meta::stream::StreamType,
metrics,
utils::{flatten, json, time::parse_timestamp_micro_from_value},
- CONFIG, DISTINCT_FIELDS,
+ DISTINCT_FIELDS,
};
use infra::schema::SchemaCache;
use syslog_loose::{Message, ProcId, Protocol};
@@ -129,9 +129,10 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result {
let mut buf: HashMap = HashMap::new();
+ let cfg = config::get_config();
let parsed_msg = syslog_loose::parse_message(msg);
let mut value = message_to_value(parsed_msg);
- value = flatten::flatten_with_level(value, CONFIG.limit.ingest_flatten_level).unwrap();
+ value = flatten::flatten_with_level(value, cfg.limit.ingest_flatten_level).unwrap();
if !local_trans.is_empty() {
value = crate::service::ingestion::apply_stream_functions(
@@ -160,7 +161,7 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result {
};
// handle timestamp
- let timestamp = match local_val.get(&CONFIG.common.column_timestamp) {
+ let timestamp = match local_val.get(&cfg.common.column_timestamp) {
Some(v) => match parse_timestamp_micro_from_value(v) {
Ok(t) => t,
Err(_) => Utc::now().timestamp_micros(),
@@ -168,14 +169,14 @@ pub async fn ingest(msg: &str, addr: SocketAddr) -> Result {
None => Utc::now().timestamp_micros(),
};
// check ingestion time
- let earlest_time = Utc::now() - Duration::try_hours(CONFIG.limit.ingest_allowed_upto).unwrap();
+ let earlest_time = Utc::now() - Duration::try_hours(cfg.limit.ingest_allowed_upto).unwrap();
if timestamp < earlest_time.timestamp_micros() {
stream_status.status.failed += 1; // to old data, just discard
stream_status.status.error = get_upto_discard_error().to_string();
}
local_val.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
diff --git a/src/service/metadata/distinct_values.rs b/src/service/metadata/distinct_values.rs
index 48a188f42f..273dd0128b 100644
--- a/src/service/metadata/distinct_values.rs
+++ b/src/service/metadata/distinct_values.rs
@@ -23,9 +23,10 @@ use std::{
use arrow_schema::{DataType, Field, Schema};
use config::{
+ get_config,
meta::stream::StreamType,
utils::{json, schema_ext::SchemaExt},
- FxIndexMap, CONFIG,
+ FxIndexMap,
};
use infra::{
errors::{Error, Result},
@@ -152,7 +153,7 @@ impl Metadata for DistinctValues {
fn generate_schema(&self) -> Arc {
Arc::new(Schema::new(vec![
Field::new(
- CONFIG.common.column_timestamp.as_str(),
+ get_config().common.column_timestamp.as_str(),
DataType::Int64,
false,
),
@@ -222,7 +223,7 @@ impl Metadata for DistinctValues {
let data = data.as_object_mut().unwrap();
data.insert("count".to_string(), json::Value::Number(count.into()));
data.insert(
- CONFIG.common.column_timestamp.clone(),
+ get_config().common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
let hour_key = ingestion::get_wal_time_key(
@@ -274,7 +275,7 @@ impl Metadata for DistinctValues {
async fn run_flush() {
let mut interval = time::interval(time::Duration::from_secs(
- CONFIG.limit.distinct_values_interval,
+ get_config().limit.distinct_values_interval,
));
interval.tick().await; // trigger the first run
loop {
diff --git a/src/service/metadata/trace_list_index.rs b/src/service/metadata/trace_list_index.rs
index fe7431207b..3a54e28e8f 100644
--- a/src/service/metadata/trace_list_index.rs
+++ b/src/service/metadata/trace_list_index.rs
@@ -24,9 +24,9 @@ use std::{
use arrow_schema::{DataType, Field, Schema};
use config::{
+ get_config,
meta::stream::{StreamPartition, StreamSettings, StreamType},
utils::{json, schema_ext::SchemaExt},
- CONFIG,
};
use infra::schema::unwrap_partition_time_level;
use once_cell::sync::Lazy;
@@ -65,7 +65,7 @@ impl Metadata for TraceListIndex {
fn generate_schema(&self) -> Arc {
Arc::new(Schema::new(vec![
Field::new(
- CONFIG.common.column_timestamp.as_str(),
+ get_config().common.column_timestamp.as_str(),
DataType::Int64,
false,
),
@@ -202,7 +202,7 @@ impl TraceListIndex {
mod tests {
use std::{collections::HashMap, sync::Arc};
- use config::{meta::stream::StreamType, utils::json, CONFIG};
+ use config::{meta::stream::StreamType, utils::json};
use infra::schema::unwrap_partition_time_level;
use crate::{
@@ -240,7 +240,7 @@ mod tests {
let mut data = json::to_value(item).unwrap();
let data = data.as_object_mut().unwrap();
data.insert(
- CONFIG.common.column_timestamp.clone(),
+ config::get_config().common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
let hour_key = ingestion::get_wal_time_key(
diff --git a/src/service/metrics/json.rs b/src/service/metrics/json.rs
index 9c53b2da1b..7ffa1d695c 100644
--- a/src/service/metrics/json.rs
+++ b/src/service/metrics/json.rs
@@ -25,7 +25,6 @@ use config::{
},
metrics,
utils::{flatten, json, schema::infer_json_schema, schema_ext::SchemaExt, time},
- CONFIG,
};
use datafusion::arrow::datatypes::Schema;
use infra::schema::{unwrap_partition_time_level, SchemaCache};
@@ -54,7 +53,9 @@ pub async fn ingest(org_id: &str, body: web::Bytes, thread_id: usize) -> Result<
return Err(anyhow::anyhow!("not an ingester"));
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Err(anyhow::anyhow!(
"Quota exceeded for this organization [{}]",
org_id
@@ -134,8 +135,9 @@ pub async fn ingest(org_id: &str, body: web::Bytes, thread_id: usize) -> Result<
let record = record.as_object_mut().unwrap();
+ let cfg = config::get_config();
// check timestamp & value
- let timestamp: i64 = match record.get(&CONFIG.common.column_timestamp) {
+ let timestamp: i64 = match record.get(&cfg.common.column_timestamp) {
None => chrono::Utc::now().timestamp_micros(),
Some(json::Value::Number(s)) => {
time::parse_i64_to_timestamp_micros(s.as_f64().unwrap() as i64)
@@ -152,7 +154,7 @@ pub async fn ingest(org_id: &str, body: web::Bytes, thread_id: usize) -> Result<
};
// reset time & value
record.insert(
- CONFIG.common.column_timestamp.clone(),
+ cfg.common.column_timestamp.clone(),
json::Value::Number(timestamp.into()),
);
record.insert(
@@ -170,7 +172,7 @@ pub async fn ingest(org_id: &str, body: web::Bytes, thread_id: usize) -> Result<
if k == NAME_LABEL
|| k == TYPE_LABEL
|| k == VALUE_LABEL
- || k == &CONFIG.common.column_timestamp
+ || k == &cfg.common.column_timestamp
{
continue;
}
diff --git a/src/service/metrics/mod.rs b/src/service/metrics/mod.rs
index 30348525d8..9ce1544318 100644
--- a/src/service/metrics/mod.rs
+++ b/src/service/metrics/mod.rs
@@ -13,7 +13,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-use config::CONFIG;
use datafusion::arrow::datatypes::Schema;
use once_cell::sync::Lazy;
use regex::Regex;
@@ -25,7 +24,13 @@ pub mod otlp_grpc;
pub mod otlp_http;
pub mod prom;
-const EXCLUDE_LABELS: [&str; 4] = [VALUE_LABEL, HASH_LABEL, "is_monotonic", "exemplars"];
+const EXCLUDE_LABELS: [&str; 5] = [
+ VALUE_LABEL,
+ HASH_LABEL,
+ "is_monotonic",
+ "exemplars",
+ "_timestamp",
+];
static RE_CORRECT_LABEL_NAME: Lazy = Lazy::new(|| Regex::new(r"[^a-zA-Z0-9_]+").unwrap());
@@ -67,8 +72,10 @@ pub fn signature_without_labels(
}
fn get_exclude_labels() -> Vec<&'static str> {
- let mut vec: Vec<&str> = EXCLUDE_LABELS.to_vec();
- vec.push(&CONFIG.common.column_timestamp);
+ let vec: Vec<&str> = EXCLUDE_LABELS.to_vec();
+ // TODO: fixed _timestamp
+ // let column_timestamp = config::get_config().common.column_timestamp.as_str();
+ // vec.push(column_timestamp);
vec
}
diff --git a/src/service/metrics/otlp_grpc.rs b/src/service/metrics/otlp_grpc.rs
index edb3ee73ff..9fa2434f0c 100644
--- a/src/service/metrics/otlp_grpc.rs
+++ b/src/service/metrics/otlp_grpc.rs
@@ -19,14 +19,13 @@ use actix_web::{http, HttpResponse};
use bytes::BytesMut;
use chrono::Utc;
use config::{
- cluster,
+ cluster, get_config,
meta::{
stream::{PartitioningDetails, StreamType},
usage::UsageType,
},
metrics,
utils::{flatten, json, schema_ext::SchemaExt},
- CONFIG,
};
use hashbrown::HashSet;
use infra::schema::{unwrap_partition_time_level, update_setting, SchemaCache};
@@ -72,7 +71,9 @@ pub async fn handle_grpc_request(
);
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Ok(HttpResponse::Forbidden().json(MetaHttpResponse::error(
http::StatusCode::FORBIDDEN.into(),
format!("Quota exceeded for this organization [{}]", org_id),
@@ -98,6 +99,7 @@ pub async fn handle_grpc_request(
let mut stream_trigger_map: HashMap> = HashMap::new();
let mut stream_partitioning_map: HashMap = HashMap::new();
+ let cfg = get_config();
for resource_metric in &request.resource_metrics {
for scope_metric in &resource_metric.scope_metrics {
for metric in &scope_metric.metrics {
@@ -214,7 +216,6 @@ pub async fn handle_grpc_request(
);
}
}
-
for mut rec in records {
// flattening
rec = flatten::flatten(rec)?;
@@ -289,7 +290,7 @@ pub async fn handle_grpc_request(
rec.as_object_mut().unwrap();
let timestamp = val_map
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap_or(Utc::now().timestamp_micros());
@@ -602,7 +603,7 @@ fn process_data_point(rec: &mut json::Value, data_point: &NumberDataPoint) {
rec[format_label_name(attr.key.as_str())] = get_val(&attr.value.as_ref());
}
rec[VALUE_LABEL] = get_metric_val(&data_point.value);
- rec[&CONFIG.common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
+ rec[&get_config().common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
rec["start_time"] = data_point.start_time_unix_nano.to_string().into();
rec["flag"] = if data_point.flags == 1 {
DataPointFlags::NoRecordedValueMask.as_str_name()
@@ -622,7 +623,7 @@ fn process_hist_data_point(
for attr in &data_point.attributes {
rec[format_label_name(attr.key.as_str())] = get_val(&attr.value.as_ref());
}
- rec[&CONFIG.common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
+ rec[&get_config().common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
rec["start_time"] = data_point.start_time_unix_nano.to_string().into();
rec["flag"] = if data_point.flags == 1 {
DataPointFlags::NoRecordedValueMask.as_str_name()
@@ -671,7 +672,7 @@ fn process_exp_hist_data_point(
for attr in &data_point.attributes {
rec[format_label_name(attr.key.as_str())] = get_val(&attr.value.as_ref());
}
- rec[&CONFIG.common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
+ rec[&get_config().common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
rec["start_time"] = data_point.start_time_unix_nano.to_string().into();
rec["flag"] = if data_point.flags == 1 {
DataPointFlags::NoRecordedValueMask.as_str_name()
@@ -736,7 +737,7 @@ fn process_summary_data_point(
for attr in &data_point.attributes {
rec[format_label_name(attr.key.as_str())] = get_val(&attr.value.as_ref());
}
- rec[&CONFIG.common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
+ rec[&get_config().common.column_timestamp] = (data_point.time_unix_nano / 1000).into();
rec["start_time"] = data_point.start_time_unix_nano.to_string().into();
rec["flag"] = if data_point.flags == 1 {
DataPointFlags::NoRecordedValueMask.as_str_name()
@@ -774,7 +775,8 @@ fn process_exemplars(rec: &mut json::Value, exemplars: &Vec) {
exemplar_rec[attr.key.as_str()] = get_val(&attr.value.as_ref());
}
exemplar_rec[VALUE_LABEL] = get_exemplar_val(&exemplar.value);
- exemplar_rec[&CONFIG.common.column_timestamp] = (exemplar.time_unix_nano / 1000).into();
+ exemplar_rec[&get_config().common.column_timestamp] =
+ (exemplar.time_unix_nano / 1000).into();
match TraceId::from_bytes(exemplar.trace_id.as_slice().try_into().unwrap_or_default()) {
TraceId::INVALID => {}
diff --git a/src/service/metrics/otlp_http.rs b/src/service/metrics/otlp_http.rs
index a73746fbd1..0ff21a80f8 100644
--- a/src/service/metrics/otlp_http.rs
+++ b/src/service/metrics/otlp_http.rs
@@ -19,14 +19,13 @@ use actix_web::{http, web, HttpResponse};
use bytes::BytesMut;
use chrono::Utc;
use config::{
- cluster,
+ cluster, get_config,
meta::{
stream::{PartitioningDetails, StreamType},
usage::UsageType,
},
metrics,
utils::{flatten, json, schema_ext::SchemaExt},
- CONFIG,
};
use hashbrown::HashSet;
use infra::schema::{unwrap_partition_time_level, update_setting, SchemaCache};
@@ -91,7 +90,9 @@ pub async fn metrics_json_handler(
);
}
- if !db::file_list::BLOCKED_ORGS.is_empty() && db::file_list::BLOCKED_ORGS.contains(&org_id) {
+ if !db::file_list::BLOCKED_ORGS.is_empty()
+ && db::file_list::BLOCKED_ORGS.contains(&org_id.to_string())
+ {
return Ok(HttpResponse::Forbidden().json(MetaHttpResponse::error(
http::StatusCode::FORBIDDEN.into(),
format!("Quota exceeded for this organization [{}]", org_id),
@@ -145,6 +146,7 @@ pub async fn metrics_json_handler(
}
};
+ let cfg = get_config();
for res_metric in res_metrics.iter() {
let mut service_att_map: json::Map = json::Map::new();
if res_metric.get("resource").is_some() {
@@ -381,7 +383,7 @@ pub async fn metrics_json_handler(
rec.as_object_mut().unwrap();
let timestamp = val_map
- .get(&CONFIG.common.column_timestamp)
+ .get(&cfg.common.column_timestamp)
.unwrap()
.as_i64()
.unwrap_or(Utc::now().timestamp_micros());
@@ -764,7 +766,7 @@ fn process_data_point(rec: &mut json::Value, data_point: &json::Map>()
.join("\", \"");
if label_names.is_empty() {
@@ -578,7 +582,7 @@ pub(crate) async fn get_series(
let mut sql_where = Vec::new();
if let Some(selector) = selector {
for mat in selector.matchers.matchers.iter() {
- if mat.name == CONFIG.common.column_timestamp
+ if mat.name == cfg.common.column_timestamp
|| mat.name == VALUE_LABEL
|| schema.field_with_name(&mat.name).is_err()
{
@@ -652,6 +656,7 @@ pub(crate) async fn get_labels(
Ok(schemas) => schemas,
};
let mut label_names = hashbrown::HashSet::new();
+ let cfg = get_config();
for schema in stream_schemas {
if let Some(ref metric_name) = opt_metric_name {
if *metric_name != schema.stream_name {
@@ -668,7 +673,7 @@ pub(crate) async fn get_labels(
.iter()
.map(|f| f.name())
.filter(|&s| {
- s != &CONFIG.common.column_timestamp && s != VALUE_LABEL && s != HASH_LABEL
+ s != &cfg.common.column_timestamp && s != VALUE_LABEL && s != HASH_LABEL
})
.cloned();
label_names.extend(field_names);
diff --git a/src/service/promql/aggregations/avg.rs b/src/service/promql/aggregations/avg.rs
index e45dd4098d..5792f18622 100644
--- a/src/service/promql/aggregations/avg.rs
+++ b/src/service/promql/aggregations/avg.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/bottomk.rs b/src/service/promql/aggregations/bottomk.rs
index eeb5cca820..78721d56fc 100644
--- a/src/service/promql/aggregations/bottomk.rs
+++ b/src/service/promql/aggregations/bottomk.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/count.rs b/src/service/promql/aggregations/count.rs
index 12d892a58f..cd8688f751 100644
--- a/src/service/promql/aggregations/count.rs
+++ b/src/service/promql/aggregations/count.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/count_values.rs b/src/service/promql/aggregations/count_values.rs
index cc62f44819..30950113b3 100644
--- a/src/service/promql/aggregations/count_values.rs
+++ b/src/service/promql/aggregations/count_values.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/group.rs b/src/service/promql/aggregations/group.rs
index 9eae677cc0..e8cce1b5fc 100644
--- a/src/service/promql/aggregations/group.rs
+++ b/src/service/promql/aggregations/group.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/max.rs b/src/service/promql/aggregations/max.rs
index d289a61058..3497cb0cd5 100644
--- a/src/service/promql/aggregations/max.rs
+++ b/src/service/promql/aggregations/max.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/min.rs b/src/service/promql/aggregations/min.rs
index a8b260d0c1..3fa94dd22a 100644
--- a/src/service/promql/aggregations/min.rs
+++ b/src/service/promql/aggregations/min.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/mod.rs b/src/service/promql/aggregations/mod.rs
index 7c1c28558e..1b332b72f8 100644
--- a/src/service/promql/aggregations/mod.rs
+++ b/src/service/promql/aggregations/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/quantile.rs b/src/service/promql/aggregations/quantile.rs
index d11fd93ba7..51101d28e6 100644
--- a/src/service/promql/aggregations/quantile.rs
+++ b/src/service/promql/aggregations/quantile.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/stddev.rs b/src/service/promql/aggregations/stddev.rs
index 2c2b66f2ca..c42c38a2d5 100644
--- a/src/service/promql/aggregations/stddev.rs
+++ b/src/service/promql/aggregations/stddev.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/stdvar.rs b/src/service/promql/aggregations/stdvar.rs
index fe1bacfb78..9303d53de2 100644
--- a/src/service/promql/aggregations/stdvar.rs
+++ b/src/service/promql/aggregations/stdvar.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/sum.rs b/src/service/promql/aggregations/sum.rs
index badd670819..38414caa24 100644
--- a/src/service/promql/aggregations/sum.rs
+++ b/src/service/promql/aggregations/sum.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/aggregations/topk.rs b/src/service/promql/aggregations/topk.rs
index 7e2552fe46..4612712a2e 100644
--- a/src/service/promql/aggregations/topk.rs
+++ b/src/service/promql/aggregations/topk.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/binaries/mod.rs b/src/service/promql/binaries/mod.rs
index 415e4f6ee2..0f78cb55fb 100644
--- a/src/service/promql/binaries/mod.rs
+++ b/src/service/promql/binaries/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/binaries/scalar_operations.rs b/src/service/promql/binaries/scalar_operations.rs
index 7d446cf799..ead44c437b 100644
--- a/src/service/promql/binaries/scalar_operations.rs
+++ b/src/service/promql/binaries/scalar_operations.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/binaries/vector_operations.rs b/src/service/promql/binaries/vector_operations.rs
index c76ef64f25..1acfc0fa5a 100644
--- a/src/service/promql/binaries/vector_operations.rs
+++ b/src/service/promql/binaries/vector_operations.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/common.rs b/src/service/promql/common.rs
index e90af6064b..76e187a579 100644
--- a/src/service/promql/common.rs
+++ b/src/service/promql/common.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/engine.rs b/src/service/promql/engine.rs
index 7bc08a0862..9ea11f8bca 100644
--- a/src/service/promql/engine.rs
+++ b/src/service/promql/engine.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,6 @@
use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration};
use async_recursion::async_recursion;
-use config::CONFIG;
use datafusion::{
arrow::{
array::{Float64Array, Int64Array, StringArray},
@@ -916,13 +915,14 @@ async fn selector_load_data_from_datafusion(
}
};
+ let cfg = config::get_config();
let mut df_group = table.clone().filter(
- col(&CONFIG.common.column_timestamp)
+ col(&cfg.common.column_timestamp)
.gt(lit(start))
- .and(col(&CONFIG.common.column_timestamp).lt_eq(lit(end))),
+ .and(col(&cfg.common.column_timestamp).lt_eq(lit(end))),
)?;
for mat in selector.matchers.matchers.iter() {
- if mat.name == CONFIG.common.column_timestamp
+ if mat.name == cfg.common.column_timestamp
|| mat.name == VALUE_LABEL
|| schema.field_with_name(&mat.name).is_err()
{
@@ -952,7 +952,7 @@ async fn selector_load_data_from_datafusion(
}
}
let batches = df_group
- .sort(vec![col(&CONFIG.common.column_timestamp).sort(true, true)])?
+ .sort(vec![col(&cfg.common.column_timestamp).sort(true, true)])?
.collect()
.await?;
@@ -965,7 +965,7 @@ async fn selector_load_data_from_datafusion(
.downcast_ref::()
.unwrap();
let time_values = batch
- .column_by_name(&CONFIG.common.column_timestamp)
+ .column_by_name(&cfg.common.column_timestamp)
.unwrap()
.as_any()
.downcast_ref::()
@@ -982,7 +982,7 @@ async fn selector_load_data_from_datafusion(
let mut labels = Vec::with_capacity(batch.num_columns());
for (k, v) in batch.schema().fields().iter().zip(batch.columns()) {
let name = k.name();
- if name == &CONFIG.common.column_timestamp
+ if name == &cfg.common.column_timestamp
|| name == HASH_LABEL
|| name == VALUE_LABEL
{
diff --git a/src/service/promql/functions/absent.rs b/src/service/promql/functions/absent.rs
index b727ffeec4..36282853aa 100644
--- a/src/service/promql/functions/absent.rs
+++ b/src/service/promql/functions/absent.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/absent_over_time.rs b/src/service/promql/functions/absent_over_time.rs
index 657aa845b5..ba554aac6a 100644
--- a/src/service/promql/functions/absent_over_time.rs
+++ b/src/service/promql/functions/absent_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/avg_over_time.rs b/src/service/promql/functions/avg_over_time.rs
index af770251b7..65f8446951 100644
--- a/src/service/promql/functions/avg_over_time.rs
+++ b/src/service/promql/functions/avg_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/changes.rs b/src/service/promql/functions/changes.rs
index e33e70717e..fd37e56cf4 100644
--- a/src/service/promql/functions/changes.rs
+++ b/src/service/promql/functions/changes.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/clamp.rs b/src/service/promql/functions/clamp.rs
index eb568e4b64..8fc82e58f1 100644
--- a/src/service/promql/functions/clamp.rs
+++ b/src/service/promql/functions/clamp.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/count_over_time.rs b/src/service/promql/functions/count_over_time.rs
index f826fb24a8..8ed032b2d9 100644
--- a/src/service/promql/functions/count_over_time.rs
+++ b/src/service/promql/functions/count_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/delta.rs b/src/service/promql/functions/delta.rs
index d2bb5951d2..370acc1235 100644
--- a/src/service/promql/functions/delta.rs
+++ b/src/service/promql/functions/delta.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/deriv.rs b/src/service/promql/functions/deriv.rs
index 3cc32e3d9e..821ad43fa1 100644
--- a/src/service/promql/functions/deriv.rs
+++ b/src/service/promql/functions/deriv.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/histogram.rs b/src/service/promql/functions/histogram.rs
index ef7aaa3bff..0079765144 100644
--- a/src/service/promql/functions/histogram.rs
+++ b/src/service/promql/functions/histogram.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/holt_winters.rs b/src/service/promql/functions/holt_winters.rs
index afccdd3f21..d4d8eee26b 100644
--- a/src/service/promql/functions/holt_winters.rs
+++ b/src/service/promql/functions/holt_winters.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/idelta.rs b/src/service/promql/functions/idelta.rs
index ba96795da9..50afbd09f7 100644
--- a/src/service/promql/functions/idelta.rs
+++ b/src/service/promql/functions/idelta.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/increase.rs b/src/service/promql/functions/increase.rs
index c79e71c9ab..0dc1d19d13 100644
--- a/src/service/promql/functions/increase.rs
+++ b/src/service/promql/functions/increase.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/irate.rs b/src/service/promql/functions/irate.rs
index 33fed0dfdb..be0d0dabe7 100644
--- a/src/service/promql/functions/irate.rs
+++ b/src/service/promql/functions/irate.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/label_join.rs b/src/service/promql/functions/label_join.rs
index 1fa49842ee..18af036c30 100644
--- a/src/service/promql/functions/label_join.rs
+++ b/src/service/promql/functions/label_join.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/label_replace.rs b/src/service/promql/functions/label_replace.rs
index c72278fb9d..51b6abde1e 100644
--- a/src/service/promql/functions/label_replace.rs
+++ b/src/service/promql/functions/label_replace.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/last_over_time.rs b/src/service/promql/functions/last_over_time.rs
index c6ef88193e..04263e0d8f 100644
--- a/src/service/promql/functions/last_over_time.rs
+++ b/src/service/promql/functions/last_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/max_over_time.rs b/src/service/promql/functions/max_over_time.rs
index 701b7c8fc9..42c57c1379 100644
--- a/src/service/promql/functions/max_over_time.rs
+++ b/src/service/promql/functions/max_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/min_over_time.rs b/src/service/promql/functions/min_over_time.rs
index 40f7c978f2..b9759afde3 100644
--- a/src/service/promql/functions/min_over_time.rs
+++ b/src/service/promql/functions/min_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/mod.rs b/src/service/promql/functions/mod.rs
index 4bbb98b4e0..e07e15bb13 100644
--- a/src/service/promql/functions/mod.rs
+++ b/src/service/promql/functions/mod.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/predict_linear.rs b/src/service/promql/functions/predict_linear.rs
index 19ccbf70cf..b441e00d21 100644
--- a/src/service/promql/functions/predict_linear.rs
+++ b/src/service/promql/functions/predict_linear.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/quantile_over_time.rs b/src/service/promql/functions/quantile_over_time.rs
index 37fb220bdf..e1bd9f6603 100644
--- a/src/service/promql/functions/quantile_over_time.rs
+++ b/src/service/promql/functions/quantile_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/rate.rs b/src/service/promql/functions/rate.rs
index 27b0605c28..929d0e1eaf 100644
--- a/src/service/promql/functions/rate.rs
+++ b/src/service/promql/functions/rate.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/resets.rs b/src/service/promql/functions/resets.rs
index c87f38c50a..960aae2b4f 100644
--- a/src/service/promql/functions/resets.rs
+++ b/src/service/promql/functions/resets.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/stddev_over_time.rs b/src/service/promql/functions/stddev_over_time.rs
index c7b69e1689..560956b3f2 100644
--- a/src/service/promql/functions/stddev_over_time.rs
+++ b/src/service/promql/functions/stddev_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/stdvar_over_time.rs b/src/service/promql/functions/stdvar_over_time.rs
index 10ea25df93..845fe52285 100644
--- a/src/service/promql/functions/stdvar_over_time.rs
+++ b/src/service/promql/functions/stdvar_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/sum_over_time.rs b/src/service/promql/functions/sum_over_time.rs
index 9342868143..a8059e7436 100644
--- a/src/service/promql/functions/sum_over_time.rs
+++ b/src/service/promql/functions/sum_over_time.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/functions/vector.rs b/src/service/promql/functions/vector.rs
index cd5e958ff0..06dd7974cb 100644
--- a/src/service/promql/functions/vector.rs
+++ b/src/service/promql/functions/vector.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/promql/search/grpc/mod.rs b/src/service/promql/search/grpc/mod.rs
index f9cba6ef3e..60c38e5389 100644
--- a/src/service/promql/search/grpc/mod.rs
+++ b/src/service/promql/search/grpc/mod.rs
@@ -19,7 +19,6 @@ use std::{
};
use async_trait::async_trait;
-use config::CONFIG;
use datafusion::{arrow::datatypes::Schema, error::DataFusionError, prelude::SessionContext};
use infra::{cache::tmpfs, errors::Result};
use promql_parser::parser;
@@ -97,7 +96,7 @@ pub async fn search(
let timeout = if req.timeout > 0 {
req.timeout as u64
} else {
- CONFIG.limit.query_timeout
+ config::get_config().limit.query_timeout
};
let mut engine = Query::new(
diff --git a/src/service/promql/search/grpc/storage.rs b/src/service/promql/search/grpc/storage.rs
index db5d41b0fb..2c2067452a 100644
--- a/src/service/promql/search/grpc/storage.rs
+++ b/src/service/promql/search/grpc/storage.rs
@@ -21,7 +21,6 @@ use config::{
search::{ScanStats, SearchType, Session as SearchSession, StorageType},
stream::{FileKey, PartitionTimeLevel, StreamPartition, StreamType},
},
- CONFIG,
};
use datafusion::{
arrow::datatypes::Schema,
@@ -220,14 +219,15 @@ async fn cache_parquet_files(
files: &[FileKey],
scan_stats: &ScanStats,
) -> Result<(file_data::CacheType, Vec)> {
- let cache_type = if CONFIG.memory_cache.enabled
- && scan_stats.compressed_size < CONFIG.memory_cache.skip_size as i64
+ let cfg = config::get_config();
+ let cache_type = if cfg.memory_cache.enabled
+ && scan_stats.compressed_size < cfg.memory_cache.skip_size as i64
{
// if scan_compressed_size < 80% of total memory cache, use memory cache
file_data::CacheType::Memory
} else if !is_local_disk_storage()
- && CONFIG.disk_cache.enabled
- && scan_stats.compressed_size < CONFIG.disk_cache.skip_size as i64
+ && cfg.disk_cache.enabled
+ && scan_stats.compressed_size < cfg.disk_cache.skip_size as i64
{
// if scan_compressed_size < 80% of total disk cache, use disk cache
file_data::CacheType::Disk
@@ -237,7 +237,7 @@ async fn cache_parquet_files(
};
let mut tasks = Vec::new();
- let semaphore = std::sync::Arc::new(Semaphore::new(CONFIG.limit.query_thread_num));
+ let semaphore = std::sync::Arc::new(Semaphore::new(cfg.limit.query_thread_num));
for file in files.iter() {
let trace_id = "";
let file_name = file.key.clone();
diff --git a/src/service/promql/search/grpc/wal.rs b/src/service/promql/search/grpc/wal.rs
index ca4e4e1fc7..347a9bf4da 100644
--- a/src/service/promql/search/grpc/wal.rs
+++ b/src/service/promql/search/grpc/wal.rs
@@ -17,11 +17,12 @@ use std::{collections::HashMap, io::Cursor, sync::Arc};
use arrow::{ipc::reader::StreamReader, record_batch::RecordBatch};
use config::{
+ get_config,
meta::{
search::{ScanStats, SearchType, Session as SearchSession, StorageType},
stream::StreamType,
},
- CONFIG, FILE_EXT_PARQUET,
+ FILE_EXT_PARQUET,
};
use datafusion::{
arrow::datatypes::Schema,
@@ -33,7 +34,12 @@ use datafusion::{
use futures::future::try_join_all;
use infra::cache::tmpfs;
use proto::cluster_rpc;
-use tonic::{codec::CompressionEncoding, metadata::MetadataValue, transport::Channel, Request};
+use tonic::{
+ codec::CompressionEncoding,
+ metadata::{MetadataKey, MetadataValue},
+ transport::Channel,
+ Request,
+};
use tracing::{info_span, Instrument};
use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -228,11 +234,12 @@ async fn get_file_list(
std::result::Result,
> = tokio::task::spawn(
async move {
+ let cfg = get_config();
let org_id: MetadataValue<_> = org_id
.parse()
.map_err(|_| DataFusionError::Execution("invalid org_id".to_string()))?;
let mut request = tonic::Request::new(req);
- // request.set_timeout(Duration::from_secs(CONFIG.grpc.timeout));
+ // request.set_timeout(Duration::from_secs(cfg.grpc.timeout));
opentelemetry::global::get_text_map_propagator(|propagator| {
propagator.inject_context(
@@ -241,12 +248,16 @@ async fn get_file_list(
)
});
+ let org_header_key: MetadataKey<_> =
+ cfg.grpc.org_header_key.parse().map_err(|_| {
+ DataFusionError::Execution("invalid org_header_key".to_string())
+ })?;
let token: MetadataValue<_> = get_internal_grpc_token()
.parse()
.map_err(|_| DataFusionError::Execution("invalid token".to_string()))?;
let channel = Channel::from_shared(node_addr)
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(cfg.grpc.connect_timeout))
.connect()
.await
.map_err(|_| {
@@ -257,15 +268,16 @@ async fn get_file_list(
move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
- .insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
+ .insert(org_header_key.clone(), org_id.clone());
Ok(req)
},
);
+ let cfg = get_config();
client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
let response: cluster_rpc::MetricsWalFileResponse = match client
.wal_file(request)
.await
diff --git a/src/service/promql/search/mod.rs b/src/service/promql/search/mod.rs
index 06edd15dfb..14c4293988 100644
--- a/src/service/promql/search/mod.rs
+++ b/src/service/promql/search/mod.rs
@@ -25,13 +25,17 @@ use config::{
stream::StreamType,
usage::{RequestStats, UsageType},
},
- CONFIG,
};
use futures::future::try_join_all;
use hashbrown::HashMap;
use infra::errors::{Error, ErrorCodes, Result};
use proto::cluster_rpc;
-use tonic::{codec::CompressionEncoding, metadata::MetadataValue, transport::Channel, Request};
+use tonic::{
+ codec::CompressionEncoding,
+ metadata::{MetadataKey, MetadataValue},
+ transport::Channel,
+ Request,
+};
use tracing::{info_span, Instrument};
use tracing_opentelemetry::OpenTelemetrySpanExt;
@@ -151,12 +155,13 @@ async fn search_in_cluster(
let grpc_span = info_span!("promql:search:cluster:grpc_search", org_id = req.org_id);
let task = tokio::task::spawn(
async move {
+ let cfg = config::get_config();
let org_id: MetadataValue<_> = req
.org_id
.parse()
.map_err(|_| Error::Message(format!("invalid org_id: {}", req.org_id)))?;
let mut request = tonic::Request::new(req);
- // request.set_timeout(Duration::from_secs(CONFIG.grpc.timeout));
+ // request.set_timeout(Duration::from_secs(cfg.grpc.timeout));
opentelemetry::global::get_text_map_propagator(|propagator| {
propagator.inject_context(
@@ -165,12 +170,13 @@ async fn search_in_cluster(
)
});
+ let org_header_key: MetadataKey<_> = cfg.grpc.org_header_key.parse().map_err(|_| Error::Message("invalid org_header_key".to_string()))?;
let token: MetadataValue<_> = cluster::get_internal_grpc_token()
.parse()
.map_err(|_| Error::Message("invalid token".to_string()))?;
let channel = Channel::from_shared(node_addr)
.unwrap()
- .connect_timeout(std::time::Duration::from_secs(CONFIG.grpc.connect_timeout))
+ .connect_timeout(std::time::Duration::from_secs(cfg.grpc.connect_timeout))
.connect()
.await
.map_err(|err| {
@@ -181,20 +187,21 @@ async fn search_in_cluster(
);
server_internal_error("connect search node error")
})?;
- let mut client = cluster_rpc::metrics_client::MetricsClient::with_interceptor(
- channel,
- move |mut req: Request<()>| {
+
+ let mut client = cluster_rpc::metrics_client::MetricsClient::with_interceptor(
+ channel,
+ move |mut req: Request<()>| {
req.metadata_mut().insert("authorization", token.clone());
req.metadata_mut()
- .insert(CONFIG.grpc.org_header_key.as_str(), org_id.clone());
+ .insert(org_header_key.clone(), org_id.clone());
Ok(req)
},
);
- client = client
+ client = client
.send_compressed(CompressionEncoding::Gzip)
.accept_compressed(CompressionEncoding::Gzip)
- .max_decoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024)
- .max_encoding_message_size(CONFIG.grpc.max_message_size * 1024 * 1024);
+ .max_decoding_message_size(cfg.grpc.max_message_size * 1024 * 1024)
+ .max_encoding_message_size(cfg.grpc.max_message_size * 1024 * 1024);
let response: cluster_rpc::MetricsQueryResponse = match client.query(request).await
{
Ok(res) => res.into_inner(),
diff --git a/src/service/promql/value.rs b/src/service/promql/value.rs
index 38f595d12b..7022c1a0e4 100644
--- a/src/service/promql/value.rs
+++ b/src/service/promql/value.rs
@@ -1,4 +1,4 @@
-// Copyright 2023 Zinc Labs Inc.
+// Copyright 2024 Zinc Labs Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
diff --git a/src/service/schema.rs b/src/service/schema.rs
index c78b44ed0c..2edf2b5852 100644
--- a/src/service/schema.rs
+++ b/src/service/schema.rs
@@ -17,9 +17,9 @@ use std::{collections::HashMap, sync::Arc};
use anyhow::Result;
use config::{
+ get_config,
meta::stream::StreamType,
utils::{json, schema::infer_json_schema_from_map, schema_ext::SchemaExt},
- CONFIG,
};
use datafusion::arrow::datatypes::{Field, Schema};
use hashbrown::HashSet;
@@ -38,7 +38,7 @@ use crate::{
pub(crate) fn get_upto_discard_error() -> anyhow::Error {
anyhow::anyhow!(
"Too old data, only last {} hours data can be ingested. Data discarded. You can adjust ingestion max time by setting the environment variable ZO_INGEST_ALLOWED_UPTO=",
- CONFIG.limit.ingest_allowed_upto
+ get_config().limit.ingest_allowed_upto
)
}
@@ -48,7 +48,7 @@ pub(crate) fn get_request_columns_limit_error(
) -> anyhow::Error {
anyhow::anyhow!(
"Got {num_fields} columns for stream {stream_name}, only {} columns accept. Data discarded. You can adjust ingestion columns limit by setting the environment variable ZO_COLS_PER_RECORD_LIMIT=