fix: service logs: aggregation query + killpill on index pull + retention period for logs on s3 (#4795)

* Search across all hosts in index. killpill on index pull

* Add retention period for service logs

* Remove console log

* monitor log_file when older than biggest retention period (job / log)

* Retention period deletes service and job logs

* Add setting to monitor logs out of s3

* Prepare sqlx

* Update ee repo ref

* Prepare sqlx

* Unused variables

backend/windmill-common/src/global_settings.rs

@@ -4,6 +4,7 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces";
 pub const BASE_URL_SETTING: &str = "base_url";
 pub const OAUTH_SETTING: &str = "oauths";
 pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
+pub const MONITOR_LOGS_ON_OBJECT_STORE_SETTING: &str = "monitor_logs_on_s3";
 pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout";
 pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
 pub const LICENSE_KEY_SETTING: &str = "license_key";

backend/windmill-common/src/lib.rs

@@ -54,6 +54,7 @@ pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
 pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
 
 pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
+pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 24 * 14; // 2 weeks retention period for logs
 
 #[macro_export]
 macro_rules! add_time {
@@ -99,6 +100,8 @@ lazy_static::lazy_static! {
 
     pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
 
+    pub static ref MONITOR_LOGS_ON_OBJECT_STORE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
+
     pub static ref INSTANCE_NAME: String = rd_string(5);
 
 }