diff --git a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json similarity index 51% rename from backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json rename to backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json index 24a5ee62dd..42bac71afd 100644 --- a/backend/.sqlx/query-94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033.json +++ b/backend/.sqlx/query-0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", + "query": "DELETE FROM log_file WHERE (hostname, log_ts) IN (\n SELECT hostname, log_ts FROM log_file\n WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval\n LIMIT $2\n ) RETURNING file_path, hostname", "describe": { "columns": [ { @@ -16,6 +16,7 @@ ], "parameters": { "Left": [ + "Int8", "Int8" ] }, @@ -24,5 +25,5 @@ false ] }, - "hash": "94da1e7feb4f58cc7ebe99752736f956d47810a94cb052fdcffb5cfe440f8033" + "hash": "0e03dc960c0a22e042e54af719ac90c4b8506acefc85ef5b2f92a7bc451b1c5e" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index cce83bbdff..6f56f0ab9a 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -466eb1830879052a5d042295256a78375bee916d +e6483ff5a289521405912d95be5c5ba064bedb38 diff --git a/backend/src/main.rs b/backend/src/main.rs index 8803157d84..c782e1ad97 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -61,8 +61,9 @@ use windmill_common::{ SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, @@ -143,7 +144,8 @@ use crate::monitor::{ reload_pip_index_url_setting, reload_retention_period_setting, reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config, + reload_sandbox_registry_auth_setting, reload_scim_token_setting, + reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -1953,6 +1955,9 @@ async fn process_notify_event( } TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await, RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await, + SERVICE_LOG_RETENTION_SECS_SETTING => { + reload_service_log_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index d640e1bda6..c0748a3798 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -70,11 +70,11 @@ use windmill_common::{ RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, - WORKSPACE_MAX_QUEUED_JOBS_SETTING, + SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, }, indexer::load_indexer_config, jobs::delete_jobs, @@ -97,10 +97,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - HUB_BASE_URL, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, - SERVICE_LOG_RETENTION_SECS, STORE_AUDIT_LOGS_S3, + DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, + METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, + OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -475,6 +475,19 @@ pub async fn initial_load( |v: Option| async move { HUB_API_SECRET.store(std::sync::Arc::new(v)) }, ); + // Outside the `server_mode` guard below: every mode reads this. A worker registers its + // rotated log files against the cutoff, and a dedicated indexer trims the search index to a + // window derived from it — neither is a server. + pass.setting(SERVICE_LOG_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_service_log_retention_secs(parse_setting_value::( + v, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + )) + }); + if server_mode { pass.setting(RETENTION_PERIOD_SECS_SETTING, true, |v| async move { JOB_RETENTION_SECS.store( @@ -1380,8 +1393,8 @@ async fn send_log_files_to_object_store( files: Vec<(NaiveDateTime, String)>, ) { let _guard = SENDING_LOG_FILES.lock().await; - let retention_cutoff = - Utc::now().naive_utc() - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + let retention_cutoff = Utc::now().naive_utc() + - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); for (ts, file_name) in files { if last_log_file_sent().is_some_and(|last| last >= ts) { continue; @@ -1661,6 +1674,13 @@ pub async fn trim_resource_versions(db: &DB) -> () { } } +/// Matches the batch the settings-page cleanup uses for the same table. +const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; +/// Batches per pass. `monitor_db` runs under a 600s timeout that cancels every maintenance +/// future in the same `join!` and reports a critical error, so a large backlog has to drain +/// across ticks rather than inside one, the way the neighbouring sweeps already do. +const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1743,23 +1763,48 @@ pub async fn delete_expired_items(db: &DB) -> () { Err(e) => tracing::error!("Error deleting cache resource {}", e.to_string()), } - match sqlx::query_as!( - LogFile, - "DELETE FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, - ) - .fetch_all(db) - .await - { - Ok(log_files_to_delete) => { + // Batched: every process rotates a log file a minute, so lowering the retention makes one + // ordinary setting change expire millions of rows at once. An unbounded `DELETE ... + // RETURNING` would materialize all of them, and their deletion futures, in this one tick. + for _ in 0..SERVICE_LOG_DELETE_MAX_BATCHES { + let batch = sqlx::query_as!( + LogFile, + "DELETE FROM log_file WHERE (hostname, log_ts) IN ( + SELECT hostname, log_ts FROM log_file + WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval + LIMIT $2 + ) RETURNING file_path, hostname", + windmill_common::service_log_retention_secs(), + SERVICE_LOG_DELETE_BATCH, + ) + .fetch_all(db) + .await; + + match batch { + Ok(log_files_to_delete) => { + if log_files_to_delete.is_empty() { + break; + } + let n = log_files_to_delete.len(); let paths = log_files_to_delete .iter() .map(|f| format!("{}/{}", f.hostname, f.file_path)) .collect(); - delete_log_files_from_disk_and_store(paths, &*TMP_WINDMILL_LOGS_SERVICE, windmill_common::tracing_init::LOGS_SERVICE).await; - + delete_log_files_from_disk_and_store( + paths, + &*TMP_WINDMILL_LOGS_SERVICE, + windmill_common::tracing_init::LOGS_SERVICE, + ) + .await; + if (n as i64) < SERVICE_LOG_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting log file: {:?}", e); + break; + } } - Err(e) => tracing::error!("Error deleting log file: {:?}", e), } let audit_retention_days = audit_log_retention_days().await; @@ -2866,6 +2911,21 @@ pub async fn reload_retention_period_setting(conn: &Connection) { } } +pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + SERVICE_LOG_RETENTION_SECS_SETTING, + "SERVICE_LOG_RETENTION_SECS", + DEFAULT_SERVICE_LOG_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_service_log_retention_secs(v), + Err(e) => tracing::error!("Error reloading service log retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, diff --git a/backend/windmill-api-settings/src/log_cleanup.rs b/backend/windmill-api-settings/src/log_cleanup.rs index ecbc38fcb7..82080cb61c 100644 --- a/backend/windmill-api-settings/src/log_cleanup.rs +++ b/backend/windmill-api-settings/src/log_cleanup.rs @@ -32,7 +32,7 @@ use windmill_common::tracing_init::{LOGS_SERVICE, TMP_WINDMILL_LOGS_SERVICE}; use windmill_common::worker::WINDMILL_DIR; use windmill_common::{ DB, INSTANCE_NAME, JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, - JOB_RETENTION_SECS_OVERRIDES_LOADED, SERVICE_LOG_RETENTION_SECS, + JOB_RETENTION_SECS_OVERRIDES_LOADED, }; use windmill_object_store::object_store_reexports::{ @@ -249,7 +249,7 @@ async fn cleanup_service_logs( // Count candidates upfront for progress reporting. let total: i64 = sqlx::query_scalar!( "SELECT COUNT(*) FROM log_file WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), ) .fetch_one(db) .await? @@ -274,7 +274,7 @@ async fn cleanup_service_logs( WHERE log_ts <= now() - ($1::bigint::text || ' s')::interval LIMIT $2 ) RETURNING file_path, hostname", - SERVICE_LOG_RETENTION_SECS, + windmill_common::service_log_retention_secs(), SERVICE_LOG_BATCH, ) .fetch_all(db) @@ -680,9 +680,10 @@ async fn cleanup_s3_orphans( ) -> error::Result<()> { let job_retention_secs = JOB_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed); let now = Utc::now(); - // Service logs always have a retention (hardcoded SERVICE_LOG_RETENTION_SECS), - // so we scan for service-log orphans regardless of JOB_RETENTION_SECS. - let service_cutoff = now - chrono::Duration::seconds(SERVICE_LOG_RETENTION_SECS); + // Service logs always have a retention, so we scan for service-log orphans regardless of + // JOB_RETENTION_SECS. + let service_cutoff = + now - chrono::Duration::seconds(windmill_common::service_log_retention_secs()); // Job-log orphans are only considered once past a job's effective retention window. That window // is the instance one OR, for an override workspace (EE), its own — and jobs orphan their logs as diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b539c1c6b3..b1ad2edd01 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const OAUTH_SETTING: &str = "oauths"; pub const AI_CONFIG_SETTING: &str = "ai_config"; pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs"; pub const RETENTION_PERIOD_SECS_OVERRIDES_SETTING: &str = "retention_period_secs_overrides"; +pub const SERVICE_LOG_RETENTION_SECS_SETTING: &str = "service_log_retention_secs"; /// Upper bound on how many per-workspace retention overrides may be configured. The periodic monitor /// sweeps each override workspace in its own transaction every pass, so this keeps a pass bounded /// (and the feature is a targeted escape hatch for a handful of special workspaces, not a bulk knob). diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 7b1ed407ab..559eba3bf9 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -94,6 +94,21 @@ pub async fn load_indexer_config(db: &DB) -> error::Result i64 { + let retention = crate::service_log_retention_secs(); + if max_index_time_window_secs > 0 { + std::cmp::min(max_index_time_window_secs, retention) + } else { + retention + } +} + pub fn get_env_var(env_var: &str) -> Option { match std::env::var(env_var).map(|x| x.parse()) { Ok(Ok(i)) => Some(i), @@ -136,3 +151,60 @@ pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { settings } + +#[cfg(test)] +mod tests { + use super::*; + + // One test rather than several: both halves share the process-wide retention, and the + // setter half writes it, which parallel tests would race. + #[test] + fn retention_rejects_unusable_values_and_the_index_window_clamps_to_it() { + use crate::{ + service_log_retention_secs, set_service_log_retention_secs, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + }; + + // See `set_service_log_retention_secs` for why the two unusable directions land apart: + // too large keeps the intent by capping, non-positive cannot and falls back. + let rejected: Vec = [0, -1, i64::MIN] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + let capped: Vec = [i64::MAX, 60 * 60 * 24 * 365 * 101] + .iter() + .map(|v| { + set_service_log_retention_secs(*v); + service_log_retention_secs() + }) + .collect(); + + set_service_log_retention_secs(60 * 60 * 24 * 3); + let retention = service_log_retention_secs(); + let windows = [ + // `0` disables the extra shrinking rather than lifting the ceiling — the trap that + // makes an unset setting look unbounded. + service_log_index_window_secs(0), + // Retention is the ceiling: the index cannot reach lines whose `log_file` row is gone. + service_log_index_window_secs(retention * 2), + service_log_index_window_secs(60), + ]; + set_service_log_retention_secs(DEFAULT_SERVICE_LOG_RETENTION_SECS); + + assert_eq!( + rejected, + vec![DEFAULT_SERVICE_LOG_RETENTION_SECS; 3], + "a value that would expire everything must fall back to the default" + ); + assert_eq!( + capped, + vec![60 * 60 * 24 * 365 * 100; 2], + "an oversized value must cap, not shorten retention to the default" + ); + assert_eq!(retention, 60 * 60 * 24 * 3); + assert_eq!(windows, [retention, retention, 60]); + } +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index cf0be2bbd8..f1784e9841 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -147,9 +147,53 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; -pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; +/// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower +/// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` +/// the cleanup queries build overflows Postgres' microsecond field. +const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a +/// cutoff. The two unusable directions are not the same mistake and must not share a landing +/// point: too large still says "keep these for a very long time", so it is capped and the +/// intent survives, whereas falling back would delete logs the operator meant to keep. A +/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at +/// or after `now` and the next sweep expires the entire history, rows and object-storage files +/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an +/// operator types by analogy with it, and what the settings UI writes into a field that was +/// merely focused — falls back to the default. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { + tracing::warn!( + "service log retention of {configured}s exceeds the maximum of \ + {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + ); + MAX_SERVICE_LOG_RETENTION_SECS + } else if configured >= 1 { + configured + } else { + tracing::warn!( + "service log retention of {configured}s would expire every service log, \ + falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + ); + DEFAULT_SERVICE_LOG_RETENTION_SECS + }; + SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long a service log line stays retrievable, in seconds. +/// +/// The outer bound on everything service-log: the `log_file` rows, the raw files in object +/// storage, the columnar store queried by retrieval, and — through +/// [`indexer::service_log_index_window_secs`] — the search index. +pub fn service_log_retention_secs() -> i64 { + SERVICE_LOG_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// Canonical form of a base URL, used as one of the inputs to the offline-license /// instance hash (`compute_instance_hash`). /// @@ -375,6 +419,10 @@ lazy_static::lazy_static! { /// workspace configured before its override could be read. pub static ref JOB_RETENTION_SECS_OVERRIDES_LOADED: AtomicBool = AtomicBool::new(false); pub static ref AUDIT_LOG_RETENTION_DAYS: AtomicI64 = AtomicI64::new(0); + /// Private on purpose: [`set_service_log_retention_secs`] is the only writer, so a value that + /// would expire every service log cannot reach a cutoff. Read it with + /// [`service_log_retention_secs`]. + static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 91e00d8c71..4a0bd58715 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1143,6 +1143,32 @@ description="Configure default timeouts and retention policies for job execution." link="https://www.windmill.dev/docs/advanced/instance_settings#jobs" /> + {:else if category == 'Service logs'} + + {#if !$values['object_store_cache_config']} +
+ + Instance object storage is not configured, so every server and worker keeps its log + files on its own disk. This page lists what each host wrote, but can only open the + files belonging to the replica serving the request — another host's are listed and + not readable — and a host's files go with it when it is replaced. Retention below + still governs the entries in the database and the files on disk. + +
+ {:else if !$enterpriseLicense} +
+ + Log files are uploaded to instance object storage, and the indexer that would ingest + them into the columnar store and delete each one afterwards is an enterprise + feature. Retention below expires the database entries and the local files; the + uploaded copies are only removed when Delete logs from s3 periodically is on + under Object Storage. + +
+ {/if} {:else if category == 'Object Storage'} = { triggersRestart: true } ], + 'Service logs': [ + { + label: 'Retention in secs', + key: 'service_log_retention_secs', + description: + 'How long a service log is kept, across every copy of it: the entry in the database, the file on the disk of the process that wrote it, and — once instance object storage is configured and the indexer has ingested it — its line in the columnar store that search and the log viewer read. Search reaches back at most this far, and less when the indexer time window under Indexer is shorter. Defaults to 14 days. There is no keep-forever setting here — leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + error: + 'Service log retention must be between 1 second and 100 years — leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + } + ], + Indexer: [ { label: '', @@ -1173,6 +1190,12 @@ export const instanceSettingsNavigationGroups = [ aiDescription: 'Instance OTEL/Prometheus settings', isEE: true }, + { + id: 'service_logs', + label: 'Service logs', + aiId: 'instance-settings-service-logs', + aiDescription: 'Service log retention settings' + }, { id: 'indexer', label: 'Indexer', @@ -1256,6 +1279,7 @@ export const tabToCategoryMap: Record = { webhooks: 'Webhooks', otel_prom: 'OTEL/Prom', indexer: 'Indexer', + service_logs: 'Service logs', telemetry: 'Telemetry', secret_storage: 'Secret Storage', object_storage: 'Object Storage', @@ -1291,6 +1315,7 @@ export const categoryToTabMap: Record = { Webhooks: 'webhooks', 'OTEL/Prom': 'otel_prom', Indexer: 'indexer', + 'Service logs': 'service_logs', Telemetry: 'telemetry', 'Secret Storage': 'secret_storage', 'Object Storage': 'object_storage',