diff --git a/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json new file mode 100644 index 0000000000..693d84173a --- /dev/null +++ b/backend/.sqlx/query-16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT trace_id FROM otel_traces ORDER BY trace_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "trace_id", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "16b174aaa944fd94458ce3108f0fec23514ae0419f77b632064b75473b5636c3" +} diff --git a/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json new file mode 100644 index 0000000000..f35dbc23e4 --- /dev/null +++ b/backend/.sqlx/query-1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM otel_traces WHERE ctid IN (\n SELECT ctid FROM otel_traces\n WHERE start_time_unix_nano < EXTRACT(\n EPOCH FROM now() - ($1::bigint::text || ' s')::interval\n )::bigint * 1000000000\n LIMIT $2\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1b244f65ee6a2607ebc1c333d4359fbbf8be5a81276a3050a42770e4a5b5aa5e" +} diff --git a/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json new file mode 100644 index 0000000000..67b672228f --- /dev/null +++ b/backend/.sqlx/query-62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano)\n VALUES ($1, $2, 'GET /', 3, $3, $3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "62ed1fe52bc2c22d199101309cbbadb9842318c4c7a1d2526ac567ce41b7fbdc" +} diff --git a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json similarity index 72% rename from backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json rename to backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json index b31e532fc2..2947dd8413 100644 --- a/backend/.sqlx/query-f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4.json +++ b/backend/.sqlx/query-cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics", + "query": "VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "f0070b36f7c4fc84dc9c23bb6c73d8ba80993a28b2c2e5df70968acf6d7cebe4" + "hash": "cfdd5ac1dfc7276fc37d49ddfe1b8880eaafb2d3fe71d75b676f1719e26f660f" } diff --git a/backend/src/main.rs b/backend/src/main.rs index 5ec78b0b06..f84812151d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -53,9 +53,9 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING, MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, - NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, - PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, - PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, + OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, + POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING, RETENTION_PERIOD_SECS_OVERRIDES_SETTING, RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, @@ -140,12 +140,12 @@ use crate::monitor::{ reload_instance_events_webhook_setting, reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting, - reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting, - reload_pip_index_url_setting, reload_retention_period_setting, - reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting, - reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting, - reload_sandbox_registry_auth_setting, reload_scim_token_setting, - reload_service_log_retention_secs_setting, reload_smtp_config, + reload_nsjail_tmpfs_size_setting, reload_otel_traces_retention_secs_setting, + reload_otel_tracing_proxy_setting, reload_pip_index_url_setting, + reload_retention_period_setting, reload_sandbox_image_cache_max_setting, + reload_sandbox_image_default_registry_setting, reload_sandbox_image_max_size_setting, + reload_sandbox_image_pull_policy_setting, reload_sandbox_registry_auth_setting, + reload_scim_token_setting, reload_service_log_retention_secs_setting, reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting, reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting, reload_worker_config, MonitorIteration, @@ -2013,6 +2013,9 @@ async fn process_notify_event( SERVICE_LOG_RETENTION_SECS_SETTING => { reload_service_log_retention_secs_setting(conn).await } + OTEL_TRACES_RETENTION_SECS_SETTING => { + reload_otel_traces_retention_secs_setting(conn).await + } RETENTION_PERIOD_SECS_OVERRIDES_SETTING => { if let Err(e) = load_retention_period_overrides(db).await { tracing::error!("Error loading per-workspace retention overrides: {e:#}"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index ce09cebe71..86e75b7de9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -64,10 +64,11 @@ use windmill_common::{ JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, + OTEL_SETTING, OTEL_TRACES_RETENTION_SECS_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, @@ -97,10 +98,10 @@ use windmill_common::{ KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ALERT_MUTE_ZOMBIE_JOB_RESTART, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, - DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, JOB_RETENTION_SECS, - JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, METRICS_DEBUG_ENABLED, - METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, - OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, + DEFAULT_OTEL_TRACES_RETENTION_SECS, DEFAULT_SERVICE_LOG_RETENTION_SECS, HUB_BASE_URL, + JOB_RETENTION_SECS, JOB_RETENTION_SECS_OVERRIDES, JOB_RETENTION_SECS_OVERRIDES_LOADED, + METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, + OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, STORE_AUDIT_LOGS_S3, }; use windmill_common::{ client::AuthedClient, @@ -514,6 +515,15 @@ pub async fn initial_load( Ordering::Relaxed, ) }); + pass.setting(OTEL_TRACES_RETENTION_SECS_SETTING, true, |v| async move { + windmill_common::set_otel_traces_retention_secs(parse_setting_value::( + v, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + )) + }); pass.setting(STORE_AUDIT_LOGS_S3_SETTING, true, |v| async move { STORE_AUDIT_LOGS_S3.store( parse_setting_value::( @@ -1682,6 +1692,57 @@ const SERVICE_LOG_DELETE_BATCH: i64 = 2_000; /// across ticks rather than inside one, the way the neighbouring sweeps already do. const SERVICE_LOG_DELETE_MAX_BATCHES: usize = 10; +/// One span per HTTP request made from a job script, so the table grows far faster than the +/// job table it is keyed against; batched for the same reason the service log sweep is. +const OTEL_TRACES_DELETE_BATCH: i64 = 10_000; +const OTEL_TRACES_DELETE_MAX_BATCHES: usize = 10; + +/// Delete HTTP request tracing spans older than `retention_secs`, returning how many went. +/// +/// `retention_secs` is a parameter rather than a read of the process-wide setting so a test can +/// pin a window without writing state the other tests in this binary run against concurrently. +async fn delete_expired_otel_traces(db: &DB, retention_secs: i64) -> u64 { + // `start_time_unix_nano` is the proto field stored verbatim, so the cutoff is built in that + // unit rather than compared against `now()`. Truncating the epoch to whole seconds first + // keeps the multiplication inside `bigint`. + // + // Batched on `ctid`, not on the `(trace_id, span_id)` primary key: with the key the planner + // hashes the LIMITed subquery and Seq Scans the whole table to probe it, which at the size + // this table reaches is the cost the batching exists to avoid. `ctid` plans as a Tid Scan, so + // each batch touches only the rows it deletes. Safe because the subquery and the delete share + // one snapshot, and spans are never updated after insert. + let mut deleted = 0; + for _ in 0..OTEL_TRACES_DELETE_MAX_BATCHES { + let batch = sqlx::query!( + "DELETE FROM otel_traces WHERE ctid IN ( + SELECT ctid FROM otel_traces + WHERE start_time_unix_nano < EXTRACT( + EPOCH FROM now() - ($1::bigint::text || ' s')::interval + )::bigint * 1000000000 + LIMIT $2 + )", + retention_secs, + OTEL_TRACES_DELETE_BATCH, + ) + .execute(db) + .await; + + match batch { + Ok(res) => { + deleted += res.rows_affected(); + if (res.rows_affected() as i64) < OTEL_TRACES_DELETE_BATCH { + break; + } + } + Err(e) => { + tracing::error!("Error deleting expired otel trace spans: {:?}", e); + break; + } + } + } + deleted +} + pub async fn delete_expired_items(db: &DB) -> () { let expired_tokens_r = sqlx::query_as!( TokenRow, @@ -1808,6 +1869,12 @@ pub async fn delete_expired_items(db: &DB) -> () { } } + let deleted_spans = + delete_expired_otel_traces(db, windmill_common::otel_traces_retention_secs()).await; + if deleted_spans > 0 { + tracing::info!("deleted {} expired otel trace spans", deleted_spans); + } + let audit_retention_days = audit_log_retention_days().await; let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24; @@ -2927,6 +2994,21 @@ pub async fn reload_service_log_retention_secs_setting(conn: &Connection) { } } +pub async fn reload_otel_traces_retention_secs_setting(conn: &Connection) { + match load_setting_value::( + conn, + OTEL_TRACES_RETENTION_SECS_SETTING, + "OTEL_TRACES_RETENTION_SECS", + DEFAULT_OTEL_TRACES_RETENTION_SECS, + |x| x, + ) + .await + { + Ok(v) => windmill_common::set_otel_traces_retention_secs(v), + Err(e) => tracing::error!("Error reloading otel traces retention period: {:?}", e), + } +} + pub async fn reload_audit_log_retention_days_setting(conn: &Connection) { match load_setting_value::( conn, @@ -4839,7 +4921,7 @@ async fn poll_git_fork_branches( } async fn vacuuming_tables(db: &Pool) -> error::Result<()> { - sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics") + sqlx::query!("VACUUM v2_job, v2_job_completed, job_result_stream_v2, job_stats, job_logs, job_perms, concurrency_key, log_file, metrics, otel_traces") .execute(db) .await?; Ok(()) @@ -6975,6 +7057,46 @@ mod zombie_worker_memory_pct_tests { } } +#[cfg(test)] +mod otel_traces_retention_tests { + use super::{delete_expired_otel_traces, DB}; + + async fn insert_span(db: &DB, id: u8, age_secs: i64) { + sqlx::query!( + "INSERT INTO otel_traces (trace_id, span_id, name, kind, start_time_unix_nano, end_time_unix_nano) + VALUES ($1, $2, 'GET /', 3, $3, $3)", + &[id; 16][..], + &[id; 8][..], + (chrono::Utc::now() - chrono::Duration::seconds(age_secs)) + .timestamp_nanos_opt() + .unwrap(), + ) + .execute(db) + .await + .unwrap(); + } + + /// The cutoff crosses two units: a retention configured in seconds against a column holding + /// nanoseconds. Getting that conversion wrong is silent in both directions — a window a + /// billion times too wide never deletes anything, one a billion times too narrow deletes + /// every span on the next tick — so pin it on either side of the boundary. + #[sqlx::test(migrations = "./migrations")] + async fn deletes_only_spans_past_the_window(db: DB) -> anyhow::Result<()> { + let day = 60 * 60 * 24; + insert_span(&db, 1, 60).await; + insert_span(&db, 2, 6 * day).await; + insert_span(&db, 3, 8 * day).await; + + assert_eq!(delete_expired_otel_traces(&db, 7 * day).await, 1); + + let kept = sqlx::query_scalar!("SELECT trace_id FROM otel_traces ORDER BY trace_id") + .fetch_all(&db) + .await?; + assert_eq!(kept, vec![vec![1u8; 16], vec![2u8; 16]]); + Ok(()) + } +} + #[cfg(test)] mod log_file_listing_tests { use super::{rotated_log_files, sorted_log_files}; diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b1ad2edd01..5c4b4bddab 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -108,6 +108,7 @@ pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; pub const OTEL_SETTING: &str = "otel"; pub const OTEL_TRACING_PROXY_SETTING: &str = "otel_tracing_proxy"; +pub const OTEL_TRACES_RETENTION_SECS_SETTING: &str = "otel_traces_retention_secs"; pub const APP_WORKSPACED_ROUTE_SETTING: &str = "app_workspaced_route"; pub const HTTP_ROUTE_WORKSPACED_ROUTE_SETTING: &str = "http_route_workspaced_route"; pub const SECRET_BACKEND_SETTING: &str = "secret_backend"; diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index f1784e9841..b4128f80d2 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -148,43 +148,76 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev"; pub const PRIVATE_HUB_MIN_VERSION: i32 = 10_000_000; pub const DEFAULT_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs +pub const DEFAULT_OTEL_TRACES_RETENTION_SECS: i64 = 60 * 60 * 24 * 7; // 1 week retention period for HTTP request spans pub const WM_DEPLOYERS_GROUP: &str = "wm_deployers"; /// A century. Every consumer has to survive `now - retention`, and the ceilings are much lower /// than an `i64`: `DateTime` subtraction panics past year 262143, and the `( s)::interval` /// the cleanup queries build overflows Postgres' microsecond field. -const MAX_SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; +const MAX_RETENTION_SECS: i64 = 60 * 60 * 24 * 365 * 100; -/// Apply a configured service log retention, in seconds. +/// Clamp a configured retention window, in seconds, to one a cutoff can be built from. /// -/// The only way into [`SERVICE_LOG_RETENTION_SECS`], so an unusable value can never reach a -/// cutoff. The two unusable directions are not the same mistake and must not share a landing -/// point: too large still says "keep these for a very long time", so it is capped and the -/// intent survives, whereas falling back would delete logs the operator meant to keep. A -/// non-positive value has no such reading — every cutoff is `now - retention`, so it lands at -/// or after `now` and the next sweep expires the entire history, rows and object-storage files -/// alike. Unlike job retention there is no "keep forever" spelling here, so `0` — what an -/// operator types by analogy with it, and what the settings UI writes into a field that was -/// merely focused — falls back to the default. -pub fn set_service_log_retention_secs(configured: i64) { - let effective = if configured > MAX_SERVICE_LOG_RETENTION_SECS { +/// Shared by the retention windows that have no "keep forever" spelling, so that an unusable +/// value can never reach a cutoff. The two unusable directions are not the same mistake and must +/// not share a landing point: too large still says "keep these for a very long time", so it is +/// capped and the intent survives, whereas falling back would delete data the operator meant to +/// keep. A non-positive value has no such reading — every cutoff is `now - retention`, so it +/// lands at or after `now` and the next sweep expires the entire history. `0` is both what an +/// operator types by analogy with job retention, where it does mean keep forever, and what the +/// settings UI writes into a field that was merely focused, so it falls back to the default. +fn clamp_retention_secs(configured: i64, default: i64, what: &str) -> i64 { + if configured > MAX_RETENTION_SECS { tracing::warn!( - "service log retention of {configured}s exceeds the maximum of \ - {MAX_SERVICE_LOG_RETENTION_SECS}s, capping it there" + "{what} retention of {configured}s exceeds the maximum of {MAX_RETENTION_SECS}s, \ + capping it there" ); - MAX_SERVICE_LOG_RETENTION_SECS + MAX_RETENTION_SECS } else if configured >= 1 { configured } else { tracing::warn!( - "service log retention of {configured}s would expire every service log, \ - falling back to the default of {DEFAULT_SERVICE_LOG_RETENTION_SECS}s" + "{what} retention of {configured}s would expire the entire history, \ + falling back to the default of {default}s" ); - DEFAULT_SERVICE_LOG_RETENTION_SECS - }; + default + } +} + +/// Apply a configured service log retention, in seconds. +/// +/// The only way into [`SERVICE_LOG_RETENTION_SECS`]. Expiry reaches every copy of a log line: +/// the row, the file on disk, and the object-storage object. +pub fn set_service_log_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_SERVICE_LOG_RETENTION_SECS, + "service log", + ); SERVICE_LOG_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); } +/// Apply a configured OTEL trace retention, in seconds. +/// +/// The only way into [`OTEL_TRACES_RETENTION_SECS`]. +pub fn set_otel_traces_retention_secs(configured: i64) { + let effective = clamp_retention_secs( + configured, + DEFAULT_OTEL_TRACES_RETENTION_SECS, + "otel traces", + ); + OTEL_TRACES_RETENTION_SECS.store(effective, std::sync::atomic::Ordering::Relaxed); +} + +/// How long an HTTP request tracing span stays in `otel_traces`, in seconds. +/// +/// Spans are keyed by the job they were captured for and read back by the job detail view, so +/// this is the outer bound on how far back that view can show a job's HTTP requests. It is +/// independent of job retention: a span can outlive its job, or be swept while the job remains. +pub fn otel_traces_retention_secs() -> i64 { + OTEL_TRACES_RETENTION_SECS.load(std::sync::atomic::Ordering::Relaxed) +} + /// How long a service log line stays retrievable, in seconds. /// /// The outer bound on everything service-log: the `log_file` rows, the raw files in object @@ -423,6 +456,10 @@ lazy_static::lazy_static! { /// would expire every service log cannot reach a cutoff. Read it with /// [`service_log_retention_secs`]. static ref SERVICE_LOG_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_SERVICE_LOG_RETENTION_SECS); + /// Private on purpose, same as [`SERVICE_LOG_RETENTION_SECS`]: + /// [`set_otel_traces_retention_secs`] is the only writer, [`otel_traces_retention_secs`] the + /// only reader. + static ref OTEL_TRACES_RETENTION_SECS: AtomicI64 = AtomicI64::new(DEFAULT_OTEL_TRACES_RETENTION_SECS); pub static ref MONITOR_LOGS_ON_OBJECT_STORE: AtomicBool = AtomicBool::new(false); diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index a85ea0a575..19b5ddb533 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -846,9 +846,7 @@ {:else if setting.fieldType == 'seconds'}
diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 24e7ac6438..eb8c4c86c4 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -30,6 +30,10 @@ export interface Setting { placeholder?: string cloudonly?: boolean ee_only?: string + /** Ceiling a `seconds` field enforces on a build without a license, when CE genuinely caps + * the value. Not implied by `ee_only`: a setting can be EE-badged because the feature it + * configures is EE while the value itself has the same range on either edition. */ + ceMaxSeconds?: number tooltip?: string key: string // If value is not specified for first element, it will automatcally use undefined @@ -283,6 +287,8 @@ export const settings: Record = { placeholder: '30', storage: 'setting', ee_only: 'You can only adjust this setting to above 30 days in the EE version', + // Mirrors CE_MAX_RETENTION_PERIOD_SECS, which the backend clamps to on write. + ceMaxSeconds: 60 * 60 * 24 * 30, cloudonly: false }, { @@ -971,6 +977,24 @@ export const settings: Record = { triggersRestart: true, defaultValue: () => ({ enabled: false, enabled_languages: [...OTEL_TRACING_PROXY_LANGUAGES] }) }, + { + label: 'HTTP Request Tracing retention in secs', + key: 'otel_traces_retention_secs', + description: + 'How long a captured HTTP request span is kept in the database, and therefore how far back the job details view can show a job its requests. Independent of the job retention period, so a span may outlive its job or be swept while the job remains. Defaults to 7 days. Leave it empty for the default.', + fieldType: 'seconds', + storage: 'setting', + cloudonly: false, + // Badged EE because only the EE proxy captures spans, but deliberately no + // `ceMaxSeconds`: a CE build still sweeps rows an EE-era instance left behind, and + // the backend accepts the same range on either edition. + ee_only: 'HTTP Request Tracing is an EE feature', + error: + 'HTTP Request Tracing retention must be between 1 second and 100 years, leave it empty for the default', + isValid: (value: any) => + value == undefined || + (typeof value === 'number' && value > 0 && value <= 60 * 60 * 24 * 365 * 100) + }, { label: 'Prometheus', description: