mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
feat: partition audit log table by day with configurable retention (#8292)
* feat: partition audit log table by day with configurable retention Introduce daily range partitioning for audit logs to replace expensive DELETE-based retention with instant DROP TABLE per partition. - Create `audit_partitioned` table alongside existing `audit` table - New inserts go to `audit_partitioned`, reads UNION ALL both tables - Monitor creates future partitions and drops expired ones - Add `audit_log_retention_days` instance setting (default 365 days) - Old `audit` table empties naturally via existing DELETE cleanup Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add audit log retention setting to Core instance settings UI Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: bump audit partitioning migration timestamp to avoid collision Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: update ee-repo-ref.txt for audit partitioning Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * feat: add RLS/grants to audit_partitioned, run partition mgmt hourly, CE default 14d - Add grants for windmill_user/windmill_admin and all 5 RLS policies - Move manage_audit_partitions to hourly via should_run(120) - Default retention: 14 days CE, 365 days EE - Download JSON button is now icon-only Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * fix: address code review — quote SQL identifiers, add workspace index, deduplicate retention logic - Quote partition names in dynamic SQL for defense in depth - Add idx_audit_partitioned_workspace(workspace_id, timestamp DESC) index - Extract audit_log_retention_days() helper to deduplicate retention logic Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: update ee-repo-ref for audit insert error handling Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> * chore: update ee-repo-ref to cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64 This commit updates the EE repository reference after PR #450 was merged in windmill-ee-private. Previous ee-repo-ref: f09284bb257d461bcbe3c50fe31eb6f1e7eafee5 New ee-repo-ref: cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64 Automated by sync-ee-ref workflow. * fix: create audit partitions on startup in initial_load Ensures partitions exist before any requests arrive, closing the gap between server start and the first hourly monitor run. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
Ruben Fiszel
GitHub
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2",
|
||||
"query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -73,5 +73,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "72d3ebb05ac1ffeb0e8d0a3146d95bb5b90e7c4d1dc2c8a6ef06eddf6678f230"
|
||||
"hash": "9229d9a9ff389cf26e480b604b83900e2d362ee934ef27284ef39f4eed440e59"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO audit\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
|
||||
"query": "INSERT INTO audit_partitioned\n (workspace_id, username, operation, action_kind, resource, parameters, email, span)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -29,5 +29,5 @@
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "ad8487a797713b3a6c10fb399c9fb8dcd940bb92e998145e250f28ccfe1c7033"
|
||||
"hash": "fbccafe6d34093a723b9d5a6ee8d618a80ceba6de2d39202e6293ef5207c31f6"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
f97defd916b982ef490d7ee69c671655bc9858b4
|
||||
cef4dfc45e6d6344c5d8d107bd2b4d1bf9bbdd64
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS audit_partitioned CASCADE;
|
||||
@@ -0,0 +1,58 @@
|
||||
-- Create a new daily-partitioned audit table alongside the existing one.
|
||||
-- New inserts go to audit_partitioned; reads UNION ALL both tables.
|
||||
-- The old audit table empties out naturally via retention cleanup.
|
||||
|
||||
CREATE TABLE audit_partitioned (
|
||||
workspace_id VARCHAR(50) NOT NULL,
|
||||
id BIGINT NOT NULL DEFAULT nextval('audit_id_seq'),
|
||||
timestamp TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
username VARCHAR(255) NOT NULL,
|
||||
operation VARCHAR(50) NOT NULL,
|
||||
action_kind ACTION_KIND NOT NULL,
|
||||
resource VARCHAR(255),
|
||||
parameters JSONB,
|
||||
email VARCHAR(255),
|
||||
span VARCHAR(255),
|
||||
PRIMARY KEY (id, timestamp)
|
||||
) PARTITION BY RANGE (timestamp);
|
||||
|
||||
-- Create daily partitions for today + 3 days
|
||||
DO $$
|
||||
DECLARE
|
||||
curr_date DATE := CURRENT_DATE;
|
||||
end_date DATE := CURRENT_DATE + INTERVAL '3 days';
|
||||
BEGIN
|
||||
WHILE curr_date <= end_date LOOP
|
||||
EXECUTE format(
|
||||
'CREATE TABLE %I PARTITION OF audit_partitioned FOR VALUES FROM (%L) TO (%L)',
|
||||
'audit_' || to_char(curr_date, 'YYYYMMDD'),
|
||||
curr_date,
|
||||
curr_date + INTERVAL '1 day'
|
||||
);
|
||||
curr_date := curr_date + INTERVAL '1 day';
|
||||
END LOOP;
|
||||
END $$;
|
||||
|
||||
-- Indexes (auto-propagated to all current and future partitions)
|
||||
CREATE INDEX ix_audit_partitioned_timestamps ON audit_partitioned (timestamp DESC);
|
||||
CREATE INDEX idx_audit_partitioned_workspace ON audit_partitioned (workspace_id, timestamp DESC);
|
||||
CREATE INDEX idx_audit_partitioned_recent_login_activities
|
||||
ON audit_partitioned (timestamp, username)
|
||||
WHERE operation IN ('users.login', 'oauth.login', 'users.token.refresh');
|
||||
|
||||
-- Grants (match the old audit table)
|
||||
GRANT ALL ON audit_partitioned TO windmill_user;
|
||||
GRANT ALL ON audit_partitioned TO windmill_admin;
|
||||
|
||||
-- RLS (match the old audit table)
|
||||
ALTER TABLE audit_partitioned ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY admin_policy ON audit_partitioned FOR ALL TO windmill_admin USING (true);
|
||||
CREATE POLICY see_own ON audit_partitioned FOR ALL TO windmill_user
|
||||
USING ((username)::text = current_setting('session.user'::text));
|
||||
CREATE POLICY schedule ON audit_partitioned FOR INSERT TO windmill_user
|
||||
WITH CHECK ((username)::text ~~ 'schedule-%'::text);
|
||||
CREATE POLICY schedule_audit ON audit_partitioned FOR INSERT TO windmill_user
|
||||
WITH CHECK ((parameters ->> 'end_user'::text) ~~ 'schedule-%'::text);
|
||||
CREATE POLICY webhook ON audit_partitioned FOR INSERT TO windmill_user
|
||||
WITH CHECK ((username)::text ~~ 'webhook-%'::text);
|
||||
+17
-12
@@ -37,12 +37,13 @@ use windmill_common::ee_oss::{
|
||||
use windmill_common::{
|
||||
agent_workers::AgentConfig,
|
||||
global_settings::{
|
||||
APP_WORKSPACED_ROUTE_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
|
||||
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
|
||||
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING,
|
||||
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING,
|
||||
ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
|
||||
EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
|
||||
APP_WORKSPACED_ROUTE_SETTING, AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING,
|
||||
BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
|
||||
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
|
||||
CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
|
||||
DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
|
||||
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
|
||||
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
|
||||
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING,
|
||||
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MAVEN_REPOS_SETTING,
|
||||
MAVEN_SETTINGS_XML_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NO_DEFAULT_MAVEN_SETTING,
|
||||
@@ -97,12 +98,13 @@ use windmill_worker::{
|
||||
use crate::monitor::{
|
||||
initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
|
||||
load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db,
|
||||
reload_app_workspaced_route_setting, reload_base_url_setting,
|
||||
reload_bunfig_install_scopes_setting, reload_critical_alert_mute_ui_setting,
|
||||
reload_critical_alerts_on_token_expiry_setting, reload_critical_error_channels_setting,
|
||||
reload_extra_pip_index_url_setting, reload_hub_api_secret_setting, reload_hub_base_url_setting,
|
||||
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
|
||||
reload_license_key, reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
|
||||
reload_app_workspaced_route_setting, reload_audit_log_retention_days_setting,
|
||||
reload_base_url_setting, reload_bunfig_install_scopes_setting,
|
||||
reload_critical_alert_mute_ui_setting, reload_critical_alerts_on_token_expiry_setting,
|
||||
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
|
||||
reload_hub_api_secret_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting,
|
||||
reload_job_isolation_setting, reload_jwt_secret_setting, reload_license_key,
|
||||
reload_npm_config_registry_setting, reload_otel_tracing_proxy_setting,
|
||||
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
|
||||
reload_smtp_config, reload_uv_index_strategy_setting, reload_worker_config, MonitorIteration,
|
||||
};
|
||||
@@ -1664,6 +1666,9 @@ async fn process_notify_event(
|
||||
}
|
||||
TIMEOUT_WAIT_RESULT_SETTING => reload_timeout_wait_result_setting(conn).await,
|
||||
RETENTION_PERIOD_SECS_SETTING => reload_retention_period_setting(conn).await,
|
||||
AUDIT_LOG_RETENTION_DAYS_SETTING => {
|
||||
reload_audit_log_retention_days_setting(conn).await
|
||||
}
|
||||
MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
|
||||
reload_delete_logs_periodically_setting(conn).await
|
||||
}
|
||||
|
||||
+119
-19
@@ -48,16 +48,17 @@ use windmill_common::{
|
||||
error,
|
||||
flow_status::{FlowStatus, FlowStatusModule},
|
||||
global_settings::{
|
||||
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING,
|
||||
CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
|
||||
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
|
||||
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
|
||||
EXTRA_PIP_INDEX_URL_SETTING, HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING,
|
||||
INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING,
|
||||
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
|
||||
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING,
|
||||
NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
|
||||
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
|
||||
AUDIT_LOG_RETENTION_DAYS_SETTING, BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,
|
||||
CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING, CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING,
|
||||
CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING,
|
||||
DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
|
||||
EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
|
||||
HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
|
||||
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING,
|
||||
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
|
||||
NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING,
|
||||
OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING,
|
||||
POWERSHELL_REPO_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
|
||||
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
|
||||
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
|
||||
UV_INDEX_STRATEGY_SETTING,
|
||||
@@ -77,9 +78,9 @@ use windmill_common::{
|
||||
DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, WINDMILL_DIR,
|
||||
WORKER_CONFIG, WORKER_GROUP,
|
||||
},
|
||||
KillpillSender, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE, CRITICAL_ALERTS_ON_TOKEN_EXPIRY,
|
||||
CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
|
||||
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
|
||||
KillpillSender, AUDIT_LOG_RETENTION_DAYS, BASE_URL, CRITICAL_ALERTS_ON_DB_OVERSIZE,
|
||||
CRITICAL_ALERTS_ON_TOKEN_EXPIRY, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB,
|
||||
DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
|
||||
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
|
||||
SERVICE_LOG_RETENTION_SECS,
|
||||
};
|
||||
@@ -323,9 +324,15 @@ pub async fn initial_load(
|
||||
|
||||
if server_mode {
|
||||
reload_retention_period_setting(&conn).await;
|
||||
reload_audit_log_retention_days_setting(&conn).await;
|
||||
reload_request_size(&conn).await;
|
||||
reload_saml_metadata_setting(&conn).await;
|
||||
reload_scim_token_setting(&conn).await;
|
||||
|
||||
// Ensure audit partitions exist before any requests arrive
|
||||
if let Some(db) = conn.as_sql() {
|
||||
manage_audit_partitions(&db, audit_log_retention_days().await).await;
|
||||
}
|
||||
}
|
||||
|
||||
if worker_mode {
|
||||
@@ -1027,12 +1034,10 @@ pub async fn delete_expired_items(db: &DB) -> () {
|
||||
Err(e) => tracing::error!("Error deleting log file: {:?}", e),
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "enterprise"))]
|
||||
let audit_retention_secs = 1 * 60 * 60 * 24 * 14;
|
||||
|
||||
#[cfg(feature = "enterprise")]
|
||||
let audit_retention_secs = 1 * 60 * 60 * 24 * 365;
|
||||
let audit_retention_days = audit_log_retention_days().await;
|
||||
let audit_retention_secs: i64 = audit_retention_days * 60 * 60 * 24;
|
||||
|
||||
// Clean up old (non-partitioned) audit table — will eventually be empty and dropped
|
||||
if let Err(e) = sqlx::query_scalar!(
|
||||
"DELETE FROM audit WHERE timestamp <= now() - ($1::bigint::text || ' s')::interval",
|
||||
audit_retention_secs,
|
||||
@@ -1040,7 +1045,7 @@ pub async fn delete_expired_items(db: &DB) -> () {
|
||||
.fetch_all(db)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error deleting audit log on CE: {:?}", e);
|
||||
tracing::error!("Error deleting audit log: {:?}", e);
|
||||
}
|
||||
|
||||
if let Err(e) = sqlx::query_scalar!(
|
||||
@@ -1565,6 +1570,22 @@ pub async fn reload_retention_period_setting(conn: &Connection) {
|
||||
tracing::error!("Error reloading retention period: {:?}", e)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_audit_log_retention_days_setting(conn: &Connection) {
|
||||
if let Err(e) = reload_setting(
|
||||
conn,
|
||||
AUDIT_LOG_RETENTION_DAYS_SETTING,
|
||||
"AUDIT_LOG_RETENTION_DAYS",
|
||||
0, // 0 means use default: 365 for EE, 14 for CE
|
||||
AUDIT_LOG_RETENTION_DAYS.clone(),
|
||||
|x| x,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error reloading audit log retention days: {:?}", e)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
|
||||
if let Err(e) = reload_setting(
|
||||
conn,
|
||||
@@ -2182,6 +2203,15 @@ pub async fn monitor_db(
|
||||
}
|
||||
};
|
||||
|
||||
// run every hour (120 iterations * 30s = 3600s)
|
||||
let manage_audit_partitions_f = async {
|
||||
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
|
||||
if let Some(db) = conn.as_sql() {
|
||||
manage_audit_partitions(&db, audit_log_retention_days().await).await;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
join!(
|
||||
expired_items_f,
|
||||
zombie_jobs_f,
|
||||
@@ -2204,6 +2234,7 @@ pub async fn monitor_db(
|
||||
native_triggers_sync_f,
|
||||
cleanup_notify_events_f,
|
||||
check_expiring_tokens_f,
|
||||
manage_audit_partitions_f,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -3432,3 +3463,72 @@ RETURNING job_id
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn audit_log_retention_days() -> i64 {
|
||||
let v = *AUDIT_LOG_RETENTION_DAYS.read().await;
|
||||
if v > 0 {
|
||||
v
|
||||
} else if cfg!(feature = "enterprise") {
|
||||
365
|
||||
} else {
|
||||
14
|
||||
}
|
||||
}
|
||||
|
||||
async fn manage_audit_partitions(db: &DB, retention_days: i64) {
|
||||
let today = chrono::Utc::now().date_naive();
|
||||
|
||||
// Create partitions for today and the next 3 days
|
||||
for days_ahead in 0..=3i64 {
|
||||
let date = today + chrono::Duration::days(days_ahead);
|
||||
let next_date = date + chrono::Duration::days(1);
|
||||
let partition_name = format!("audit_{}", date.format("%Y%m%d"));
|
||||
let quoted_name = format!("\"{}\"", partition_name.replace('"', "\"\""));
|
||||
let sql = format!(
|
||||
"CREATE TABLE IF NOT EXISTS {quoted_name} PARTITION OF audit_partitioned \
|
||||
FOR VALUES FROM ('{date}') TO ('{next_date}')"
|
||||
);
|
||||
if let Err(e) = sqlx::query(&sql).execute(db).await {
|
||||
if !e.to_string().contains("already exists") {
|
||||
tracing::error!("Error creating audit partition {partition_name}: {e:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drop expired partitions
|
||||
let cutoff_date = today - chrono::Duration::days(retention_days);
|
||||
|
||||
let partitions = sqlx::query_scalar::<_, String>(
|
||||
"SELECT c.relname::text \
|
||||
FROM pg_inherits i \
|
||||
JOIN pg_class c ON c.oid = i.inhrelid \
|
||||
WHERE i.inhparent = 'audit_partitioned'::regclass",
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await;
|
||||
|
||||
match partitions {
|
||||
Ok(partitions) => {
|
||||
for partition_name in partitions {
|
||||
if let Some(date_str) = partition_name.strip_prefix("audit_") {
|
||||
if let Ok(date) = chrono::NaiveDate::parse_from_str(date_str, "%Y%m%d") {
|
||||
if date < cutoff_date {
|
||||
let quoted_name =
|
||||
format!("\"{}\"", partition_name.replace('"', "\"\""));
|
||||
let sql = format!("DROP TABLE IF EXISTS {quoted_name}");
|
||||
match sqlx::query(&sql).execute(db).await {
|
||||
Ok(_) => tracing::info!(
|
||||
"Dropped expired audit partition {partition_name}"
|
||||
),
|
||||
Err(e) => tracing::error!(
|
||||
"Error dropping audit partition {partition_name}: {e:?}"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => tracing::error!("Error listing audit partitions: {e:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -393,7 +393,7 @@ async fn list_users_as_super_admin(
|
||||
let rows = if active_only.is_some_and(|x| x) {
|
||||
sqlx::query_as!(
|
||||
GlobalUserInfo,
|
||||
"WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
|
||||
"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
|
||||
authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)
|
||||
SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user
|
||||
FROM password
|
||||
|
||||
@@ -4,6 +4,7 @@ pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces";
|
||||
pub const BASE_URL_SETTING: &str = "base_url";
|
||||
pub const OAUTH_SETTING: &str = "oauths";
|
||||
pub const RETENTION_PERIOD_SECS_SETTING: &str = "retention_period_secs";
|
||||
pub const AUDIT_LOG_RETENTION_DAYS_SETTING: &str = "audit_log_retention_days";
|
||||
pub const MONITOR_LOGS_ON_OBJECT_STORE_SETTING: &str = "monitor_logs_on_s3";
|
||||
pub const JOB_DEFAULT_TIMEOUT_SECS_SETTING: &str = "job_default_timeout";
|
||||
pub const REQUEST_SIZE_LIMIT_SETTING: &str = "request_size_limit_mb";
|
||||
|
||||
@@ -205,6 +205,7 @@ lazy_static::lazy_static! {
|
||||
pub static ref CRITICAL_ALERTS_ON_DB_OVERSIZE: Arc<RwLock<Option<f32>>> = Arc::new(RwLock::new(None));
|
||||
|
||||
pub static ref JOB_RETENTION_SECS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
|
||||
pub static ref AUDIT_LOG_RETENTION_DAYS: Arc<RwLock<i64>> = Arc::new(RwLock::new(0));
|
||||
|
||||
pub static ref MONITOR_LOGS_ON_OBJECT_STORE: Arc<RwLock<bool>> = Arc::new(RwLock::new(false));
|
||||
|
||||
|
||||
@@ -465,12 +465,9 @@
|
||||
on:click={downloadAuditLogsAsJson}
|
||||
unifiedSize="md"
|
||||
title="Downloads currently displayed logs only (up to {perPage} entries)"
|
||||
>
|
||||
<div class="flex flex-row gap-1 items-center">
|
||||
<Download size={14} />
|
||||
Download JSON
|
||||
</div>
|
||||
</Button>
|
||||
startIcon={{ icon: Download }}
|
||||
iconOnly
|
||||
/>
|
||||
<Button
|
||||
variant="accent"
|
||||
on:click={() => {
|
||||
|
||||
@@ -193,6 +193,16 @@ export const settings: Record<string, Setting[]> = {
|
||||
storage: 'setting',
|
||||
ee_only: '',
|
||||
hideInQuickSetup: true
|
||||
},
|
||||
{
|
||||
label: 'Audit log retention (days)',
|
||||
key: 'audit_log_retention_days',
|
||||
description: 'How long to keep audit log entries in the database. Default: 365 days.',
|
||||
fieldType: 'number',
|
||||
placeholder: '365',
|
||||
storage: 'setting',
|
||||
ee_only: '',
|
||||
hideInQuickSetup: true
|
||||
}
|
||||
],
|
||||
Jobs: [
|
||||
|
||||
Reference in New Issue
Block a user