diff --git a/README.md b/README.md index 5f9d9eb916..51c9c99eb4 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,7 @@ it being synced automatically everyday. | GLOBAL_CACHE_INTERVAL | 10\*60 | (Enterprise Edition only) Interval in seconds in between bucket sync of the cache. This interval \* 2 is the time at which you're guaranteed all the worker's caches are synced together. | Worker | | WORKER_TAGS | 'deno,go,python3,bash,flow,hub,dependency' | The worker groups assigned to that workers | Worker | | CUSTOM_TAGS | None | The custom tags assignable to scripts. | Server | +| JOB_RETENTION_SECS | 60*60*24\*60 //60 days | The time in seconds after which jobs get deleted. Set to 0 or -1 to never delete | Server | ## Run a local dev setup diff --git a/backend/sqlx-data.json b/backend/sqlx-data.json index 4a40ee17bf..bdfdb99c32 100644 --- a/backend/sqlx-data.json +++ b/backend/sqlx-data.json @@ -1945,6 +1945,26 @@ }, "query": "INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2" }, + "502781c4e2fc692db7a66b850450a1934b2403a5cff0421fe9609f9b99d8ee95": { + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Uuid" + } + ], + "nullable": [ + false + ], + "parameters": { + "Left": [ + "Int4" + ] + } + }, + "query": "DELETE FROM completed_job WHERE started_at + ((duration_ms/1000 + $1) || ' s')::interval <= now() RETURNING id" + }, "5061c0d054bf4f028e7fe51a8f9389024c6ae4492755cadac0f7167e5300bda0": { "describe": { "columns": [], diff --git a/backend/src/main.rs b/backend/src/main.rs index f1c4cd00fc..80a321d523 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -16,6 +16,7 @@ use monitor::handle_zombie_jobs_periodically; use sqlx::{Pool, Postgres}; use tokio::{ fs::{metadata, DirBuilder}, + join, sync::RwLock, }; use windmill_common::{utils::rd_string, METRICS_ADDR}; @@ -164,6 +165,7 @@ Windmill Community Edition {GIT_VERSION} "GLOBAL_CACHE_INTERVAL", "WORKER_TAGS", "CUSTOM_TAGS", + "JOB_RETENTION_SECS", ]); if server_mode || num_workers > 0 { @@ -246,9 +248,11 @@ pub fn monitor_db let rx2 = rx.resubscribe(); let base_internal_url = base_internal_url.to_string(); tokio::spawn(async move { - handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq).await + join!( + handle_zombie_jobs_periodically(&db1, rx, &base_internal_url, rsmq), + windmill_api::delete_expired_items_perdiodically(&db2, rx2) + ); }); - tokio::spawn(async move { windmill_api::delete_expired_items_perdiodically(&db2, rx2).await }); } pub async fn run_workers( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e68fa92100..5759fe58c7 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -144,7 +144,7 @@ pub struct Metrics { } -pub const DEFAULT_TIMEOUT: u16 = 300; +pub const DEFAULT_TIMEOUT: u64 = 300; pub const DEFAULT_SLEEP_QUEUE: u64 = 50; lazy_static::lazy_static! { @@ -213,12 +213,12 @@ lazy_static::lazy_static! { "Total number of seconds since the worker has started" ); - static ref TIMEOUT: u16 = std::env::var("TIMEOUT") + static ref TIMEOUT: u64 = std::env::var("TIMEOUT") .ok() - .and_then(|x| x.parse::().ok()) - .unwrap_or(DEFAULT_TIMEOUT as u16); + .and_then(|x| x.parse::().ok()) + .unwrap_or(DEFAULT_TIMEOUT); - static ref TIMEOUT_DURATION: Duration = Duration::from_secs(*TIMEOUT as u64); + static ref TIMEOUT_DURATION: Duration = Duration::from_secs(*TIMEOUT); pub static ref SESSION_TOKEN_EXPIRY: i32 = (*TIMEOUT as i32) * 2;