From d86ce2e3e34e79d5cdaecc0b1864c1e9203e08ee Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 31 Oct 2025 07:36:38 +0000 Subject: [PATCH] fix: better handle same worker zombie job --- ...a4625bb7fbd0974c69745c31c7562563944f.json} | 4 +- ...9e6733943278998d3df62a9486607827e59ec.json | 71 ------------------- ...df749fa3bbbf2c0e43d5d4ea148b5bc990067.json | 70 ------------------ ...308b763293a27adf30d02df605317791d8126.json | 71 ------------------- backend/src/monitor.rs | 2 +- 5 files changed, 3 insertions(+), 215 deletions(-) rename backend/.sqlx/{query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json => query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json} (72%) delete mode 100644 backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json delete mode 100644 backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json delete mode 100644 backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json diff --git a/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json b/backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json similarity index 72% rename from backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json rename to backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json index 321831d724..e728045afa 100644 --- a/backend/.sqlx/query-3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8.json +++ b/backend/.sqlx/query-469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + "query": "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval\n AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker", "describe": { "columns": [ { @@ -22,5 +22,5 @@ null ] }, - "hash": "3e261911cc4c5289da49865f54350613f9e651540a279bd7d75e5e7d79f676a8" + "hash": "469f8b7f691e621cce78b83994b4a4625bb7fbd0974c69745c31c7562563944f" } diff --git a/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json b/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json deleted file mode 100644 index f3961847f4..0000000000 --- a/backend/.sqlx/query-5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH active_users AS (SELECT distinct username as email FROM audit WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n ORDER BY super_admin DESC, devops DESC\n LIMIT $1 OFFSET $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "operator_only", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "login_type", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "verified", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "super_admin", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "devops", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "company", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "username", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - null, - null, - false, - false, - false, - true, - true, - true - ] - }, - "hash": "5430f7728c1e9b539cc8aad29ca9e6733943278998d3df62a9486607827e59ec" -} diff --git a/backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json b/backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json deleted file mode 100644 index 7918b97267..0000000000 --- a/backend/.sqlx/query-df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only FROM password WHERE email = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "login_type", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "super_admin", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "devops", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "verified", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "company", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "operator_only", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null, - false, - false, - false, - true, - true, - true, - null - ] - }, - "hash": "df5b933f81ca7e3bbb3fb522baedf749fa3bbbf2c0e43d5d4ea148b5bc990067" -} diff --git a/backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json b/backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json deleted file mode 100644 index 0153bbf579..0000000000 --- a/backend/.sqlx/query-f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT $1 OFFSET $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "login_type", - "type_info": "Text" - }, - { - "ordinal": 2, - "name": "verified", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "super_admin", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "devops", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "company", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "username", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "operator_only", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - null, - false, - false, - false, - true, - true, - true, - null - ] - }, - "hash": "f07a705df1a988827e099d146f5308b763293a27adf30d02df605317791d8126" -} diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 1a18588802..fdd5cbd148 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -2174,7 +2174,7 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, worker let same_worker_timeout_jobs = { let long_same_worker_jobs = sqlx::query!( "SELECT worker, array_agg(v2_job_queue.id) as ids FROM v2_job_queue LEFT JOIN v2_job ON v2_job_queue.id = v2_job.id LEFT JOIN v2_job_runtime ON v2_job_queue.id = v2_job_runtime.id WHERE v2_job_queue.created_at < now() - ('60 seconds')::interval - AND running = true AND ping IS NULL AND same_worker = true AND worker IS NOT NULL GROUP BY worker", + AND running = true AND (ping IS NULL OR ping < now() - ('60 seconds')::interval) AND same_worker = true AND worker IS NOT NULL GROUP BY worker", ) .fetch_all(db) .await