diff --git a/backend/.sqlx/query-bc0bd9659bb8b58ed5967754b1c403e77ebb044063f0659485f44b1d04bb6b4d.json b/backend/.sqlx/query-6f27f1d150df2903b66f29af0072e85a030134e440a712e54136c8a4eae36fc7.json similarity index 65% rename from backend/.sqlx/query-bc0bd9659bb8b58ed5967754b1c403e77ebb044063f0659485f44b1d04bb6b4d.json rename to backend/.sqlx/query-6f27f1d150df2903b66f29af0072e85a030134e440a712e54136c8a4eae36fc7.json index 1592041803..3095b77bd8 100644 --- a/backend/.sqlx/query-bc0bd9659bb8b58ed5967754b1c403e77ebb044063f0659485f44b1d04bb6b4d.json +++ b/backend/.sqlx/query-6f27f1d150df2903b66f29af0072e85a030134e440a712e54136c8a4eae36fc7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table)\n RETURNING id", + "query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag, priority)\n (SELECT uuid, $1, $2, $3, 0 FROM uuid_table)\n RETURNING id", "describe": { "columns": [ { @@ -21,5 +21,5 @@ false ] }, - "hash": "bc0bd9659bb8b58ed5967754b1c403e77ebb044063f0659485f44b1d04bb6b4d" + "hash": "6f27f1d150df2903b66f29af0072e85a030134e440a712e54136c8a4eae36fc7" } diff --git a/backend/.sqlx/query-c0c89487db7422a95551b070315020afda72512882226eb0ec362881b500956d.json b/backend/.sqlx/query-c0c89487db7422a95551b070315020afda72512882226eb0ec362881b500956d.json new file mode 100644 index 0000000000..6b57e81e8d --- /dev/null +++ b/backend/.sqlx/query-c0c89487db7422a95551b070315020afda72512882226eb0ec362881b500956d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT priority FROM v2_job WHERE id = $1 AND workspace_id = $2 AND priority < 0", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "priority", + "type_info": "Int2" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c0c89487db7422a95551b070315020afda72512882226eb0ec362881b500956d" +} diff --git a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json b/backend/.sqlx/query-df06e6136337d19e9f2753a92414aac10a3f7e2218e043829f40c8d155ec6688.json similarity index 88% rename from backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json rename to backend/.sqlx/query-df06e6136337d19e9f2753a92414aac10a3f7e2218e043829f40c8d155ec6688.json index d26e9f7c87..ff16110438 100644 --- a/backend/.sqlx/query-7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8.json +++ b/backend/.sqlx/query-df06e6136337d19e9f2753a92414aac10a3f7e2218e043829f40c8d155ec6688.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $16)\n )\n INSERT INTO v2_job\n (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,\n script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,\n concurrency_time_window_s, timeout, args)\n (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n ('{ \"uuid\": \"' || uuid || '\" }')::jsonb FROM uuid_table)\n RETURNING id AS \"id!\"", + "query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $16)\n )\n INSERT INTO v2_job\n (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,\n script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,\n concurrency_time_window_s, timeout, args, priority)\n (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, 0 FROM uuid_table)\n RETURNING id AS \"id!\"", "describe": { "columns": [ { @@ -96,5 +96,5 @@ false ] }, - "hash": "7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8" + "hash": "df06e6136337d19e9f2753a92414aac10a3f7e2218e043829f40c8d155ec6688" } diff --git a/backend/migrations/20261105210250_queue_priority_without_null.down.sql b/backend/migrations/20261105210250_queue_priority_without_null.down.sql new file mode 100644 index 0000000000..c8521a1e18 --- /dev/null +++ b/backend/migrations/20261105210250_queue_priority_without_null.down.sql @@ -0,0 +1,3 @@ +-- Flow steps and dependency jobs at 1 still sort ahead of NULL, so only the jobs without +-- priority are moved back. +UPDATE v2_job_queue SET priority = NULL WHERE priority = 0; diff --git a/backend/migrations/20261105210250_queue_priority_without_null.up.sql b/backend/migrations/20261105210250_queue_priority_without_null.up.sql new file mode 100644 index 0000000000..e7229a815b --- /dev/null +++ b/backend/migrations/20261105210250_queue_priority_without_null.up.sql @@ -0,0 +1,7 @@ +-- Workers pull `ORDER BY priority DESC NULLS LAST`. A job without priority is pushed as 0 and +-- flow steps and dependency jobs as 1, so that a negative (low) priority sorts after the +-- former. Moves the jobs already queued to those values so they keep their place; a NULL left +-- behind would sort after every low-priority job. +UPDATE v2_job_queue +SET priority = CASE WHEN priority IS NULL THEN 0 ELSE 1 END +WHERE priority IS NULL OR priority = 0; diff --git a/backend/windmill-api-jobs/src/jobs_export.rs b/backend/windmill-api-jobs/src/jobs_export.rs index c426497cf0..09a6f976ef 100644 --- a/backend/windmill-api-jobs/src/jobs_export.rs +++ b/backend/windmill-api-jobs/src/jobs_export.rs @@ -548,7 +548,8 @@ pub async fn import_queued_jobs( job.worker, job.extras as _, &job.tag, - job.priority + // A job without priority sorts as 0 (see `push`); an older export carries NULL. + job.priority.unwrap_or(0) ) .execute(&mut *tx) .await?; diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index d84e84d61d..2eaa13ff9d 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -10290,9 +10290,9 @@ async fn add_batch_jobs( INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind, script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit, - concurrency_time_window_s, timeout, args) + concurrency_time_window_s, timeout, args, priority) (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, - ('{ "uuid": "' || uuid || '" }')::jsonb FROM uuid_table) + ('{ "uuid": "' || uuid || '" }')::jsonb, 0 FROM uuid_table) RETURNING id AS "id!""#, w_id, raw_code, @@ -10319,8 +10319,8 @@ async fn add_batch_jobs( select unnest($4::uuid[]) as uuid ) INSERT INTO v2_job_queue - (id, workspace_id, scheduled_for, tag) - (SELECT uuid, $1, $2, $3 FROM uuid_table) + (id, workspace_id, scheduled_for, tag, priority) + (SELECT uuid, $1, $2, $3, 0 FROM uuid_table) RETURNING id"#, w_id, Utc::now(), diff --git a/backend/windmill-common/src/queue.rs b/backend/windmill-common/src/queue.rs index 2008e1ce93..cae326822e 100644 --- a/backend/windmill-common/src/queue.rs +++ b/backend/windmill-common/src/queue.rs @@ -2,6 +2,67 @@ use std::collections::HashMap; use sqlx::{Pool, Postgres}; +/// The priority a job is pushed with. Workers pull `ORDER BY priority DESC NULLS LAST`, so a +/// job without priority is stored as 0 rather than NULL, which leaves negative values (low +/// priority) below it. Flow steps and dependency jobs run at 1 or above, ahead of jobs without +/// priority, to drain the queue faster; a low flow keeps its steps low. +pub fn pushed_job_priority(priority: Option, flow_step: bool, dependency_job: bool) -> i16 { + match priority { + _ if dependency_job => 1, + Some(p) if p < 0 => p, + p if flow_step => p.unwrap_or(0).max(1), + p => p.unwrap_or(0), + } +} + +/// A job's stored priority, as an override for the jobs it starts. 0 and 1 are what a job +/// without priority and a flow step (a loop iteration or branch included) are stored as (see +/// [`pushed_job_priority`]): passing them on would replace the started runnable's own priority, +/// a Low subflow's included, instead of leaving it in effect. +pub fn passed_on_priority(priority: Option) -> Option { + priority.filter(|p| !matches!(p, 0 | 1)) +} + +#[cfg(test)] +mod tests { + use super::{passed_on_priority, pushed_job_priority}; + + #[test] + fn low_priority_sorts_after_jobs_without_priority() { + // (priority, flow step, dependency job) -> stored priority + let cases = [ + ((None, false, false), 0), + ((Some(100), false, false), 100), + ((Some(-100), false, false), -100), + ((None, true, false), 1), + ((Some(0), true, false), 1), + ((Some(50), true, false), 50), + ((Some(-100), true, false), -100), + ((None, false, true), 1), + ]; + for ((p, step, dep), expected) in cases { + assert_eq!( + pushed_job_priority(p, step, dep), + expected, + "{p:?} step={step} dep={dep}" + ); + } + } + + #[test] + fn stored_defaults_are_not_passed_on() { + for (stored, passed_on) in [ + (None, None), + (Some(0), None), + (Some(1), None), + (Some(50), Some(50)), + (Some(-100), Some(-100)), + ] { + assert_eq!(passed_on_priority(stored), passed_on, "{stored:?}"); + } + } +} + pub async fn get_queue_counts(db: &Pool) -> HashMap { sqlx::query!( "SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2a044484b1..961c4d1270 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -6480,6 +6480,36 @@ async fn fetch_pushed_flow_version( } } +/// A job with low (negative) priority passes it to every job it starts, all the way down, so a +/// background flow cannot get ahead of other work through the jobs its scripts start. It wins +/// over the priority the run endpoints give a job with a parent, which would otherwise put every +/// job such a flow starts ahead of all ordinary jobs. Flow steps get their flow's priority as an +/// override already and are not looked up here. +#[cfg(feature = "enterprise")] +async fn inherited_low_priority( + tx: &mut PushIsolationLevel<'_>, + db: &Pool, + workspace_id: &str, + parent_job: Option, + flow_step_id: &Option, +) -> error::Result> { + let Some(parent_job) = parent_job.filter(|_| flow_step_id.is_none()) else { + return Ok(None); + }; + let query = sqlx::query_scalar!( + "SELECT priority FROM v2_job WHERE id = $1 AND workspace_id = $2 AND priority < 0", + parent_job, + workspace_id, + ); + let priority = match tx { + PushIsolationLevel::Transaction(ntx) => query.fetch_optional(&mut **ntx).await?, + PushIsolationLevel::Isolated(..) | PushIsolationLevel::IsolatedRoot(_) => { + query.fetch_optional(db).await? + } + }; + Ok(priority.flatten()) +} + // #[instrument(level = "trace", skip_all)] async fn push_inner<'c, 'd>( db: &Pool, @@ -7612,6 +7642,10 @@ async fn push_inner<'c, 'd>( final_priority = if *CLOUD_HOSTED { // for cloud hosted instance, priority queues is disabled None + } else if let Some(low) = + inherited_low_priority(&mut tx, db, workspace_id, parent_job, &flow_step_id).await? + { + Some(low) } else if _priority_override.is_some() { _priority_override } else { @@ -7619,16 +7653,13 @@ async fn push_inner<'c, 'd>( _low_level_priority }; // else it remains empty, i.e. no priority } - // prioritize flow steps to drain the queue faster - let final_priority = if flow_step_id.is_some() && final_priority.is_none() { - Some(0) - } else if job_kind == JobKind::Dependencies && !build_binary_only { + let final_priority = Some(windmill_common::queue::pushed_job_priority( + final_priority, + flow_step_id.is_some(), // A binary build is a background optimization, not a deploy the user waits on, so // it does not get the dependency-job jump ahead of ordinary jobs on its tag. - Some(0) - } else { - final_priority - }; + job_kind == JobKind::Dependencies && !build_binary_only, + )); let is_running = same_worker || running; if let Some(flow) = raw_flow.as_ref() { diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index ed1f5ad79c..8c108a6278 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -767,7 +767,9 @@ async fn enqueue_windmill_tool( .await? .map(|x| x.into()); - let job_priority = tool_module.priority.or(ctx.job.priority); + let job_priority = tool_module + .priority + .or(windmill_common::queue::passed_on_priority(ctx.job.priority)); let tx = PushIsolationLevel::Transaction(tx); let (uuid, tx) = push( diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f2d683bcbf..c1e1d95616 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -4044,8 +4044,11 @@ pub async fn handle_wac_v2_output( step.tag.clone().or_else(|| Some(job.tag.clone())), step.timeout.or(job.timeout), None, // flow_step_id - step.priority - .or(if own_runnable { None } else { job.priority }), + step.priority.or(if own_runnable { + None + } else { + windmill_common::queue::passed_on_priority(job.priority) + }), None, // authed false, // running job.permissioned_as_end_user_email.clone(), diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 78da513502..a93355fda7 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -4816,13 +4816,13 @@ async fn push_next_flow_job( // compute job-to-be-pushed priority // The job definition itself might have its own priority, but as we're running // it from a flow here, it inherits first the flow step priority and second the - // flow priority. - let new_job_priority_override = if module.priority.is_some() { - module.priority - } else if flow_job.priority.is_some() { - flow_job.priority + // flow priority. A low (negative) flow priority wins over the step's own: a background + // flow must not get ahead of other work through one of its steps. + let flow_priority = windmill_common::queue::passed_on_priority(flow_job.priority); + let new_job_priority_override = if flow_priority.is_some_and(|p| p < 0) { + flow_priority } else { - None + module.priority.or(flow_priority) }; // The `flow_input` the step's input transforms read, which a `$flow_expr[flow_input...]` diff --git a/frontend/src/lib/components/itemSettings/PrioritySetting.svelte b/frontend/src/lib/components/itemSettings/PrioritySetting.svelte index 99a36543e4..8387e95bc1 100644 --- a/frontend/src/lib/components/itemSettings/PrioritySetting.svelte +++ b/frontend/src/lib/components/itemSettings/PrioritySetting.svelte @@ -12,6 +12,9 @@ } let { kind, priority = $bindable() }: Props = $props() + + // Any negative value sorts after jobs without priority, which are stored as 0. + const LOW_PRIORITY = -100 @@ -50,4 +53,15 @@ /> + (priority = on ? LOW_PRIORITY : undefined)} + /> {/if} diff --git a/frontend/src/lib/components/runs/RunBadges.svelte b/frontend/src/lib/components/runs/RunBadges.svelte index a04b6e646a..2c31ed852c 100644 --- a/frontend/src/lib/components/runs/RunBadges.svelte +++ b/frontend/src/lib/components/runs/RunBadges.svelte @@ -67,7 +67,8 @@ persistent {/if} -{#if job && 'priority' in job} + +{#if job && 'priority' in job && job.priority != undefined && job.priority !== 0}
priority: {job.priority}
diff --git a/frontend/src/lib/components/scriptSettings.test.ts b/frontend/src/lib/components/scriptSettings.test.ts index 92d8722a2f..34af208dfd 100644 --- a/frontend/src/lib/components/scriptSettings.test.ts +++ b/frontend/src/lib/components/scriptSettings.test.ts @@ -27,6 +27,11 @@ describe('getActiveScriptSettingsBadges', () => { expect(keys).toEqual([]) }) + it('surfaces a negative priority as low priority', () => { + const keys = getActiveScriptSettingsBadges({ priority: -100 }).map((b) => b.key) + expect(keys).toEqual(['low_priority']) + }) + it('treats non-positive concurrency limits, timeouts and cache ttl as inactive (legacy zero rows)', () => { const keys = getActiveScriptSettingsBadges({ concurrent_limit: 0, diff --git a/frontend/src/lib/components/scriptSettings.ts b/frontend/src/lib/components/scriptSettings.ts index c2e9eb809f..770406c3af 100644 --- a/frontend/src/lib/components/scriptSettings.ts +++ b/frontend/src/lib/components/scriptSettings.ts @@ -1,4 +1,5 @@ import { + ChevronsDown, ChevronsUp, CircleStop, Combine, @@ -169,6 +170,15 @@ export function getActiveScriptSettingsBadges( icon: ChevronsUp, detail: `Priority ${settings.priority}` }) + } else if (settings.priority != undefined && settings.priority < 0) { + badges.push({ + key: 'low_priority', + short: 'Low priority', + setting: 'low_priority', + label: 'Low priority', + icon: ChevronsDown, + detail: 'Runs when no other job is waiting' + }) } if (settings.restart_unless_cancelled) { badges.push({