mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-20 08:08:58 +00:00
feat: inherited low priority for background flows and scripts (#12256)
* feat: add inherited low priority for background flows and scripts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: keep a low flow priority over step priorities and align the -1 boundary Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs: note the rolling-upgrade ordering and the renamed queue index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * refactor: order low priority by stored values instead of a coalesce index Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: keep a runnable's own priority when its caller has none Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: do not pass a flow step's default priority on to the jobs it starts Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
4d9bb44686
commit
3c5a5caf97
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag)\n (SELECT uuid, $1, $2, $3 FROM uuid_table)\n RETURNING id",
|
||||
"query": "WITH uuid_table as (\n select unnest($4::uuid[]) as uuid\n )\n INSERT INTO v2_job_queue\n (id, workspace_id, scheduled_for, tag, priority)\n (SELECT uuid, $1, $2, $3, 0 FROM uuid_table)\n RETURNING id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -21,5 +21,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "bc0bd9659bb8b58ed5967754b1c403e77ebb044063f0659485f44b1d04bb6b4d"
|
||||
"hash": "6f27f1d150df2903b66f29af0072e85a030134e440a712e54136c8a4eae36fc7"
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT priority FROM v2_job WHERE id = $1 AND workspace_id = $2 AND priority < 0",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "priority",
|
||||
"type_info": "Int2"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "c0c89487db7422a95551b070315020afda72512882226eb0ec362881b500956d"
|
||||
}
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $16)\n )\n INSERT INTO v2_job\n (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,\n script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,\n concurrency_time_window_s, timeout, args)\n (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n ('{ \"uuid\": \"' || uuid || '\" }')::jsonb FROM uuid_table)\n RETURNING id AS \"id!\"",
|
||||
"query": "WITH uuid_table as (\n select gen_random_uuid() as uuid from generate_series(1, $16)\n )\n INSERT INTO v2_job\n (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,\n script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,\n concurrency_time_window_s, timeout, args, priority)\n (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,\n ('{ \"uuid\": \"' || uuid || '\" }')::jsonb, 0 FROM uuid_table)\n RETURNING id AS \"id!\"",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -96,5 +96,5 @@
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "7bb8ff3426ec73672fb572cd1ce36f495a84e0ad7d60fd39eddcccbc129d43c8"
|
||||
"hash": "df06e6136337d19e9f2753a92414aac10a3f7e2218e043829f40c8d155ec6688"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Flow steps and dependency jobs at 1 still sort ahead of NULL, so only the jobs without
|
||||
-- priority are moved back.
|
||||
UPDATE v2_job_queue SET priority = NULL WHERE priority = 0;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- Workers pull `ORDER BY priority DESC NULLS LAST`. A job without priority is pushed as 0 and
|
||||
-- flow steps and dependency jobs as 1, so that a negative (low) priority sorts after the
|
||||
-- former. Moves the jobs already queued to those values so they keep their place; a NULL left
|
||||
-- behind would sort after every low-priority job.
|
||||
UPDATE v2_job_queue
|
||||
SET priority = CASE WHEN priority IS NULL THEN 0 ELSE 1 END
|
||||
WHERE priority IS NULL OR priority = 0;
|
||||
@@ -548,7 +548,8 @@ pub async fn import_queued_jobs(
|
||||
job.worker,
|
||||
job.extras as _,
|
||||
&job.tag,
|
||||
job.priority
|
||||
// A job without priority sorts as 0 (see `push`); an older export carries NULL.
|
||||
job.priority.unwrap_or(0)
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
@@ -10290,9 +10290,9 @@ async fn add_batch_jobs(
|
||||
INSERT INTO v2_job
|
||||
(id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind,
|
||||
script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit,
|
||||
concurrency_time_window_s, timeout, args)
|
||||
concurrency_time_window_s, timeout, args, priority)
|
||||
(SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15,
|
||||
('{ "uuid": "' || uuid || '" }')::jsonb FROM uuid_table)
|
||||
('{ "uuid": "' || uuid || '" }')::jsonb, 0 FROM uuid_table)
|
||||
RETURNING id AS "id!""#,
|
||||
w_id,
|
||||
raw_code,
|
||||
@@ -10319,8 +10319,8 @@ async fn add_batch_jobs(
|
||||
select unnest($4::uuid[]) as uuid
|
||||
)
|
||||
INSERT INTO v2_job_queue
|
||||
(id, workspace_id, scheduled_for, tag)
|
||||
(SELECT uuid, $1, $2, $3 FROM uuid_table)
|
||||
(id, workspace_id, scheduled_for, tag, priority)
|
||||
(SELECT uuid, $1, $2, $3, 0 FROM uuid_table)
|
||||
RETURNING id"#,
|
||||
w_id,
|
||||
Utc::now(),
|
||||
|
||||
@@ -2,6 +2,67 @@ use std::collections::HashMap;
|
||||
|
||||
use sqlx::{Pool, Postgres};
|
||||
|
||||
/// The priority a job is pushed with. Workers pull `ORDER BY priority DESC NULLS LAST`, so a
|
||||
/// job without priority is stored as 0 rather than NULL, which leaves negative values (low
|
||||
/// priority) below it. Flow steps and dependency jobs run at 1 or above, ahead of jobs without
|
||||
/// priority, to drain the queue faster; a low flow keeps its steps low.
|
||||
pub fn pushed_job_priority(priority: Option<i16>, flow_step: bool, dependency_job: bool) -> i16 {
|
||||
match priority {
|
||||
_ if dependency_job => 1,
|
||||
Some(p) if p < 0 => p,
|
||||
p if flow_step => p.unwrap_or(0).max(1),
|
||||
p => p.unwrap_or(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// A job's stored priority, as an override for the jobs it starts. 0 and 1 are what a job
|
||||
/// without priority and a flow step (a loop iteration or branch included) are stored as (see
|
||||
/// [`pushed_job_priority`]): passing them on would replace the started runnable's own priority,
|
||||
/// a Low subflow's included, instead of leaving it in effect.
|
||||
pub fn passed_on_priority(priority: Option<i16>) -> Option<i16> {
|
||||
priority.filter(|p| !matches!(p, 0 | 1))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{passed_on_priority, pushed_job_priority};
|
||||
|
||||
#[test]
|
||||
fn low_priority_sorts_after_jobs_without_priority() {
|
||||
// (priority, flow step, dependency job) -> stored priority
|
||||
let cases = [
|
||||
((None, false, false), 0),
|
||||
((Some(100), false, false), 100),
|
||||
((Some(-100), false, false), -100),
|
||||
((None, true, false), 1),
|
||||
((Some(0), true, false), 1),
|
||||
((Some(50), true, false), 50),
|
||||
((Some(-100), true, false), -100),
|
||||
((None, false, true), 1),
|
||||
];
|
||||
for ((p, step, dep), expected) in cases {
|
||||
assert_eq!(
|
||||
pushed_job_priority(p, step, dep),
|
||||
expected,
|
||||
"{p:?} step={step} dep={dep}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stored_defaults_are_not_passed_on() {
|
||||
for (stored, passed_on) in [
|
||||
(None, None),
|
||||
(Some(0), None),
|
||||
(Some(1), None),
|
||||
(Some(50), Some(50)),
|
||||
(Some(-100), Some(-100)),
|
||||
] {
|
||||
assert_eq!(passed_on_priority(stored), passed_on, "{stored:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_queue_counts(db: &Pool<Postgres>) -> HashMap<String, u32> {
|
||||
sqlx::query!(
|
||||
"SELECT tag AS \"tag!\", count(*) AS \"count!\" FROM v2_job_queue WHERE
|
||||
|
||||
@@ -6480,6 +6480,36 @@ async fn fetch_pushed_flow_version(
|
||||
}
|
||||
}
|
||||
|
||||
/// A job with low (negative) priority passes it to every job it starts, all the way down, so a
|
||||
/// background flow cannot get ahead of other work through the jobs its scripts start. It wins
|
||||
/// over the priority the run endpoints give a job with a parent, which would otherwise put every
|
||||
/// job such a flow starts ahead of all ordinary jobs. Flow steps get their flow's priority as an
|
||||
/// override already and are not looked up here.
|
||||
#[cfg(feature = "enterprise")]
|
||||
async fn inherited_low_priority(
|
||||
tx: &mut PushIsolationLevel<'_>,
|
||||
db: &Pool<Postgres>,
|
||||
workspace_id: &str,
|
||||
parent_job: Option<Uuid>,
|
||||
flow_step_id: &Option<String>,
|
||||
) -> error::Result<Option<i16>> {
|
||||
let Some(parent_job) = parent_job.filter(|_| flow_step_id.is_none()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let query = sqlx::query_scalar!(
|
||||
"SELECT priority FROM v2_job WHERE id = $1 AND workspace_id = $2 AND priority < 0",
|
||||
parent_job,
|
||||
workspace_id,
|
||||
);
|
||||
let priority = match tx {
|
||||
PushIsolationLevel::Transaction(ntx) => query.fetch_optional(&mut **ntx).await?,
|
||||
PushIsolationLevel::Isolated(..) | PushIsolationLevel::IsolatedRoot(_) => {
|
||||
query.fetch_optional(db).await?
|
||||
}
|
||||
};
|
||||
Ok(priority.flatten())
|
||||
}
|
||||
|
||||
// #[instrument(level = "trace", skip_all)]
|
||||
async fn push_inner<'c, 'd>(
|
||||
db: &Pool<Postgres>,
|
||||
@@ -7612,6 +7642,10 @@ async fn push_inner<'c, 'd>(
|
||||
final_priority = if *CLOUD_HOSTED {
|
||||
// for cloud hosted instance, priority queues is disabled
|
||||
None
|
||||
} else if let Some(low) =
|
||||
inherited_low_priority(&mut tx, db, workspace_id, parent_job, &flow_step_id).await?
|
||||
{
|
||||
Some(low)
|
||||
} else if _priority_override.is_some() {
|
||||
_priority_override
|
||||
} else {
|
||||
@@ -7619,16 +7653,13 @@ async fn push_inner<'c, 'd>(
|
||||
_low_level_priority
|
||||
}; // else it remains empty, i.e. no priority
|
||||
}
|
||||
// prioritize flow steps to drain the queue faster
|
||||
let final_priority = if flow_step_id.is_some() && final_priority.is_none() {
|
||||
Some(0)
|
||||
} else if job_kind == JobKind::Dependencies && !build_binary_only {
|
||||
let final_priority = Some(windmill_common::queue::pushed_job_priority(
|
||||
final_priority,
|
||||
flow_step_id.is_some(),
|
||||
// A binary build is a background optimization, not a deploy the user waits on, so
|
||||
// it does not get the dependency-job jump ahead of ordinary jobs on its tag.
|
||||
Some(0)
|
||||
} else {
|
||||
final_priority
|
||||
};
|
||||
job_kind == JobKind::Dependencies && !build_binary_only,
|
||||
));
|
||||
|
||||
let is_running = same_worker || running;
|
||||
if let Some(flow) = raw_flow.as_ref() {
|
||||
|
||||
@@ -767,7 +767,9 @@ async fn enqueue_windmill_tool(
|
||||
.await?
|
||||
.map(|x| x.into());
|
||||
|
||||
let job_priority = tool_module.priority.or(ctx.job.priority);
|
||||
let job_priority = tool_module
|
||||
.priority
|
||||
.or(windmill_common::queue::passed_on_priority(ctx.job.priority));
|
||||
|
||||
let tx = PushIsolationLevel::Transaction(tx);
|
||||
let (uuid, tx) = push(
|
||||
|
||||
@@ -4044,8 +4044,11 @@ pub async fn handle_wac_v2_output(
|
||||
step.tag.clone().or_else(|| Some(job.tag.clone())),
|
||||
step.timeout.or(job.timeout),
|
||||
None, // flow_step_id
|
||||
step.priority
|
||||
.or(if own_runnable { None } else { job.priority }),
|
||||
step.priority.or(if own_runnable {
|
||||
None
|
||||
} else {
|
||||
windmill_common::queue::passed_on_priority(job.priority)
|
||||
}),
|
||||
None, // authed
|
||||
false, // running
|
||||
job.permissioned_as_end_user_email.clone(),
|
||||
|
||||
@@ -4816,13 +4816,13 @@ async fn push_next_flow_job(
|
||||
// compute job-to-be-pushed priority
|
||||
// The job definition itself might have its own priority, but as we're running
|
||||
// it from a flow here, it inherits first the flow step priority and second the
|
||||
// flow priority.
|
||||
let new_job_priority_override = if module.priority.is_some() {
|
||||
module.priority
|
||||
} else if flow_job.priority.is_some() {
|
||||
flow_job.priority
|
||||
// flow priority. A low (negative) flow priority wins over the step's own: a background
|
||||
// flow must not get ahead of other work through one of its steps.
|
||||
let flow_priority = windmill_common::queue::passed_on_priority(flow_job.priority);
|
||||
let new_job_priority_override = if flow_priority.is_some_and(|p| p < 0) {
|
||||
flow_priority
|
||||
} else {
|
||||
None
|
||||
module.priority.or(flow_priority)
|
||||
};
|
||||
|
||||
// The `flow_input` the step's input transforms read, which a `$flow_expr[flow_input...]`
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
}
|
||||
|
||||
let { kind, priority = $bindable() }: Props = $props()
|
||||
|
||||
// Any negative value sorts after jobs without priority, which are stored as 0.
|
||||
const LOW_PRIORITY = -100
|
||||
</script>
|
||||
|
||||
<!-- Not available on the cloud, where the toggle could only ever be disabled. -->
|
||||
@@ -50,4 +53,15 @@
|
||||
/>
|
||||
</Label>
|
||||
</SettingToggle>
|
||||
<SettingToggle
|
||||
setting="low_priority"
|
||||
label="Low priority"
|
||||
tooltip={kind === 'flow'
|
||||
? 'The jobs of this flow, of its steps and every job they start run only when no other job is waiting. Use it for large background flows so they do not hold up the rest of the workspace. A script that waits for a job it started keeps its worker until that job runs.'
|
||||
: 'The jobs of this script and every job they start run only when no other job is waiting. A script that waits for a job it started keeps its worker until that job runs.'}
|
||||
eeOnly
|
||||
disabled={!$enterpriseLicense}
|
||||
checked={priority !== undefined && priority < 0}
|
||||
onchange={(on) => (priority = on ? LOW_PRIORITY : undefined)}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
@@ -67,7 +67,8 @@
|
||||
<Badge color="red" {large}>persistent</Badge>
|
||||
</button>
|
||||
{/if}
|
||||
{#if job && 'priority' in job}
|
||||
<!-- 0 is how a job without priority is stored. -->
|
||||
{#if job && 'priority' in job && job.priority != undefined && job.priority !== 0}
|
||||
<div>
|
||||
<Badge color="blue" {large}>priority: {job.priority}</Badge>
|
||||
</div>
|
||||
|
||||
@@ -27,6 +27,11 @@ describe('getActiveScriptSettingsBadges', () => {
|
||||
expect(keys).toEqual([])
|
||||
})
|
||||
|
||||
it('surfaces a negative priority as low priority', () => {
|
||||
const keys = getActiveScriptSettingsBadges({ priority: -100 }).map((b) => b.key)
|
||||
expect(keys).toEqual(['low_priority'])
|
||||
})
|
||||
|
||||
it('treats non-positive concurrency limits, timeouts and cache ttl as inactive (legacy zero rows)', () => {
|
||||
const keys = getActiveScriptSettingsBadges({
|
||||
concurrent_limit: 0,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
ChevronsDown,
|
||||
ChevronsUp,
|
||||
CircleStop,
|
||||
Combine,
|
||||
@@ -169,6 +170,15 @@ export function getActiveScriptSettingsBadges(
|
||||
icon: ChevronsUp,
|
||||
detail: `Priority ${settings.priority}`
|
||||
})
|
||||
} else if (settings.priority != undefined && settings.priority < 0) {
|
||||
badges.push({
|
||||
key: 'low_priority',
|
||||
short: 'Low priority',
|
||||
setting: 'low_priority',
|
||||
label: 'Low priority',
|
||||
icon: ChevronsDown,
|
||||
detail: 'Runs when no other job is waiting'
|
||||
})
|
||||
}
|
||||
if (settings.restart_unless_cancelled) {
|
||||
badges.push({
|
||||
|
||||
Reference in New Issue
Block a user