fix: refill every free concurrency slot on completion, at constant cost (#12261)

* perf: top up deferred concurrency jobs with a bounded index walk

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: compute the wake bounds in bigint

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: filter wake candidates before bounding the walk

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: skip the wake when the completed job is limited per time window

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* revert: wake on every completion that frees a slot, whatever its time window

The completed job's window does not decide whether a waiter without one can take its slot.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-11-06 04:51:39 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 32e67db1c3
commit a07aef30d5
7 changed files with 96 additions and 54 deletions
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "WITH head AS (\n SELECT q.running, q.scheduled_for <= now() AS due\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND q.canceled_by IS NULL\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 2 * $2::bigint + 1\n ), taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*) FROM head WHERE due AND NOT running) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0)\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now()\n FROM next\n WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running\n AND v2_job_queue.scheduled_for > now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": []
},
"hash": "254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7"
}
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*)\n FROM concurrency_key due_ck\n JOIN v2_job_queue due ON due.id = due_ck.job_id\n WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL\n AND NOT due.running AND due.canceled_by IS NULL\n AND due.scheduled_for <= now()) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 1\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c"
}
@@ -0,0 +1 @@
DROP INDEX IF EXISTS concurrency_key_waiting_idx;
@@ -0,0 +1,11 @@
-- Serves wake_deferred_concurrency_job in windmill-queue/src/jobs.rs, which walks the jobs a
-- concurrency limit deferred on one key in the order they were first due and stops after a
-- few. Without it, every completion of a concurrency-limited job sorts the key's whole backlog.
-- Dropped before it is built because the OVERRIDDEN_MIGRATIONS rewrite in
-- windmill-api/src/db.rs runs both CONCURRENTLY: an interrupted concurrent build leaves an
-- invalid index that IF NOT EXISTS would skip rebuilding.
DROP INDEX IF EXISTS concurrency_key_waiting_idx;
CREATE INDEX IF NOT EXISTS concurrency_key_waiting_idx
ON concurrency_key (key, original_scheduled_for, job_id)
WHERE ended_at IS NULL AND original_scheduled_for IS NOT NULL;
+22 -15
View File
@@ -53,36 +53,43 @@ async fn due(db: &Pool<Postgres>, id: Uuid) -> anyhow::Result<bool> {
)
}
/// Completions wake the jobs the limit deferred first, as many as the free slots, never
/// a job scheduled for later or one limited per time window.
/// One completion tops the key up: it wakes the jobs the limit deferred first, as many as
/// the free slots, and a later completion wakes none while they are due. Never a job
/// scheduled for later or one limited per time window, and older time-window jobs sharing
/// the key, more of them than the head the wake counts in, do not hide the eligible ones.
#[sqlx::test(fixtures("base"))]
async fn completions_wake_the_first_deferred_jobs(db: Pool<Postgres>) -> anyhow::Result<()> {
let windowed = waiting_job(&db, Some(40), Some(10)).await?;
async fn completion_fills_the_free_slots(db: Pool<Postgres>) -> anyhow::Result<()> {
let mut windowed = vec![];
for i in 0..(2 * LIMIT + 2) as i64 {
windowed.push(waiting_job(&db, Some(100 - i), Some(10)).await?);
}
let first = waiting_job(&db, Some(30), None).await?;
let second = waiting_job(&db, Some(20), None).await?;
let third = waiting_job(&db, Some(10), None).await?;
let scheduled_later = waiting_job(&db, None, None).await?;
for _ in 0..3 {
wake_deferred_concurrency_job(&db, KEY).await?;
}
wake_deferred_concurrency_job(&db, KEY, LIMIT).await?;
let second_woken_at_once = due(&db, second).await?;
wake_deferred_concurrency_job(&db, KEY, LIMIT).await?;
assert!(
second_woken_at_once,
"a free slot was left to the deferral estimate"
);
assert!(
due(&db, first).await?,
"the first deferred job was not woken"
);
assert!(
due(&db, second).await?,
"a free slot was left to the deferral estimate"
);
assert!(
!due(&db, third).await?,
"more jobs were woken than slots are free"
);
assert!(
!due(&db, windowed).await?,
"a job limited per time window was woken"
);
for w in windowed {
assert!(
!due(&db, w).await?,
"a job limited per time window was woken"
);
}
assert!(
!due(&db, scheduled_later).await?,
"a job scheduled for later was woken"
+3
View File
@@ -120,6 +120,9 @@ lazy_static::lazy_static! {
(20261015114712, include_str!(
"../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql"
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
(20261105203203, include_str!(
"../../migrations/20261105203203_concurrency_key_waiting_index.up.sql"
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
// The widening fails every query the previous release's still-running
// servers prepared on the column; `variables::widen_variable_value` does it
// once they are gone.
+44 -25
View File
@@ -1601,16 +1601,16 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// Resolve the concurrency-limit settings on the pool *before* opening the
// completion transaction: doing it inside the tx would hold a second
// simultaneous connection from the small per-worker pool.
let settings_concurrent_limit =
windmill_common::runnable_settings::prefetch_cached_from_handle(
completed_job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit;
let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit)
|| has_active_concurrency_limit(
windmill_common::runnable_settings::prefetch_cached_from_handle(
completed_job.runnable_settings_handle,
db,
)
.await?
.1
.concurrent_limit,
);
|| has_active_concurrency_limit(settings_concurrent_limit);
// A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that
// the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape
@@ -1868,7 +1868,11 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
// After the commit, so the woken job finds the slot this one freed.
#[cfg(all(feature = "enterprise", feature = "private"))]
if let Some(key) = released_concurrency_key {
if let Err(e) = wake_deferred_concurrency_job(db, &key).await {
let limit = completed_job
.concurrent_limit
.max(settings_concurrent_limit)
.unwrap_or(0);
if let Err(e) = wake_deferred_concurrency_job(db, &key, limit).await {
tracing::error!("Could not wake a deferred job for concurrency key {key}: {e:#}");
}
}
@@ -5309,25 +5313,36 @@ pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result<
Ok(true)
}
/// Make due now the job of `key` that a concurrency limit deferred first, so the slot a
/// completed job freed is taken rather than left empty until the deferral's estimate.
/// No-op while the jobs of the key already due fill the free slots, and for a job limited
/// per time window, whose slot a completion does not free. Queue bookkeeping, not an
/// authorization boundary: only for the completion path, never with a key taken from a request.
/// Make due now the earliest-deferred jobs of `key`, as many as the slots no admitted or
/// already-due job holds. It tops up to the limit: waking one job per completion never
/// refills a slot whose woken job was lost, and the key stays below its limit.
/// Already-due jobs are counted in the first `2 * limit + 1` deferred jobs not limited per
/// time window: wakes go to the earliest of those, so they come first, and missing one can
/// only over-wake, which admission re-defers. Both walks filter before they stop, so older
/// time-window jobs or ones at a full lower limit never hide an eligible job on a shared key.
/// `limit` is the completed job's. Runs whatever the completed job's own time window: its slot
/// can go to a waiter that has none. Queue bookkeeping, not an authorization boundary.
#[cfg(all(feature = "enterprise", feature = "private"))]
pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result<()> {
pub async fn wake_deferred_concurrency_job(db: &DB, key: &str, limit: i32) -> error::Result<()> {
sqlx::query!(
"WITH taken AS (
"WITH head AS (
SELECT q.running, q.scheduled_for <= now() AS due
FROM concurrency_key ck
JOIN v2_job_queue q ON q.id = ck.job_id
JOIN v2_job j ON j.id = ck.job_id
LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle
LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings
WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL
AND q.canceled_by IS NULL
AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0
ORDER BY ck.original_scheduled_for, ck.job_id
LIMIT 2 * $2::bigint + 1
), taken AS (
SELECT
(SELECT count(*)
FROM concurrency_counter, jsonb_object_keys(job_uuids)
WHERE concurrency_id = $1)
+ (SELECT count(*)
FROM concurrency_key due_ck
JOIN v2_job_queue due ON due.id = due_ck.job_id
WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL
AND NOT due.running AND due.canceled_by IS NULL
AND due.scheduled_for <= now()) AS n
+ (SELECT count(*) FROM head WHERE due AND NOT running) AS n
), next AS (
SELECT q.id
FROM concurrency_key ck
@@ -5340,11 +5355,15 @@ pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result<
AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0
AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)
ORDER BY ck.original_scheduled_for, ck.job_id
LIMIT 1
LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0)
FOR UPDATE OF q SKIP LOCKED
)
UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id",
UPDATE v2_job_queue SET scheduled_for = now()
FROM next
WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running
AND v2_job_queue.scheduled_for > now()",
key,
i64::from(limit),
)
.execute(db)
.await?;