mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
fix: refill every free concurrency slot on completion, at constant cost (#12261)
* perf: top up deferred concurrency jobs with a bounded index walk Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: compute the wake bounds in bigint Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: filter wake candidates before bounding the walk Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: skip the wake when the completed job is limited per time window Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * revert: wake on every completion that frees a slot, whatever its time window The completed job's window does not decide whether a waiter without one can take its slot. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
32e67db1c3
commit
a07aef30d5
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH head AS (\n SELECT q.running, q.scheduled_for <= now() AS due\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND q.canceled_by IS NULL\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 2 * $2::bigint + 1\n ), taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*) FROM head WHERE due AND NOT running) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0)\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now()\n FROM next\n WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running\n AND v2_job_queue.scheduled_for > now()",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "254033e08ae82503662b598b6949e3d59bd4df85375c7fd35b0dfe16283fa8e7"
|
||||
}
|
||||
-14
@@ -1,14 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH taken AS (\n SELECT\n (SELECT count(*)\n FROM concurrency_counter, jsonb_object_keys(job_uuids)\n WHERE concurrency_id = $1)\n + (SELECT count(*)\n FROM concurrency_key due_ck\n JOIN v2_job_queue due ON due.id = due_ck.job_id\n WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL\n AND NOT due.running AND due.canceled_by IS NULL\n AND due.scheduled_for <= now()) AS n\n ), next AS (\n SELECT q.id\n FROM concurrency_key ck\n JOIN v2_job_queue q ON q.id = ck.job_id\n JOIN v2_job j ON j.id = ck.job_id\n LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle\n LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings\n WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL\n AND NOT q.running AND q.canceled_by IS NULL AND q.scheduled_for > now()\n AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0\n AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)\n ORDER BY ck.original_scheduled_for, ck.job_id\n LIMIT 1\n FOR UPDATE OF q SKIP LOCKED\n )\n UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "f388d7cf331a0aeb054f20b0fc319fb76d423193411bbd8ac80a1eac8eb1448c"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
DROP INDEX IF EXISTS concurrency_key_waiting_idx;
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Serves wake_deferred_concurrency_job in windmill-queue/src/jobs.rs, which walks the jobs a
|
||||
-- concurrency limit deferred on one key in the order they were first due and stops after a
|
||||
-- few. Without it, every completion of a concurrency-limited job sorts the key's whole backlog.
|
||||
-- Dropped before it is built because the OVERRIDDEN_MIGRATIONS rewrite in
|
||||
-- windmill-api/src/db.rs runs both CONCURRENTLY: an interrupted concurrent build leaves an
|
||||
-- invalid index that IF NOT EXISTS would skip rebuilding.
|
||||
DROP INDEX IF EXISTS concurrency_key_waiting_idx;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS concurrency_key_waiting_idx
|
||||
ON concurrency_key (key, original_scheduled_for, job_id)
|
||||
WHERE ended_at IS NULL AND original_scheduled_for IS NOT NULL;
|
||||
@@ -53,36 +53,43 @@ async fn due(db: &Pool<Postgres>, id: Uuid) -> anyhow::Result<bool> {
|
||||
)
|
||||
}
|
||||
|
||||
/// Completions wake the jobs the limit deferred first, as many as the free slots, never
|
||||
/// a job scheduled for later or one limited per time window.
|
||||
/// One completion tops the key up: it wakes the jobs the limit deferred first, as many as
|
||||
/// the free slots, and a later completion wakes none while they are due. Never a job
|
||||
/// scheduled for later or one limited per time window, and older time-window jobs sharing
|
||||
/// the key, more of them than the head the wake counts in, do not hide the eligible ones.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn completions_wake_the_first_deferred_jobs(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let windowed = waiting_job(&db, Some(40), Some(10)).await?;
|
||||
async fn completion_fills_the_free_slots(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let mut windowed = vec![];
|
||||
for i in 0..(2 * LIMIT + 2) as i64 {
|
||||
windowed.push(waiting_job(&db, Some(100 - i), Some(10)).await?);
|
||||
}
|
||||
let first = waiting_job(&db, Some(30), None).await?;
|
||||
let second = waiting_job(&db, Some(20), None).await?;
|
||||
let third = waiting_job(&db, Some(10), None).await?;
|
||||
let scheduled_later = waiting_job(&db, None, None).await?;
|
||||
|
||||
for _ in 0..3 {
|
||||
wake_deferred_concurrency_job(&db, KEY).await?;
|
||||
}
|
||||
wake_deferred_concurrency_job(&db, KEY, LIMIT).await?;
|
||||
let second_woken_at_once = due(&db, second).await?;
|
||||
wake_deferred_concurrency_job(&db, KEY, LIMIT).await?;
|
||||
|
||||
assert!(
|
||||
second_woken_at_once,
|
||||
"a free slot was left to the deferral estimate"
|
||||
);
|
||||
assert!(
|
||||
due(&db, first).await?,
|
||||
"the first deferred job was not woken"
|
||||
);
|
||||
assert!(
|
||||
due(&db, second).await?,
|
||||
"a free slot was left to the deferral estimate"
|
||||
);
|
||||
assert!(
|
||||
!due(&db, third).await?,
|
||||
"more jobs were woken than slots are free"
|
||||
);
|
||||
assert!(
|
||||
!due(&db, windowed).await?,
|
||||
"a job limited per time window was woken"
|
||||
);
|
||||
for w in windowed {
|
||||
assert!(
|
||||
!due(&db, w).await?,
|
||||
"a job limited per time window was woken"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
!due(&db, scheduled_later).await?,
|
||||
"a job scheduled for later was woken"
|
||||
|
||||
@@ -120,6 +120,9 @@ lazy_static::lazy_static! {
|
||||
(20261015114712, include_str!(
|
||||
"../../migrations/20261015114712_queue_workspace_scheduled_for_index.up.sql"
|
||||
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
|
||||
(20261105203203, include_str!(
|
||||
"../../migrations/20261105203203_concurrency_key_waiting_index.up.sql"
|
||||
).replace("CREATE INDEX", "CREATE INDEX CONCURRENTLY").replace("DROP INDEX", "DROP INDEX CONCURRENTLY")),
|
||||
// The widening fails every query the previous release's still-running
|
||||
// servers prepared on the column; `variables::widen_variable_value` does it
|
||||
// once they are gone.
|
||||
|
||||
@@ -1601,16 +1601,16 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
// Resolve the concurrency-limit settings on the pool *before* opening the
|
||||
// completion transaction: doing it inside the tx would hold a second
|
||||
// simultaneous connection from the small per-worker pool.
|
||||
let settings_concurrent_limit =
|
||||
windmill_common::runnable_settings::prefetch_cached_from_handle(
|
||||
completed_job.runnable_settings_handle,
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
.1
|
||||
.concurrent_limit;
|
||||
let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit)
|
||||
|| has_active_concurrency_limit(
|
||||
windmill_common::runnable_settings::prefetch_cached_from_handle(
|
||||
completed_job.runnable_settings_handle,
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
.1
|
||||
.concurrent_limit,
|
||||
);
|
||||
|| has_active_concurrency_limit(settings_concurrent_limit);
|
||||
|
||||
// A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that
|
||||
// the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape
|
||||
@@ -1868,7 +1868,11 @@ async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
|
||||
// After the commit, so the woken job finds the slot this one freed.
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
if let Some(key) = released_concurrency_key {
|
||||
if let Err(e) = wake_deferred_concurrency_job(db, &key).await {
|
||||
let limit = completed_job
|
||||
.concurrent_limit
|
||||
.max(settings_concurrent_limit)
|
||||
.unwrap_or(0);
|
||||
if let Err(e) = wake_deferred_concurrency_job(db, &key, limit).await {
|
||||
tracing::error!("Could not wake a deferred job for concurrency key {key}: {e:#}");
|
||||
}
|
||||
}
|
||||
@@ -5309,25 +5313,36 @@ pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result<
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Make due now the job of `key` that a concurrency limit deferred first, so the slot a
|
||||
/// completed job freed is taken rather than left empty until the deferral's estimate.
|
||||
/// No-op while the jobs of the key already due fill the free slots, and for a job limited
|
||||
/// per time window, whose slot a completion does not free. Queue bookkeeping, not an
|
||||
/// authorization boundary: only for the completion path, never with a key taken from a request.
|
||||
/// Make due now the earliest-deferred jobs of `key`, as many as the slots no admitted or
|
||||
/// already-due job holds. It tops up to the limit: waking one job per completion never
|
||||
/// refills a slot whose woken job was lost, and the key stays below its limit.
|
||||
/// Already-due jobs are counted in the first `2 * limit + 1` deferred jobs not limited per
|
||||
/// time window: wakes go to the earliest of those, so they come first, and missing one can
|
||||
/// only over-wake, which admission re-defers. Both walks filter before they stop, so older
|
||||
/// time-window jobs or ones at a full lower limit never hide an eligible job on a shared key.
|
||||
/// `limit` is the completed job's. Runs whatever the completed job's own time window: its slot
|
||||
/// can go to a waiter that has none. Queue bookkeeping, not an authorization boundary.
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result<()> {
|
||||
pub async fn wake_deferred_concurrency_job(db: &DB, key: &str, limit: i32) -> error::Result<()> {
|
||||
sqlx::query!(
|
||||
"WITH taken AS (
|
||||
"WITH head AS (
|
||||
SELECT q.running, q.scheduled_for <= now() AS due
|
||||
FROM concurrency_key ck
|
||||
JOIN v2_job_queue q ON q.id = ck.job_id
|
||||
JOIN v2_job j ON j.id = ck.job_id
|
||||
LEFT JOIN runnable_settings rs ON rs.hash = q.runnable_settings_handle
|
||||
LEFT JOIN concurrency_settings cs ON cs.hash = rs.concurrency_settings
|
||||
WHERE ck.key = $1 AND ck.ended_at IS NULL AND ck.original_scheduled_for IS NOT NULL
|
||||
AND q.canceled_by IS NULL
|
||||
AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0
|
||||
ORDER BY ck.original_scheduled_for, ck.job_id
|
||||
LIMIT 2 * $2::bigint + 1
|
||||
), taken AS (
|
||||
SELECT
|
||||
(SELECT count(*)
|
||||
FROM concurrency_counter, jsonb_object_keys(job_uuids)
|
||||
WHERE concurrency_id = $1)
|
||||
+ (SELECT count(*)
|
||||
FROM concurrency_key due_ck
|
||||
JOIN v2_job_queue due ON due.id = due_ck.job_id
|
||||
WHERE due_ck.key = $1 AND due_ck.ended_at IS NULL
|
||||
AND NOT due.running AND due.canceled_by IS NULL
|
||||
AND due.scheduled_for <= now()) AS n
|
||||
+ (SELECT count(*) FROM head WHERE due AND NOT running) AS n
|
||||
), next AS (
|
||||
SELECT q.id
|
||||
FROM concurrency_key ck
|
||||
@@ -5340,11 +5355,15 @@ pub async fn wake_deferred_concurrency_job(db: &DB, key: &str) -> error::Result<
|
||||
AND COALESCE(cs.concurrency_time_window_s, j.concurrency_time_window_s, 0) = 0
|
||||
AND (SELECT n FROM taken) < COALESCE(cs.concurrent_limit, j.concurrent_limit, 0)
|
||||
ORDER BY ck.original_scheduled_for, ck.job_id
|
||||
LIMIT 1
|
||||
LIMIT GREATEST($2::bigint - (SELECT n FROM taken), 0)
|
||||
FOR UPDATE OF q SKIP LOCKED
|
||||
)
|
||||
UPDATE v2_job_queue SET scheduled_for = now() FROM next WHERE v2_job_queue.id = next.id",
|
||||
UPDATE v2_job_queue SET scheduled_for = now()
|
||||
FROM next
|
||||
WHERE v2_job_queue.id = next.id AND NOT v2_job_queue.running
|
||||
AND v2_job_queue.scheduled_for > now()",
|
||||
key,
|
||||
i64::from(limit),
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
|
||||
Reference in New Issue
Block a user