mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-06 16:02:23 +00:00
feat(debounce): claim-based exactly-once batch consumption
Eliminates the rare duplicate/loss when two survivors land on one debounce batch (a narrow push/pull race), without locking the worker pull hot path. - migration: v2_job_debounce_batch gains consumed_at + consumed_by. - pull side (maybe_apply_debouncing): instead of deleting the batch on consume, a survivor atomically claims its own row + any unclaimed siblings (stamping consumed_by = itself) and accumulates exactly the rows it claimed. A second survivor of the same batch finds its row already consumed by another job and runs empty (no duplicate); a re-pulled survivor recognizes its own prior claim and keeps its accumulated args; a never-batched job (CE/legacy) keeps its own args. Non-accumulate debounce paths still hard-delete their batch rows. - complete_debounced_job (EE companion) never completes a running predecessor, so its in-flight run is not killed (no loss); the claim then prevents the duplicate the guard would otherwise allow. - monitor: GC sweep deletes consumed batch rows past a 1h grace. Together with the running-survivor guard this makes debounce accumulation exactly-once. Adds tests: batch_consumed_exactly_once, repull_keeps_accumulated. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
f336674312
commit
320984b92e
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT args->>$2 FROM v2_job WHERE id = ANY($1)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "?column?",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"UuidArray",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "0d6c3399bc637e2c599534c2cafacf623cd6e5fde2d075a4f1f0d4852efcedca"
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH mine AS (\n SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1\n ), claim_self AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE id = $1 AND consumed_at IS NULL\n RETURNING debounce_batch\n ), claim_rest AS (\n UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1\n WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)\n AND id <> $1 AND consumed_at IS NULL\n RETURNING id\n )\n SELECT\n EXISTS (SELECT 1 FROM mine) AS \"had_row!\",\n (SELECT debounce_batch FROM claim_self) AS claimed_batch,\n (SELECT consumed_by FROM mine) AS prev_consumed_by,\n ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "had_row!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "claimed_batch",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "prev_consumed_by",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "claimed_ids!",
|
||||
"type_info": "UuidArray"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "57f375e89d63ac118c5c6767487af401e6d800488ade04facb6af14578c30a89"
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH del AS (\n DELETE FROM v2_job_debounce_batch\n WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'\n RETURNING 1\n ) SELECT count(*) FROM del",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "count",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "70f01b322765442de8888b6d9b79984da751f9ae10c8b15ada925bd371501e18"
|
||||
}
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH target AS (\n SELECT id FROM v2_job_queue WHERE id = $1 AND NOT running FOR UPDATE\n ), completed AS (\n INSERT INTO v2_job_completed\n (workspace_id, id, started_at, duration_ms, result,\n flow_status, workflow_as_code_status, status, worker)\n SELECT\n q.workspace_id, q.id, q.started_at,\n (EXTRACT('epoch' FROM now()) - EXTRACT('epoch' FROM COALESCE(q.started_at, now()))) * 1000,\n $3::text::jsonb,\n s.flow_status,\n s.workflow_as_code_status,\n 'skipped'::job_status,\n q.worker\n FROM v2_job_queue q\n LEFT JOIN v2_job_status s ON s.id = q.id\n WHERE q.id IN (SELECT id FROM target)\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = EXCLUDED.result\n RETURNING 1 AS x\n ), _deleted AS (\n DELETE FROM v2_job_queue WHERE id IN (SELECT id FROM target)\n ), _logged AS (\n INSERT INTO job_logs (logs, job_id, workspace_id)\n SELECT $4, $1, $2 WHERE EXISTS (SELECT 1 FROM target)\n ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)\n )\n SELECT x FROM completed\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "x",
|
||||
"type_info": "Int4"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Varchar",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "b9ce07cda12e2540f428d4c8e5bbfa8763c03dfe3f0c74edf958667035fb5135"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO v2_job_debounce_batch (id, debounce_batch)\n SELECT $1, debounce_batch FROM v2_job_debounce_batch WHERE id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "bf5db4bba06a2af085577f36446bcb900ac113c14584fa933d2a94318e47a354"
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (\n SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1\n )",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "e050734d7642b26f8859982c55ec2c8b1fc14a8de665b15a2f2dfd6e0b5b7fdf"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
aa82e3f706803380b36c7950ff82297a76a71e2e
|
||||
30d740e619fad219108ec4b4c6a9d67c1ab42d46
|
||||
@@ -0,0 +1,4 @@
|
||||
DROP INDEX IF EXISTS idx_v2_job_debounce_batch_consumed_at;
|
||||
ALTER TABLE v2_job_debounce_batch
|
||||
DROP COLUMN IF EXISTS consumed_at,
|
||||
DROP COLUMN IF EXISTS consumed_by;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Claim-based, exactly-once consumption of debounce batches.
|
||||
-- A batch row is "claimed" by the survivor that accumulates its args. Stamping the
|
||||
-- row consumed (instead of deleting it) lets a later-pulled survivor of the same
|
||||
-- batch tell "my contribution was already processed" (consumed_at set -> no-op)
|
||||
-- apart from "I was never batched" (no row at all; CE / legacy -> run my own args).
|
||||
-- consumed_at doubles as the GC timestamp. NULL = not yet consumed.
|
||||
-- consumed_by records which job claimed the row, so a job that is re-pulled (e.g.
|
||||
-- crash recovery) can tell its own prior claim (keep its accumulated args) apart from
|
||||
-- a sibling survivor having swept it in (run empty).
|
||||
ALTER TABLE v2_job_debounce_batch
|
||||
ADD COLUMN IF NOT EXISTS consumed_at TIMESTAMP WITH TIME ZONE,
|
||||
ADD COLUMN IF NOT EXISTS consumed_by UUID;
|
||||
|
||||
-- Keeps the GC sweep (delete consumed rows past a grace period) cheap.
|
||||
CREATE INDEX IF NOT EXISTS idx_v2_job_debounce_batch_consumed_at
|
||||
ON v2_job_debounce_batch (consumed_at)
|
||||
WHERE consumed_at IS NOT NULL;
|
||||
@@ -2691,6 +2691,9 @@ pub async fn monitor_db(
|
||||
if let Err(e) = cleanup_debounce_orphaned_keys(&db).await {
|
||||
tracing::error!("Error cleaning up debounce keys: {:?}", e);
|
||||
}
|
||||
if let Err(e) = cleanup_consumed_debounce_batches(&db).await {
|
||||
tracing::error!("Error cleaning up consumed debounce batches: {:?}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -4393,6 +4396,31 @@ RETURNING key,job_id
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// GC for claim-based debounce batches: once a batch row has been consumed (its
|
||||
/// args accumulated into some survivor's run), it only lingers to let a later-pulled
|
||||
/// survivor of the same batch tell "already consumed" from "never batched". A generous
|
||||
/// grace period (>> any debounce window) makes that decision safe; after it, the rows
|
||||
/// are dead weight. A re-pulled survivor whose row was GC'd correctly falls back to its
|
||||
/// own (already-accumulated, persisted) args, so the grace period is not correctness-
|
||||
/// critical.
|
||||
async fn cleanup_consumed_debounce_batches(db: &DB) -> error::Result<()> {
|
||||
let deleted = sqlx::query_scalar!(
|
||||
"WITH del AS (
|
||||
DELETE FROM v2_job_debounce_batch
|
||||
WHERE consumed_at IS NOT NULL AND consumed_at < now() - interval '1 hour'
|
||||
RETURNING 1
|
||||
) SELECT count(*) FROM del"
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await?
|
||||
.unwrap_or(0);
|
||||
|
||||
if deleted > 0 {
|
||||
tracing::info!("Cleaned up {deleted} consumed debounce batch rows");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup_debounce_keys_for_completed_jobs(db: &DB) -> error::Result<()> {
|
||||
// If min version doesn't support runnable settings, clean up debounce keys for completed jobs
|
||||
if !windmill_common::min_version::MIN_VERSION_SUPPORTS_RUNNABLE_SETTINGS_V0
|
||||
|
||||
@@ -3172,87 +3172,77 @@ impl PulledJobResult {
|
||||
if let Some(args) = &mut j.args {
|
||||
args.remove(field_name);
|
||||
}
|
||||
|
||||
// No accumulation on this path: just clean up the batch rows.
|
||||
sqlx::query!(
|
||||
"DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (
|
||||
SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
|
||||
)",
|
||||
j_id,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
} else if let Some(arg_name_to_accumulate) =
|
||||
// TODO: Maybe support multiple arguments in future
|
||||
debounce_args_to_accumulate.as_ref().and_then(|v| v.get(0))
|
||||
{
|
||||
tracing::debug!(
|
||||
job_id = %j_id,
|
||||
job_kind = ?kind,
|
||||
arg_name = arg_name_to_accumulate,
|
||||
"Accumulating debounced arguments from batch"
|
||||
);
|
||||
let mut accumulated_arg: Vec<Box<RawValue>> = vec![];
|
||||
for str_o in sqlx::query_scalar!(
|
||||
"WITH ids AS (
|
||||
SELECT id as job_id FROM v2_job_debounce_batch WHERE debounce_batch = (
|
||||
SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
|
||||
)
|
||||
) SELECT args->>$2 FROM ids LEFT JOIN v2_job ON v2_job.id = ids.job_id
|
||||
// Claim this job's contribution to its debounce batch exactly once.
|
||||
// Instead of deleting the batch rows, mark them consumed (stamping
|
||||
// consumed_by = this job). A batch normally has a single survivor that
|
||||
// sweeps every row; only a narrow push/pull race can leave two survivors
|
||||
// on one batch. The claim lets the second survivor tell apart:
|
||||
// - already swept in by the other survivor -> run empty (no duplicate),
|
||||
// - its own earlier claim on re-pull -> keep its accumulated args,
|
||||
// - never batched (CE / workers behind v2) -> keep its own args.
|
||||
// Consumed rows are GC'd by the monitor.
|
||||
let claim = sqlx::query!(
|
||||
"WITH mine AS (
|
||||
SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1
|
||||
), claim_self AS (
|
||||
UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1
|
||||
WHERE id = $1 AND consumed_at IS NULL
|
||||
RETURNING debounce_batch
|
||||
), claim_rest AS (
|
||||
UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1
|
||||
WHERE debounce_batch = (SELECT debounce_batch FROM claim_self)
|
||||
AND id <> $1 AND consumed_at IS NULL
|
||||
RETURNING id
|
||||
)
|
||||
SELECT
|
||||
EXISTS (SELECT 1 FROM mine) AS \"had_row!\",
|
||||
(SELECT debounce_batch FROM claim_self) AS claimed_batch,
|
||||
(SELECT consumed_by FROM mine) AS prev_consumed_by,
|
||||
ARRAY(SELECT id FROM claim_rest) AS \"claimed_ids!\"
|
||||
",
|
||||
j_id,
|
||||
arg_name_to_accumulate,
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await?
|
||||
.into_iter()
|
||||
{
|
||||
if let Some(s) = str_o.as_ref() {
|
||||
match serde_json::from_str::<Vec<Box<RawValue>>>(s) {
|
||||
Ok(ref mut vec) => accumulated_arg.append(vec),
|
||||
Err(_) => {
|
||||
// Value is not an array — wrap the scalar into a
|
||||
// single-element array. This supports union types
|
||||
// like T | T[] where the caller may pass a bare T.
|
||||
match RawValue::from_string(s.to_string()) {
|
||||
Ok(raw) => accumulated_arg.push(raw),
|
||||
Err(e) => {
|
||||
return Err(error::Error::ArgumentErr(format!("cannot consolidate argument `{arg_name_to_accumulate}`: value is neither a valid list nor a valid JSON value\nUnwrapped Error: {e}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
|
||||
tracing::debug!(
|
||||
job_id = %j_id,
|
||||
arg_name = arg_name_to_accumulate,
|
||||
accumulated_count = accumulated_arg.len(),
|
||||
"Accumulated arguments from debounced jobs in batch"
|
||||
);
|
||||
|
||||
// If the batch query returned no entries (e.g. CE where
|
||||
// v2_job_debounce_batch is never populated), keep the
|
||||
// original value unchanged instead of replacing it with [].
|
||||
if !accumulated_arg.is_empty() {
|
||||
let new_value = to_raw_value(&accumulated_arg);
|
||||
|
||||
let original_value = j
|
||||
.args
|
||||
.as_ref()
|
||||
.and_then(|a| a.get(arg_name_to_accumulate))
|
||||
.map(|v| v.get().to_string())
|
||||
.unwrap_or_else(|| "null".to_string());
|
||||
|
||||
append_logs(
|
||||
&j_id,
|
||||
&j.workspace_id,
|
||||
format!(
|
||||
"Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n",
|
||||
&new_value
|
||||
),
|
||||
&(db.into()),
|
||||
)
|
||||
.await;
|
||||
let consumed_by_other =
|
||||
claim.claimed_batch.is_none() && claim.prev_consumed_by != Some(j_id);
|
||||
|
||||
if !claim.had_row {
|
||||
// Never batched (CE / workers behind v2): keep the job's own args.
|
||||
tracing::debug!(
|
||||
job_id = %j_id,
|
||||
"Debounce: no batch row, keeping original args"
|
||||
);
|
||||
} else if consumed_by_other {
|
||||
// Another survivor of this batch already accumulated this job's
|
||||
// contribution; run as a no-op so its items are not reprocessed.
|
||||
tracing::info!(
|
||||
job_id = %j_id,
|
||||
arg_name = arg_name_to_accumulate,
|
||||
"Debounce: contribution already consumed by a concurrent survivor, running empty"
|
||||
);
|
||||
j.args
|
||||
.get_or_insert(Json(Default::default()))
|
||||
.as_mut()
|
||||
.insert(arg_name_to_accumulate.to_owned(), new_value);
|
||||
|
||||
// Persist accumulated args to v2_job so that flow steps
|
||||
// re-reading from the DB (via get_mini_pulled_job) see them
|
||||
.insert(
|
||||
arg_name_to_accumulate.to_owned(),
|
||||
to_raw_value(&Vec::<Box<RawValue>>::new()),
|
||||
);
|
||||
if let Some(ref args) = j.args {
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job SET args = $2 WHERE id = $1",
|
||||
@@ -3262,18 +3252,102 @@ impl PulledJobResult {
|
||||
.execute(db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if claim.claimed_batch.is_some() {
|
||||
// We claimed our own row (+ any unclaimed siblings): accumulate the
|
||||
// args of exactly the rows we own.
|
||||
let mut ids: Vec<Uuid> = Vec::with_capacity(claim.claimed_ids.len() + 1);
|
||||
ids.push(j_id);
|
||||
ids.extend(claim.claimed_ids.iter().copied());
|
||||
|
||||
// Clean up the debounce batch entries now that the job has been pulled
|
||||
sqlx::query!(
|
||||
"DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (
|
||||
SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
|
||||
)",
|
||||
j_id,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
tracing::debug!(
|
||||
job_id = %j_id,
|
||||
job_kind = ?kind,
|
||||
arg_name = arg_name_to_accumulate,
|
||||
claimed = ids.len(),
|
||||
"Accumulating debounced arguments from claimed batch rows"
|
||||
);
|
||||
|
||||
let mut accumulated_arg: Vec<Box<RawValue>> = vec![];
|
||||
for str_o in sqlx::query_scalar!(
|
||||
"SELECT args->>$2 FROM v2_job WHERE id = ANY($1)",
|
||||
&ids,
|
||||
arg_name_to_accumulate,
|
||||
)
|
||||
.fetch_all(db)
|
||||
.await?
|
||||
.into_iter()
|
||||
{
|
||||
if let Some(s) = str_o.as_ref() {
|
||||
match serde_json::from_str::<Vec<Box<RawValue>>>(s) {
|
||||
Ok(ref mut vec) => accumulated_arg.append(vec),
|
||||
Err(_) => {
|
||||
// Value is not an array — wrap the scalar into a
|
||||
// single-element array. This supports union types
|
||||
// like T | T[] where the caller may pass a bare T.
|
||||
match RawValue::from_string(s.to_string()) {
|
||||
Ok(raw) => accumulated_arg.push(raw),
|
||||
Err(e) => {
|
||||
return Err(error::Error::ArgumentErr(format!("cannot consolidate argument `{arg_name_to_accumulate}`: value is neither a valid list nor a valid JSON value\nUnwrapped Error: {e}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !accumulated_arg.is_empty() {
|
||||
let new_value = to_raw_value(&accumulated_arg);
|
||||
|
||||
let original_value = j
|
||||
.args
|
||||
.as_ref()
|
||||
.and_then(|a| a.get(arg_name_to_accumulate))
|
||||
.map(|v| v.get().to_string())
|
||||
.unwrap_or_else(|| "null".to_string());
|
||||
|
||||
append_logs(
|
||||
&j_id,
|
||||
&j.workspace_id,
|
||||
format!(
|
||||
"Accumulating debounced argument `{arg_name_to_accumulate}`:\n original: {original_value}\n accumulated: {}\n\n",
|
||||
&new_value
|
||||
),
|
||||
&(db.into()),
|
||||
)
|
||||
.await;
|
||||
|
||||
j.args
|
||||
.get_or_insert(Json(Default::default()))
|
||||
.as_mut()
|
||||
.insert(arg_name_to_accumulate.to_owned(), new_value);
|
||||
|
||||
// Persist accumulated args to v2_job so that flow steps
|
||||
// re-reading from the DB (via get_mini_pulled_job) see them
|
||||
if let Some(ref args) = j.args {
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job SET args = $2 WHERE id = $1",
|
||||
j_id,
|
||||
args as &Json<HashMap<String, Box<RawValue>>>,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
// else: claimed_batch is None but it was our own prior claim (re-pull) —
|
||||
// keep the args we already persisted on the first pull.
|
||||
} else {
|
||||
// Debounced but no args to accumulate (plain debounce / dependency job):
|
||||
// consume the batch by removing this job's rows.
|
||||
sqlx::query!(
|
||||
"DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (
|
||||
SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
|
||||
)",
|
||||
j_id,
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Handle dependency job debouncing cleanup when a job is pulled for execution
|
||||
if is_djob_to_debounce {
|
||||
|
||||
@@ -4575,6 +4575,137 @@ mod debounce {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Claim-based exactly-once: if two survivors end up on the same batch (only
|
||||
/// possible in a narrow push/pull race), the args of each member are accumulated
|
||||
/// into exactly ONE run. The survivor that claims the batch first accumulates
|
||||
/// everyone; the second survivor finds its contribution already consumed and runs
|
||||
/// empty — no item is dropped and none is processed twice.
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
|
||||
async fn test_debounce_batch_consumed_exactly_once(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let key = "exactly_once_key";
|
||||
let settings = DebouncingSettings {
|
||||
debounce_delay_s: Some(5),
|
||||
debounce_key: Some(key.to_string()),
|
||||
debounce_args_to_accumulate: Some(vec!["items".to_string()]),
|
||||
..Default::default()
|
||||
};
|
||||
let rs_handle = setup_debouncing_settings(&db, &settings).await;
|
||||
|
||||
// J1 superseded, J2 the (first) survivor of batch B = {J1, J2}.
|
||||
let j1 = Uuid::new_v4();
|
||||
let j2 = Uuid::new_v4();
|
||||
push_debounced_script(&db, j1, vec![1], &settings, rs_handle).await;
|
||||
let j2_args = push_debounced_script(&db, j2, vec![2], &settings, rs_handle).await;
|
||||
|
||||
// Simulate the race outcome: a second survivor J3 ended up on the SAME batch B.
|
||||
let j3 = Uuid::new_v4();
|
||||
let j3_args = serde_json::json!({ "items": [3] });
|
||||
insert_script_job_with_args(&db, j3, "test-workspace", "f/test/script", &j3_args).await;
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job_queue SET runnable_settings_handle = $1 WHERE id = $2",
|
||||
rs_handle,
|
||||
j3,
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
sqlx::query!(
|
||||
"INSERT INTO v2_job_debounce_batch (id, debounce_batch)
|
||||
SELECT $1, debounce_batch FROM v2_job_debounce_batch WHERE id = $2",
|
||||
j3,
|
||||
j2,
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
// J2 pulled first: claims the whole batch, accumulates everyone's items.
|
||||
let mut j2_res = make_pulled_job_result(
|
||||
j2,
|
||||
"test-workspace",
|
||||
"f/test/script",
|
||||
&j2_args,
|
||||
JobKind::Script,
|
||||
"deno",
|
||||
rs_handle,
|
||||
);
|
||||
j2_res.maybe_apply_debouncing(&db).await?;
|
||||
assert!(j2_res.job.is_some(), "J2 runs");
|
||||
assert_accumulated_items(&j2_res, &[1, 2, 3], "items");
|
||||
|
||||
// J3 pulled next: its contribution was already consumed by J2 -> runs empty,
|
||||
// so [3] is not processed a second time.
|
||||
let mut j3_res = make_pulled_job_result(
|
||||
j3,
|
||||
"test-workspace",
|
||||
"f/test/script",
|
||||
&j3_args,
|
||||
JobKind::Script,
|
||||
"deno",
|
||||
rs_handle,
|
||||
);
|
||||
j3_res.maybe_apply_debouncing(&db).await?;
|
||||
let job = j3_res.job.as_ref().expect("J3 still runs (empty)");
|
||||
let items: Vec<serde_json::Value> =
|
||||
serde_json::from_str(job.job.args.as_ref().unwrap().get("items").unwrap().get())?;
|
||||
assert!(
|
||||
items.is_empty(),
|
||||
"J3's items must be empty (already consumed by J2), got {items:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A survivor re-pulled (e.g. crash recovery) must NOT mistake its own earlier
|
||||
/// claim for a sibling's and wipe its accumulated args. consumed_by = self is
|
||||
/// distinguished from consumed_by = another job.
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
|
||||
async fn test_debounce_repull_keeps_accumulated(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
let key = "repull_key";
|
||||
let settings = DebouncingSettings {
|
||||
debounce_delay_s: Some(5),
|
||||
debounce_key: Some(key.to_string()),
|
||||
debounce_args_to_accumulate: Some(vec!["items".to_string()]),
|
||||
..Default::default()
|
||||
};
|
||||
let rs_handle = setup_debouncing_settings(&db, &settings).await;
|
||||
|
||||
let j1 = Uuid::new_v4();
|
||||
let j2 = Uuid::new_v4();
|
||||
push_debounced_script(&db, j1, vec![1], &settings, rs_handle).await;
|
||||
let j2_args = push_debounced_script(&db, j2, vec![2], &settings, rs_handle).await;
|
||||
|
||||
// First pull: J2 claims its batch and accumulates [1, 2].
|
||||
let mut first = make_pulled_job_result(
|
||||
j2,
|
||||
"test-workspace",
|
||||
"f/test/script",
|
||||
&j2_args,
|
||||
JobKind::Script,
|
||||
"deno",
|
||||
rs_handle,
|
||||
);
|
||||
first.maybe_apply_debouncing(&db).await?;
|
||||
assert_accumulated_items(&first, &[1, 2], "items");
|
||||
|
||||
// Re-pull with the args persisted by the first pull: J2 sees its OWN prior claim
|
||||
// (consumed_by = j2), so it keeps the accumulated args rather than running empty.
|
||||
let persisted = first.job.as_ref().unwrap().job.args.as_ref().unwrap();
|
||||
let persisted_json = serde_json::to_value(persisted).unwrap();
|
||||
let mut second = make_pulled_job_result(
|
||||
j2,
|
||||
"test-workspace",
|
||||
"f/test/script",
|
||||
&persisted_json,
|
||||
JobKind::Script,
|
||||
"deno",
|
||||
rs_handle,
|
||||
);
|
||||
second.maybe_apply_debouncing(&db).await?;
|
||||
assert!(second.job.is_some(), "re-pulled J2 still runs");
|
||||
assert_accumulated_items(&second, &[1, 2], "items");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test: Push-time (script) debounce with max_total_debounces_amount=2.
|
||||
/// 5 calls, each sending {x: [i]}. Expected:
|
||||
/// Call 1: debounced (scheduled_for set)
|
||||
|
||||
Reference in New Issue
Block a user