windmill/backend/windmill-queue/src/jobs.rs

/*
 * Author: Ruben Fiszel
 * Copyright: Windmill Labs, Inc 2022
 * This file and its contents are licensed under the AGPLv3 License.
 * Please see the included NOTICE for copyright information and
 * LICENSE-AGPL for a copy of the license.
 */

use std::future::Future;
use std::time::Duration;
use std::{collections::HashMap, sync::Arc, vec};

use anyhow::Context;
use async_recursion::async_recursion;
use chrono::{DateTime, Utc};
use futures::future::TryFutureExt;
use itertools::Itertools;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
use quick_cache::sync::Cache;
use regex::Regex;
use reqwest::Client;
use serde::Deserialize;
use serde::{ser::SerializeMap, Serialize};
use serde_json::{json, value::RawValue};
use sqlx::{types::Json, Acquire, Pool, Postgres, Transaction};
use sqlx::{Encode, PgExecutor};
use std::borrow::Cow;
use tokio::sync::mpsc::Sender;
use tokio::sync::oneshot;
use tokio::task::JoinHandle;
use tokio::time::timeout;
use tokio::{sync::RwLock, time::sleep};
use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::audit_oss::audit_log;
use windmill_audit::ActionKind;
#[cfg(feature = "benchmark")]
use windmill_common::add_time;
use windmill_common::audit::AuditAuthor;
use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
use windmill_common::jobs::{
    script_path_to_payload, JobTriggerKind, TriggerKindLabel, EMAIL_ERROR_HANDLER_USER_EMAIL,
    MODULES_ARG,
};
use windmill_common::min_version::{
    MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2,
};
use windmill_common::runnable_settings::{
    ConcurrencySettings, ConcurrencySettingsWithCustom, DebouncingSettings, RetrySettings,
    RunnableSettings, RunnableSettingsTrait,
};
use windmill_common::triggers::TriggerMetadata;
use windmill_common::utils::{calculate_hash, configure_client, now_from_db, strip_json_nul};
use windmill_common::worker::Connection;

use windmill_common::otel_oss::{
    otel_incr_queue_delete_count, otel_incr_queue_pull_count, otel_incr_queue_push_count,
    otel_incr_worker_execution_failed,
};
use windmill_common::{
    auth::permissioned_as_to_username,
    cache::{self, FlowData},
    db::{Authed, UserDB},
    error::{self, Error},
    flow_status::{
        BranchAllStatus, FlowCleanupModule, FlowStatus, FlowStatusModule, FlowStatusModuleWParent,
        Iterator as FlowIterator, JobResult, RestartedFrom, RetryStatus, MAX_RETRY_ATTEMPTS,
        MAX_RETRY_INTERVAL,
    },
    flows::{
        add_virtual_items_if_necessary, FlowModule, FlowModuleValue, FlowValue, InputTransform,
        Retry, StopAfterIf,
    },
    jobs::{get_payload_tag_from_prefixed_path, JobKind, JobPayload, RawCode},
    min_version::{MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440},
    schedule::Schedule,
    scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
    users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
    utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt},
    worker::{
        to_raw_value, CLOUD_HOSTED, DEPENDENCY_JOB_TAG, DISABLE_FLOW_SCRIPT, NO_LOGS,
        PREVIEW_TAGS_OVERRIDE, WORKER_PULL_QUERIES, WORKER_PULL_QUERIES_FAIRNESS,
        WORKER_SUSPENDED_PULL_QUERY, WORKSPACE_FAIRNESS_OVERLOADED,
    },
    DB, METRICS_ENABLED,
};

use backon::ConstantBuilder;
use backon::{BackoffBuilder, Retryable};

use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status};
use crate::schedule::{get_schedule_opt, push_scheduled_job};
use crate::tags::per_workspace_tag;
#[cfg(feature = "cloud")]
use windmill_common::users::SUPERADMIN_SYNC_EMAIL;

#[cfg(feature = "prometheus")]
lazy_static::lazy_static! {

    // TODO: these aren't synced, they should be moved into the queue abstraction once/if that happens.
    static ref QUEUE_PUSH_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
        "queue_push_count",
        "Total number of jobs pushed to the queue."
    )
    .unwrap();

    static ref QUEUE_DELETE_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
        "queue_delete_count",
        "Total number of jobs deleted from the queue."
    )
    .unwrap();

    pub static ref QUEUE_PULL_COUNT: prometheus::IntCounter = prometheus::register_int_counter!(
        "queue_pull_count",
        "Total number of jobs pulled from the queue."
    )
    .unwrap();

    pub static ref WORKER_EXECUTION_FAILED: Arc<RwLock<HashMap<String, IntCounter>>> = Arc::new(RwLock::new(HashMap::new()));

}

#[cfg(feature = "failpoints")]
pub mod schedule_failpoints {
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    pub enum ScheduleFailPoint {
        SavepointCreate,
        Push,
        PushQuotaExceeded,
        SavepointCommit,
        ScheduleDisable,
    }

    tokio::task_local! {
        pub static ACTIVE: ScheduleFailPoint;
    }

    pub fn is_active(point: ScheduleFailPoint) -> bool {
        ACTIVE.try_with(|fp| *fp == point).unwrap_or(false)
    }
}

lazy_static::lazy_static! {
    pub static ref HTTP_CLIENT: Client = configure_client(reqwest::ClientBuilder::new()
        .user_agent("windmill/beta")
        .timeout(std::time::Duration::from_secs(20))
        .connect_timeout(std::time::Duration::from_secs(10)))
        .build().unwrap();

    pub static ref WMDEBUG_NO_DEBOUNCING: bool = std::env::var("WMDEBUG_NO_DEBOUNCING").is_ok();
    pub static ref WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: bool = std::env::var("WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT").is_ok();

    static ref JOB_ARGS_AUDIT_LOGS: bool = std::env::var("JOB_ARGS_AUDIT_LOGS")
        .ok()
        .and_then(|x| x.parse().ok())
        .unwrap_or(false);

}

#[cfg(feature = "cloud")]
const MAX_FREE_EXECS: i32 = 1000;
#[cfg(feature = "cloud")]
const MAX_FREE_CONCURRENT_RUNS: i32 = 30;

const ERROR_HANDLER_USERNAME: &str = "error_handler";
pub const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler";
const GLOBAL_ERROR_HANDLER_USERNAME: &str = "global";
const SUCCESS_HANDLER_USERNAME: &str = "success_handler";

pub const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
pub const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
pub const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
pub const SUCCESS_HANDLER_USER_GROUP: &str = "g/success_handler";
pub const SUCCESS_HANDLER_USER_EMAIL: &str = "success_handler@windmill.dev";

#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CanceledBy {
    pub username: Option<String>,
    pub reason: Option<String>,
}

/// What only the worker that ran a job knows about it, for the per-runnable rollup: an
/// agent worker's completion is processed by the server, whose own worker group is not
/// the one the job ran in.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobResourceUsage {
    pub worker_group: String,
    pub cpu_time_ms: Option<i64>,
}

#[derive(Debug, Serialize, Deserialize)]
pub struct JobCompleted {
    pub job: MiniCompletedJob,
    pub preprocessed_args: Option<HashMap<String, Box<RawValue>>>,
    pub result: Arc<Box<RawValue>>,
    pub result_columns: Option<Vec<String>>,
    pub mem_peak: i32,
    pub success: bool,
    pub cached_res_path: Option<String>,
    pub token: String,
    pub canceled_by: Option<CanceledBy>,
    pub duration: Option<i64>,
    pub has_stream: Option<bool>,
    pub from_cache: Option<bool>,
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub resource_usage: Option<JobResourceUsage>,
    #[serde(skip)]
    pub flow_runners: Option<Arc<FlowRunners>>,
    #[serde(skip)]
    pub done_tx: Option<oneshot::Sender<()>>,
}

pub async fn cancel_single_job<'c>(
    username: &str,
    reason: Option<String>,
    job_running: QueuedJobV2,
    w_id: &str,
    mut tx: Transaction<'c, Postgres>,
    db: &Pool<Postgres>,
    force_cancel: bool,
) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
    let id = job_running.id;
    if force_cancel || (job_running.parent_job.is_none() && !job_running.running) {
        let username = username.to_string();
        let w_id = w_id.to_string();
        let db = db.clone();
        tracing::info!("cancelling job {:?}", job_running.id);
        tokio::task::spawn(async move {
            let reason: String = reason
                .clone()
                .unwrap_or_else(|| "unexplicited reasons".to_string());
            let e = serde_json::json!({"message": format!("Job canceled: {reason} by {username}"), "name": "Canceled", "reason": reason, "canceler": username});

            append_logs(
                &job_running.id,
                w_id.to_string(),
                format!("canceled by {username}: (force cancel: {force_cancel})"),
                &Connection::from(db.clone()),
            )
            .await;
            let memory_peak = job_running.memory_peak.unwrap_or(0);
            let add_job = add_completed_job_error(
                &db,
                &MiniCompletedJob::from(job_running),
                memory_peak,
                Some(CanceledBy { username: Some(username.to_string()), reason: Some(reason) }),
                e,
                "server",
                false,
                None,
            )
            .await;

            if let Err(e) = add_job {
                tracing::error!("Failed to add canceled job: {}", e);
            }
        });
    } else {
        let id: Option<Uuid> = sqlx::query_scalar!(
            "UPDATE v2_job_queue SET canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = $3 AND workspace_id = $4 AND (canceled_by IS NULL OR canceled_reason != $2) RETURNING id",
            username,
            reason,
            job_running.id,
            w_id
        )
        .fetch_optional(&mut *tx)
        .await?;
        if let Some(id) = id {
            tracing::info!("Soft cancelling job {}", id);
        }
    }

    Ok((tx, Some(id)))
}

pub async fn cancel_job<'c>(
    username: &str,
    reason: Option<String>,
    id: Uuid,
    w_id: &str,
    mut tx: Transaction<'c, Postgres>,
    db: &Pool<Postgres>,
    force_cancel: bool,
    require_anonymous: bool,
) -> error::Result<(Transaction<'c, Postgres>, Option<Uuid>)> {
    //TODO fetch mini completed job instead of QueuedJob
    let job = get_queued_job_v2(&mut *tx, &id).await?;

    if job.is_none() {
        return Ok((tx, None));
    }

    let mut job = job.unwrap();

    if require_anonymous && job.created_by != "anonymous" {
        return Err(Error::BadRequest(
            "You are not logged in and this job was not created by an anonymous user like you so you cannot cancel it".to_string(),
        ));
    }

    if job.workspace_id != w_id {
        return Err(Error::BadRequest(
            "You are not authorized to cancel this job belonging to another workspace".to_string(),
        ));
    }

    if force_cancel {
        // if force canceling a flow step, make sure we force cancel from the highest parent
        loop {
            if job.parent_job.is_none() {
                break;
            }
            match get_queued_job_v2(&mut *tx, &job.parent_job.unwrap()).await? {
                Some(j) => {
                    job = j;
                }
                None => break,
            }
        }
    }

    // prevent cancelling a future tick of a schedule
    if let Some(schedule_path) = job.schedule_path().as_ref() {
        let now = now_from_db(&mut *tx).await?;
        if job.scheduled_for > now {
            return Err(Error::BadRequest(
                format!(
                    "Cannot cancel a future tick of a schedule, cancel the schedule direcly ({})",
                    schedule_path
                )
                .to_string(),
            ));
        }
    }

    let job = job;

    // get all children using recursive CTE
    let mut jobs_to_cancel = sqlx::query!(
        r#"
WITH RECURSIVE job_tree AS (
    -- Base case: direct children of the given parent job
    SELECT id, parent_job, 1 AS depth
    FROM v2_job_queue
    INNER JOIN v2_job USING (id)
    WHERE parent_job = $1 AND v2_job.workspace_id = $2

    UNION ALL

    -- Recursive case: fetch children of previously found jobs
    SELECT q.id, j.parent_job, t.depth + 1
    FROM v2_job_queue q
    INNER JOIN v2_job j USING (id)
    INNER JOIN job_tree t ON t.id = j.parent_job
    WHERE j.workspace_id = $2 AND t.depth < 500  -- Limit recursion depth to 500
)
SELECT id AS id, depth
FROM job_tree
ORDER BY depth, id
        "#,
        job.id,
        w_id
    )
    .fetch_all(&mut *tx)
    .await?
    .into_iter()
    .filter_map(|r| r.id.clone())
    .collect_vec();

    // A native retry is the child of its first attempt, which has completed by the time the
    // retry is queued, so the walk above stops before reaching it.
    let tree = [&[job.id][..], &jobs_to_cancel[..]].concat();
    let pending_retries = sqlx::query_scalar!(
        "SELECT q.id FROM v2_job_queue q
            JOIN native_retry_attempt n ON n.job_id = q.id
            JOIN v2_job r ON r.id = q.id
            JOIN v2_job a ON a.id = r.parent_job
            WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)",
        tree.as_slice(),
        w_id
    )
    .fetch_all(&mut *tx)
    .await?;
    jobs_to_cancel.extend(pending_retries);

    jobs_to_cancel.reverse();
    if !jobs_to_cancel.is_empty() {
        tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len());
    }

    let (ntx, _) =
        cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel).await?;
    tx = ntx;

    if !force_cancel {
        // cancel children in batch first
        if !jobs_to_cancel.is_empty() {
            let updated = sqlx::query_scalar!(
            "UPDATE v2_job_queue SET canceled_by = $1, canceled_reason = $2, scheduled_for = now(), suspend = 0 WHERE id = ANY($3) AND workspace_id = $4 AND (canceled_by IS NULL OR canceled_reason != $2) RETURNING id",
            username,
            reason,
            jobs_to_cancel.as_slice(),
            w_id
        )
        .fetch_all(&mut *tx)
        .await?;

            // Remove any jobs that were successfully updated
            jobs_to_cancel.retain(|id| !updated.contains(&id));
        }
    }
    for job_id in jobs_to_cancel {
        let job = get_queued_job_v2(&mut *tx, &job_id).await?;

        if let Some(job) = job {
            let (ntx, _) =
                cancel_single_job(username, reason.clone(), job, w_id, tx, db, force_cancel)
                    .await?;
            tx = ntx;
        }
    }
    Ok((tx, Some(id)))
}

/* TODO retry this? */
#[tracing::instrument(level = "trace", skip_all)]
pub async fn append_logs(
    job_id: &uuid::Uuid,
    workspace: impl AsRef<str>,
    logs: impl AsRef<str>,
    conn: &Connection,
) {
    if logs.as_ref().is_empty() {
        return;
    }

    if job_id.is_nil() {
        tracing::info!("local job: {}", logs.as_ref());
        return;
    }

    if *NO_LOGS {
        tracing::info!("NO LOGS [{job_id}]: {}", logs.as_ref());
        return;
    }
    match conn {
        Connection::Sql(pool) => {
            if let Err(err) = sqlx::query!(
                "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, EXCLUDED.logs)",
                logs.as_ref(),
                job_id,
                workspace.as_ref(),
            )
            .execute(pool)
            .warn_after_seconds(1)
            .await
            {
                tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}");
            }
        }
        Connection::Http(client) => {
            if let Err(e) = client
            .post::<_, String>(
                &format!("/api/w/{}/agent_workers/push_logs/{}", workspace.as_ref(), job_id),
                None,
                &logs.as_ref(),
            )
            .await {
                tracing::error!(%job_id, %e, "error sending logs for  job {job_id}: {e}");
            };
        }
    }
}

pub const PERIODIC_SCRIPT_TAG: &str = "periodic_bash_script";
pub const INIT_SCRIPT_TAG: &str = "init_script";
pub const INIT_SCRIPT_PATH_PREFIX: &str = "init_script_";
pub const PERIODIC_SCRIPT_PATH_PREFIX: &str = "periodic_script_";

pub async fn push_init_job<'c>(
    db: &Pool<Postgres>,
    content: String,
    worker_name: &str,
) -> error::Result<Uuid> {
    let tx = PushIsolationLevel::IsolatedRoot(db.clone());
    let ehm = HashMap::new();
    let (uuid, inner_tx) = push(
        &db,
        tx,
        "admins",
        windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode {
            hash: None,
            content,
            path: Some(format!("{INIT_SCRIPT_PATH_PREFIX}{worker_name}")),
            language: ScriptLang::Bash,
            lock: None,
            cache_ttl: None,
            cache_ignore_s3_path: None,
            dedicated_worker: None,
            concurrency_settings: ConcurrencySettingsWithCustom::default(),
            debouncing_settings: DebouncingSettings::default(),
            modules: None,
            tag: None,
        }),
        PushArgs::from(&ehm),
        worker_name,
        "worker@windmill.dev",
        SUPERADMIN_SECRET_EMAIL.to_string(),
        Some("worker_init_job"),
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        false,
        true,
        None,
        true,
        Some(INIT_SCRIPT_TAG.to_string()),
        None,
        None,
        None,
        None,
        false,
        None,
        None,
        None,
        None,
    )
    .await?;
    inner_tx.commit().await?;
    Ok(uuid)
}

pub async fn push_periodic_bash_job<'c>(
    db: &Pool<Postgres>,
    content: String,
    worker_name: &str,
) -> error::Result<Uuid> {
    let tx = PushIsolationLevel::IsolatedRoot(db.clone());
    let ehm = HashMap::new();
    let timestamp = chrono::Utc::now().timestamp();
    let (uuid, inner_tx) = push(
        &db,
        tx,
        "admins",
        windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode {
            hash: None,
            content,
            path: Some(format!(
                "{PERIODIC_SCRIPT_PATH_PREFIX}{}_{}",
                worker_name, timestamp
            )),
            language: ScriptLang::Bash,
            lock: None,
            cache_ttl: None,
            cache_ignore_s3_path: None,
            dedicated_worker: None,
            concurrency_settings: ConcurrencySettingsWithCustom::default(),
            debouncing_settings: DebouncingSettings::default(),
            modules: None,
            tag: None,
        }),
        PushArgs::from(&ehm),
        worker_name,
        "worker@windmill.dev",
        SUPERADMIN_SECRET_EMAIL.to_string(),
        Some("worker_periodic_script_job"),
        None,
        None,
        None,
        None,
        None,
        None,
        None,
        false,
        true,
        None,
        true,
        Some(PERIODIC_SCRIPT_TAG.to_string()),
        None,
        None,
        None,
        None,
        false,
        None,
        None,
        None,
        None,
    )
    .await?;
    inner_tx.commit().await?;
    Ok(uuid)
}

pub async fn cancel_persistent_script_jobs<'c>(
    username: &str,
    reason: Option<String>,
    script_path: &str,
    w_id: &str,
    db: &Pool<Postgres>,
) -> error::Result<Vec<Uuid>> {
    // There can be only one perpetual script run per 10 seconds window. We execute the cancel twice with 5s interval
    // to avoid the case of a job _about to be restarted_ when the first cancel is run.
    let cancelled_job_ids_first_batch =
        cancel_persistent_script_jobs_internal(username, reason.clone(), script_path, w_id, db)
            .await?;
    sleep(std::time::Duration::from_secs(5)).await;
    let cancelled_job_ids_second_batch =
        cancel_persistent_script_jobs_internal(username, reason.clone(), script_path, w_id, db)
            .await?;
    return Ok(cancelled_job_ids_first_batch
        .into_iter()
        .chain(cancelled_job_ids_second_batch)
        .collect_vec());
}

async fn cancel_persistent_script_jobs_internal<'c>(
    username: &str,
    reason: Option<String>,
    script_path: &str,
    w_id: &str,
    db: &Pool<Postgres>,
) -> error::Result<Vec<Uuid>> {
    let mut tx = db.begin().await?;

    // we could have retrieved the job IDs in the first query where we retrieve the hashes, but just in case a job was inserted in the queue right in-between the two above query, we re-do the fetch here
    // Only the loops: a dependency job of this script shares its path, and a run of a version that
    // does not restart itself ends on its own.
    let jobs_to_cancel = sqlx::query_scalar::<_, Uuid>(
        "SELECT j.id FROM v2_job_queue q JOIN v2_job j USING (id) \
         JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id \
         WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' \
         AND j.flow_step_id IS NULL AND q.canceled_by IS NULL AND s.restart_unless_cancelled",
    )
    .bind(w_id)
    .bind(script_path)
    .fetch_all(&mut *tx)
    .await?;

    // Then we cancel all the jobs currently in the queue, one by one
    for queued_job_id in jobs_to_cancel.clone() {
        let (new_tx, _) = cancel_job(
            username,
            reason.clone(),
            queued_job_id,
            w_id,
            tx,
            db,
            false,
            false,
        )
        .await?;
        tx = new_tx;
    }
    tx.commit().await?;

    return Ok(jobs_to_cancel);
}

/// Moves the perpetual runs at `script_path` to the version a deploy just made runnable: each one
/// is canceled and pushed again on that version with the arguments it ran with. A deploy that
/// leaves the script non-perpetual moves nothing, so turning perpetual off keeps the runs going as
/// it does today.
///
/// Carries the authority of the deploy, which every caller has authorized: it cancels and pushes
/// runs at `script_path` without an `Authed` of its own. A replacement runs as the deployed
/// version's identity when it names one and otherwise as the identity of the run it replaces, and
/// a run only moves to a tag that identity may use.
///
/// Errors are logged, never returned: a deploy stands whatever happens to the runs of its earlier
/// versions.
pub async fn restart_perpetual_runs_on_new_version(
    db: &Pool<Postgres>,
    w_id: &str,
    script_path: &str,
    deployed_by: &str,
) {
    let loops = match restart_perpetual_runs_at_path(db, w_id, script_path, deployed_by).await {
        Ok(loops) => loops,
        Err(e) => {
            tracing::error!(
                "Could not restart the perpetual runs of {script_path} on the deployed version: {e:#}"
            );
            // The second pass is the retry.
            true
        }
    };
    if !loops {
        return;
    }
    // A run that ended just before its cancel restarts itself on its own version, and only a
    // cancel this won is replaced, so that run is still on the earlier version. It is queued
    // again within the 10s a perpetual restart is throttled to, which this second pass then
    // catches.
    let (db, w_id, script_path, deployed_by) = (
        db.clone(),
        w_id.to_string(),
        script_path.to_string(),
        deployed_by.to_string(),
    );
    tokio::spawn(async move {
        sleep(std::time::Duration::from_secs(5)).await;
        if let Err(e) = restart_perpetual_runs_at_path(&db, &w_id, &script_path, &deployed_by).await
        {
            tracing::error!(
                "Could not restart the perpetual runs of {script_path} on the deployed version: {e:#}"
            );
        }
    });
}

/// Whether the deployed version loops, which is what the second pass is for.
async fn restart_perpetual_runs_at_path(
    db: &Pool<Postgres>,
    w_id: &str,
    script_path: &str,
    deployed_by: &str,
) -> error::Result<bool> {
    // Built the way a run of this path is built anywhere else, so the next run takes the deployed
    // version's tag, timeout, language and identity. Whether its preprocessor runs is decided per
    // run, below.
    let (mut payload, tag, _, _, timeout, on_behalf_of) =
        script_path_to_payload(script_path, None, db.clone(), w_id, None).await?;
    // A replacement continues a loop rather than answering a trigger, and every perpetual restart
    // is pushed without debouncing for that reason. Deployed settings here would debounce the
    // loops at this path against each other and collapse those that share arguments into one.
    if let JobPayload::ScriptHash { debouncing_settings, .. } = &mut payload {
        *debouncing_settings = DebouncingSettings::default();
    }
    let JobPayload::ScriptHash { hash, dedicated_worker, .. } = &payload else {
        return Ok(false);
    };
    let (hash, dedicated_worker) = (*hash, *dedicated_worker);
    let perpetual = sqlx::query_scalar!(
        "SELECT restart_unless_cancelled FROM script WHERE hash = $1 AND workspace_id = $2",
        hash.0,
        w_id
    )
    .fetch_optional(db)
    .await?
    .flatten()
    .unwrap_or(false);
    if !perpetual {
        return Ok(false);
    }

    let runs = sqlx::query_as!(
        PerpetualRunToRestart,
        "SELECT q.id AS \"id!\", j.created_by, j.permissioned_as, j.permissioned_as_email, \
         j.trigger, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.preprocessed, \
         j.args AS \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\", \
         p.job_token_scopes AS \"job_token_scopes?\" \
         FROM v2_job_queue q JOIN v2_job j USING (id) \
         JOIN script s ON s.workspace_id = j.workspace_id AND s.hash = j.runnable_id \
         LEFT JOIN job_perms p ON p.job_id = q.id \
         WHERE j.workspace_id = $1 AND j.runnable_path = $2 AND j.kind = 'script' \
         AND j.flow_step_id IS NULL AND j.runnable_id != $3 AND q.canceled_by IS NULL \
         AND s.restart_unless_cancelled",
        w_id,
        script_path,
        hash.0
    )
    .fetch_all(db)
    .await?;

    for run in runs {
        let id = run.id;
        // Per run, so that a run this fails on leaves the others to move.
        if let Err(e) = restart_perpetual_run(
            db,
            w_id,
            script_path,
            deployed_by,
            RestartOnVersion {
                payload: &payload,
                tag: tag.as_deref(),
                timeout,
                dedicated_worker,
                on_behalf_of: on_behalf_of.as_ref(),
            },
            run,
        )
        .await
        {
            tracing::error!(
                "Could not restart perpetual run {id} on the version deployed at {script_path}: {e:#}"
            );
        }
    }
    Ok(true)
}

/// A worker older than `job_token_scopes` ignores a step's or agent tool's own restriction, so
/// a flow that sets one is refused while such a worker is live, whichever way the flow arrived
/// (a deploy, a restart, a preview, a standalone agent, an eval). Free when every worker is
/// current.
async fn refuse_step_scopes_on_outdated_workers(value: &FlowValue) -> Result<(), Error> {
    let gate = &windmill_common::min_version::MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES;
    if !gate.met().await
        && windmill_common::scopes::validate_flow_step_job_token_scopes(value).unwrap_or(true)
    {
        gate.assert().await?;
    }
    Ok(())
}

/// A run of an earlier version at the path, and what its replacement inherits from it.
struct PerpetualRunToRestart {
    id: Uuid,
    created_by: String,
    permissioned_as: String,
    permissioned_as_email: String,
    trigger: Option<String>,
    trigger_kind: Option<TriggerKindLabel>,
    /// `Some(false)` while the run still carries the arguments it was started with: only its own
    /// completion swaps in what a preprocessor returned.
    preprocessed: Option<bool>,
    args: Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
    /// Its effective token scopes: the replacement never holds a wider token.
    job_token_scopes: Option<Vec<String>>,
}

/// What every run at the path moves to.
struct RestartOnVersion<'a> {
    payload: &'a JobPayload,
    tag: Option<&'a str>,
    timeout: Option<i32>,
    dedicated_worker: Option<bool>,
    on_behalf_of: Option<&'a windmill_common::jobs::OnBehalfOf>,
}

async fn restart_perpetual_run(
    db: &Pool<Postgres>,
    w_id: &str,
    script_path: &str,
    deployed_by: &str,
    version: RestartOnVersion<'_>,
    run: PerpetualRunToRestart,
) -> error::Result<()> {
    let RestartOnVersion { payload, tag, timeout, dedicated_worker, on_behalf_of } = version;
    let (email, permissioned_as) = match on_behalf_of {
        Some(obo) => (obo.email.clone(), obo.permissioned_as.clone()),
        None => (
            run.permissioned_as_email.clone(),
            run.permissioned_as.clone(),
        ),
    };
    let args = run.args.clone().map(|args| args.0).unwrap_or_default();
    // The run's own tag was checked when the loop started; the deployed version's has not
    // been checked against the identity that would run it. Checked the way a push checks one,
    // so a `$args[...]` tag resolves from the arguments this run carries. A dedicated worker's
    // tag is the script's own and names no worker group to gain access to.
    if dedicated_worker != Some(true) {
        if let Some(tag) = tag.filter(|tag| !tag.is_empty()) {
            let is_super_admin = windmill_common::auth::is_super_admin_email(db, &email).await?;
            if let Err(e) = check_tag_available_for_push(
                db,
                w_id,
                tag,
                &PushArgs::from(&args),
                is_super_admin,
                None,
            )
            .await
            {
                tracing::warn!(
                    "Perpetual run {} stays on its version: the deployed version of \
                     {script_path} has tag {tag}: {e}",
                    run.id
                );
                return Ok(());
            }
        }
    }
    // A run whose own preprocessor has not run yet carries what started it, so the replacement has
    // to run one: pushed without, those arguments reach `main` and every iteration after it.
    let mut payload = payload.clone();
    if let JobPayload::ScriptHash { apply_preprocessor, .. } = &mut payload {
        *apply_preprocessor = *apply_preprocessor && run.preprocessed == Some(false);
    }
    let mut tx = db.begin().await?;
    // Claiming the run and queueing its replacement in one transaction: a push that fails
    // leaves the run looping on its own version rather than canceled with nothing to follow
    // it, and a concurrent deploy cannot claim a run this one already has. A worker completes
    // a run canceled this way when it next pulls it.
    let claimed = sqlx::query_scalar!(
        "UPDATE v2_job_queue SET canceled_by = $1, canceled_reason = $2, scheduled_for = now(), \
         suspend = 0 WHERE id = $3 AND workspace_id = $4 AND canceled_by IS NULL RETURNING id",
        deployed_by,
        format!("a new version of {script_path} was deployed"),
        run.id,
        w_id
    )
    .fetch_optional(&mut *tx)
    .await?;
    if claimed.is_none() {
        // It ended or was canceled since the scan. Its own restart, if it had one, is a run of
        // an earlier version the next pass picks up.
        return Ok(());
    }
    let scope_ceiling = run.job_token_scopes;
    let (_, tx) = push(
        db,
        PushIsolationLevel::Transaction(tx),
        w_id,
        payload,
        PushArgs::from(&args),
        &run.created_by,
        &email,
        permissioned_as,
        Some(&format!("deploy.restart.{}", run.id)),
        None,
        None,
        schedule_path(&run.trigger_kind, &run.trigger),
        None,
        None,
        None,
        None,
        false,
        false,
        None,
        true,
        tag.map(str::to_string),
        timeout,
        None,
        None,
        None,
        false,
        None,
        None,
        None,
        scope_ceiling.as_deref(),
    )
    .await?;
    tx.commit().await?;
    // Now that the replacement is queued: the children the run left behind, and, for a run no
    // worker would pull, its completion.
    match cancel_job(
        deployed_by,
        Some(format!("a new version of {script_path} was deployed")),
        run.id,
        w_id,
        db.begin().await?,
        db,
        false,
        false,
    )
    .await
    {
        Ok((tx, _)) => tx.commit().await?,
        Err(e) => {
            tracing::error!("Could not finish canceling perpetual run {}: {e:#}", run.id)
        }
    }
    Ok(())
}

#[derive(Serialize, Debug)]
pub struct WrappedError {
    pub error: serde_json::Value,
}

pub trait ValidableJson {
    fn is_valid_json(&self) -> bool;
    fn wm_labels(&self) -> Option<Vec<String>>;
    fn wm_failure(&self) -> Option<String>;
    fn result_metadata(&self) -> ResultMetadata;
    fn size(&self) -> usize;
    /// The result as JSON text, for binding into the `jsonb` `result` column.
    /// `Box<RawValue>` is already serialized and returns a zero-cost borrow;
    /// other impls serialize on demand. Callers pass this through
    /// `strip_json_nul` before the INSERT, since a genuine NUL escape would
    /// abort the write with 22P05.
    fn serialized_json(&self) -> Cow<'_, str>;
}

/// The Windmill-specific markers we look for inside a job's result.
/// `wm_failure` retags a successful run as a failure with the
/// given message; `wm_labels` adds runtime labels to the job row.
#[derive(serde::Deserialize, Default, Debug, Clone)]
pub struct ResultMetadata {
    pub wm_labels: Option<Vec<String>>,
    pub wm_failure: Option<String>,
}

/// Parses a marker struct out of a job result, which only an object can carry.
/// A derived `Deserialize` also accepts an array, filling fields by position, so
/// without the check a result like `[[], "boom"]` reads as `wm_failure: "boom"`.
pub fn parse_result_object<T: serde::de::DeserializeOwned>(result: &str) -> Option<T> {
    if !result.trim_start().starts_with('{') {
        return None;
    }
    serde_json::from_str(result).ok()
}

/// Sentinel `error.name` we inject into a result when retagging a successful
/// run as a failure due to `wm_failure`. Used downstream to detect that
/// the result is already in the standard `{ error: { name, message }, ... }`
/// shape and must not be wrapped a second time by `WrappedError`.
pub const MANUAL_FAILURE_ERROR_NAME: &str = "ManualFailure";

/// Returns true when the result already carries our injected
/// `error: { name: "ManualFailure", ... }` marker — i.e. it was shaped by
/// `process_jc`'s wm_failure path. A real runtime failure whose raw
/// result happens to contain a `wm_failure` field but no such error
/// key returns false (and so still goes through the standard wrap path).
pub fn is_pre_shaped_wm_failure_result(result: &str) -> bool {
    #[derive(serde::Deserialize)]
    struct Marker {
        error: Option<NameOnly>,
    }
    #[derive(serde::Deserialize)]
    struct NameOnly {
        name: String,
    }
    parse_result_object::<Marker>(result)
        .and_then(|m| m.error)
        .map(|e| e.name == MANUAL_FAILURE_ERROR_NAME)
        .unwrap_or(false)
}

impl ValidableJson for WrappedError {
    fn is_valid_json(&self) -> bool {
        true
    }

    fn wm_labels(&self) -> Option<Vec<String>> {
        None
    }

    fn wm_failure(&self) -> Option<String> {
        None
    }

    fn result_metadata(&self) -> ResultMetadata {
        ResultMetadata::default()
    }

    fn size(&self) -> usize {
        0
    }

    fn serialized_json(&self) -> Cow<'_, str> {
        Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string()))
    }
}

impl ValidableJson for Box<RawValue> {
    fn is_valid_json(&self) -> bool {
        !self.get().is_empty()
    }

    fn wm_labels(&self) -> Option<Vec<String>> {
        self.result_metadata().wm_labels
    }

    fn wm_failure(&self) -> Option<String> {
        self.result_metadata().wm_failure
    }

    fn result_metadata(&self) -> ResultMetadata {
        parse_result_object::<ResultMetadata>(self.get()).unwrap_or_default()
    }

    fn size(&self) -> usize {
        self.get().len()
    }

    fn serialized_json(&self) -> Cow<'_, str> {
        // Already serialized JSON text — borrow it, no re-serialization.
        Cow::Borrowed(self.get())
    }
}

impl<T: ValidableJson> ValidableJson for Arc<T> {
    fn is_valid_json(&self) -> bool {
        T::is_valid_json(&self)
    }

    fn wm_labels(&self) -> Option<Vec<String>> {
        T::wm_labels(&self)
    }

    fn wm_failure(&self) -> Option<String> {
        T::wm_failure(&self)
    }

    fn result_metadata(&self) -> ResultMetadata {
        T::result_metadata(&self)
    }

    fn size(&self) -> usize {
        T::size(&self)
    }

    fn serialized_json(&self) -> Cow<'_, str> {
        T::serialized_json(&self)
    }
}

impl ValidableJson for serde_json::Value {
    fn is_valid_json(&self) -> bool {
        true
    }

    fn wm_labels(&self) -> Option<Vec<String>> {
        self.result_metadata().wm_labels
    }

    fn wm_failure(&self) -> Option<String> {
        self.result_metadata().wm_failure
    }

    fn result_metadata(&self) -> ResultMetadata {
        // An array would decode positionally, see `parse_result_object`.
        if !self.is_object() {
            return ResultMetadata::default();
        }
        serde_json::from_value::<ResultMetadata>(self.clone()).unwrap_or_default()
    }

    fn size(&self) -> usize {
        self.size_hint()
    }

    fn serialized_json(&self) -> Cow<'_, str> {
        Cow::Owned(serde_json::to_string(self).unwrap_or_else(|_| "{}".to_string()))
    }
}

impl<T: ValidableJson> ValidableJson for Json<T> {
    fn is_valid_json(&self) -> bool {
        self.0.is_valid_json()
    }

    fn wm_labels(&self) -> Option<Vec<String>> {
        self.0.wm_labels()
    }

    fn wm_failure(&self) -> Option<String> {
        self.0.wm_failure()
    }

    fn result_metadata(&self) -> ResultMetadata {
        self.0.result_metadata()
    }

    fn size(&self) -> usize {
        self.0.size()
    }

    fn serialized_json(&self) -> Cow<'_, str> {
        self.0.serialized_json()
    }
}

pub async fn register_metric<T, F, F2, R>(
    l: &Arc<RwLock<HashMap<String, T>>>,
    s: &str,
    fnew: F2,
    f: F,
) -> Option<R>
where
    F: FnOnce(&T) -> R,
    F2: FnOnce(&str) -> (T, R),
{
    if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
        let lock = l.read().await;

        let counter = lock.get(s);
        if let Some(counter) = counter {
            let r = f(counter);
            drop(lock);
            Some(r)
        } else {
            drop(lock);
            let (metric, r) = fnew(s);
            let mut m = l.write().await;
            (*m).insert(s.to_string(), metric);
            Some(r)
        }
    } else {
        None
    }
}

async fn record_failure_metrics(completed_job: &MiniCompletedJob, _worker_name: &str) {
    #[cfg(feature = "prometheus")]
    register_metric(
        &WORKER_EXECUTION_FAILED,
        &completed_job.tag,
        |s| {
            let counter = prometheus::register_int_counter!(prometheus::Opts::new(
                "worker_execution_failed",
                "Number of jobs having failed"
            )
            .const_label("name", _worker_name)
            .const_label("tag", s))
            .expect("register prometheus metric");
            counter.inc();
            (counter, ())
        },
        |c| c.inc(),
    )
    .await;

    otel_incr_worker_execution_failed(&completed_job.tag);
}

/// Tag a completed job as a failure while storing the result as-is, without
/// the standard `WrappedError` `{ error: ... }` wrap. Use for jobs whose result
/// is already shaped (e.g. when `wm_failure` injected a top-level
/// `error` key, while preserving sibling fields like `windmill_status_code`).
///
/// This is a worker-internal helper called by trusted result-processing code
/// after the worker has authenticated and pulled the job. Callers MUST verify
/// upstream auth (i.e. the job was legitimately pulled by this worker) — this
/// function performs no authorization check itself, mirroring the contract of
/// `add_completed_job_error`.
pub async fn add_completed_job_pre_shaped_failure<T: Serialize + Send + Sync + ValidableJson>(
    db: &Pool<Postgres>,
    completed_job: &MiniCompletedJob,
    mem_peak: i32,
    canceled_by: Option<CanceledBy>,
    result: Json<&T>,
    worker_name: &str,
    flow_is_done: bool,
    duration: Option<i64>,
) -> Result<CompletedSpan, Error> {
    record_failure_metrics(completed_job, worker_name).await;

    tracing::error!(
        "job {} in {} did not succeed (wm_failure)",
        completed_job.id,
        completed_job.workspace_id,
    );
    let (_, span) = add_completed_job(
        db,
        completed_job,
        false,
        false,
        result,
        None,
        mem_peak,
        canceled_by,
        flow_is_done,
        duration,
        false,
    )
    .warn_after_seconds(10)
    .await?;
    Ok(span)
}

pub async fn add_completed_job_error(
    db: &Pool<Postgres>,
    completed_job: &MiniCompletedJob,
    mem_peak: i32,
    canceled_by: Option<CanceledBy>,
    e: serde_json::Value,
    worker_name: &str,
    flow_is_done: bool,
    duration: Option<i64>,
) -> Result<WrappedError, Error> {
    add_completed_job_error_with_span(
        db,
        completed_job,
        mem_peak,
        canceled_by,
        e,
        worker_name,
        flow_is_done,
        duration,
    )
    .await
    .map(|(wrapped, _)| wrapped)
}

/// `add_completed_job_error`, also handing back what the completed row holds.
pub async fn add_completed_job_error_with_span(
    db: &Pool<Postgres>,
    completed_job: &MiniCompletedJob,
    mem_peak: i32,
    canceled_by: Option<CanceledBy>,
    e: serde_json::Value,
    worker_name: &str,
    flow_is_done: bool,
    duration: Option<i64>,
) -> Result<(WrappedError, CompletedSpan), Error> {
    record_failure_metrics(completed_job, worker_name).await;

    let result = WrappedError { error: e };
    tracing::error!(
        "job {} in {} did not succeed: {}",
        completed_job.id,
        completed_job.workspace_id,
        serde_json::to_string(&result).unwrap_or_else(|_| "".to_string())
    );
    let (_, span) = add_completed_job(
        db,
        &completed_job,
        false,
        false,
        Json(&result),
        None,
        mem_peak,
        canceled_by,
        flow_is_done,
        duration,
        false,
    )
    .warn_after_seconds(10)
    .await?;
    Ok((result, span))
}

lazy_static::lazy_static! {
    pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
    pub static ref MAX_RESULT_SIZE_MB: usize = std::env::var("MAX_RESULT_SIZE_MB").unwrap_or("500".to_string()).parse().unwrap_or(500);

    // Cache for perpetual-restart settings (restart_unless_cancelled, timeout) - keyed by (hash, workspace_id)
    static ref RESTART_UNLESS_CANCELLED_CACHE: Cache<(i64, String), (bool, Option<i32>, Option<Vec<String>>)> = Cache::new(10000);

    // Cache for workspace error handler settings with 60s TTL
    // Key: workspace_id, Value: (error_handler, error_handler_extra_args, error_handler_muted_on_cancel, error_handler_muted_on_user_path, report_to_instance_alerts, expiry_timestamp)
    static ref WORKSPACE_ERROR_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, bool, bool, bool, i64)> = Cache::new(1000);

    // Best-effort per-worker throttle for the instance-channel fallback: a flapping runnable
    // would otherwise turn every failure into outbound Slack/SMTP traffic on channels shared by
    // the whole instance. Key: workspace_id, Value: (last_sent_epoch, failures suppressed since)
    static ref INSTANCE_ALERT_THROTTLE: Cache<String, (i64, u64)> = Cache::new(1000);

    // Cache for workspace success handler settings with 60s TTL
    // Key: workspace_id, Value: (success_handler, success_handler_extra_args, expiry_timestamp)
    static ref WORKSPACE_SUCCESS_HANDLER_CACHE: Cache<String, (Option<String>, Option<Json<Box<RawValue>>>, i64)> = Cache::new(1000);
}

const WORKSPACE_HANDLER_CACHE_TTL_SECONDS: i64 = 60;
const INSTANCE_ALERT_COOLDOWN_SECONDS: i64 = 60;

pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
    db: &Pool<Postgres>,
    completed_job: &MiniCompletedJob,
    success: bool,
    skipped: bool,
    result: Json<&T>,
    result_columns: Option<Vec<String>>,
    mem_peak: i32,
    canceled_by: Option<CanceledBy>,
    flow_is_done: bool,
    duration: Option<i64>,
    from_cache: bool,
) -> Result<(Uuid, CompletedSpan), Error> {
    // tracing::error!("Start");
    // let start = tokio::time::Instant::now();

    // add_time!(bench, "add_completed_job start");
    if !result.is_valid_json() {
        return Err(Error::internal_err(
            "Result of job is invalid json (empty)".to_string(),
        ));
    }

    // Native script retry: a failed `Script` or `Script_Hub` job that carries a retry policy and
    // has attempts left gets its next attempt enqueued here — before the queue
    // row (which holds the attempt counter) is removed by commit. The failed
    // attempt is still recorded as a completed job below. `maybe_enqueue_…`
    // self-guards on kind/cancellation/policy, so the success path is unaffected.
    let retry_pending = if !success && !skipped && !from_cache {
        // Serialized lazily, and only when a `retry_if` policy actually needs it.
        let result_fn = || serde_json::value::to_raw_value(&result).ok();
        match maybe_enqueue_native_script_retry(db, completed_job, &canceled_by, &result_fn).await {
            Ok(enqueued) => enqueued,
            Err(e) => {
                tracing::error!(
                    "native retry enqueue failed for {}: {e:#}",
                    completed_job.id
                );
                false
            }
        }
    } else {
        false
    };

    let result_columns = result_columns.as_ref();
    let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| {
        commit_completed_job(
            db,
            completed_job,
            success,
            skipped,
            result,
            result_columns,
            mem_peak,
            &canceled_by,
            flow_is_done,
            duration,
            from_cache,
            retry_pending,
        )
        .warn_after_seconds(10)
    })
    .retry(
        ConstantBuilder::default()
            .with_delay(std::time::Duration::from_secs(3))
            .with_max_times(10)
            .build(),
    )
    .when(|err| {
        !matches!(err, Error::QuotaExceeded(_))
            && !matches!(err, Error::ResultTooLarge(_))
            && !matches!(err, Error::AlreadyCompleted(_))
            && !matches!(err, Error::NotFound(_))
    })
    .notify(|err, dur| {
        tracing::error!("Could not insert completed job, retrying in {dur:#?}, err: {err:#?}");
    })
    .sleep(tokio::time::sleep)
    .await?;

    if wac_parent_ready {
        windmill_common::wac::WAC_SUSPEND_READY.store(true, std::sync::atomic::Ordering::Relaxed);
    }

    // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout
    if let Some(job_id) = opt_uuid {
        return Ok((job_id, span));
    }

    // Auto-resolve a retry chain that ultimately worked, from whichever of the two
    // completions lands last (see resolve_retry_chain_if_succeeded): a success that has a
    // parent (so is a possible retry attempt), or a failure that just enqueued a retry.
    // `retry_pending` already implies a non-flow-step `Script` or `Script_Hub`.
    let resolve_root = if success && !skipped && !completed_job.is_flow_step() {
        matches!(completed_job.kind, JobKind::Script | JobKind::Script_Hub)
            .then(|| completed_job.parent_job)
            .flatten()
    } else if !success && !skipped && retry_pending {
        let is_retry_attempt = sqlx::query_scalar!(
            "SELECT attempt FROM native_retry_attempt WHERE job_id = $1",
            completed_job.id,
        )
        .fetch_optional(db)
        .await
        .unwrap_or_else(|e| {
            tracing::error!("reading the retry marker of {}: {e:#}", completed_job.id);
            None
        })
        .is_some();
        Some(native_retry_root(completed_job, is_retry_attempt))
    } else {
        None
    };
    if let Some(root) = resolve_root {
        if let Err(e) = resolve_retry_chain_if_succeeded(
            db,
            root,
            &completed_job.workspace_id,
            completed_job.runnable_id,
        )
        .await
        {
            tracing::error!("Error auto-resolving native retry chain {root}: {e:#}");
        }
    }

    #[cfg(feature = "cloud")]
    apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms);

    #[cfg(all(feature = "enterprise", feature = "private"))]
    crate::jobs_ee::apply_completed_job_error_handlers(
        db,
        completed_job,
        success,
        result,
        &canceled_by,
        _skip_downstream_error_handlers,
    )
    .await;

    restart_job_if_perpetual(db, completed_job, &canceled_by).await?;

    // tracing::error!("4 {:?}", start.elapsed());

    Ok((completed_job.id, span))
}

async fn commit_completed_job<T: Serialize + Send + Sync + ValidableJson>(
    db: &Pool<Postgres>,
    completed_job: &MiniCompletedJob,
    success: bool,
    skipped: bool,
    result: Json<&T>,
    result_columns: Option<&Vec<String>>,
    mem_peak: i32,
    canceled_by: &Option<CanceledBy>,
    flow_is_done: bool,
    duration: Option<i64>,
    from_cache: bool,
    // True when a native script retry was enqueued for this failed attempt, i.e.
    // this is not the terminal attempt — schedule completion handlers must wait.
    retry_pending: bool,
) -> windmill_common::error::Result<(Option<Uuid>, CompletedSpan, bool, bool)> {
    // let start = std::time::Instant::now();

    let job_id = completed_job.id;
    // tracing::error!("1 {:?}", start.elapsed());

    // tracing::debug!(
    //     "completed job {} {}",
    //     queued_job.id,
    //     serde_json::to_string(&result).unwrap_or_else(|_| "".to_string())
    // );

    let mem_peak = mem_peak;
    // add_time!(bench, "add_completed_job query START");

    if let Some(value) = check_result_size(db, completed_job, result).await {
        return value;
    }

    // Resolve the concurrency-limit settings on the pool *before* opening the
    // completion transaction: doing it inside the tx would hold a second
    // simultaneous connection from the small per-worker pool.
    let has_concurrent_limit = has_active_concurrency_limit(completed_job.concurrent_limit)
        || has_active_concurrency_limit(
            windmill_common::runnable_settings::prefetch_cached_from_handle(
                completed_job.runnable_settings_handle,
                db,
            )
            .await?
            .1
            .concurrent_limit,
        );

    // A genuine NUL (U+0000) in the result serializes to a `\u0000` escape that
    // the jsonb `result` column rejects with 22P05 ("unsupported Unicode escape
    // sequence"), which would abort the whole completion INSERT. Strip it before
    // binding — near-zero cost when clean: a single scan, and for an
    // already-serialized `RawValue` result the serialization itself is a borrow.
    let serialized_result = result.serialized_json();
    let sanitized_result = strip_json_nul(serialized_result.as_ref());

    let labels = result.wm_labels();
    let is_scheduled =
        completed_job.schedule_path().is_some() && completed_job.runnable_path.is_some();
    let wac_parent = (!completed_job.is_flow_step())
        .then_some(completed_job.parent_job)
        .flatten();
    let monitor_parent = (completed_job.is_flow_step() && flow_is_done)
        .then_some(completed_job.parent_job)
        .flatten();
    let completion = Completion {
        completed_job,
        success,
        skipped,
        result: sanitized_result.as_ref(),
        result_columns,
        mem_peak,
        canceled_by,
        duration,
    };

    if labels.is_none()
        && !has_concurrent_limit
        && wac_parent.is_none()
        && monitor_parent.is_none()
        && !is_scheduled
    {
        let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else {
            return Err(not_in_queue_error(db, job_id).await);
        };
        log_completed_job(completed_job, completed.duration_ms, success);
        return Ok((None, completed, false, false));
    }

    let mut tx = db.begin().warn_after_seconds(10).await?;

    // The parent's rows are locked ahead of the child's own queue row (see
    // `record_child_completion` for the order this must keep), so the duration it stamps is read
    // before the completion: the one the completed row will hold. `now()` is fixed for the
    // transaction, and a completed row already there keeps its own duration. For a child that
    // itself parked it is the last run's, matching the last start its timeline entry carries.
    let mut wac_parent_ready = false;
    if let Some(parent_job) = wac_parent {
        let Some(duration) = sqlx::query_scalar!(
            "SELECT COALESCE(c.duration_ms, COALESCE($2::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(q.started_at, now()))))*1000)::bigint) AS \"duration_ms!\"
             FROM v2_job_queue q LEFT JOIN v2_job_completed c ON c.id = q.id WHERE q.id = $1",
            job_id,
            duration,
        )
        .fetch_optional(&mut *tx)
        .warn_after_seconds(10)
        .await?
        else {
            return Err(not_in_queue_error(&mut *tx, job_id).await);
        };
        wac_parent_ready = windmill_common::wac::record_child_completion(
            &mut tx,
            &parent_job,
            &completed_job.id,
            success,
            duration,
            sanitized_result.as_ref(),
        )
        .warn_after_seconds(10)
        .await?;
    }

    let Some(completed) = completion.execute(&mut *tx).await? else {
        return Err(not_in_queue_error(&mut *tx, job_id).await);
    };

    if let Some(mut labels) = labels {
        // A `\u0000` inside a wm_labels entry decodes to a real NUL that the
        // `text[]` column rejects, which would abort this same transaction (and
        // roll back the sanitized result insert) exactly like an unsanitized
        // result. Strip it so the labels match the sanitized result.
        for label in &mut labels {
            label.retain(|c| c != '\0');
        }
        sqlx::query!(
            "UPDATE v2_job SET labels = (
                    SELECT array_agg(DISTINCT all_labels)
                    FROM unnest(coalesce(labels, ARRAY[]::TEXT[]) || $2) all_labels
                ) WHERE id = $1",
            job_id,
            labels as Vec<String>
        )
        .execute(&mut *tx)
        .warn_after_seconds(10)
        .await
        .map_err(|e| Error::InternalErr(format!("Could not update job labels: {e:#}")))?;
    }

    let mut _skip_downstream_error_handlers = false;
    if completed_job.is_flow_step() {
        if let Some(parent_job) = monitor_parent {
            let r = sqlx::query_scalar!(
                "UPDATE parallel_monitor_lock SET last_ping = now() WHERE parent_flow_id = $1 and job_id = $2 RETURNING 1",
                parent_job,
                &completed_job.id
            ).fetch_optional(&mut *tx).warn_after_seconds(10).await?;
            if r.is_some() {
                tracing::info!(
                        "parallel flow iteration is done, setting parallel monitor last ping lock for job {}",
                        &completed_job.id
                    );
            }
        }
    } else {
        if completed_job.schedule_path().is_some() && completed_job.runnable_path.is_some() {
            let schedule_path = completed_job.schedule_path().unwrap();
            let script_path = completed_job.runnable_path.as_ref().unwrap();

            let schedule = get_schedule_opt(&mut *tx, &completed_job.workspace_id, &schedule_path)
                .warn_after_seconds(10)
                .await?;

            if let Some(schedule) = schedule {
                #[cfg(feature = "enterprise")]
                {
                    _skip_downstream_error_handlers = schedule.ws_error_handler_muted;
                }

                // for scripts, always try to schedule next tick
                // for flows, only try to schedule next tick here if flow failed and because first handle_flow failed (step = 0, modules[0] = {type: 'Failure', 'job': uuid::nil()})
                // or job was cancelled before first handle_flow was called (step = 0, modules = [] OR modules[0].type == 'WaitingForPriorSteps')
                // otherwise flow rescheduling is done inside handle_flow
                // Native retry attempts carry the schedule trigger (so the
                // terminal attempt can drive handlers) but `parent_job` is set —
                // they must not each push the next cron tick (the root already
                // did), so gate next-tick on a top-level (`parent_job IS NULL`)
                // occurrence.
                let schedule_next_tick = completed_job.parent_job.is_none()
                    && (!completed_job.is_flow()
                        || from_cache
                        || !success
                            && sqlx::query_scalar!(
                                "SELECT
                                flow_status->>'step' = '0'
                                AND (
                                    jsonb_array_length(flow_status->'modules') = 0
                                    OR flow_status->'modules'->0->>'type' = 'WaitingForPriorSteps'
                                    OR (
                                        flow_status->'modules'->0->>'type' = 'Failure'
                                        AND flow_status->'modules'->0->>'job' = $1
                                    )
                                )
                            FROM v2_job_completed WHERE id = $2 AND workspace_id = $3",
                                Uuid::nil().to_string(),
                                &completed_job.id,
                                &completed_job.workspace_id
                            )
                            .fetch_optional(&mut *tx)
                            .warn_after_seconds(10)
                            .await?
                            .flatten()
                            .unwrap_or(false));

                if schedule_next_tick {
                    let (returned_tx, schedule_push_err) =
                        try_schedule_next_job(db, tx, completed_job, &schedule, &script_path).await;
                    tx = returned_tx;
                    if let Some(err) = schedule_push_err {
                        return Err(err);
                    }
                }

                // Defer schedule completion handlers (on_failure/on_success/
                // on_recovery) while a native retry is pending: only the terminal
                // attempt should drive them. apply_schedule_handlers resolves
                // per-occurrence failure/recovery status across the whole retry
                // chain, so multi-count/exact handler policies work even though
                // each attempt is its own completed job.
                #[cfg(all(feature = "enterprise", feature = "private"))]
                if !retry_pending {
                    if let Err(err) = crate::jobs_ee::apply_schedule_handlers(
                        db,
                        &schedule,
                        &script_path,
                        &completed_job.workspace_id,
                        success,
                        result,
                        job_id,
                        // Current occurrence's root: the terminal native-retry
                        // attempt's parent, else the job itself.
                        completed_job.parent_job.unwrap_or(job_id),
                        completed_job.started_at.unwrap_or(chrono::Utc::now()),
                        completed_job.priority,
                    )
                    .warn_after_seconds(10)
                    .await
                    {
                        if !success {
                            tracing::error!("Could not apply schedule error handler: {}", err);
                            let base_url = windmill_common::BASE_URL.load();
                            let w_id: &String = &completed_job.workspace_id;
                            if !matches!(err, Error::QuotaExceeded(_)) {
                                report_error_to_workspace_handler_or_critical_side_channel(
                                        &completed_job,
                                        db,
                                        format!(
                                            "Failed to push schedule error handler job to handle failed job ({base_url}/run/{}?workspace={w_id}): {}",
                                            completed_job.id,
                                            err
                                        ),
                                    )
                                    .warn_after_seconds(10)
                                    .await;
                            }
                        } else {
                            tracing::error!("Could not apply schedule recovery handler: {}", err);
                        }
                    };
                }
                #[cfg(not(all(feature = "enterprise", feature = "private")))]
                let _ = retry_pending;
            } else {
                tracing::error!(
                        "Schedule {schedule_path} in {} not found. Impossible to schedule again and apply schedule handlers",
                        &completed_job.workspace_id
                    );
            }
        }
    }

    if has_concurrent_limit {
        let concurrency_key = sqlx::query_scalar!(
            "SELECT key FROM concurrency_key WHERE job_id = $1",
            &completed_job.id
        )
        .fetch_optional(&mut *tx)
        .warn_after_seconds(10)
        .await
        .map_err(|e| {
            Error::internal_err(format!(
                "Could not get concurrency key for job {}: {e:#}",
                completed_job.id
            ))
        })?;
        if *DISABLE_CONCURRENCY_LIMIT || concurrency_key.is_none() {
            tracing::warn!("Concurrency limit is disabled, skipping");
        } else {
            let concurrency_key = concurrency_key.unwrap();
            sqlx::query_scalar!(
                "UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1",
                concurrency_key,
                completed_job.id.hyphenated().to_string(),
            )
            .execute(&mut *tx)
            .warn_after_seconds(10)
            .await
            .map_err(|e| {
                Error::internal_err(format!(
                    "Could not decrement concurrency counter for job_id={}: {e:#}",
                    completed_job.id
                ))
            })?;
        }

        if let Err(e) = sqlx::query_scalar!(
            "UPDATE concurrency_key SET ended_at = now() WHERE job_id = $1",
            completed_job.id,
        )
        .execute(&mut *tx)
        .warn_after_seconds(10)
        .await
        {
            tracing::error!(
                "Could not update concurrency_key ended_at for job_id={}: {e:#}",
                completed_job.id,
            );
        }
        tracing::debug!("decremented concurrency counter");
    }

    tx.commit().warn_after_seconds(10).await?;

    log_completed_job(completed_job, completed.duration_ms, success);
    // tracing::info!("completed job: {:?}", start.elapsed().as_micros());
    Ok((
        None,
        completed,
        _skip_downstream_error_handlers,
        wac_parent_ready,
    ))
}

/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's.
/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered;
/// it differs from `duration_ms` only for a job that parked.
#[derive(Debug, Clone, Copy)]
pub struct CompletedSpan {
    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
    pub duration_ms: i64,
    pub last_run_ms: i64,
}

impl CompletedSpan {
    /// Whether the row spans more than the run that started at `last_started_at`, i.e. the
    /// job parked and resumed. A flow records such a step with the row's span, so the step
    /// reads like the job does.
    pub fn outlasts_run(&self, last_started_at: Option<chrono::DateTime<chrono::Utc>>) -> bool {
        matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last)
    }
}

/// What a job's completion writes, for `Completion::execute`.
struct Completion<'a> {
    completed_job: &'a MiniCompletedJob,
    success: bool,
    skipped: bool,
    result: &'a str,
    result_columns: Option<&'a Vec<String>>,
    mem_peak: i32,
    canceled_by: &'a Option<CanceledBy>,
    duration: Option<i64>,
}

impl Completion<'_> {
    /// Moves the job from the queue to the completed jobs and refreshes a flow step's parent
    /// ping, as one statement. Returns `None` when the job was no longer in the queue.
    ///
    /// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's
    /// `started_at` and `duration_ms` span the whole workflow from its first start, and the
    /// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from
    /// completed rows.
    ///
    /// The completion takes its cancellation from the queue row it deletes, not only from
    /// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still
    /// being written, so the deleted row is the final word on whether the job was canceled.
    ///
    /// It locks the queue row before the completed row's key. Any other writer completing a job
    /// (the monitor's zombie fallback, debounce) must take them in the same order, or the two
    /// deadlock.
    async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result<Option<CompletedSpan>> {
        let Completion {
            completed_job,
            success,
            skipped,
            result,
            result_columns,
            mem_peak,
            canceled_by,
            duration,
        } = *self;
        #[cfg(feature = "prometheus")]
        if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
            QUEUE_DELETE_COUNT.inc();
        }
        otel_incr_queue_delete_count();
        let err = |e: sqlx::Error| {
            Error::internal_err(format!(
                "Could not add completed job {}: {e:#}",
                completed_job.id
            ))
        };
        // A step's completion is progress of its flow, and keeps the flow from being reaped as a
        // zombie. Any other completion runs the statement without the ping: Postgres sets up every
        // write of a plan, so an unused ping would cost about a tenth of the completion. The two
        // statements differ only by the ping; a change to the delete or the insert goes in both.
        let Some(parent_to_ping) = completed_job
            .is_flow_step()
            .then_some(completed_job.parent_job)
            .flatten()
        else {
            return sqlx::query_as!(
                CompletedSpan,
                "WITH deleted AS (
                    DELETE FROM v2_job_queue WHERE id = $1
                    RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
                        (extras->>'wac_first_started_at')::timestamptz AS first_started_at,
                        COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
                ), completed AS (
                    INSERT INTO v2_job_completed AS cj
                        ( workspace_id
                        , id
                        , started_at
                        , duration_ms
                        , extras
                        , result
                        , result_columns
                        , canceled_by
                        , canceled_reason
                        , flow_status
                        , workflow_as_code_status
                        , memory_peak
                        , status
                        , worker
                        )
                    SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
                        CASE WHEN d.first_started_at IS NULL THEN d.run_ms
                            ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
                        CASE WHEN d.first_started_at IS NOT NULL
                            THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
                        $3::text::jsonb, $10,
                        CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
                        CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
                        s.flow_status, s.workflow_as_code_status, $8,
                        CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status
                            WHEN $7::BOOL THEN 'skipped'::job_status
                            WHEN $2::BOOL THEN 'success'::job_status
                            ELSE 'failure'::job_status END,
                        d.worker
                    FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id
                    ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,
                        canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
                            THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
                        canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
                            THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
                    RETURNING cj.started_at, cj.duration_ms,
                        COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
                )
                SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
                    last_run_ms AS \"last_run_ms!\" FROM completed",
                /* $1 */ completed_job.id,
                /* $2 */ success,
                /* $3 */ result,
                /* $4 */ canceled_by.is_some(),
                /* $5 */ canceled_by.as_ref().and_then(|cb| cb.username.as_deref()),
                /* $6 */ canceled_by.as_ref().and_then(|cb| cb.reason.as_deref()),
                /* $7 */ skipped,
                /* $8 */ if mem_peak > 0 { Some(mem_peak) } else { None },
                /* $9 */ duration,
                /* $10 */ result_columns as Option<&Vec<String>>,
            )
            .fetch_optional(&mut *conn)
            .warn_after_seconds(10)
            .await
            .map_err(err);
        };
        // A canceled flow is pinged too: it is completed by its next transition like any other
        // flow, and the zombie flow monitor needs the ping to finish the cancel if that
        // transition is lost.
        sqlx::query_as!(
        CompletedSpan,
        "WITH deleted AS (
            DELETE FROM v2_job_queue WHERE id = $1
            RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,
                (extras->>'wac_first_started_at')::timestamptz AS first_started_at,
                COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms
        ), completed AS (
            INSERT INTO v2_job_completed AS cj
                ( workspace_id
                , id
                , started_at
                , duration_ms
                , extras
                , result
                , result_columns
                , canceled_by
                , canceled_reason
                , flow_status
                , workflow_as_code_status
                , memory_peak
                , status
                , worker
                )
            SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),
                CASE WHEN d.first_started_at IS NULL THEN d.run_ms
                    ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,
                CASE WHEN d.first_started_at IS NOT NULL
                    THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,
                $3::text::jsonb, $10,
                CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,
                CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,
                s.flow_status, s.workflow_as_code_status, $8,
                CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status
                    WHEN $7::BOOL THEN 'skipped'::job_status
                    WHEN $2::BOOL THEN 'success'::job_status
                    ELSE 'failure'::job_status END,
                d.worker
            FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id
            ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,
                canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
                    THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,
                canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL
                    THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END
            RETURNING cj.started_at, cj.duration_ms,
                COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms
        ), parent_ping AS (
            UPDATE v2_job_runtime r SET ping = now()
            FROM v2_job_queue q
            WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12
                AND EXISTS (SELECT 1 FROM completed)
        )
        SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",
            last_run_ms AS \"last_run_ms!\" FROM completed",
        /* $1 */ completed_job.id,
        /* $2 */ success,
        /* $3 */ result,
        /* $4 */ canceled_by.is_some(),
        /* $5 */ canceled_by.as_ref().and_then(|cb| cb.username.as_deref()),
        /* $6 */ canceled_by.as_ref().and_then(|cb| cb.reason.as_deref()),
        /* $7 */ skipped,
        /* $8 */ if mem_peak > 0 { Some(mem_peak) } else { None },
        /* $9 */ duration,
        /* $10 */ result_columns as Option<&Vec<String>>,
        /* $11 */ parent_to_ping,
        /* $12 */ &completed_job.workspace_id,
    )
    .fetch_optional(&mut *conn)
    .warn_after_seconds(10)
    .await
    .map_err(err)
    }
}

/// The error for a completion that found no queue row to complete.
async fn not_in_queue_error<'e>(conn: impl PgExecutor<'e>, job_id: Uuid) -> Error {
    let already_inserted = sqlx::query_scalar!(
        "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1)",
        job_id
    )
    .fetch_one(conn)
    .warn_after_seconds(10)
    .await;
    match already_inserted {
        Err(e) => Error::internal_err(format!("Could not add completed job {job_id}: {e:#}")),
        Ok(Some(true)) => {
            Error::AlreadyCompleted(format!("The queued job {job_id} is already completed."))
        }
        Ok(_) => Error::AlreadyCompleted(format!(
            "There is no queued job anymore for {job_id} but there is no completed job either."
        )),
    }
}

fn log_completed_job(completed_job: &MiniCompletedJob, duration: i64, success: bool) {
    let job_id = completed_job.id;
    tracing::info!(
        %job_id,
        root_job = ?completed_job.flow_innermost_root_job.map(|x| x.to_string()).unwrap_or_else(|| String::new()),
        path = &completed_job.runnable_path,
        job_kind = ?completed_job.kind,
        started_at = ?completed_job.started_at.map(|x| x.to_string()).unwrap_or_else(|| String::new()),
        duration = ?duration,
        permissioned_as = ?completed_job.permissioned_as,
        email = ?completed_job.permissioned_as_email,
        created_by = completed_job.created_by,
        is_flow_step = completed_job.is_flow_step(),
        language = completed_job.script_lang.map(|x| x.as_str()).unwrap_or_default(),
        scheduled_for = ?completed_job.scheduled_for,
        workspace_id = ?completed_job.workspace_id,
        success,
        "inserted completed job: {} (success: {success})",
        completed_job.id
    );
}

async fn check_result_size<T: ValidableJson>(
    db: &Pool<Postgres>,
    queued_job: &MiniCompletedJob,
    result: Json<&T>,
) -> Option<Result<(Option<Uuid>, CompletedSpan, bool, bool), Error>> {
    let result_size = result.size() / 1024 / 1024;
    if result_size > 2 {
        if result_size > *MAX_RESULT_SIZE_MB {
            tracing::error!(
                "Result of job {} is too large: {}MB > MAX_RESULT_SIZE_MB={}MB",
                queued_job.id,
                result_size,
                *MAX_RESULT_SIZE_MB
            );
            return Some(Err(Error::ResultTooLarge(format!("Result of job {} is too large: {}MB > MAX_RESULT_SIZE_MB={}MB.\nUse external storages such as the Windmill Object Storage to store large results: https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill", queued_job.id, result_size, *MAX_RESULT_SIZE_MB))));
        }
        append_logs(
            &queued_job.id,
            &queued_job.workspace_id,
            format!("Warning: Result of job {} is large: {}MB.\nRecommended max size is 2MB.\nPrefer using external storages such as the Windmill Object Storage to store large results: https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill", queued_job.id, result_size),
            &db.into(),
        )
        .await;
        if *CLOUD_HOSTED {
            return Some(Err(Error::ResultTooLarge(format!("Result of job {} is too large for multi-tenant cloud: {}MB (max 2MB).\nUse external storages such as the Windmill Object Storage to store large results: https://www.windmill.dev/docs/core_concepts/object_storage_in_windmill", queued_job.id, result_size))));
        } else {
            tracing::warn!(
                "Result of job {} is larger than 2MB: {}MB. Not recommended.",
                queued_job.id,
                result_size
            );
        }
    }
    None
}

async fn restart_job_if_perpetual(
    db: &Pool<Postgres>,
    queued_job: &MiniCompletedJob,
    canceled_by: &Option<CanceledBy>,
) -> Result<(), Error> {
    if !queued_job.is_flow_step() && queued_job.kind == JobKind::Script && canceled_by.is_none() {
        if let Some(hash) = queued_job.runnable_id {
            (|| restart_job_if_perpetual_inner(db, queued_job, hash))
                .retry(
                    ConstantBuilder::default()
                        .with_delay(std::time::Duration::from_secs(3))
                        .with_max_times(5)
                        .build(),
                )
                .notify(|err, dur| {
                    tracing::error!(
                    "Could not apply perpetual job restart, retrying in {dur:#?}, err: {err:#?}"
                );
                })
                .sleep(tokio::time::sleep)
                .await?;
        }
    }
    Ok(())
}

async fn restart_job_if_perpetual_inner(
    db: &Pool<Postgres>,
    queued_job: &MiniCompletedJob,
    hash: ScriptHash,
) -> Result<(), Error> {
    let cache_key = (hash.0, queued_job.workspace_id.clone());

    let (restart, script_timeout, script_job_token_scopes) = if let Some(cached) =
        RESTART_UNLESS_CANCELLED_CACHE.get(&cache_key)
    {
        cached
    } else {
        let row = sqlx::query!(
            "SELECT restart_unless_cancelled, timeout, job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2",
            hash.0,
            &queued_job.workspace_id
        )
        .fetch_optional(db)
        .await?;

        let restart = row
            .as_ref()
            .and_then(|r| r.restart_unless_cancelled)
            .unwrap_or(false);
        let script_timeout = row.as_ref().and_then(|r| r.timeout);
        let script_job_token_scopes = row.and_then(|r| r.job_token_scopes);

        let cached = (restart, script_timeout, script_job_token_scopes);
        RESTART_UNLESS_CANCELLED_CACHE.insert(cache_key, cached.clone());
        cached
    };

    if restart {
        // Not `canceled_by`: a worker reads the queue row on a widening interval, up to every 5s
        // once a job has run for a minute, so a cancel landing after its last read reaches a
        // completion carrying none. `commit_completed_job` records it from the queue row it
        // deletes, so the completed row is what says whether this loop was stopped.
        let canceled = sqlx::query_scalar!(
            "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_completed \
             WHERE id = $1 AND workspace_id = $2",
            queued_job.id,
            &queued_job.workspace_id
        )
        .fetch_optional(db)
        .await?
        .unwrap_or(false);
        if canceled {
            return Ok(());
        }
        let tx = PushIsolationLevel::IsolatedRoot(db.clone());

        // perpetual jobs can run one job per 10s max. If the job was faster than 10s, schedule the next one with the appropriate delay
        let now = now_from_db(db).await?;
        let scheduled_for = if now
            .signed_duration_since(queued_job.started_at.unwrap_or(now))
            .num_seconds()
            < 10
        {
            let next_run =
                queued_job.started_at.unwrap_or(now) + chrono::Duration::try_seconds(10).unwrap();
            tracing::warn!("Perpetual script {:?} is running too fast, only 1 job per 10s it supported. Scheduling next run for {:?}", queued_job.runnable_path, next_run);
            Some(next_run)
        } else {
            None
        };

        let args = sqlx::query_scalar!(
            "SELECT args as \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2",
            queued_job.id,
            queued_job.workspace_id
        )
        .fetch_optional(db)
        .await?
        .flatten()
        .unwrap_or_default();
        // The replacement never holds a wider token than the run it replaces.
        let scope_ceiling = queued_job.job_token_scopes.clone();
        let (_uuid, tx) = push(
            db,
            tx,
            &queued_job.workspace_id,
            JobPayload::ScriptHash {
                hash,
                path: queued_job.runnable_path.clone().unwrap_or_default(),
                cache_ttl: queued_job.cache_ttl,
                cache_ignore_s3_path: queued_job.cache_ignore_s3_path,
                dedicated_worker: None,
                language: queued_job
                    .script_lang
                    .clone()
                    .unwrap_or_else(|| ScriptLang::Deno),
                priority: queued_job.priority,
                apply_preprocessor: false,
                concurrency_settings: ConcurrencySettings {
                    concurrency_key: custom_concurrency_key(db, &queued_job.id).await?,
                    concurrent_limit: None,
                    concurrency_time_window_s: None,
                },
                // TODO(debouncing): handle properly
                debouncing_settings: DebouncingSettings::default(),
                labels: None, // labels already set on original job
                job_token_scopes: script_job_token_scopes,
            },
            PushArgs::from(&args.0),
            &queued_job.created_by,
            &queued_job.permissioned_as_email,
            queued_job.permissioned_as.clone(),
            Some(&format!("add.completed.job{}", queued_job.id)),
            None,
            scheduled_for,
            queued_job.schedule_path(),
            None,
            None,
            None,
            None,
            false,
            false,
            None,
            true,
            Some(queued_job.tag.clone()),
            script_timeout,
            None,
            queued_job.priority,
            None,
            false,
            None,
            None,
            None,
            scope_ceiling.as_deref(),
        )
        .await?;
        tx.commit().await?;
    }
    Ok(())
}

/// Marks the failures a succeeding native retry attempt superseded as resolved, so
/// triage surfaces stop showing red for a chain that ultimately worked. `automatic` is set
/// so the UI can say so without reading `resolved_by`, which is NULL for every manual CE
/// resolution too; `ON CONFLICT DO NOTHING` keeps a human's note intact.
///
/// Membership of the chain must be *proven* per row, never inferred from `parent_job`
/// alone: the success-side caller passes the succeeding job's `parent_job`, which for a
/// job launched with an explicit parent (WAC inline children, SDK-launched children) is
/// the *calling* job, and its other failed children are unrelated. Resolving those would
/// hide exactly the failures this feature exists to surface, so each row must be either
/// - a job carrying a `native_retry_attempt` marker under `root` (provably an attempt), or
/// - `root` itself as the original attempt: unmarked, and proven a chain root by the
///   marked success under it, since only a chain root is ever a marked job's parent
///   (see [`native_retry_root`]).
///
/// Both arms also require the same `runnable_id` as the succeeding attempt, since every
/// attempt in a chain runs the same runnable.
///
/// Nothing is trusted of the caller (this writes through a privileged pool): the gate is
/// "some marked attempt under `root` running `runnable_id` has succeeded", evaluated in
/// SQL, so a call for a chain that has not succeeded resolves nothing.
///
/// Call this from *both* completion paths. A retry is enqueued before its predecessor's
/// failure row is committed, so with a zero delay the retry can succeed while that row is
/// still absent; the success-side call would then find nothing to resolve. Calling again
/// when a failure commits with a retry pending makes the two commit orders converge.
///
/// `ON CONFLICT DO NOTHING` makes a repeat call idempotent while a resolution *exists*, but
/// it is not inert: once a human unresolves a chain member, a later sweep for the same
/// chain resolves it again. Reaching that needs another completion under the same `root`
/// (a WAC inline sibling, or a replayed completion), so it is rare rather than impossible.
/// Making an unresolve durable against it needs a tombstone the read path can see, which
/// this deliberately does not add.
pub async fn resolve_retry_chain_if_succeeded(
    db: &Pool<Postgres>,
    root: Uuid,
    workspace_id: &str,
    runnable_id: Option<ScriptHash>,
) -> Result<(), Error> {
    sqlx::query!(
        "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)
            SELECT c.id, c.workspace_id, NULL, NULL, true
                FROM v2_job_completed c
                JOIN v2_job j ON j.id = c.id
                WHERE c.status = 'failure'
                    AND c.workspace_id = $2
                    AND j.flow_step_id IS NULL
                    AND j.runnable_id IS NOT DISTINCT FROM $3
                    AND EXISTS (
                        SELECT 1 FROM v2_job_completed sc
                        JOIN v2_job sj ON sj.id = sc.id
                        JOIN native_retry_attempt nra ON nra.job_id = sc.id
                        WHERE sc.status = 'success'
                            AND sc.workspace_id = $2
                            AND sj.parent_job = $1
                            AND sj.runnable_id IS NOT DISTINCT FROM $3
                    )
                    AND (
                        (j.parent_job = $1
                            AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))
                        OR (c.id = $1
                            AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))
                    )
            ON CONFLICT (job_id) DO NOTHING",
        root,
        workspace_id,
        runnable_id.map(|h| h.0),
    )
    .execute(db)
    .await?;
    Ok(())
}

/// Evaluate a `retry_if` JS expression. `result`/`previous_result` are the
/// failure output and `flow_input` the job args. Defaults to retrying on eval
/// error (an unevaluable gate shouldn't silently swallow retries).
#[cfg(feature = "quickjs")]
async fn eval_retry_if(
    expr: &str,
    result: Option<&serde_json::value::RawValue>,
    args: &HashMap<String, Box<serde_json::value::RawValue>>,
) -> bool {
    let result_val = result
        .and_then(|r| serde_json::from_str::<serde_json::Value>(r.get()).ok())
        .unwrap_or(serde_json::Value::Null);
    let mut globals = HashMap::new();
    globals.insert("result".to_string(), result_val.clone());
    globals.insert("previous_result".to_string(), result_val);
    globals.insert(
        "flow_input".to_string(),
        serde_json::to_value(args).unwrap_or(serde_json::Value::Null),
    );
    match windmill_jseval::eval_simple_js(format!("Boolean({expr})"), globals).await {
        Ok(v) => v.get() == "true",
        Err(e) => {
            tracing::warn!("Failed to evaluate retry_if expression, retrying anyway: {e:#}");
            true
        }
    }
}

/// `retry_if` is unsupported on a worker built without the `quickjs` feature
/// (the expression cannot be evaluated). Such a worker can't run JS jobs either,
/// so this is not reached in practice; we fail closed and do not retry.
#[cfg(not(feature = "quickjs"))]
async fn eval_retry_if(
    _expr: &str,
    _result: Option<&serde_json::value::RawValue>,
    _args: &HashMap<String, Box<serde_json::value::RawValue>>,
) -> bool {
    tracing::warn!("retry_if is unsupported without the quickjs feature; not retrying");
    false
}

/// The first attempt of `job`'s native retry chain, which every later attempt is pushed
/// with as `parent_job` and whose id seeds theirs. A first attempt may have a parent of its
/// own (a script dispatched from another job), so only a marked attempt's parent is the root:
/// rooting at that outer parent would merge the chains of its retried children.
fn native_retry_root(job: &MiniCompletedJob, is_retry_attempt: bool) -> Uuid {
    if is_retry_attempt {
        job.parent_job.unwrap_or(job.id)
    } else {
        job.id
    }
}

/// Native script retry. When a failed `Script` or `Script_Hub` job carries a retry
/// policy (via `runnable_settings_handle`) and has attempts left, enqueue a fresh
/// attempt of the same script after the policy's backoff delay — instead of having
/// wrapped it in a one-step flow. Each attempt is a job of the same kind; the attempt
/// counter lives in the `native_retry_attempt` marker, written here and read only
/// on the next failure (never on the hot job-pull path).
///
/// Returns `true` if a retry was enqueued.
///
/// Authorization: this performs no auth check by design. It is `pub` only so the
/// integration test can reach it; the sole production caller is the worker
/// job-completion path (`add_completed_job`), which passes a `MiniCompletedJob`
/// built from a real, already-persisted completed job — its workspace/identity
/// fields come from the DB, not from request input. Callers MUST uphold this:
/// never invoke it with caller-supplied or unauthorized job identity.
pub async fn maybe_enqueue_native_script_retry(
    db: &Pool<Postgres>,
    job: &MiniCompletedJob,
    canceled_by: &Option<CanceledBy>,
    // Lazily serialize the failure result: only `retry_if` policies need it, so
    // the common (no-retry_if) failure never pays the serialization cost.
    result_fn: &(dyn Fn() -> Option<Box<serde_json::value::RawValue>> + Sync),
) -> Result<bool, Error> {
    // Only plain top-level scripts retry natively; cancellation always wins.
    if canceled_by.is_some()
        || !matches!(job.kind, JobKind::Script | JobKind::Script_Hub)
        || job.is_flow_step()
    {
        return Ok(false);
    }

    let Some(retry_settings) = windmill_common::runnable_settings::prefetch_retry_from_handle(
        job.runnable_settings_handle,
        db,
    )
    .await?
    else {
        return Ok(false);
    };
    let policy: Retry = retry_settings.into();
    if !policy.has_attempts() {
        return Ok(false);
    }
    // Cancelling a job that has a parent (every retry attempt does) only flags its queue row,
    // and the worker that pulls it completes it without a `canceled_by`: read the flag here.
    let soft_canceled = sqlx::query_scalar!(
        "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1",
        job.id,
    )
    .fetch_optional(db)
    .await?
    .unwrap_or(false);
    if soft_canceled {
        return Ok(false);
    }

    // Attempt counter for this job: its `native_retry_attempt` marker, or 0 for
    // the first (un-marked) attempt. The marker is persistent (unlike the queue
    // row), so it doubles as the explicit "this job is a retry attempt" signal
    // consumers key off of.
    let prev_attempts = sqlx::query_scalar!(
        "SELECT attempt FROM native_retry_attempt WHERE job_id = $1",
        job.id,
    )
    .fetch_optional(db)
    .await?
    .unwrap_or(0) as u32;
    let root = native_retry_root(job, prev_attempts > 0);
    // Scheduled chains keep the schedule trigger so the terminal attempt drives
    // the schedule completion handlers (on_failure/on_success); `parent_job`
    // keeps every retry out of the per-occurrence handler counting queries.
    let trigger = job
        .schedule_path()
        .map(|sp| TriggerMetadata::new(Some(sp), JobTriggerKind::Schedule));

    let Some(delay) = policy.interval(prev_attempts, false) else {
        // Attempts exhausted — let the failure finalize normally.
        return Ok(false);
    };
    // Cap the backoff to match the flow-runtime retry path (evaluate_retry).
    let delay = std::cmp::min(delay, MAX_RETRY_INTERVAL);
    let scheduled_for = chrono::Utc::now()
        + chrono::Duration::from_std(delay).unwrap_or_else(|_| chrono::Duration::zero());

    let args = sqlx::query_scalar!(
        "SELECT args as \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\" FROM v2_job WHERE id = $1 AND workspace_id = $2",
        job.id,
        job.workspace_id,
    )
    .fetch_optional(db)
    .await?
    .flatten()
    .unwrap_or_default();

    // Optional `retry_if`: gate the retry on a JS expression over the failure
    // `result` and `flow_input` (the job args). Evaluated by `eval_retry_if`,
    // which on a worker built without the `quickjs` feature cannot evaluate the
    // expression and fails closed (no retry).
    if let Some(retry_if) = policy.retry_if.as_ref() {
        let result = result_fn();
        if !eval_retry_if(&retry_if.expr, result.as_deref(), &args.0).await {
            return Ok(false);
        }
    }

    // Re-push as a one-step-flow request; `push` materializes it back into a
    // native retryable `Script` carrying the policy again. `parent_job = root`
    // links the chain (and excludes retries from schedule-handler counting); the
    // schedule trigger, when present, lets the terminal attempt fire handlers.
    //
    // Idempotent retry id: deterministic per (root, next attempt). If a worker
    // dies between this push and the current attempt's finalization, the reaper
    // re-handles the un-finalized attempt and we land here again with the same
    // id, so the retry is enqueued exactly once (no double-retry).
    let retry_job_id = {
        use std::hash::{Hash, Hasher};
        let next_attempt = prev_attempts + 1;
        let mut high = std::hash::DefaultHasher::new();
        root.hash(&mut high);
        next_attempt.hash(&mut high);
        let mut low = std::hash::DefaultHasher::new();
        "native-retry".hash(&mut low);
        next_attempt.hash(&mut low);
        root.hash(&mut low);
        Uuid::from_u64_pair(high.finish(), low.finish())
    };
    // If that retry already exists (the crash-and-reaper-replay case above),
    // report it as pending WITHOUT re-pushing. Deriving `retry_pending` from the
    // push *result* would otherwise flip to false on the duplicate-id error and
    // let the schedule completion handlers fire for this non-terminal attempt.
    if sqlx::query_scalar!("SELECT 1 FROM v2_job WHERE id = $1", retry_job_id)
        .fetch_optional(db)
        .await?
        .is_some()
    {
        return Ok(true);
    }
    // Carry forward the failed attempt's concurrency/debouncing settings (same
    // runnable_settings_handle as the retry policy) so a retry of a concurrency-
    // limited script still inserts its concurrency_key and respects the limit,
    // rather than running unbounded with only the retry policy.
    let (debouncing_settings, concurrency_settings) =
        windmill_common::runnable_settings::prefetch_cached_from_handle(
            job.runnable_settings_handle,
            db,
        )
        .await?;
    let tx = PushIsolationLevel::IsolatedRoot(db.clone());
    // The retry never holds a wider token than the attempt it replaces.
    let scope_ceiling = job.job_token_scopes.clone();
    let (new_id, mut tx) = match push(
        db,
        tx,
        &job.workspace_id,
        JobPayload::SingleStepFlow {
            path: job.runnable_path.clone().unwrap_or_default(),
            hash: job.runnable_id,
            flow_version: None,
            language: job.script_lang.clone(),
            args: HashMap::new(),
            retry: Some(policy),
            error_handler_path: None,
            error_handler_args: None,
            skip_handler: None,
            cache_ttl: job.cache_ttl,
            cache_ignore_s3_path: job.cache_ignore_s3_path,
            priority: job.priority,
            tag_override: Some(job.tag.clone()),
            trigger_path: None,
            apply_preprocessor: false,
            concurrency_settings,
            debouncing_settings,
        },
        PushArgs::from(&args.0),
        &job.created_by,
        &job.permissioned_as_email,
        job.permissioned_as.clone(),
        Some(&format!("retry.{}", job.id)),
        None,
        Some(scheduled_for),
        None,
        Some(root),
        None,
        None,
        Some(retry_job_id),
        false,
        false,
        None,
        true,
        Some(job.tag.clone()),
        None,
        None,
        job.priority,
        None,
        false,
        None,
        trigger,
        None,
        scope_ceiling.as_deref(),
    )
    .await
    {
        Ok(v) => v,
        Err(e) => {
            // Race with a concurrent completion of the same attempt: it may have
            // inserted the deterministic retry id between our pre-check and this
            // push, so the push fails on the duplicate id. The retry IS pending —
            // re-check and report it as such instead of propagating the error
            // (which would flip `retry_pending` to false and fire the schedule
            // completion handlers for this non-terminal attempt).
            if sqlx::query_scalar!("SELECT 1 FROM v2_job WHERE id = $1", retry_job_id)
                .fetch_optional(db)
                .await?
                .is_some()
            {
                return Ok(true);
            }
            return Err(e);
        }
    };

    sqlx::query!(
        "INSERT INTO native_retry_attempt (job_id, attempt) VALUES ($1, $2)",
        new_id,
        (prev_attempts + 1) as i32,
    )
    .execute(&mut *tx)
    .await?;

    tx.commit().await?;
    tracing::info!(
        "Native retry: enqueued attempt {} of script {:?} (root {root}) in {}s as {new_id}",
        prev_attempts + 1,
        job.runnable_path,
        delay.as_secs(),
    );
    Ok(true)
}

#[cfg(feature = "cloud")]
fn apply_completed_job_cloud_usage(
    db: &Pool<Postgres>,
    queued_job: &MiniCompletedJob,
    _duration: i64,
) {
    if !queued_job.is_flow() {
        meter_execution_seconds(
            db,
            &queued_job.workspace_id,
            &queued_job.permissioned_as_email,
            _duration,
        );
    }
}

/// Charge `_duration` of execution time to the cloud usage meters: the workspace's
/// monthly row, plus the per-user row on non-premium plans.
///
/// The unit is one finished **segment**, not one job. A Workflow-as-Code parent parks on
/// a sleep, an approval or its children and resumes with a fresh timer, so its compute
/// arrives here as several calls; metering only the one at completion would drop
/// everything it ran before its first park.
///
/// Fire-and-forget, like every other write to `usage`: billing must never hold up the
/// job that produced it.
///
/// `w_id` and `email` are billed as given and authorize nothing on their own — take them
/// from a job the caller already holds, never from request input.
#[cfg(feature = "cloud")]
pub fn meter_execution_seconds(db: &Pool<Postgres>, w_id: &str, email: &str, _duration: i64) {
    if *CLOUD_HOSTED && _duration > 1000 {
        let db = db.clone();
        let w_id = w_id.to_string();
        let email = email.to_string();
        let w_id2 = w_id.clone();
        let email2 = email.clone();
        tokio::task::spawn(async move {
            let additional_usage = _duration / 1000;
            let result = tokio::time::timeout(std::time::Duration::from_secs(10), async move {
                // Fork/dev execution-seconds meter against the root (billing) workspace; resolves to
                // `w_id` itself off-fork.
                let billing_w_id =
                    windmill_common::workspaces::get_billing_workspace_id(&db, &w_id)
                        .await
                        .unwrap_or_else(|e| {
                            tracing::error!("Failed to resolve billing workspace for {w_id}: {e:#}");
                            w_id.clone()
                        });
                // Update workspace usage
                let workspace_result = sqlx::query!(
                    "INSERT INTO usage (id, is_workspace, month_, usage)
                    VALUES ($1, TRUE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2)
                    ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage",
                    &billing_w_id,
                    additional_usage as i32
                )
                .execute(&db)
                .await;

                if let Err(e) = workspace_result {
                    tracing::error!("Failed to update workspace usage for {}: {:#}", billing_w_id, e);
                }

                match windmill_common::workspaces::get_team_plan_status(&db, &billing_w_id).await {
                    Ok(team_plan_status) => {
                        // Update user usage for non-premium workspaces
                        if !team_plan_status.premium {
                            let user_result = sqlx::query!(
                                "INSERT INTO usage (id, is_workspace, month_, usage)
                                VALUES ($1, FALSE, EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date), $2)
                                ON CONFLICT (id, is_workspace, month_) DO UPDATE SET usage = usage.usage + EXCLUDED.usage",
                                &email,
                                additional_usage as i32
                            )
                            .execute(&db)
                            .await;

                            if let Err(e) = user_result {
                                tracing::error!("Failed to update user usage for {}: {:#}", email, e);
                            }
                        }
                    },
                    Err(err) => {
                        tracing::error!("Failed to get team plan status to update usage for workspace {w_id}: {err:#}");
                    }
                };
            }).await;

            if let Err(_) = result {
                tracing::error!(
                    "Could not update usage for workspace {} and permissioned as {}, stopped after 10s",
                    w_id2,
                    email2
                );
            }
        });
    }
}

pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>(
    queued_job: &MiniCompletedJob,
    db: &Pool<Postgres>,
    result: Json<&T>,
) -> Result<(), Error> {
    if let Some(ref global_error_handler) = *GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE {
        let prefixed_global_error_handler_path = if global_error_handler.starts_with("script/")
            || global_error_handler.starts_with("flow/")
        {
            global_error_handler.clone()
        } else {
            format!("script/{}", global_error_handler)
        };
        push_error_handler(
            db,
            queued_job.id,
            queued_job.schedule_path(),
            queued_job.runnable_path.clone(),
            queued_job.is_flow(),
            &queued_job.workspace_id,
            &prefixed_global_error_handler_path,
            result,
            None,
            queued_job.started_at,
            None,
            &queued_job.permissioned_as_email,
            None,
            true,
            None,
        )
        .await?;
    }

    Ok(())
}

pub async fn report_error_to_workspace_handler_or_critical_side_channel(
    queued_job: &MiniCompletedJob,
    db: &Pool<Postgres>,
    error_message: String,
) -> () {
    let w_id = &queued_job.workspace_id;
    let row_result = sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>)>(
        r#"
                SELECT
                    error_handler->>'path',
                    (error_handler->'extra_args')::text::json
                FROM
                    workspace_settings
                WHERE
                    workspace_id = $1
                "#,
    )
    .bind(&w_id)
    .fetch_optional(db)
    .await
    .ok()
    .flatten()
    .unwrap_or((None, None));

    let (error_handler, error_handler_extra_args) = row_result;

    if let Some(error_handler) = error_handler {
        if let Err(err) = push_error_handler(
            db,
            queued_job.id,
            queued_job.schedule_path(),
            queued_job.runnable_path.clone(),
            queued_job.is_flow(),
            w_id,
            &error_handler,
            Json(&json!({
                "error": {
                    "message": error_message
                }
            })),
            None,
            queued_job.started_at,
            error_handler_extra_args,
            &queued_job.permissioned_as_email,
            None,
            false,
            None,
        )
        .await
        {
            tracing::error!(
                "Could not push workspace error handler trying to handle critical error ({error_message}) of failed job ({}): {}",
                queued_job.id,
                err
            );
            report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
        }
    } else {
        report_critical_error(error_message, db.clone(), Some(&w_id), None).await;
    }
}

async fn fetch_error_handler_from_db(
    db: &Pool<Postgres>,
    w_id: &str,
) -> Result<
    (
        Option<String>,
        Option<Json<Box<RawValue>>>,
        bool,
        bool,
        bool,
    ),
    Error,
> {
    sqlx::query_as::<
        _,
        (
            Option<String>,
            Option<Json<Box<RawValue>>>,
            Option<bool>,
            Option<bool>,
            bool,
        ),
    >(
        r#"
        SELECT
            error_handler->>'path',
            (error_handler->'extra_args')::text::json,
            (error_handler->>'muted_on_cancel')::boolean,
            (error_handler->>'muted_on_user_path')::boolean,
            ws.error_handler_fallback_to_instance_alerts AND w.parent_workspace_id IS NULL
        FROM workspace_settings ws
        JOIN workspace w ON w.id = ws.workspace_id
        WHERE ws.workspace_id = $1
        "#,
    )
    .bind(w_id)
    .fetch_optional(db)
    .await
    .context("fetching error handler info from workspace_settings")?
    .map(
        |(path, extra_args, muted_on_cancel, muted_on_user_path, report_to_instance_alerts)| {
            (
                path,
                extra_args,
                muted_on_cancel.unwrap_or(false),
                muted_on_user_path.unwrap_or(false),
                report_to_instance_alerts,
            )
        },
    )
    .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))
}

pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>(
    queued_job: &MiniCompletedJob,
    is_canceled: bool,
    db: &Pool<Postgres>,
    result: Json<&'a T>,
) -> Result<(), Error> {
    let w_id = &queued_job.workspace_id;

    let now = chrono::Utc::now().timestamp();
    let (
        error_handler,
        error_handler_extra_args,
        error_handler_muted_on_cancel,
        error_handler_muted_on_user_path,
        report_to_instance_alerts,
    ) = if let Some(cached) = WORKSPACE_ERROR_HANDLER_CACHE.get(w_id) {
        if cached.5 > now {
            (
                cached.0.clone(),
                cached.1.clone(),
                cached.2,
                cached.3,
                cached.4,
            )
        } else {
            let row = fetch_error_handler_from_db(db, w_id).await?;
            let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
            WORKSPACE_ERROR_HANDLER_CACHE.insert(
                w_id.clone(),
                (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry),
            );
            row
        }
    } else {
        let row = fetch_error_handler_from_db(db, w_id).await?;
        let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
        WORKSPACE_ERROR_HANDLER_CACHE.insert(
            w_id.clone(),
            (row.0.clone(), row.1.clone(), row.2, row.3, row.4, expiry),
        );
        row
    };

    // Nothing to do for the vast majority of workspaces, and returning here keeps the
    // per-runnable mute lookup below off the path of every failed job.
    if error_handler.is_none() && !report_to_instance_alerts {
        return Ok(());
    }

    if is_canceled && error_handler_muted_on_cancel {
        return Ok(());
    }

    // Skip error handler for scripts/flows starting with u/
    if error_handler_muted_on_user_path {
        if let Some(ref path) = queued_job.runnable_path {
            if path.starts_with("u/") {
                return Ok(());
            }
        }
    }

    let ws_error_handler_muted: Option<bool> = match queued_job.kind {
        JobKind::Script => {
            sqlx::query_scalar!(
                "SELECT ws_error_handler_muted FROM script WHERE workspace_id = $1 AND hash = $2",
                queued_job.workspace_id,
                queued_job.runnable_id.map(|x| x.0),
            )
            .fetch_optional(db)
            .await?
        }
        JobKind::Flow => {
            sqlx::query_scalar!(
                "SELECT ws_error_handler_muted FROM flow WHERE workspace_id = $1 AND path = $2",
                queued_job.workspace_id,
                queued_job.runnable_path.clone(),
            )
            .fetch_optional(db)
            .await?
        }
        _ => None,
    };

    if ws_error_handler_muted.unwrap_or(false) {
        return Ok(());
    }

    if let Some(error_handler) = error_handler {
        tracing::info!("workspace error handled for job {}", &queued_job.id);

        push_error_handler(
            db,
            queued_job.id,
            queued_job.schedule_path(),
            queued_job.runnable_path.clone(),
            queued_job.is_flow(),
            &queued_job.workspace_id,
            &error_handler,
            result,
            None,
            queued_job.started_at,
            error_handler_extra_args,
            &queued_job.permissioned_as_email,
            None,
            false,
            None,
        )
        .await?;
    } else if !is_canceled {
        // A cancellation is a human action rather than an operational failure, and unlike the
        // handler path this one has no per-workspace toggle to opt out of reporting them.
        let suppressed = match INSTANCE_ALERT_THROTTLE.get(w_id) {
            Some((last_sent, suppressed)) if now - last_sent < INSTANCE_ALERT_COOLDOWN_SECONDS => {
                INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (last_sent, suppressed + 1));
                None
            }
            entry => {
                INSTANCE_ALERT_THROTTLE.insert(w_id.clone(), (now, 0));
                Some(entry.map(|(_, suppressed)| suppressed).unwrap_or(0))
            }
        };
        if let Some(suppressed) = suppressed {
            tracing::info!(
                "reporting failed job {} to the instance critical alert channels",
                &queued_job.id
            );
            let base_url = windmill_common::BASE_URL.load();
            let rollup = if suppressed > 0 {
                format!(
                    " (and {suppressed} more failure(s) in the preceding {INSTANCE_ALERT_COOLDOWN_SECONDS}s)"
                )
            } else {
                String::new()
            };
            windmill_common::utils::send_workspace_error_to_instance_channels(
                format!(
                    "Job {} failed in workspace {w_id} ({base_url}/run/{}?workspace={w_id}){rollup}",
                    queued_job.runnable_path.as_deref().unwrap_or("preview"),
                    queued_job.id
                ),
                db,
            )
            .await;
        }
    }
    Ok(())
}

async fn fetch_success_handler_from_db(
    db: &Pool<Postgres>,
    w_id: &str,
) -> Result<(Option<String>, Option<Json<Box<RawValue>>>), Error> {
    sqlx::query_as::<_, (Option<String>, Option<Json<Box<RawValue>>>)>(
        r#"
        SELECT
            success_handler->>'path',
            (success_handler->'extra_args')::text::json
        FROM workspace_settings
        WHERE workspace_id = $1
        "#,
    )
    .bind(w_id)
    .fetch_optional(db)
    .await
    .context("fetching success handler info from workspace_settings")?
    .ok_or_else(|| Error::internal_err(format!("no workspace settings for id {w_id}")))
}

pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>(
    queued_job: &MiniCompletedJob,
    db: &Pool<Postgres>,
    result: Json<&'a T>,
) -> Result<(), Error> {
    let w_id = &queued_job.workspace_id;

    let now = chrono::Utc::now().timestamp();
    let (success_handler, success_handler_extra_args) =
        if let Some(cached) = WORKSPACE_SUCCESS_HANDLER_CACHE.get(w_id) {
            if cached.2 > now {
                (cached.0.clone(), cached.1.clone())
            } else {
                let row = fetch_success_handler_from_db(db, w_id).await?;
                let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
                WORKSPACE_SUCCESS_HANDLER_CACHE
                    .insert(w_id.clone(), (row.0.clone(), row.1.clone(), expiry));
                row
            }
        } else {
            let row = fetch_success_handler_from_db(db, w_id).await?;
            let expiry = now + WORKSPACE_HANDLER_CACHE_TTL_SECONDS;
            WORKSPACE_SUCCESS_HANDLER_CACHE
                .insert(w_id.clone(), (row.0.clone(), row.1.clone(), expiry));
            row
        };

    if let Some(success_handler) = success_handler {
        tracing::info!("workspace success handler for job {}", &queued_job.id);

        push_success_handler(
            db,
            queued_job.id,
            queued_job.schedule_path(),
            queued_job.runnable_path.clone(),
            queued_job.is_flow(),
            &queued_job.workspace_id,
            &success_handler,
            result,
            queued_job.started_at,
            success_handler_extra_args,
            &queued_job.permissioned_as_email,
            None,
        )
        .await?;
    }
    Ok(())
}

/// The event for a schedule the server disabled on its own.
pub fn schedule_auto_disable_event<'a>(
    workspace_id: &'a str,
    path: &'a str,
    error: &str,
) -> windmill_common::trigger_history::TriggerHistoryEvent<'a> {
    windmill_common::trigger_history::TriggerHistoryEvent::server_disable(
        workspace_id,
        windmill_common::trigger_history::SCHEDULE_TRIGGER_KIND,
        path,
        serde_json::json!({ "enabled": { "old": true, "new": false } }),
        error,
    )
}

/// Disable a schedule the server can no longer arm, and record that it did.
///
/// Contract on `record_in_disable_tx`. Here `tx` is the job-completion
/// transaction, so the savepoint also keeps a failed insert from poisoning it.
///
/// Returns `Err` only when the disable itself failed; a lost history row comes
/// back through `history_lost` for the caller to report.
async fn disable_schedule_and_record(
    tx: &mut Transaction<'_, Postgres>,
    schedule: &Schedule,
    err: &Error,
    history_lost: &mut Option<String>,
) -> Result<u64, Error> {
    let disable_result = sqlx::query!(
        "UPDATE schedule SET enabled = false, error = $1 WHERE workspace_id = $2 AND path = $3 AND enabled = true",
        err.to_string(),
        &schedule.workspace_id,
        &schedule.path
    )
    .execute(&mut **tx)
    .await;

    #[cfg(feature = "failpoints")]
    let disable_result = if schedule_failpoints::is_active(
        schedule_failpoints::ScheduleFailPoint::ScheduleDisable,
    ) {
        Err(sqlx::Error::Protocol(
            "failpoint: schedule disable".to_string(),
        ))
    } else {
        disable_result
    };

    let rows = disable_result?.rows_affected();
    // Zero rows means a user disabled the schedule first: no transition of ours
    // to record.
    if rows == 0 {
        return Ok(0);
    }

    let event =
        schedule_auto_disable_event(&schedule.workspace_id, &schedule.path, &err.to_string());
    *history_lost = windmill_common::trigger_history::record_in_disable_tx(tx, event).await;

    Ok(rows)
}

pub async fn try_schedule_next_job<'c>(
    db: &Pool<Postgres>,
    mut tx: Transaction<'c, Postgres>,
    job: &MiniCompletedJob,
    schedule: &Schedule,
    script_path: &str,
) -> (Transaction<'c, Postgres>, Option<Error>) {
    if !schedule.enabled {
        tracing::info!(
            "Schedule {} in {} is disabled. Not scheduling again.",
            schedule.path,
            &job.workspace_id
        );
        return (tx, None);
    }

    if script_path != schedule.script_path {
        tracing::warn!(
            "Schedule {} in {} has a different script path than the job. Not scheduling again",
            schedule.path,
            &job.workspace_id
        );
        return (tx, None);
    }

    tracing::info!(
        "Schedule {} scheduling next job for {} in {}",
        schedule.path,
        schedule.script_path,
        &job.workspace_id
    );

    let permissioned_as = schedule.permissioned_as.clone();
    let email = match windmill_common::users::get_email_from_permissioned_as(
        &permissioned_as,
        &job.workspace_id,
        &mut *tx,
    )
    .await
    {
        Ok(email) => email,
        Err(e) => return (tx, Some(e)),
    };
    let schedule_authed = windmill_common::auth::fetch_authed_from_permissioned_as(
        &permissioned_as,
        &email,
        &job.workspace_id,
        &mut *tx,
    )
    .await
    .ok();

    let mut push_err = None;

    #[cfg(feature = "failpoints")]
    if schedule_failpoints::is_active(schedule_failpoints::ScheduleFailPoint::SavepointCreate) {
        push_err = Some(Error::internal_err(
            "failpoint: savepoint create".to_string(),
        ));
    }

    if push_err.is_none() {
        let savepoint_result = tx.begin().await;
        match savepoint_result {
            Ok(savepoint) => {
                let push_result = match tokio::time::timeout(
                    std::time::Duration::from_secs(5),
                    push_scheduled_job(
                        db,
                        savepoint,
                        schedule,
                        schedule_authed.as_ref(),
                        Some(job.scheduled_for),
                    ),
                )
                .await
                {
                    Ok(result) => result,
                    Err(_elapsed) => Err(Error::internal_err(
                        "push_scheduled_job timed out after 5s".to_string(),
                    )),
                };
                #[cfg(feature = "failpoints")]
                let push_result =
                    if schedule_failpoints::is_active(schedule_failpoints::ScheduleFailPoint::Push)
                    {
                        if let Ok(sp) = push_result {
                            sp.rollback().await.ok();
                        }
                        Err(Error::internal_err("failpoint: push".to_string()))
                    } else if schedule_failpoints::is_active(
                        schedule_failpoints::ScheduleFailPoint::PushQuotaExceeded,
                    ) {
                        if let Ok(sp) = push_result {
                            sp.rollback().await.ok();
                        }
                        Err(Error::QuotaExceeded(
                            "failpoint: push quota exceeded".to_string(),
                        ))
                    } else {
                        push_result
                    };
                match push_result {
                    Ok(savepoint) => {
                        #[cfg(feature = "failpoints")]
                        let savepoint_commit_fail = schedule_failpoints::is_active(
                            schedule_failpoints::ScheduleFailPoint::SavepointCommit,
                        );
                        #[cfg(not(feature = "failpoints"))]
                        let savepoint_commit_fail = false;

                        if savepoint_commit_fail {
                            savepoint.rollback().await.ok();
                            push_err = Some(Error::internal_err(
                                "failpoint: savepoint commit".to_string(),
                            ));
                        } else {
                            match savepoint.commit().await {
                                Ok(()) => {}
                                Err(e) => {
                                    push_err = Some(Error::internal_err(format!(
                                        "Could not commit savepoint: {e:#}"
                                    )));
                                }
                            }
                        }
                    }
                    Err(err) if matches!(err, Error::QuotaExceeded(_)) => {
                        push_err = Some(err);
                    }
                    Err(err) => {
                        tracing::warn!(
                            "Could not push next scheduled job for {}: {err}",
                            schedule.path,
                        );
                        push_err = Some(err);
                    }
                }
            }
            Err(e) => {
                tracing::error!("Could not create savepoint for schedule push: {e:#}",);
                push_err = Some(Error::internal_err(format!(
                    "Could not create savepoint: {e:#}"
                )));
            }
        }
    }

    if let Some(ref err) = push_err {
        if matches!(err, Error::QuotaExceeded(_) | Error::NotFound(_)) {
            tracing::error!(
                "Could not push next scheduled job for {}: {err}. Disabling schedule.",
                schedule.path
            );
            let mut history_lost = None;
            match disable_schedule_and_record(&mut tx, schedule, err, &mut history_lost).await {
                Err(disable_err) => {
                    report_error_to_workspace_handler_or_critical_side_channel(
                        job,
                        db,
                        format!(
                            "Could not push next scheduled job for {} and could not disable schedule: {disable_err}",
                            schedule.path,
                        ),
                    )
                    .await;
                }
                Ok(_) => push_err = None,
            }
            if let Some(history_err) = history_lost {
                report_error_to_workspace_handler_or_critical_side_channel(
                    job,
                    db,
                    format!(
                        "Disabled schedule {} but could not record it in the trigger history: {history_err}",
                        schedule.path,
                    ),
                )
                .await;
            }
        }
    }

    (tx, push_err)
}

const ERROR_HANDLER_PATH_EMAIL: &str = "/workspace-or-error-handler-email";

/// Every handler version the frontend has offered as a preset (`frontend/src/lib/hubPaths.json`
/// and its history, which stored schedules still point at). Matched whole: a hub script is
/// resolved by its numeric id alone, so the name after it proves nothing about its code.
const PRESET_HANDLER_PATHS: [&str; 23] = [
    "hub/6512/workspace-or-schedule-error-handler-slack",
    "hub/5792/workspace-or-schedule-error-handler-slack",
    "hub/9079/workspace-or-schedule-error-handler-slack",
    "hub/9206/workspace-or-schedule-error-handler-slack",
    "hub/19741/workspace-or-schedule-error-handler-slack",
    "hub/28241/workspace-or-schedule-error-handler-slack",
    "hub/28794/workspace-or-schedule-error-handler-slack",
    "hub/2431/slack/schedule-error-handler-slack",
    "hub/2430/slack/schedule-recovery-handler-slack",
    "hub/9067/slack/schedule-recovery-handler-slack",
    "hub/9080/slack/schedule-recovery-handler-slack",
    "hub/28239/slack/schedule-recovery-handler-slack",
    "hub/28791/slack/schedule-recovery-handler-slack",
    "hub/9069/slack/schedule-success-handler-slack",
    "hub/9072/slack/schedule-success-handler-slack",
    "hub/28220/slack/schedule-success-handler-slack",
    "hub/28240/slack/schedule-success-handler-slack",
    "hub/28793/slack/schedule-success-handler-slack",
    "hub/11598/workspace-or-schedule-error-handler-teams",
    "hub/19742/workspace-or-schedule-error-handler-teams",
    "hub/11593/schedule-recovery-handler-teams",
    "hub/11596/schedule-success-handler-teams",
    "hub/19795/workspace-or-error-handler-email",
];

/// The preset handler Windmill ships on the hub that `handler_path` names, `None` for user code.
fn preset_handler(handler_path: &str) -> Option<&'static str> {
    let path = handler_path.strip_prefix("script/").unwrap_or(handler_path);
    PRESET_HANDLER_PATHS.into_iter().find(|p| *p == path)
}

/// Whether `handler_path` is a preset hub handler, which runs as the shared handler identity
/// (Slack bot token, instance SMTP) instead of as the schedule or user that set it.
pub fn is_preset_handler_path(handler_path: &str) -> bool {
    preset_handler(handler_path).is_some()
}

/// `(email, permissioned_as)` a schedule handler runs as. The handler identity reads the
/// workspace Slack bot token and may send through the instance SMTP, and a schedule's handlers
/// are set by anyone who can write the schedule: only the preset hub handlers get it, custom
/// code runs as the schedule itself.
pub async fn schedule_handler_identity(
    db: &Pool<Postgres>,
    w_id: &str,
    handler_path: &str,
    schedule_permissioned_as: &str,
    preset_email: &str,
) -> error::Result<(String, String)> {
    if is_preset_handler_path(handler_path) {
        return Ok((
            preset_email.to_string(),
            ERROR_HANDLER_USER_GROUP.to_string(),
        ));
    }
    if handler_path.contains("hub/") && handler_path.contains("-handler-") {
        tracing::warn!(
            "schedule handler {handler_path} in {w_id} is named like a preset but is not a pinned \
             preset version: it runs as the schedule ({schedule_permissioned_as})"
        );
    }
    let email =
        windmill_common::users::get_email_from_permissioned_as(schedule_permissioned_as, w_id, db)
            .await?;
    Ok((email, schedule_permissioned_as.to_string()))
}

async fn error_handler_identity(
    db: &Pool<Postgres>,
    w_id: &str,
    error_handler_path: &str,
    is_global_error_handler: bool,
    schedule_permissioned_as: Option<&str>,
) -> error::Result<(String, String)> {
    if is_global_error_handler {
        return Ok((
            SUPERADMIN_SECRET_EMAIL.to_string(),
            SUPERADMIN_SECRET_EMAIL.to_string(),
        ));
    }
    if let Some(schedule_permissioned_as) = schedule_permissioned_as {
        return schedule_handler_identity(
            db,
            w_id,
            error_handler_path,
            schedule_permissioned_as,
            SCHEDULE_ERROR_HANDLER_USER_EMAIL,
        )
        .await;
    }
    let email = if preset_handler(error_handler_path)
        .is_some_and(|p| p.ends_with(ERROR_HANDLER_PATH_EMAIL))
    {
        EMAIL_ERROR_HANDLER_USER_EMAIL
    } else {
        ERROR_HANDLER_USER_EMAIL
    };
    Ok((email.to_string(), ERROR_HANDLER_USER_GROUP.to_string()))
}

pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>(
    db: &Pool<Postgres>,
    job_id: Uuid,
    schedule_path: Option<String>,
    script_path: Option<String>,
    is_flow: bool,
    w_id: &str,
    on_failure_path: &str,
    result: Json<&'a T>,
    failed_times: Option<i32>,
    started_at: Option<DateTime<Utc>>,
    extra_args: Option<Json<Box<RawValue>>>,
    email: &str,
    // The schedule's `permissioned_as` when this is a schedule's own error handler.
    schedule_permissioned_as: Option<&str>,
    is_global_error_handler: bool,
    priority: Option<i16>,
) -> windmill_common::error::Result<Uuid> {
    let handler_w_id = if is_global_error_handler {
        "admins"
    } else {
        w_id
    };
    let (payload, tag, on_behalf_of) =
        get_payload_tag_from_prefixed_path(on_failure_path, db, handler_w_id).await?;

    let mut extra = HashMap::new();
    if let Some(schedule_path) = schedule_path {
        extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path));
    }
    extra.insert("workspace_id".to_string(), to_raw_value(&w_id));
    extra.insert("job_id".to_string(), to_raw_value(&job_id));
    extra.insert("path".to_string(), to_raw_value(&script_path));
    extra.insert("is_flow".to_string(), to_raw_value(&is_flow));
    extra.insert("started_at".to_string(), to_raw_value(&started_at));
    extra.insert("email".to_string(), to_raw_value(&email));

    if let Some(failed_times) = failed_times {
        extra.insert("failed_times".to_string(), to_raw_value(&failed_times));
    }

    if let Some(args_v) = extra_args {
        if let Ok(args_m) = serde_json::from_str::<HashMap<String, Box<RawValue>>>(args_v.get()) {
            extra.extend(args_m);
        } else {
            return Err(error::Error::ExecutionErr(
                "args of scripts needs to be dict".to_string(),
            ));
        }
    }

    let result = sanitize_result(result);

    let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of {
        (on_behalf_of.email, on_behalf_of.permissioned_as)
    } else {
        error_handler_identity(
            db,
            w_id,
            on_failure_path,
            is_global_error_handler,
            schedule_permissioned_as,
        )
        .await?
    };

    let tx = PushIsolationLevel::IsolatedRoot(db.clone());
    let (uuid, tx) = push(
        &db,
        tx,
        handler_w_id,
        payload,
        PushArgs { extra: Some(extra), args: &result },
        if is_global_error_handler {
            GLOBAL_ERROR_HANDLER_USERNAME
        } else if schedule_permissioned_as.is_some() {
            SCHEDULE_ERROR_HANDLER_USERNAME
        } else {
            ERROR_HANDLER_USERNAME
        },
        &email,
        permissioned_as,
        Some(&format!("error.handler.{job_id}")),
        None,
        None,
        None,
        Some(job_id),
        None,
        Some(job_id),
        None,
        false,
        false,
        None,
        true,
        tag,
        None,
        None,
        priority,
        None,
        false,
        None,
        None,
        None,
        None,
    )
    .await?;
    tx.commit().await?;
    return Ok(uuid);
}

pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>(
    db: &Pool<Postgres>,
    job_id: Uuid,
    schedule_path: Option<String>,
    script_path: Option<String>,
    is_flow: bool,
    w_id: &str,
    on_success_path: &str,
    result: Json<&'a T>,
    started_at: Option<DateTime<Utc>>,
    extra_args: Option<Json<Box<RawValue>>>,
    email: &str,
    priority: Option<i16>,
) -> windmill_common::error::Result<Uuid> {
    let (payload, tag, on_behalf_of) =
        get_payload_tag_from_prefixed_path(on_success_path, db, w_id).await?;

    let mut extra = HashMap::new();
    if let Some(schedule_path) = schedule_path {
        extra.insert("schedule_path".to_string(), to_raw_value(&schedule_path));
    }
    extra.insert("workspace_id".to_string(), to_raw_value(&w_id));
    extra.insert("job_id".to_string(), to_raw_value(&job_id));
    extra.insert("path".to_string(), to_raw_value(&script_path));
    extra.insert("is_flow".to_string(), to_raw_value(&is_flow));
    extra.insert("started_at".to_string(), to_raw_value(&started_at));
    extra.insert("email".to_string(), to_raw_value(&email));

    if let Some(args_v) = extra_args {
        if let Ok(args_m) = serde_json::from_str::<HashMap<String, Box<RawValue>>>(args_v.get()) {
            extra.extend(args_m);
        } else {
            return Err(error::Error::ExecutionErr(
                "args of scripts needs to be dict".to_string(),
            ));
        }
    }

    let result = sanitize_result(result);

    let (email, permissioned_as) = if let Some(on_behalf_of) = on_behalf_of.as_ref() {
        (
            on_behalf_of.email.as_str(),
            on_behalf_of.permissioned_as.clone(),
        )
    } else {
        (
            SUCCESS_HANDLER_USER_EMAIL,
            SUCCESS_HANDLER_USER_GROUP.to_string(),
        )
    };

    let tx = PushIsolationLevel::IsolatedRoot(db.clone());
    let (uuid, tx) = push(
        &db,
        tx,
        w_id,
        payload,
        PushArgs { extra: Some(extra), args: &result },
        SUCCESS_HANDLER_USERNAME,
        email,
        permissioned_as,
        Some(&format!("success.handler.{job_id}")),
        None,
        None,
        None,
        Some(job_id), // parent_job
        Some(job_id), // root_job
        Some(job_id), // flow_innermost_root_job
        None,
        false,
        false,
        None,
        true,
        tag,
        None,
        None,
        priority,
        None,
        false,
        None,
        None,
        None,
        None,
    )
    .await?;
    tx.commit().await?;
    return Ok(uuid);
}

fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<String, Box<RawValue>> {
    let as_str = serde_json::to_string(result.0).unwrap_or_else(|_| "{}".to_string());
    serde_json::from_str::<HashMap<String, Box<RawValue>>>(&as_str)
        .unwrap_or_else(|_| [("error".to_string(), RawValue::from_string(as_str).unwrap())].into())
}

// #[derive(Serialize)]
// pub struct RecoveryValue<T> {
//     error_started_at: chrono::DateTime<Utc>,
//     schedule_path: String,
//     path: String,
//     is_flow: boolean,
//     extra_args: serde_json::Value
// }

#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)]
pub struct MiniPulledJob {
    pub workspace_id: String,
    pub id: Uuid,
    pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
    pub parent_job: Option<Uuid>,
    pub created_by: String,
    pub scheduled_for: chrono::DateTime<chrono::Utc>,
    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
    pub runnable_path: Option<String>,
    pub kind: JobKind,
    pub runnable_id: Option<ScriptHash>,
    pub canceled_reason: Option<String>,
    pub canceled_by: Option<String>,
    pub permissioned_as: String,
    pub permissioned_as_email: String,
    pub flow_status: Option<Json<Box<RawValue>>>,
    pub tag: String,
    pub script_lang: Option<ScriptLang>,
    pub same_worker: bool,
    pub pre_run_error: Option<String>,
    pub concurrent_limit: Option<i32>,
    pub concurrency_time_window_s: Option<i32>,
    pub flow_innermost_root_job: Option<Uuid>,
    pub root_job: Option<Uuid>,
    pub timeout: Option<i32>,
    pub flow_step_id: Option<String>,
    pub cache_ttl: Option<i32>,
    pub cache_ignore_s3_path: Option<bool>,
    pub priority: Option<i16>,
    pub preprocessed: Option<bool>,
    pub script_entrypoint_override: Option<String>,
    pub trigger: Option<String>,
    pub trigger_kind: Option<TriggerKindLabel>,
    pub visible_to_owner: bool,
    pub permissioned_as_end_user_email: Option<String>,
    pub runnable_settings_handle: Option<i64>,
    /// The job's effective token scopes (`job_perms.job_token_scopes`), minted into its token.
    /// No `sqlx(default)`: a query that forgets the column must fail rather than mint an
    /// unrestricted token.
    #[serde(default)]
    pub job_token_scopes: Option<Vec<String>>,
}

impl MiniPulledJob {
    pub fn new_inline(
        workspace_id: String,
        args: Option<HashMap<String, Box<RawValue>>>,
        created_by: String,
        permissioned_as: String,
        permissioned_as_email: String,
        runnable_path: Option<String>,
        kind: JobKind,
        runnable_id: Option<ScriptHash>,
        tag: String,
        script_lang: Option<ScriptLang>,
    ) -> Self {
        Self {
            workspace_id,
            id: Uuid::new_v4(),
            args: args.map(Json),
            parent_job: None,
            created_by,
            scheduled_for: chrono::Utc::now(),
            started_at: None,
            runnable_path,
            kind,
            runnable_id,
            canceled_reason: None,
            canceled_by: None,
            permissioned_as,
            permissioned_as_email,
            flow_status: None,
            tag,
            script_lang,
            same_worker: true,
            pre_run_error: None,
            flow_innermost_root_job: None,
            root_job: None,
            timeout: None,
            flow_step_id: None,
            cache_ttl: None,
            cache_ignore_s3_path: None,
            priority: None,
            preprocessed: None,
            script_entrypoint_override: None,
            trigger: None,
            trigger_kind: None,
            visible_to_owner: false,
            permissioned_as_end_user_email: None,
            runnable_settings_handle: None,
            concurrent_limit: None,
            concurrency_time_window_s: None,
            job_token_scopes: None,
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MiniCompletedJob {
    pub id: Uuid,
    pub workspace_id: String,
    pub runnable_id: Option<ScriptHash>,
    pub scheduled_for: chrono::DateTime<chrono::Utc>,
    pub parent_job: Option<Uuid>,
    // pub root_job: Option<Uuid>,
    pub flow_innermost_root_job: Option<Uuid>,
    pub runnable_path: Option<String>,
    pub kind: JobKind,
    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
    pub permissioned_as: String,
    pub created_by: String,
    pub script_lang: Option<ScriptLang>,
    pub permissioned_as_email: String,
    pub flow_step_id: Option<String>,
    pub trigger_kind: Option<TriggerKindLabel>,
    pub trigger: Option<String>,
    pub priority: Option<i16>,
    pub concurrent_limit: Option<i32>,
    pub tag: String,
    pub cache_ttl: Option<i32>,
    pub cache_ignore_s3_path: Option<bool>,
    pub runnable_settings_handle: Option<i64>,
    /// A `dependencies` job that only rebuilt a binary. It deployed nothing, so consumers
    /// that react to a dependency job as "a new version is live" must skip it.
    ///
    /// Defaulted because this struct is deserialized from the `JobCompleted` payload an
    /// agent worker POSTs: an agent older than this field omits it, and without a default
    /// the server would reject every completion it sends, not just build jobs.
    #[serde(default)]
    pub build_binary_only: bool,
    /// The job's effective token scopes, carried from the pull: a re-run (retry, perpetual
    /// restart) caps itself with them after the job's `job_perms` row may have been swept.
    #[serde(default)]
    pub job_token_scopes: Option<Vec<String>>,
}

impl From<QueuedJobV2> for MiniCompletedJob {
    fn from(job: QueuedJobV2) -> Self {
        MiniCompletedJob {
            id: job.id,
            workspace_id: job.workspace_id,
            runnable_id: job.runnable_id,
            scheduled_for: job.scheduled_for,
            parent_job: job.parent_job,
            flow_innermost_root_job: job.flow_innermost_root_job,
            runnable_path: job.runnable_path,
            kind: job.kind,
            started_at: job.started_at,
            permissioned_as: job.permissioned_as,
            created_by: job.created_by,
            script_lang: job.script_lang,
            permissioned_as_email: job.permissioned_as_email,
            flow_step_id: job.flow_step_id,
            trigger_kind: job.trigger_kind.map(Into::into),
            trigger: job.trigger,
            priority: job.priority,
            concurrent_limit: job.concurrent_limit,
            tag: job.tag,
            cache_ttl: job.cache_ttl,
            cache_ignore_s3_path: job.cache_ignore_s3_path,
            runnable_settings_handle: job.runnable_settings_handle,
            // `QueuedJobV2` carries no args, which is what marks a binary-build job.
            build_binary_only: false,
            // Nor scopes: a caller whose completion can re-run the job (the monitor's zombie
            // recovery) fills them in from `job_perms` while the job is still queued.
            job_token_scopes: None,
        }
    }
}

impl From<MiniPulledJob> for MiniCompletedJob {
    fn from(job: MiniPulledJob) -> Self {
        MiniCompletedJob {
            id: job.id,
            workspace_id: job.workspace_id,
            runnable_id: job.runnable_id,
            scheduled_for: job.scheduled_for,
            parent_job: job.parent_job,
            // root_job: job.root_job,,
            flow_innermost_root_job: job.flow_innermost_root_job,
            runnable_path: job.runnable_path,
            kind: job.kind,
            started_at: job.started_at,
            permissioned_as: job.permissioned_as,
            created_by: job.created_by,
            script_lang: job.script_lang,
            permissioned_as_email: job.permissioned_as_email,
            flow_step_id: job.flow_step_id,
            trigger_kind: job.trigger_kind,
            trigger: job.trigger,
            priority: job.priority,
            concurrent_limit: job.concurrent_limit,
            tag: job.tag,
            cache_ttl: job.cache_ttl,
            cache_ignore_s3_path: job.cache_ignore_s3_path,
            runnable_settings_handle: job.runnable_settings_handle,
            job_token_scopes: job.job_token_scopes.clone(),
            build_binary_only: crate::binary_prebuild::is_build_binary_job(
                job.args.as_ref().map(|x| &x.0),
            ),
        }
    }
}

impl From<Arc<MiniPulledJob>> for MiniCompletedJob {
    fn from(job: Arc<MiniPulledJob>) -> Self {
        MiniCompletedJob {
            id: job.id,
            workspace_id: job.workspace_id.clone(),
            runnable_id: job.runnable_id,
            scheduled_for: job.scheduled_for,
            parent_job: job.parent_job,
            flow_innermost_root_job: job.flow_innermost_root_job,
            runnable_path: job.runnable_path.clone(),
            kind: job.kind,
            started_at: job.started_at,
            permissioned_as: job.permissioned_as.clone(),
            created_by: job.created_by.clone(),
            script_lang: job.script_lang,
            permissioned_as_email: job.permissioned_as_email.clone(),
            flow_step_id: job.flow_step_id.clone(),
            trigger_kind: job.trigger_kind.clone(),
            trigger: job.trigger.clone(),
            priority: job.priority,
            concurrent_limit: job.concurrent_limit,
            tag: job.tag.clone(),
            cache_ttl: job.cache_ttl,
            cache_ignore_s3_path: job.cache_ignore_s3_path,
            runnable_settings_handle: job.runnable_settings_handle,
            job_token_scopes: job.job_token_scopes.clone(),
            build_binary_only: crate::binary_prebuild::is_build_binary_job(
                job.args.as_ref().map(|x| &x.0),
            ),
        }
    }
}

impl MiniCompletedJob {
    pub fn is_flow_step(&self) -> bool {
        self.flow_step_id.is_some()
    }
    pub fn schedule_path(&self) -> Option<String> {
        schedule_path(&self.trigger_kind, &self.trigger)
    }

    pub fn is_flow(&self) -> bool {
        self.kind.is_flow()
    }

    pub fn is_dependency(&self) -> bool {
        self.kind.is_dependency()
    }
}

fn schedule_path(
    trigger_kind: &Option<TriggerKindLabel>,
    trigger: &Option<String>,
) -> Option<String> {
    if trigger_kind
        .as_ref()
        .is_some_and(|t| t.is(JobTriggerKind::Schedule))
    {
        trigger.clone()
    } else {
        None
    }
}

#[derive(Serialize, Deserialize, Debug, Clone)]
struct FlowStatusChatInputEnabled {
    chat_input_enabled: Option<bool>,
}

impl MiniPulledJob {
    pub fn runnable_path(&self) -> &str {
        self.runnable_path
            .as_ref()
            .map(String::as_str)
            .unwrap_or("tmp/main")
    }

    pub fn is_flow_step(&self) -> bool {
        self.flow_step_id.is_some()
    }

    pub fn is_canceled(&self) -> bool {
        self.canceled_by.is_some()
    }

    pub fn parse_flow_status(&self) -> Option<FlowStatus> {
        // tracing::error!("parse_flow_status: {:?}", self.flow_status);

        self.flow_status
            .as_ref()
            .and_then(|v| serde_json::from_str::<FlowStatus>((**v).get()).ok())
    }

    pub fn parse_chat_input_enabled(&self) -> Option<bool> {
        self.flow_status
            .as_ref()
            .and_then(|v| serde_json::from_str::<FlowStatusChatInputEnabled>((**v).get()).ok())
            .and_then(|f| f.chat_input_enabled)
    }

    pub fn is_flow(&self) -> bool {
        self.kind.is_flow()
    }

    pub fn is_dependency(&self) -> bool {
        self.kind.is_dependency()
    }

    pub fn schedule_path(&self) -> Option<String> {
        schedule_path(&self.trigger_kind, &self.trigger)
    }

    pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> {
        if self.is_flow_step() {
            let _ = update_flow_status_in_progress(
                db,
                &self.workspace_id,
                self.parent_job
                    .ok_or_else(|| Error::internal_err(format!("expected parent job")))?,
                self.id,
            )
            .warn_after_seconds(5)
            .await?;
        } else if let Some(parent_job) = self.parent_job {
            let _ = update_workflow_as_code_status(db, &self.id, &parent_job).await?;
        }
        Ok(())
    }
}

#[derive(sqlx::FromRow, Debug, Clone)]
pub struct PulledJob {
    #[sqlx(flatten)]
    pub job: MiniPulledJob,
    pub raw_code: Option<String>,
    pub raw_lock: Option<String>,
    pub raw_flow: Option<Json<Box<RawValue>>>,
    pub parent_runnable_path: Option<String>,
    pub permissioned_as_email: Option<String>,
    pub permissioned_as_username: Option<String>,
    pub permissioned_as_is_admin: Option<bool>,
    pub permissioned_as_is_operator: Option<bool>,
    pub permissioned_as_groups: Option<Vec<String>>,
    pub permissioned_as_folders: Option<Vec<serde_json::Value>>,
}

// NOTE:
// Precomputed by the server
// Used to offload work from agent workers to server
#[derive(Debug, Serialize, Deserialize)]
pub enum PrecomputedAgentInfo {
    Bun {
        local: String,
        remote: String,
    },
    Python {
        // V1, not used anymore. Exists for compat.
        // TODO: Needs to be removed eventually
        py_version: Option<u32>,
        py_version_v2: Option<String>,
        requirements: Option<String>,
    },
}

#[derive(Debug, Serialize, Deserialize)]
pub struct JobAndPerms {
    pub job: MiniPulledJob,
    pub raw_code: Option<String>,
    pub raw_flow: Option<Json<Box<RawValue>>>,
    pub raw_lock: Option<String>,
    pub parent_runnable_path: Option<String>,
    pub token: String,
    pub precomputed_agent_info: Option<PrecomputedAgentInfo>,
    #[serde(skip)]
    pub flow_runners: Option<Arc<FlowRunners>>,
}
impl PulledJob {
    pub async fn get_job_and_perms(self, db: &DB) -> JobAndPerms {
        let job_perms = match (
            self.permissioned_as_email,
            self.permissioned_as_username,
            self.permissioned_as_is_admin,
            self.permissioned_as_is_operator,
            self.permissioned_as_groups,
            self.permissioned_as_folders,
        ) {
            (
                Some(email),
                Some(username),
                Some(is_admin),
                Some(is_operator),
                Some(groups),
                Some(folders),
            ) => Some(JobPerms {
                email,
                username,
                is_admin,
                is_operator,
                groups,
                folders,
                end_user_email: self.job.permissioned_as_end_user_email.clone(),
                job_token_scopes: self.job.job_token_scopes.clone(),
            }),
            _ => None,
        };

        let token = create_token(&db, &self.job, job_perms).await;
        JobAndPerms {
            job: self.job,
            raw_code: self.raw_code,
            raw_flow: self.raw_flow,
            raw_lock: self.raw_lock,
            parent_runnable_path: self.parent_runnable_path,
            token,
            precomputed_agent_info: None,
            flow_runners: None,
        }
    }
}

// struct Permission
pub async fn create_token(db: &DB, job: &MiniPulledJob, perms: Option<JobPerms>) -> String {
    // skipping test runs
    if job.workspace_id != "" {
        let label = windmill_common::auth::ephemeral_script_token_label(
            &job.permissioned_as,
            &job.created_by,
        );
        windmill_common::auth::create_token_for_owner(
            db,
            &job.workspace_id,
            &job.permissioned_as,
            &label,
            windmill_common::auth::job_token_expiry_secs(db, &job.workspace_id).await,
            &job.permissioned_as_email,
            &job.id,
            perms,
            Some(format!(
                "job-span-{}",
                job.flow_innermost_root_job.unwrap_or(job.id)
            )),
        )
        .warn_after_seconds(5)
        .await
        .expect("could not create job token")
    } else {
        return "".to_string();
    }
}

impl std::ops::Deref for PulledJob {
    type Target = MiniPulledJob;
    fn deref(&self) -> &Self::Target {
        &self.job
    }
}

impl std::ops::DerefMut for PulledJob {
    fn deref_mut(&mut self) -> &mut Self::Target {
        &mut self.job
    }
}

lazy_static::lazy_static! {
    pub static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
}

pub async fn get_mini_pulled_job<'c>(
    e: impl PgExecutor<'c>,
    job_id: &Uuid,
) -> windmill_common::error::Result<Option<MiniPulledJob>> {
    let job = sqlx::query_as!(
        MiniPulledJob,
        "SELECT
        v2_job_queue.workspace_id,
        v2_job_queue.id,
        v2_job.args as \"args: sqlx::types::Json<HashMap<String, Box<RawValue>>>\",
        v2_job.parent_job,
        v2_job.created_by,
        v2_job_queue.started_at,
        v2_job_queue.runnable_settings_handle,
        scheduled_for,
        runnable_path,
        kind as \"kind: JobKind\",
        runnable_id as \"runnable_id: ScriptHash\",
        canceled_reason,
        canceled_by,
        permissioned_as,
        permissioned_as_email,
        flow_status as \"flow_status: sqlx::types::Json<Box<RawValue>>\",
        v2_job.tag,
        script_lang as \"script_lang: ScriptLang\",
        same_worker,
        pre_run_error,
        concurrent_limit,
        concurrency_time_window_s,
        flow_innermost_root_job,
        root_job,
        timeout,
        flow_step_id,
        cache_ttl,
        cache_ignore_s3_path,
        v2_job_queue.priority,
        preprocessed,
        script_entrypoint_override,
        trigger,
        trigger_kind as \"trigger_kind: TriggerKindLabel\",
        visible_to_owner,
        NULL as permissioned_as_end_user_email,
        job_perms.job_token_scopes
        FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id
        LEFT JOIN job_perms ON job_perms.job_id = v2_job_queue.id WHERE v2_job_queue.id = $1",
        job_id,
    )
    .fetch_optional(e)
    .await?;
    Ok(job)
}

pub struct QueuedJobV2 {
    pub id: Uuid,
    pub workspace_id: String,
    pub runnable_id: Option<ScriptHash>,
    pub scheduled_for: chrono::DateTime<chrono::Utc>,
    pub parent_job: Option<Uuid>,
    // pub root_job: Option<Uuid>,
    pub flow_innermost_root_job: Option<Uuid>,
    pub runnable_path: Option<String>,
    pub kind: JobKind,
    pub started_at: Option<chrono::DateTime<chrono::Utc>>,
    pub permissioned_as: String,
    pub created_by: String,
    pub script_lang: Option<ScriptLang>,
    pub permissioned_as_email: String,
    pub flow_step_id: Option<String>,
    pub trigger_kind: Option<TriggerKindLabel>,
    pub trigger: Option<String>,
    pub priority: Option<i16>,
    pub concurrent_limit: Option<i32>,
    pub tag: String,
    pub cache_ttl: Option<i32>,
    pub cache_ignore_s3_path: Option<bool>,
    pub runnable_settings_handle: Option<i64>,
    pub last_ping: Option<chrono::DateTime<chrono::Utc>>,
    pub worker: Option<String>,
    pub memory_peak: Option<i32>,
    pub running: bool,
}

impl QueuedJobV2 {
    pub fn schedule_path(&self) -> Option<String> {
        schedule_path(&self.trigger_kind, &self.trigger)
    }
}

pub async fn get_queued_job_v2<'c>(
    e: impl PgExecutor<'c>,
    job_id: &Uuid,
) -> error::Result<Option<QueuedJobV2>> {
    let job = sqlx::query_as!(
        QueuedJobV2,
        r#"SELECT
                id,
                q.runnable_settings_handle,
                q.workspace_id,
                j.runnable_id as "runnable_id: ScriptHash",
                scheduled_for,
                parent_job,
                flow_innermost_root_job,
                runnable_path,
                kind as "kind: JobKind",
                started_at,
                permissioned_as,
                created_by,
                script_lang as "script_lang: ScriptLang",
                permissioned_as_email,
                flow_step_id,
                trigger_kind as "trigger_kind: TriggerKindLabel",
                trigger,
                q.priority,
                concurrent_limit,
                q.tag,
                cache_ttl,
                cache_ignore_s3_path,
                r.ping as last_ping,
                worker,
                memory_peak,
                running
            FROM v2_job_queue q
                JOIN v2_job j USING (id)
                LEFT JOIN v2_job_runtime r USING (id)
                LEFT JOIN v2_job_status s USING (id)
            WHERE j.id = $1"#,
        job_id,
    )
    .fetch_optional(e)
    .await?;
    Ok(job)
}

#[derive(Debug)]
pub struct PulledJobResult {
    pub job: Option<PulledJob>,
    pub suspended: bool,
    pub missing_concurrency_key: bool,
    pub error_while_preprocessing: Option<String>,
}

#[derive(thiserror::Error, Debug)]
pub enum PulledJobResultToJobErr {
    #[error("missing concurrency key")]
    MissingConcurrencyKey(JobCompleted),
    #[error("pulled job preprocessor error: {}", .0.result)]
    ErrorWhilePreprocessing(JobCompleted),
}

impl PulledJobResult {
    pub fn to_pulled_job(self) -> Result<Option<PulledJob>, PulledJobResultToJobErr> {
        match self {
            PulledJobResult { job: Some(job), missing_concurrency_key: true, .. } => Err(
                PulledJobResultToJobErr::MissingConcurrencyKey(JobCompleted {
                    preprocessed_args: None,
                    job: MiniCompletedJob::from(job.job),
                    success: false,
                    result: Arc::new(windmill_common::worker::to_raw_value(&json!({
                        "name": "InternalErr",
                        "message": "The job has a concurrency limit but concurrency key couldn't be found. This is an unexpected behavior that should never happen. Please report this to support."}
                    ))),
                    result_columns: None,
                    mem_peak: 0,
                    resource_usage: None,
                    cached_res_path: None,
                    token: "".to_string(),
                    canceled_by: None,
                    duration: None,
                    has_stream: Some(false),
                    from_cache: None,
                    flow_runners: None,
                    done_tx: None,
                }),
            ),
            PulledJobResult { job: Some(job), error_while_preprocessing: Some(e), .. } => Err(
                PulledJobResultToJobErr::ErrorWhilePreprocessing(JobCompleted {
                    preprocessed_args: None,
                    job: MiniCompletedJob::from(job.job),
                    success: false,
                    result: Arc::new(windmill_common::worker::to_raw_value(&json!({
                        "name": "Pulled job preprocessing error",
                        "message": e
                    }))),
                    result_columns: None,
                    mem_peak: 0,
                    resource_usage: None,
                    cached_res_path: None,
                    token: "".to_string(),
                    canceled_by: None,
                    duration: None,
                    has_stream: Some(false),
                    from_cache: None,
                    flow_runners: None,
                    done_tx: None,
                }),
            ),
            PulledJobResult { job, .. } => Ok(job),
        }
    }

    /// Generic preprocess function
    /// Can be used for any kind of preprocessing
    pub async fn maybe_apply_debouncing(&mut self, db: &DB) -> error::Result<()> {
        let PulledJobResult { job: Some(ref mut j), .. } = self else {
            return Ok(());
        };

        let DebouncingSettings { debounce_delay_s, debounce_args_to_accumulate, .. } =
            windmill_common::runnable_settings::prefetch_cached_from_handle(
                j.runnable_settings_handle,
                db,
            )
            .await?
            .0;

        let (kind, j_id) = (j.kind, j.id);
        let is_djob_to_debounce = kind.is_dependency()
            && j.args
                .as_ref()
                .and_then(|x| x.get("triggered_by_relative_import"))
                .is_some();

        let has_debounce_args = debounce_args_to_accumulate
            .as_ref()
            .map_or(false, |v| !v.is_empty());

        if (is_djob_to_debounce
            || debounce_delay_s.filter(|x| *x > 0).is_some()
            || has_debounce_args)
            && MIN_VERSION_SUPPORTS_DEBOUNCING.met().await
            && !*WMDEBUG_NO_DEBOUNCING
        {
            let needs_debounce = sqlx::query_scalar!(
                "WITH _ AS (
                    DELETE FROM debounce_key WHERE job_id = $1
                ) SELECT status = 'skipped' FROM v2_job_completed WHERE id = $1",
                j_id,
            )
            .fetch_optional(db)
            .await?
            .flatten()
            .unwrap_or_default();

            if needs_debounce {
                tracing::info!(
                    job_id = %j_id,
                    "Late debounce: job was already debounced by another job, skipping execution"
                );
                self.job = None;
                return Ok(());
            }

            if matches!(kind, JobKind::FlowDependencies | JobKind::AppDependencies)
                && !MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
                && !*WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT
            {
                // Simply disable optimization for apps and flows if min version doesn't support debouncing v2
                let field_name = match kind {
                    JobKind::FlowDependencies => "nodes_to_relock",
                    JobKind::AppDependencies => "components_to_relock",
                    _ => unreachable!(),
                };

                tracing::debug!(
                    job_id = %j_id,
                    job_kind = ?kind,
                    field_removed = field_name,
                    "Removing optimization field: workers behind v2 debouncing version, disabling relock optimization"
                );

                if let Some(args) = &mut j.args {
                    args.remove(field_name);
                }

                // No accumulation on this path: just clean up the batch rows.
                sqlx::query!(
                    "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (
                        SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
                    )",
                    j_id,
                )
                .execute(db)
                .await?;
            } else if let Some(arg_name_to_accumulate) =
                // TODO: Maybe support multiple arguments in future
                debounce_args_to_accumulate.as_ref().and_then(|v| v.get(0))
            {
                // Claim this job's contribution to its debounce batch exactly once.
                // Instead of deleting the batch rows, mark them consumed (stamping
                // consumed_by = this job). A batch normally has a single survivor that
                // sweeps every row; only a narrow push/pull race can leave two survivors
                // on one batch. The claim lets the second survivor tell apart:
                //   - already swept in by the other survivor -> run empty (no duplicate),
                //   - its own earlier claim on re-pull -> keep its accumulated args,
                //   - never batched (CE / workers behind v2) -> keep its own args.
                // Consumed rows are GC'd by the monitor.
                // Claim + accumulate + persist atomically: a crash between stamping the
                // batch rows consumed_by=self and persisting the merged args would
                // otherwise let a zombie re-pull see its own prior claim and keep only
                // its own args (dropping the siblings it had claimed). One transaction
                // makes the claim and the merged-args write commit together (or neither).
                let mut tx = db.begin().await?;
                // Emitted AFTER the transaction commits — writing logs via a second pool
                // connection while the claim tx + row locks are held risks pool-exhaustion
                // stalls under concurrent debounced pulls.
                let mut accumulation_log: Option<String> = None;
                let claim = sqlx::query!(
                    "WITH mine AS (
                        SELECT debounce_batch, consumed_by FROM v2_job_debounce_batch WHERE id = $1
                    ), claimed AS (
                        -- Claim the whole batch in ONE update so concurrent same-batch
                        -- survivors lock rows in identical scan order (no lock-ordering
                        -- deadlock); each re-evaluates `consumed_at IS NULL` under EvalPlanQual
                        -- and skips rows the other already took. A claim therefore consumes
                        -- every still-unclaimed row of the batch atomically.
                        UPDATE v2_job_debounce_batch SET consumed_at = now(), consumed_by = $1
                        WHERE debounce_batch = (SELECT debounce_batch FROM mine)
                          AND consumed_at IS NULL
                        RETURNING id
                    )
                    SELECT
                        EXISTS (SELECT 1 FROM mine) AS \"had_row!\",
                        (SELECT consumed_by FROM mine) AS prev_consumed_by,
                        ARRAY(SELECT id FROM claimed) AS \"claimed_ids!\",
                        EXISTS (SELECT 1 FROM claimed WHERE id = $1) AS \"claimed_self!\"
                    ",
                    j_id,
                )
                .fetch_one(&mut *tx)
                .await?;

                if !claim.had_row {
                    // Never batched (CE / workers behind v2): keep the job's own args.
                    tracing::debug!(
                        job_id = %j_id,
                        "Debounce: no batch row, keeping original args"
                    );
                } else if claim.claimed_self {
                    // We claimed our own row; since a claim takes the whole batch, this also
                    // swept any not-yet-claimed siblings. Accumulate exactly the rows we own.
                    let ids = claim.claimed_ids;

                    tracing::debug!(
                        job_id = %j_id,
                        job_kind = ?kind,
                        arg_name = arg_name_to_accumulate,
                        claimed = ids.len(),
                        "Accumulating debounced arguments from claimed batch rows"
                    );

                    let mut accumulated_arg: Vec<Box<RawValue>> = vec![];
                    for str_o in sqlx::query_scalar!(
                        "SELECT args->>$2 FROM v2_job WHERE id = ANY($1)",
                        &ids,
                        arg_name_to_accumulate,
                    )
                    .fetch_all(&mut *tx)
                    .await?
                    .into_iter()
                    {
                        if let Some(s) = str_o.as_ref() {
                            match serde_json::from_str::<Vec<Box<RawValue>>>(s) {
                                Ok(ref mut vec) => accumulated_arg.append(vec),
                                Err(_) => {
                                    // Value is not an array — wrap the scalar into a
                                    // single-element array. This supports union types
                                    // like T | T[] where the caller may pass a bare T.
                                    match RawValue::from_string(s.to_string()) {
                                        Ok(raw) => accumulated_arg.push(raw),
                                        Err(e) => {
                                            return Err(error::Error::ArgumentErr(format!("cannot consolidate argument `{arg_name_to_accumulate}`: value is neither a valid list nor a valid JSON value\nUnwrapped Error: {e}")));
                                        }
                                    }
                                }
                            }
                        }
                    }

                    if !accumulated_arg.is_empty() {
                        let new_value = to_raw_value(&accumulated_arg);

                        let original_value = j
                            .args
                            .as_ref()
                            .and_then(|a| a.get(arg_name_to_accumulate))
                            .map(|v| v.get().to_string())
                            .unwrap_or_else(|| "null".to_string());

                        accumulation_log = Some(format!(
                            "Accumulating debounced argument `{arg_name_to_accumulate}`:\n  original: {original_value}\n  accumulated: {}\n\n",
                            &new_value
                        ));

                        j.args
                            .get_or_insert(Json(Default::default()))
                            .as_mut()
                            .insert(arg_name_to_accumulate.to_owned(), new_value);

                        // Persist accumulated args to v2_job so that flow steps
                        // re-reading from the DB (via get_mini_pulled_job) see them
                        if let Some(ref args) = j.args {
                            sqlx::query!(
                                "UPDATE v2_job SET args = $2 WHERE id = $1",
                                j_id,
                                args as &Json<HashMap<String, Box<RawValue>>>,
                            )
                            .execute(&mut *tx)
                            .await?;
                        }
                    }
                } else if claim.prev_consumed_by == Some(j_id) {
                    // Our own prior claim seen again on a re-pull (e.g. crash recovery):
                    // keep the args we already persisted on the first pull.
                } else {
                    // Another survivor already accumulated this job's contribution
                    // (consumed_by a different job); run as a no-op so its items are not
                    // reprocessed.
                    tracing::info!(
                        job_id = %j_id,
                        arg_name = arg_name_to_accumulate,
                        "Debounce: contribution already consumed by a concurrent survivor, running empty"
                    );
                    j.args
                        .get_or_insert(Json(Default::default()))
                        .as_mut()
                        .insert(
                            arg_name_to_accumulate.to_owned(),
                            to_raw_value(&Vec::<Box<RawValue>>::new()),
                        );
                    if let Some(ref args) = j.args {
                        sqlx::query!(
                            "UPDATE v2_job SET args = $2 WHERE id = $1",
                            j_id,
                            args as &Json<HashMap<String, Box<RawValue>>>,
                        )
                        .execute(&mut *tx)
                        .await?;
                    }
                }
                tx.commit().await?;

                if let Some(msg) = accumulation_log {
                    append_logs(&j_id, &j.workspace_id, msg, &(db.into())).await;
                }
            } else {
                // Debounced but no args to accumulate (plain debounce / dependency job):
                // consume the batch by removing this job's rows.
                sqlx::query!(
                    "DELETE FROM v2_job_debounce_batch WHERE debounce_batch = (
                        SELECT debounce_batch FROM v2_job_debounce_batch WHERE id = $1
                    )",
                    j_id,
                )
                .execute(db)
                .await?;
            }

            // Handle dependency job debouncing cleanup when a job is pulled for execution
            if is_djob_to_debounce {
                clone_runnable(j, db).await?;
            }
        }

        Ok(())
    }
}

async fn clone_runnable(j: &mut PulledJob, db: &DB) -> error::Result<()> {
    let Some(base_hash) = j.runnable_id else {
        return Err(Error::InternalErr(
            "Missing runnable_id for dependency job triggered by relative import".to_string(),
        ));
    };

    tracing::debug!(
        job_id = %j.id,
        base_hash = %base_hash,
        job_kind = ?j.kind,
        runnable_path = ?j.runnable_path,
        "Creating new version for dependency job triggered by relative import"
    );

    {
        let maybe_new_id = match j.kind {
            // A script gets its new version from the worker, once the generated lock is known
            // to differ from the live version's: minting one here would deploy, and walk the
            // importers of, a version whose lock turns out byte-identical to its parent's.
            JobKind::Dependencies => *base_hash,
            JobKind::FlowDependencies => {
                sqlx::query_scalar!(
                    "INSERT INTO flow_version
                    (workspace_id, path, value, schema, created_by)

                    SELECT workspace_id, path, value, schema, created_by
                    FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3

                    RETURNING id
                    ",
                    j.runnable_path(),
                    j.workspace_id,
                    *base_hash,
                )
                .fetch_one(db)
                .await?
            }
            JobKind::AppDependencies => {
                sqlx::query_scalar!(
                    "INSERT INTO app_version
                        (app_id, value, created_by, raw_app)
                    SELECT app_id, value, created_by, raw_app
                    FROM app_version WHERE id = $1
                    RETURNING id",
                    *base_hash
                )
                .fetch_one(db)
                .await?
            }
            _ => *base_hash,
        };

        j.runnable_id.replace(maybe_new_id.into());
    }

    Ok(())
}

// TODO: Factorize
/// Pull the job from queue
pub async fn pull(
    db: &Pool<Postgres>,
    // Whether or not try to pull from suspended jobs first
    suspend_first: bool,
    worker_name: &str,
    // Execute queries supplied by caller instead of generic one
    query_o: Option<&(String, String)>,
    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<PulledJobResult> {
    let mut pull_loop_count = 0;
    loop {
        pull_loop_count += 1;
        // Cap to bound DB work per pull cycle. Each iteration on an over-limit job runs
        // the full apply_concurrency_limit query stack (~7 queries), so without a tight
        // cap a single pull() can issue thousands of queries when the queue is full of
        // jobs sharing one over-limit concurrency key. Capping at 10 lets workers skip
        // a few stale jobs in healthy conditions while preventing storm amplification.
        if pull_loop_count > 10 {
            tracing::warn!(
                "Pull job loop count exceeded 10, backing off (likely concurrency re-queue storm)"
            );
            return Ok(PulledJobResult {
                job: None,
                suspended: false,
                missing_concurrency_key: false,
                error_while_preprocessing: None,
            });
        }

        if let Some((query_suspended, query_no_suspend)) = query_o {
            let njob = {
                let job = if query_suspended.is_empty() {
                    None
                } else {
                    timeout(
                        Duration::from_secs(15),
                        sqlx::query_as::<_, PulledJob>(query_suspended)
                            .bind(worker_name)
                            .fetch_optional(db),
                    )
                    .await??
                };

                let (job, suspended) = if let Some(job) = job {
                    (Some(job), true)
                } else {
                    let job = timeout(
                        Duration::from_secs(15),
                        sqlx::query_as::<_, PulledJob>(query_no_suspend)
                            .bind(worker_name)
                            .fetch_optional(db),
                    )
                    .await??;

                    (job, false)
                };

                if let Some(job) = job.as_ref() {
                    if (job.is_flow() || job.is_dependency())
                        && !(job.kind.is_preview()
                            && PREVIEW_TAGS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed))
                    {
                        let tag = match configured_dependency_job_tag(
                            &job.kind,
                            job.script_lang.as_ref(),
                            // the job sits on a shell worker's tag, so its own tag is lost
                            false,
                            &job.workspace_id,
                            db,
                        )
                        .await
                        {
                            Some(tag) => tag,
                            None => {
                                let effective_ws = per_workspace_tag(&job.workspace_id, db).await;
                                let base_tag = if job.is_flow() {
                                    "flow".to_string()
                                } else {
                                    "dependency".to_string()
                                };
                                match &effective_ws {
                                    Some(ws) => format!("{}-{}", base_tag, ws),
                                    None => base_tag,
                                }
                            }
                        };
                        sqlx::query!(
                            "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2",
                            tag,
                            job.id
                        )
                        .execute(db)
                        .await?;
                        continue;
                    }
                }

                #[cfg(feature = "private")]
                let concurrency_settings = if let Some(ref j) = job {
                    windmill_common::runnable_settings::prefetch_cached_from_handle(
                        j.runnable_settings_handle,
                        db,
                    )
                    .await?
                    .1
                    .maybe_fallback(
                        None,
                        j.concurrent_limit,
                        j.concurrency_time_window_s,
                    )
                } else {
                    Default::default()
                };

                let pulled_job_result = match job {
                    #[cfg(feature = "private")]
                    Some(job)
                        if has_active_concurrency_limit(concurrency_settings.concurrent_limit)
                            // Concurrency limit is available for either enterprise job or dependency job
                            && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING)) =>
                    {
                        timeout(
                            Duration::from_secs(15),
                            crate::jobs_ee::apply_concurrency_limit(
                                db,
                                pull_loop_count,
                                suspended,
                                job,
                                &concurrency_settings,
                            ),
                        )
                        .await??
                        .unwrap_or(PulledJobResult {
                            job: None,
                            suspended,
                            missing_concurrency_key: false,
                            error_while_preprocessing: None,
                        })
                    }
                    _ => PulledJobResult {
                        job,
                        suspended,
                        missing_concurrency_key: false,
                        error_while_preprocessing: None,
                    },
                };

                Ok::<_, Error>(pulled_job_result)
            }?;

            return Ok(njob);
        };

        let (job, suspended) = timeout(
            Duration::from_secs(15),
            pull_single_job_and_mark_as_running_no_concurrency_limit(
                db,
                suspend_first,
                worker_name,
                #[cfg(feature = "benchmark")]
                bench,
            ),
        )
        .await??;
        let Some(job) = job else {
            return Ok(PulledJobResult {
                job: None,
                suspended,
                missing_concurrency_key: false,
                error_while_preprocessing: None,
            });
        };

        let concurrency_settings = windmill_common::runnable_settings::prefetch_cached_from_handle(
            job.runnable_settings_handle,
            db,
        )
        .await?
        .1
        .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s);

        let has_concurent_limit =
            has_active_concurrency_limit(concurrency_settings.concurrent_limit);

        #[cfg(not(feature = "enterprise"))]
        if has_concurent_limit && !job.is_dependency() {
            tracing::error!("Concurrent limits are an EE feature only, ignoring constraints")
        }

        #[cfg(not(feature = "enterprise"))]
        let has_concurent_limit = job.is_dependency()
            && has_active_concurrency_limit(job.concurrent_limit)
            && cfg!(feature = "private")
            && !*WMDEBUG_NO_DEBOUNCING;
        // if we don't have private flag, we don't have concurrency limit

        // concurrency check. If more than X jobs for this path are already running, we re-queue and pull another job from the queue
        let pulled_job = job;
        if pulled_job.runnable_path.is_none()
            || !has_concurent_limit
            || pulled_job.canceled_by.is_some()
        {
            #[cfg(feature = "prometheus")]
            if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
                QUEUE_PULL_COUNT.inc();
            }
            otel_incr_queue_pull_count();
            return Ok(PulledJobResult {
                job: Some(pulled_job),
                suspended,
                missing_concurrency_key: false,
                error_while_preprocessing: None,
            });
        }

        #[cfg(feature = "private")]
        if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DEBOUNCING) {
            if let Some(pulled_job_res) = timeout(
                Duration::from_secs(15),
                crate::jobs_ee::apply_concurrency_limit(
                    db,
                    pull_loop_count,
                    suspended,
                    pulled_job,
                    &concurrency_settings,
                ),
            )
            .await??
            {
                return Ok(pulled_job_res);
            }
        }
    }
}

async fn pull_single_job_and_mark_as_running_no_concurrency_limit<'c>(
    db: &Pool<Postgres>,
    suspend_first: bool,
    worker_name: &str,
    #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<(Option<PulledJob>, bool)> {
    let job_and_suspended: (Option<PulledJob>, bool) = {
        /* Jobs can be started if they:
         * - haven't been started before,
         *   running = false
         * - are flows with a step that needed resume,
         *   suspend_until is non-null
         *   and suspend = 0 when the resume messages are received
         *   or suspend_until <= now() if it has timed out */
        let query = WORKER_SUSPENDED_PULL_QUERY.load();

        if query.is_empty() {
            tracing::warn!("No suspended pull queries available");
            tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
            return Ok((None, false));
        }
        let r = if suspend_first {
            // tracing::info!("Pulling job with query: {}", query);
            sqlx::query_as::<_, PulledJob>(&query)
                .bind(worker_name)
                .fetch_optional(db)
                .await?
        } else {
            None
        };

        if r.is_none() {
            // #[cfg(feature = "benchmark")]
            // let instant = Instant::now();
            let mut highest_priority_job: Option<PulledJob> = None;

            let queries = WORKER_PULL_QUERIES.load();

            if queries.is_empty() {
                tracing::warn!("No pull queries available");
                tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
                return Ok((None, false));
            }

            // Workspace fairness (Enterprise): if the fairness refresh has flagged
            // any overloaded workspaces, this pull is randomly routed between two
            // pull queries:
            //   * with probability `(cap + ε)/100`, the standard query (capped
            //     workspaces are admissible — they win via FIFO when noisy)
            //   * with probability `1 - (cap + ε)/100`, the fairness query (the
            //     overloaded workspace_ids are filtered out)
            // Over many pulls this converges to a steady share around the cap,
            // without the on/off oscillation a binary cap/uncap dispatch produces.
            // If the chosen query returns nothing, we always fall back to the
            // standard query so workers don't idle when only capped jobs remain.
            // Lazy refresh fires from the same place: runs at most once per
            // process per refresh interval, never blocks this pull.
            crate::workspace_fairness::maybe_refresh_overloaded(db);
            let overloaded = WORKSPACE_FAIRNESS_OVERLOADED.load_full();
            let fairness_active = !overloaded.is_empty();

            if fairness_active && !crate::workspace_fairness::should_admit_capped() {
                let fairness_queries = WORKER_PULL_QUERIES_FAIRNESS.load();
                let overloaded_slice: &[String] = overloaded.as_slice();
                for query in fairness_queries.iter() {
                    #[cfg(feature = "benchmark")]
                    add_time!(bench, "pre pull (fairness)");

                    let r = sqlx::query_as::<_, PulledJob>(query)
                        .bind(worker_name)
                        .bind(overloaded_slice)
                        .fetch_optional(db)
                        .await?;

                    #[cfg(feature = "benchmark")]
                    add_time!(bench, "post pull (fairness)");

                    if let Some(pulled_job) = r {
                        highest_priority_job = Some(pulled_job);
                        break;
                    }
                }
            }

            if highest_priority_job.is_none() {
                // Standard pull path. Also acts as the fallback when fairness
                // filtered out every candidate: prefer running a capped
                // workspace's job over leaving a worker idle. (The cap is
                // re-asserted statistically on subsequent pulls.)
                for query in queries.iter() {
                    // tracing::info!("Pulling job with query: {}", query);
                    // let instant = std::time::Instant::now();

                    #[cfg(feature = "benchmark")]
                    add_time!(bench, "pre pull");

                    let r = sqlx::query_as::<_, PulledJob>(query)
                        .bind(worker_name)
                        .fetch_optional(db)
                        .await?;

                    #[cfg(feature = "benchmark")]
                    add_time!(bench, "post pull");

                    if let Some(pulled_job) = r {
                        // tracing::info!("pulled job: {:?}", instant.elapsed().as_micros());

                        highest_priority_job = Some(pulled_job);
                        break;
                    }
                    // else continue pulling for lower priority tags
                }
            }

            // #[cfg(feature = "benchmark")]
            // println!("pull query: {:?}", instant.elapsed());
            (highest_priority_job, false)
        } else {
            (r, true)
        }
    };
    Ok(job_and_suspended)
}

/// A concurrency limit is only active when it caps at 1+ slots. `Some(0)` (or negative) is
/// a disabled limit, not a zero-slot one — see [`ConcurrencySettings::normalized`]. The gate
/// checks must use this instead of `.is_some()` so a legacy stored `0` behaves as disabled.
pub fn has_active_concurrency_limit(concurrent_limit: Option<i32>) -> bool {
    concurrent_limit.is_some_and(|n| n > 0)
}

/// Admit a job already owned by a worker without releasing its queue reservation.
/// The caller must maintain its heartbeat while waiting and complete it on failure.
pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result<bool> {
    #[cfg(all(feature = "private", feature = "enterprise"))]
    {
        let settings = windmill_common::runnable_settings::prefetch_cached_from_handle(
            job.runnable_settings_handle,
            db,
        )
        .await?
        .1
        .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s);
        if has_active_concurrency_limit(settings.concurrent_limit)
            && !*DISABLE_CONCURRENCY_LIMIT
            && job.canceled_by.is_none()
        {
            let key = concurrency_key(db, &job.id).await?.ok_or_else(|| {
                Error::internal_err(format!("No concurrency key found for job {}", job.id))
            })?;
            if !key.is_empty() {
                return Ok(crate::jobs_ee::update_concurrency_counter(
                    db,
                    &job.id,
                    key,
                    serde_json::json!({ job.id.to_string(): {} }),
                    job.id.to_string(),
                    settings.concurrency_time_window_s.unwrap_or(0),
                    settings.concurrent_limit.unwrap_or_default(),
                )
                .await?
                .0);
            }
        }
    }
    #[cfg(not(all(feature = "private", feature = "enterprise")))]
    let _ = (db, job);
    Ok(true)
}

pub async fn custom_concurrency_key(
    db: &Pool<Postgres>,
    job_id: &Uuid,
) -> Result<Option<String>, sqlx::Error> {
    let fut = async || {
        sqlx::query_scalar!("SELECT key FROM concurrency_key WHERE job_id = $1", job_id)
            .fetch_optional(db) // this should no longer be fetch optional
            .await
    };
    fut.retry(
        ConstantBuilder::default()
            .with_delay(std::time::Duration::from_secs(3))
            .with_max_times(5)
            .build(),
    )
    .notify(|err, dur| {
        tracing::error!(
            "Could not get concurrency key for job {job_id}, retrying in {dur:#?}, err: {err:#?}"
        );
    })
    .await
}

pub async fn concurrency_key(
    db: &Pool<Postgres>,
    id: &Uuid,
) -> windmill_common::error::Result<Option<String>> {
    custom_concurrency_key(db, id)
        .await
        .map(|x| {
            if x.is_none() {
                tracing::info!("No concurrency key found for job {id}");
            }
            return x;
        })
        .map_err(|e| {
            Error::internal_err(format!("Could not get concurrency key for job {id}: {e:#}"))
        })
}

pub async fn custom_debounce_key(
    db: &Pool<Postgres>,
    job_id: &Uuid,
) -> Result<Option<String>, sqlx::Error> {
    let fut = async || {
        sqlx::query_scalar!("SELECT key FROM debounce_key WHERE job_id = $1", job_id)
            .fetch_optional(db) // this should no longer be fetch optional
            .await
    };
    fut.retry(
        ConstantBuilder::default()
            .with_delay(std::time::Duration::from_secs(3))
            .with_max_times(5)
            .build(),
    )
    .notify(|err, dur| {
        tracing::error!(
            "Could not get debounce key for job {job_id}, retrying in {dur:#?}, err: {err:#?}"
        );
    })
    .await
}

/// Concurrency lane for a git-sync callback: the workspace's lane, narrowed by
/// `suffix` (the repo, plus its branch where one is knowable).
///
/// Both destination columns are VARCHAR(255) and `concurrency_key.key` is written
/// inside `push`, so an over-long key aborts the push and the sync job is never
/// created — the suffix is hashed rather than allowed to overflow.
/// `reserved_prefix_len` is what a caller-side prefix will consume afterwards.
fn git_sync_concurrency_key(
    workspace_id: &str,
    suffix: Option<String>,
    reserved_prefix_len: usize,
) -> String {
    const MAX_CONCURRENCY_KEY_LEN: usize = 255;
    let max_key_len = MAX_CONCURRENCY_KEY_LEN.saturating_sub(reserved_prefix_len);
    match suffix {
        Some(suffix) => {
            let full = format!("{workspace_id}:git_sync:{suffix}");
            if full.len() <= max_key_len {
                full
            } else {
                // SHA-256 hex (64) over the whole suffix, so distinct repos stay
                // distinct and the result fits any workspace id (VARCHAR(50)).
                format!("{workspace_id}:git_sync:{}", calculate_hash(&suffix))
            }
        }
        None => format!("{workspace_id}:git_sync"),
    }
}

pub fn resolve_debounce_key<'b>(
    unresolved_debounce_key: Option<String>,
    runnable_path: &Option<String>,
    workspace_id: &str,
    job_kind: JobKind,
    args: &PushArgs<'b>,
    args_to_ignore_if_default: Option<&String>,
) -> String {
    let original_debounce_key = unresolved_debounce_key
        .map(|x| crate::interpolate_args(x, &args, workspace_id))
        .unwrap_or(format!(
            "{}#args:{}",
            crate::fullpath_with_workspace(workspace_id, runnable_path.as_ref(), &job_kind),
            args.args
                .iter()
                .filter_map(|(k, v)| {
                    if args_to_ignore_if_default
                        .map(|name| name == k)
                        .unwrap_or_default()
                    {
                        None
                    } else {
                        Some(v.to_string())
                    }
                })
                // TODO: disable sorted?
                .sorted()
                .collect_vec()
                .join(":"),
        ));

    tracing::debug!(
        "Original debounce key (len={}): {}",
        original_debounce_key.len(),
        original_debounce_key
    );

    // If debounce_key is not too long (< 255 chars), keep it as is, otherwise hash it.
    // On cloud, we prepend "{workspace_id}:" so we must reserve space for that prefix
    // to avoid exceeding the VARCHAR(255) column limit.
    const MAX_DEBOUNCE_KEY_LENGTH: usize = 255;

    #[cfg(feature = "cloud")]
    let prefix = format!("{workspace_id}:");
    #[cfg(not(feature = "cloud"))]
    let prefix = String::new();

    let max_key_length = MAX_DEBOUNCE_KEY_LENGTH - prefix.len();

    let resolved = if original_debounce_key.len() <= max_key_length {
        original_debounce_key
    } else {
        let hash = calculate_hash(&original_debounce_key);
        tracing::debug!(
            "Debounce key too long ({}), using hash: {}",
            original_debounce_key.len(),
            hash
        );
        if hash.len() > max_key_length {
            hash[..max_key_length].to_string()
        } else {
            hash
        }
    };

    let resolved = format!("{prefix}{resolved}");

    tracing::debug!("Final debounce key (len={}): {}", resolved.len(), resolved);
    resolved
}

#[derive(Debug)]
pub enum DebouncingLimitsReport {
    MaxCountExceeded,
    TimeExceeded,
    CanDebounce,
}

pub async fn check_debouncing_within_limits(
    (current_time, current_amount): (DateTime<Utc>, i32),
    (allowed_time, allowed_amount): (Option<i32>, Option<i32>),
    job_id: &Uuid,
    runnable_path: &Option<String>,
) -> DebouncingLimitsReport {
    use DebouncingLimitsReport::*;

    let no_legacy_compat = MIN_VERSION_SUPPORTS_DEBOUNCING_V2.met().await
        || *WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT;

    if !no_legacy_compat {
        tracing::warn!(
            job_id = %job_id,
            runnable_path = ?runnable_path,
            allowed_time = ?allowed_time,
            allowed_amount = ?allowed_amount,
            "Debouncing limits not enforced: workers are behind minimum version for v2 debouncing (require >= 1.597.0)"
        );
    }

    let elapsed_seconds = (Utc::now() - current_time).num_seconds() as i32;

    tracing::debug!(
        job_id = %job_id,
        runnable_path = ?runnable_path,
        current_amount = current_amount,
        allowed_amount = ?allowed_amount,
        elapsed_seconds = elapsed_seconds,
        allowed_time = ?allowed_time,
        first_started_at = %current_time,
        no_legacy_compat = no_legacy_compat,
        "Checking debouncing limits"
    );

    if allowed_amount
        .filter(|&a| a > 0)
        .map(|allowed_amount| current_amount + 1 >= allowed_amount)
        .unwrap_or_default()
        && no_legacy_compat
    {
        tracing::info!(
            job_id = %job_id,
            runnable_path = ?runnable_path,
            current_amount = current_amount,
            allowed_amount = allowed_amount,
            "Debouncing limit exceeded: max debounce count reached"
        );
        MaxCountExceeded
    } else if allowed_time
        .map(|allowed_time| elapsed_seconds > allowed_time)
        .unwrap_or_default()
        && no_legacy_compat
    {
        tracing::info!(
            job_id = %job_id,
            runnable_path = ?runnable_path,
            elapsed_seconds = elapsed_seconds,
            allowed_time = allowed_time,
            first_started_at = %current_time,
            "Debouncing limit exceeded: max debounce time window reached"
        );
        TimeExceeded
    } else {
        tracing::debug!(
            job_id = %job_id,
            runnable_path = ?runnable_path,
            current_amount = current_amount,
            elapsed_seconds = elapsed_seconds,
            "Debouncing within limits: can continue debouncing"
        );
        CanDebounce
    }
}

/// Whether the tag's queue name is computed from the job's arguments, so that a caller holding
/// arguments it could not build knows the tag cannot be built either.
pub fn tag_reads_args(tag: &str) -> bool {
    RE_ARG_TAG.is_match(tag)
}

/// Whether the tag reads the flow's state (`$flow_expr[results.a.foo]`), which only the flow
/// runtime can resolve, right before pushing the step. A malformed placeholder counts too, so it
/// is rejected or dropped instead of queueing the job on its literal text.
pub fn tag_reads_flow_expr(tag: &str) -> bool {
    tag.contains("$flow_expr[")
}

/// Renders the value at the dotted `path` below `root` as a dynamic tag component, shared by
/// `$args[...]` and `$flow_expr[...]`: its JSON text with surrounding quotes trimmed, and empty
/// once a segment is missing. Only object keys are followed, never array indexes.
pub fn render_tag_path(root: Option<&RawValue>, path: &str) -> String {
    let mut value = root.map(|x| x.get()).unwrap_or_default().to_string();
    for part in path.split('.').filter(|p| !p.is_empty()) {
        match serde_json::from_str::<serde_json::Value>(&value) {
            Ok(obj) => value = obj.get(part).map(|v| v.to_string()).unwrap_or_default(),
            Err(_) => {
                value = String::new();
                break;
            }
        }
    }
    value.trim_matches('"').to_string()
}

pub fn interpolate_args(x: String, args: &PushArgs, workspace_id: &str) -> String {
    // Save this value to avoid parsing twice
    let workspaced = x.as_str().replace("$workspace", workspace_id).to_string();
    if RE_ARG_TAG.is_match(&workspaced) {
        let mut interpolated = workspaced.clone();
        for cap in RE_ARG_TAG.captures_iter(&workspaced) {
            let arg_name = cap.get(1).unwrap().as_str();
            let (root, rest) = arg_name.split_once('.').unwrap_or((arg_name, ""));
            // `push` strips a caller's `_MODULES` only after a run handler has authorized the
            // tag, so reading it here would let the authorized tag and the queued one differ.
            let root_value = (root != MODULES_ARG)
                .then(|| {
                    args.args
                        .get(root)
                        .or(args.extra.as_ref().and_then(|x| x.get(root)))
                })
                .flatten();
            let arg_value = render_tag_path(root_value.map(|x| &**x), rest);
            interpolated =
                interpolated.replace(format!("$args[{}]", arg_name).as_str(), &arg_value);
        }
        interpolated
    } else {
        workspaced
    }
}

/// The queue an explicit `tag` sends a job pushed with `args` to, or `None` when `push` drops the
/// tag and the job runs on its default one.
pub async fn resolve_push_tag(
    tag: &str,
    args: &PushArgs<'_>,
    workspace_id: &str,
    db: &DB,
) -> Option<String> {
    // The flow runtime resolves a step's `$flow_expr[...]` before pushing it, so one still here
    // was pushed with no flow state to read (a step test, a dependency job) and would name a
    // queue no worker serves: the job runs on its default tag instead.
    if tag.is_empty() || tag_reads_flow_expr(tag) {
        return None;
    }
    // `$workspace` must resolve the same way the default tags do, or an explicit tag and a default
    // tag from the same workspace address two different worker pools. Resolving costs a lookup,
    // so pay it only for tags that actually interpolate `$workspace`.
    let tag_ws = if tag.contains("$workspace") {
        crate::tags::tag_workspace_id(workspace_id, db).await
    } else {
        workspace_id.to_string()
    };
    Some(interpolate_args(tag.to_string(), args, &tag_ws))
}

/// The tag a dependency job is routed to when the instance's `dependency_job_tag` setting is
/// set, whatever tag the script, flow or app itself runs on. `None` for any other job kind, for
/// bunnative (its bundle must be built on a worker with the bun tag), for a binary prebuild
/// pushed with an explicit `auto_build_binary_tag`, or when the setting is unset. An untagged
/// prebuild follows its dependency job onto the configured tag.
pub async fn configured_dependency_job_tag(
    job_kind: &JobKind,
    language: Option<&ScriptLang>,
    tagged_binary_prebuild: bool,
    workspace_id: &str,
    db: &DB,
) -> Option<String> {
    if !job_kind.is_dependency()
        || tagged_binary_prebuild
        || language == Some(&ScriptLang::Bunnative)
    {
        return None;
    }
    let tag = DEPENDENCY_JOB_TAG.load_full();
    let tag = tag.as_deref()?;
    resolve_push_tag(tag, &PushArgs::from(&HashMap::new()), workspace_id, db).await
}

/// Refuses a `tag` the caller chose that the instance's custom tags do not let `w_id` use,
/// judging the queue it resolves to. `args` must be the ones the job is pushed with: resolving
/// with any others checks a queue the job does not land on.
pub async fn check_tag_available_for_push(
    db: &DB,
    w_id: &str,
    tag: &str,
    args: &PushArgs<'_>,
    is_super_admin: bool,
    scope_tags: Option<Vec<&str>>,
) -> Result<(), Error> {
    check_tag_written_as_available_for_push(db, w_id, tag, tag, args, is_super_admin, scope_tags)
        .await
}

/// [`check_tag_available_for_push`] for a `tag` the flow runtime already partly resolved from
/// `written_tag`, the step's tag as its author wrote it.
pub async fn check_tag_written_as_available_for_push(
    db: &DB,
    w_id: &str,
    written_tag: &str,
    tag: &str,
    args: &PushArgs<'_>,
    is_super_admin: bool,
    scope_tags: Option<Vec<&str>>,
) -> Result<(), Error> {
    let Some(resolved_tag) = resolve_push_tag(tag, args, w_id, db).await else {
        return Ok(());
    };
    windmill_common::jobs::check_tag_available_for_workspace_internal(
        db,
        w_id,
        written_tag,
        Some(&resolved_tag),
        crate::tags::tag_workspace_id(w_id, db),
        is_super_admin,
        scope_tags,
    )
    .await
}

pub fn fullpath_with_workspace(
    workspace_id: &str,
    script_path: Option<&String>,
    job_kind: &JobKind,
) -> String {
    let path = script_path.map(String::as_str).unwrap_or("tmp/main");
    let is_flow = job_kind.is_flow();
    format!(
        "{}/{}/{}",
        workspace_id,
        if is_flow { "flow" } else { "script" },
        path,
    )
}

pub async fn get_result_by_id(
    db: Pool<Postgres>,
    w_id: String,
    flow_id: Uuid,
    node_id: String,
    json_path: Option<String>,
) -> error::Result<Box<RawValue>> {
    match get_result_by_id_from_running_flow(
        &db,
        w_id.as_str(),
        &flow_id,
        node_id.as_str(),
        json_path.clone(),
    )
    .await
    {
        Ok(res) => Ok(res),
        Err(e) => {
            let root = sqlx::query!(
                "SELECT
                    id As \"id!\",
                    flow_status->'restarted_from'->'flow_job_id' AS \"restarted_from: Json<Uuid>\"
                FROM v2_job_status
                WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $1), $1) = id",
                flow_id
            )
            .fetch_optional(&db)
            .await?;
            match root {
                Some(root) => {
                    let restarted_from_id = not_found_if_none(
                        root.restarted_from,
                        "Id not found in the result's mapping of the root job and root job had no restarted from information",
                        format!("parent: {}, root: {}, id: {}, error: {e:#}", flow_id, root.id, node_id),
                    )?;

                    get_result_by_id_from_original_flow(
                        &db,
                        w_id.as_str(),
                        &restarted_from_id,
                        node_id.as_str(),
                        json_path.clone(),
                    )
                    .await
                }
                None => {
                    get_result_by_id_from_original_flow(
                        &db,
                        w_id.as_str(),
                        &flow_id,
                        node_id.as_str(),
                        json_path.clone(),
                    )
                    .await
                }
            }
        }
    }
}

pub async fn get_result_and_success_by_id_from_flow(
    db: &Pool<Postgres>,
    w_id: &str,
    flow_id: &Uuid,
    node_id: &str,
    json_path: Option<String>,
) -> error::Result<(Box<RawValue>, bool)> {
    // used exclusively for sync webhook/routes, do not handle restarted_from like get_result_by_id
    let mut job_result =
        match get_result_by_id_from_running_flow_inner(db, w_id, flow_id, node_id).await {
            Ok(res) => Some(res),
            Err(err) => match err {
                Error::NotFound(_) => None,
                _ => return Err(err),
            },
        };

    let mut completed = false;

    if job_result.is_none() {
        job_result =
            match get_result_by_id_from_original_flow_inner(db, w_id, flow_id, node_id).await {
                Ok(res) => Some(res),
                Err(err) => match err {
                    Error::NotFound(_) => None,
                    _ => return Err(err),
                },
            };
        completed = job_result.is_some();
    }

    let job_result = not_found_if_none(
        job_result,
        "Node result",
        format!("flow: {}, node: {}", flow_id, node_id),
    )?;

    let success = match &job_result {
        JobResult::SingleJob(job_id) => {
            sqlx::query_scalar!(
                "SELECT status = 'success' OR status = 'skipped' AS \"success!\"
                FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
                job_id,
                w_id
            )
            .fetch_one(db)
            .await?
        }
        JobResult::ListJob(_) => {
            let query = format!(
                r#"WITH modules AS (
                    SELECT jsonb_array_elements(flow_status->'modules') AS module
                    FROM {}
                    WHERE id = $1
                )
                SELECT module->>'type' = 'Success'
                FROM modules
                WHERE module->>'id' = $2"#,
                if completed {
                    "v2_job_completed"
                } else {
                    "v2_job_status"
                }
            );
            sqlx::query_scalar(&query)
                .bind(flow_id)
                .bind(node_id)
                .fetch_optional(db)
                .await?
                .ok_or_else(|| {
                    error::Error::internal_err(format!(
                        "Could not get success from flow job status"
                    ))
                })?
        }
    };

    let result = extract_result_from_job_result(db, w_id, job_result, json_path).await?;

    Ok((result, success))
}

#[async_recursion]
pub async fn get_result_by_id_from_running_flow(
    db: &Pool<Postgres>,
    w_id: &str,
    flow_id: &Uuid,
    node_id: &str,
    json_path: Option<String>,
) -> error::Result<Box<RawValue>> {
    let result_id = get_result_by_id_from_running_flow_inner(db, w_id, flow_id, node_id).await?;

    extract_result_from_job_result(db, w_id, result_id, json_path).await
}

#[async_recursion]
pub async fn get_result_by_id_from_running_flow_inner(
    db: &Pool<Postgres>,
    w_id: &str,
    flow_id: &Uuid,
    node_id: &str,
) -> error::Result<JobResult> {
    let flow_job_result = sqlx::query!(
        "SELECT flow_leaf_jobs->$1::text AS \"leaf_jobs: Json<Box<RawValue>>\", v2_job.parent_job
        FROM v2_job_status
        LEFT JOIN v2_job ON v2_job.id = v2_job_status.id AND v2_job.workspace_id = $3
        WHERE COALESCE((SELECT flow_innermost_root_job FROM v2_job WHERE id = $2), $2) = v2_job_status.id",
        node_id,
        flow_id,
        w_id,
    )
    .fetch_optional(db)
    .await?;

    // tracing::error!("flow_job_result: {:?} {:?}", flow_job_result, flow_id);
    let flow_job_result = windmill_common::utils::not_found_if_none(
        flow_job_result,
        "Root job of parent runnnig flow",
        format!("parent: {}, id: {}", flow_id, node_id),
    )?;
    // tracing::error!("flow_job_result: {:?}, {:?}", flow_job_result.leaf_jobs, flow_job_result.parent_job);

    let job_result = flow_job_result
        .leaf_jobs
        .map(|x| serde_json::from_str(x.get()).ok())
        .flatten();

    if job_result.is_none() && flow_job_result.parent_job.is_some() {
        let parent_job = flow_job_result.parent_job.unwrap();
        let root_job = sqlx::query_scalar!(
            "SELECT flow_innermost_root_job FROM v2_job WHERE id = $1",
            parent_job
        )
        .fetch_optional(db)
        .await?
        .flatten()
        .unwrap_or(parent_job);
        return get_result_by_id_from_running_flow_inner(db, w_id, &root_job, node_id).await;
    }

    let result_id = windmill_common::utils::not_found_if_none(
        job_result,
        "Flow result by id",
        format!("parent: {}, id: {}", flow_id, node_id),
    )?;

    Ok(result_id)
}

async fn get_completed_flow_node_result_rec(
    db: &Pool<Postgres>,
    w_id: &str,
    created_at: DateTime<Utc>,
    subflows: Vec<(Uuid, FlowStatus)>,
    node_id: &str,
) -> error::Result<Option<JobResult>> {
    for (id, flow_status) in subflows {
        if let Some(node_status) = flow_status
            .modules
            .iter()
            .find(|module| module.id() == node_id)
        {
            return match (node_status.job(), node_status.flow_jobs()) {
                (Some(leaf_job_uuid), None) => Ok(Some(JobResult::SingleJob(leaf_job_uuid))),
                (Some(_), Some(jobs)) => Ok(Some(JobResult::ListJob(jobs))),
                _ => Err(error::Error::NotFound(format!(
                    "Flow result by id not found going top-down in subflows (currently: {}), (id: {})",
                    id,
                    node_id,
                ))),
            };
        } else {
            let subflows = sqlx::query!(
                "SELECT j.id, jc.flow_status AS \"flow_status!: Json<FlowStatus>\"
                FROM v2_job j
                JOIN v2_job_completed jc ON j.id = jc.id
                WHERE j.parent_job = $1 AND j.workspace_id = $2 AND j.created_at >= $3 AND jc.flow_status IS NOT NULL",
                id,
                w_id,
                created_at
            )
            .map(|record| (record.id, record.flow_status.0))
            .fetch_all(db)
            .await?;
            match Box::pin(get_completed_flow_node_result_rec(
                db, w_id, created_at, subflows, node_id,
            ))
            .await?
            {
                Some(res) => return Ok(Some(res)),
                None => continue,
            };
        }
    }

    Ok(None)
}

async fn get_result_by_id_from_original_flow_inner(
    db: &Pool<Postgres>,
    w_id: &str,
    completed_flow_id: &Uuid,
    node_id: &str,
) -> error::Result<JobResult> {
    let flow_job = sqlx::query!(
        "SELECT jc.id, jc.flow_status AS \"flow_status!: Json<FlowStatus>\", j.created_at
        FROM v2_job_completed jc
        JOIN v2_job j ON j.id = jc.id
        WHERE jc.id = $1 AND jc.workspace_id = $2 AND jc.flow_status IS NOT NULL",
        completed_flow_id,
        w_id
    )
    .map(|record| (record.id, record.flow_status.0, record.created_at))
    .fetch_optional(db)
    .await?;

    let (id, flow_status, created_at) = not_found_if_none(
        flow_job,
        "Root completed job",
        format!("root: {}, id: {}", completed_flow_id, node_id),
    )?;

    match get_completed_flow_node_result_rec(db, w_id, created_at, vec![(id, flow_status)], node_id)
        .await?
    {
        Some(res) => Ok(res),
        None => Err(Error::NotFound(format!(
            "Flow result by id not found going top-down from {}, (id: {})",
            completed_flow_id, node_id
        ))),
    }
}

async fn get_result_by_id_from_original_flow(
    db: &Pool<Postgres>,
    w_id: &str,
    completed_flow_id: &Uuid,
    node_id: &str,
    json_path: Option<String>,
) -> error::Result<Box<RawValue>> {
    let job_result =
        get_result_by_id_from_original_flow_inner(db, w_id, completed_flow_id, node_id).await?;
    extract_result_from_job_result(db, w_id, job_result, json_path).await
}

async fn extract_result_from_job_result(
    db: &Pool<Postgres>,
    w_id: &str,
    job_result: JobResult,
    json_path: Option<String>,
) -> error::Result<Box<RawValue>> {
    match job_result {
        JobResult::ListJob(job_ids) => match json_path {
            Some(json_path) => {
                let mut parts = json_path.split(".");

                let Some(idx) = parts.next().map(|x| x.parse::<usize>().ok()).flatten() else {
                    return Ok(to_raw_value(&serde_json::Value::Null));
                };
                let Some(job_id) = job_ids.get(idx).cloned() else {
                    return Ok(to_raw_value(&serde_json::Value::Null));
                };
                Ok(sqlx::query_scalar!(
                    "SELECT result #> $3 AS \"result: Json<Box<RawValue>>\"
                    FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
                    job_id,
                    w_id,
                    parts.collect::<Vec<_>>() as Vec<&str>
                )
                .fetch_optional(db)
                .await?
                .flatten()
                .map(|x| x.0)
                .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)))
            }
            None => {
                let rows = sqlx::query!(
                    "SELECT id, result  AS \"result: Json<Box<RawValue>>\"
                    FROM v2_job_completed WHERE id = ANY($1) AND workspace_id = $2",
                    job_ids.as_slice(),
                    w_id
                )
                .fetch_all(db)
                .await?
                .into_iter()
                .filter_map(|x| x.result.map(|y| (x.id, y)))
                .collect::<HashMap<Uuid, Json<Box<RawValue>>>>();
                let result = job_ids
                    .into_iter()
                    .map(|id| {
                        rows.get(&id)
                            .map(|x| x.0.clone())
                            .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null))
                    })
                    .collect::<Vec<_>>();
                Ok(to_raw_value(&result))
            }
        },
        JobResult::SingleJob(x) => Ok(sqlx::query!(
            "SELECT result #> $3 AS \"result: Json<Box<RawValue>>\"
            FROM v2_job_completed WHERE id = $1 AND workspace_id = $2",
            x,
            w_id,
            json_path
                .as_ref()
                .map(|x| x.split(".").collect::<Vec<_>>())
                .unwrap_or_default() as Vec<&str>,
        )
        .fetch_optional(db)
        .await?
        .map(|r| r.result.map(|x| x.0))
        .flatten()
        .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null))),
    }
}

pub async fn job_is_complete(db: &DB, id: Uuid, w_id: &str) -> error::Result<bool> {
    Ok(sqlx::query_scalar!(
        "SELECT EXISTS(SELECT 1 FROM v2_job_completed WHERE id = $1 AND workspace_id = $2)",
        id,
        w_id
    )
    .fetch_one(db)
    .await?
    .unwrap_or(false))
}

pub fn get_mini_completed_job<'a, 'e, A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a>(
    id: &'a Uuid,
    w_id: &'a str,
    db: A,
) -> impl Future<Output = error::Result<Option<MiniCompletedJob>>> + Send + 'a {
    async move {
        let mut conn = db.acquire().await?;
        sqlx::query_as!(
            MiniCompletedJob,
            "SELECT
            j.id, j.workspace_id, j.runnable_id AS \"runnable_id: ScriptHash\", q.scheduled_for, q.started_at, j.parent_job, j.flow_innermost_root_job, j.runnable_path, j.kind as \"kind!: JobKind\", j.permissioned_as,
            j.created_by, j.script_lang AS \"script_lang: ScriptLang\", j.permissioned_as_email, j.flow_step_id, j.trigger_kind AS \"trigger_kind: TriggerKindLabel\", j.trigger, j.priority, j.concurrent_limit, j.tag, j.cache_ttl, q.cache_ignore_s3_path, q.runnable_settings_handle,
            COALESCE(j.args->'build_binary_only' = 'true'::jsonb, false) AS \"build_binary_only!\",
            p.job_token_scopes AS \"job_token_scopes?\"
            FROM v2_job j LEFT JOIN v2_job_queue q ON j.id = q.id
            LEFT JOIN job_perms p ON p.job_id = j.id
            WHERE j.id = $1 AND j.workspace_id = $2",
            id,
            w_id
        )
        .fetch_optional(&mut *conn)
        .await
        .map_err(Into::into)
    }
}

pub enum PushIsolationLevel<'c> {
    IsolatedRoot(DB),
    Isolated(UserDB, Authed),
    Transaction(Transaction<'c, Postgres>),
}

impl<'c> PushIsolationLevel<'c> {
    pub async fn into_tx(self) -> error::Result<Transaction<'c, Postgres>> {
        match self {
            PushIsolationLevel::Isolated(db, authed) => Ok((db.begin(&authed).await?).into()),
            PushIsolationLevel::IsolatedRoot(db) => Ok(db.begin().await?),
            PushIsolationLevel::Transaction(tx) => Ok(tx),
        }
    }
}

#[macro_export]
macro_rules! fetch_scalar_isolated {
    ( $query:expr, $tx:expr) => {
        match $tx {
            PushIsolationLevel::IsolatedRoot(db) => {
                let r = $query.fetch_optional(&db).await;
                $tx = PushIsolationLevel::IsolatedRoot(db);
                r
            }
            PushIsolationLevel::Isolated(db, user) => {
                let mut ntx = db.clone().begin(&user).await?;
                let r = $query.fetch_optional(&mut *ntx).await;
                $tx = PushIsolationLevel::Isolated(db, user);
                r
            }
            PushIsolationLevel::Transaction(mut tx) => {
                let r = $query.fetch_optional(&mut *tx).await;
                $tx = PushIsolationLevel::Transaction(tx);
                r
            }
        }
    };
}

use sqlx::types::JsonRawValue;

#[derive(Debug, Clone, Default)]
pub struct PushArgsOwned {
    pub extra: Option<HashMap<String, Box<RawValue>>>,
    pub args: HashMap<String, Box<RawValue>>,
}

#[derive(Debug)]
pub struct PushArgs<'c> {
    pub extra: Option<HashMap<String, Box<RawValue>>>,
    pub args: &'c HashMap<String, Box<RawValue>>,
}

impl<'c> From<&'c HashMap<String, Box<RawValue>>> for PushArgs<'c> {
    fn from(args: &'c HashMap<String, Box<RawValue>>) -> Self {
        PushArgs { extra: None, args }
    }
}

impl<'c> Serialize for PushArgs<'c> {
    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
    where
        S: serde::Serializer,
    {
        let mut map = serializer.serialize_map(Some(
            self.args.len() + self.extra.as_ref().map(|x| x.len()).unwrap_or_default(),
        ))?;
        let mut in_extra = vec![];
        if let Some(extra) = &self.extra {
            for (k, v) in extra {
                map.serialize_entry(k, v)?;
                in_extra.push(k);
            }
        }
        for (k, v) in self.args {
            if !in_extra.contains(&k) {
                map.serialize_entry(k, v)?;
            }
        }
        map.end()
    }
}

impl<'c> From<&PushArgs<'c>> for HashMap<String, Box<RawValue>> {
    fn from(args: &PushArgs<'c>) -> Self {
        let mut map = HashMap::new();
        map.extend(args.args.clone().into_iter());
        if let Some(ref extra) = args.extra {
            map.extend(extra.clone().into_iter());
        }
        map
    }
}

impl PushArgsOwned {
    pub fn empty() -> Self {
        PushArgsOwned { extra: None, args: HashMap::new() }
    }
}

pub fn empty_result() -> Box<RawValue> {
    return JsonRawValue::from_string("{}".to_string()).unwrap();
}

// impl<T> From<PushArgsInner<T>> for PushArgs<T> {
//     fn from(value: PushArgsInner<T>) -> Self {
//         PushArgs::Unwrapped(value)
//     }
// }

lazy_static::lazy_static! {
    pub static ref RE_ARG_TAG: Regex = Regex::new(r#"\$args\[((?:\w+\.)*\w+)\]"#).unwrap();
    pub static ref RE_FLOW_EXPR_TAG: Regex =
        Regex::new(r#"\$flow_expr\[((?:\w+\.)*\w+)\]"#).unwrap();
}

#[cfg(feature = "cloud")]
lazy_static::lazy_static! {
    // Cache for superadmin status: email -> (is_super_admin, expiry_timestamp)
    static ref SUPERADMIN_CACHE: Arc<RwLock<HashMap<String, (bool, std::time::Instant)>>> =
        Arc::new(RwLock::new(HashMap::new()));
}

#[cfg(feature = "cloud")]
const SUPERADMIN_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60);

#[cfg(feature = "cloud")]
async fn is_superadmin_cached(db: &Pool<Postgres>, email: &str) -> Result<bool, Error> {
    let now = std::time::Instant::now();

    // Try to get from cache first
    {
        let cache = SUPERADMIN_CACHE.read().await;
        if let Some((is_super_admin, expiry)) = cache.get(email) {
            if *expiry > now {
                return Ok(*is_super_admin);
            }
        }
    }

    // Cache miss or expired, fetch from database
    let is_super_admin =
        sqlx::query_scalar!("SELECT super_admin FROM password WHERE email = $1", email)
            .fetch_optional(db)
            .await?
            .unwrap_or(false);

    // Update cache
    {
        let mut cache = SUPERADMIN_CACHE.write().await;
        cache.insert(
            email.to_string(),
            (is_super_admin, now + SUPERADMIN_CACHE_TTL),
        );
    }

    Ok(is_super_admin)
}

#[cfg(feature = "cloud")]
async fn check_usage_limits(
    db: &Pool<Postgres>,
    workspace_id: &str,
    email: &str,
    check_user_usage: bool,
) -> Result<(i32, Option<i32>), Error> {
    // Get current workspace usage with a simple SELECT (no row lock)
    let workspace_usage = sqlx::query_scalar!(
        "SELECT usage FROM usage
        WHERE id = $1
        AND is_workspace = TRUE
        AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)",
        workspace_id
    )
    .fetch_optional(db)
    .await
    .map_err(|e| Error::internal_err(format!("fetching workspace usage: {e:#}")))?
    .unwrap_or(0);

    // Get current user usage (only for non-premium workspaces)
    let user_usage = if check_user_usage {
        sqlx::query_scalar!(
            "SELECT usage FROM usage
            WHERE id = $1
            AND is_workspace = FALSE
            AND month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)",
            email
        )
        .fetch_optional(db)
        .await
        .map_err(|e| Error::internal_err(format!("fetching user usage: {e:#}")))?
    } else {
        None
    };

    Ok((workspace_usage, user_usage))
}

#[cfg(feature = "cloud")]
use crate::cloud_usage::increment_usage_async;

// Thin wrapper that boxes the future to reduce async state machine sizes in callers.
// Without this, the ~13KB future of push_inner is inlined into every caller's state machine,
// causing stack overflows in deeply nested async call chains (e.g. flow execution).
pub async fn push<'c, 'd>(
    db: &Pool<Postgres>,
    tx: PushIsolationLevel<'c>,
    workspace_id: &str,
    job_payload: JobPayload,
    args: PushArgs<'d>,
    user: &str,
    email: &str,
    permissioned_as: String,
    token_prefix: Option<&str>,
    // Identifies the caller in the audit trail — distinct from the `end_user_email` below,
    // which the job itself reads. The API sources it from `ApiAuthed::username_override`;
    // a token label reaches audit through here alone, since `user` is the token owner.
    // Callers that know no caller beyond `user` pass None.
    audit_end_user: Option<&str>,
    scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
    schedule_path: Option<String>,
    parent_job: Option<Uuid>,
    root_job: Option<Uuid>,
    flow_innermost_root_job: Option<Uuid>,
    job_id: Option<Uuid>,
    _is_flow_step: bool,
    same_worker: bool,
    pre_run_error: Option<&windmill_common::error::Error>,
    visible_to_owner: bool,
    tag: Option<String>,
    custom_timeout: Option<i32>,
    flow_step_id: Option<String>,
    _priority_override: Option<i16>,
    authed: Option<&Authed>,
    running: bool,
    end_user_email: Option<String>,
    trigger: Option<TriggerMetadata>,
    suspended_mode: Option<bool>,
    // Caps the new job's token on top of the target's own `job_token_scopes`: the scopes of
    // the job (or job token) this push acts for, `None` when nothing above it restricts it.
    // A job may never hold a token wider than the job that created it.
    scope_ceiling: Option<&[String]>,
) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
    Box::pin(push_inner(
        db,
        tx,
        workspace_id,
        job_payload,
        args,
        user,
        email,
        permissioned_as,
        token_prefix,
        audit_end_user,
        scheduled_for_o,
        schedule_path,
        parent_job,
        root_job,
        flow_innermost_root_job,
        job_id,
        _is_flow_step,
        same_worker,
        pre_run_error,
        visible_to_owner,
        tag,
        custom_timeout,
        flow_step_id,
        _priority_override,
        authed,
        running,
        end_user_email,
        trigger,
        suspended_mode,
        scope_ceiling,
    ))
    .await
}

// #[instrument(level = "trace", skip_all)]
async fn push_inner<'c, 'd>(
    db: &Pool<Postgres>,
    mut tx: PushIsolationLevel<'c>,
    workspace_id: &str,
    job_payload: JobPayload,
    args: PushArgs<'d>,
    user: &str,
    mut email: &str,
    mut permissioned_as: String,
    token_prefix: Option<&str>,
    audit_end_user: Option<&str>,
    #[allow(unused_mut)] mut scheduled_for_o: Option<chrono::DateTime<chrono::Utc>>,
    schedule_path: Option<String>, //should be removed in favor of the trigger param below
    parent_job: Option<Uuid>,
    root_job: Option<Uuid>,
    flow_innermost_root_job: Option<Uuid>,
    job_id: Option<Uuid>,
    _is_flow_step: bool,
    mut same_worker: bool, // whether the job will be executed on the same worker: if true, the job will be set to running but started_at will not be set.
    pre_run_error: Option<&windmill_common::error::Error>,
    visible_to_owner: bool,
    mut tag: Option<String>,
    custom_timeout: Option<i32>,
    flow_step_id: Option<String>,
    _priority_override: Option<i16>,
    authed: Option<&Authed>,
    running: bool, // whether the job is already running: only set this to true if you don't want the job to be picked up by a worker from the queue. It will also set started_at to now.
    end_user_email: Option<String>,
    trigger: Option<TriggerMetadata>,
    suspended_mode: Option<bool>,
    scope_ceiling: Option<&[String]>,
) -> Result<(Uuid, Transaction<'c, Postgres>), Error> {
    // The worker builds a preview's `_MODULES` arg into the job as its module code. Every
    // caller-reachable value lands in `args` or `extra` (webhook query and headers go to
    // `extra`, WAC children copy their parent's args), so it is dropped from both and only
    // the `JobPayload::Code` arm below sets it, from the server-side `RawCode::modules`.
    let args_without_modules;
    let mut args = {
        let mut extra = args.extra;
        if let Some(extra) = extra.as_mut() {
            extra.remove(MODULES_ARG);
        }
        let args = if args.args.contains_key(MODULES_ARG) {
            let mut stripped = args.args.clone();
            stripped.remove(MODULES_ARG);
            args_without_modules = stripped;
            &args_without_modules
        } else {
            args.args
        };
        PushArgs { extra, args }
    };

    #[cfg(feature = "cloud")]
    if *CLOUD_HOSTED {
        // A fork/dev workspace draws its plan and usage from the root (billing) workspace, so its
        // executions are metered against the parent's quota/bill. Resolves to `workspace_id` itself
        // for a standalone workspace (no behavior change off-fork).
        let billing_w_id =
            windmill_common::workspaces::get_billing_workspace_id(db, workspace_id).await?;
        let team_plan_status =
            windmill_common::workspaces::get_team_plan_status(db, &billing_w_id).await?;
        // we track only non flow steps
        let (workspace_usage, user_usage) = if !matches!(
            job_payload,
            JobPayload::Flow { .. } | JobPayload::RawFlow { .. }
        ) {
            // Check current usage with SELECT (fast, no row locks)
            // Only check user usage for non-premium workspaces
            // `email` here and in the per-user checks below can be a cached dispatch address, up
            // to one notify poll stale; accepted, see `get_email_from_permissioned_as`.
            let (current_workspace_usage, current_user_usage) =
                check_usage_limits(db, &billing_w_id, email, !team_plan_status.premium).await?;

            // Spawn async task to update usage counters in the background
            increment_usage_async(
                db.clone(),
                billing_w_id.clone(),
                if !team_plan_status.premium {
                    Some(email.to_string())
                } else {
                    None
                },
            );

            // Return the current usage + 1 to account for this job
            let workspace_usage_with_new_job = current_workspace_usage + 1;
            let user_usage_with_new_job = if !team_plan_status.premium {
                Some(current_user_usage.unwrap_or(0) + 1)
            } else {
                None
            };

            (Some(workspace_usage_with_new_job), user_usage_with_new_job)
        } else {
            (None, None)
        };

        if !team_plan_status.premium || team_plan_status.is_past_due {
            let is_super_admin = is_superadmin_cached(db, email).await?;

            #[cfg(feature = "private")]
            let recovery_email = crate::jobs_ee::SCHEDULE_RECOVERY_HANDLER_USER_EMAIL;
            #[cfg(not(feature = "private"))]
            let recovery_email = "recovery@windmill.dev";

            if !is_super_admin {
                if !team_plan_status.premium
                    && email != ERROR_HANDLER_USER_EMAIL
                    && email != SCHEDULE_ERROR_HANDLER_USER_EMAIL
                    && email != recovery_email
                    && email != "worker@windmill.dev"
                    && email != SUPERADMIN_SECRET_EMAIL
                    && permissioned_as != SUPERADMIN_SYNC_EMAIL
                    && email != SUPERADMIN_NOTIFICATION_EMAIL
                {
                    let user_usage = if let Some(user_usage) = user_usage {
                        user_usage
                    } else {
                        sqlx::query_scalar!(
                            "SELECT usage.usage + 1 FROM usage
                            WHERE is_workspace IS FALSE AND
                            month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
                            AND id = $1",
                            email
                        )
                        .fetch_optional(db)
                        .await?
                        .flatten()
                        .unwrap_or(1)
                    };

                    if user_usage > MAX_FREE_EXECS
                        && !matches!(job_payload, JobPayload::Dependencies { .. })
                        && !matches!(job_payload, JobPayload::FlowDependencies { .. })
                        && !matches!(job_payload, JobPayload::AppDependencies { .. })
                    {
                        return Err(error::Error::QuotaExceeded(format!(
                            "User {email} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                        )));
                    }

                    let in_queue = sqlx::query_scalar!(
                        "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.permissioned_as_email = $1",
                        email
                    )
                    .fetch_one(db)
                    .await?
                    .unwrap_or(0);

                    if in_queue > MAX_FREE_EXECS as i64 {
                        return Err(error::Error::QuotaExceeded(format!(
                            "User {email} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                        )));
                    }

                    let concurrent_runs = sqlx::query_scalar!(
                        "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE q.running = true AND j.permissioned_as_email = $1",
                        email
                    )
                    .fetch_one(db)
                    .await?
                    .unwrap_or(0);

                    if concurrent_runs > MAX_FREE_CONCURRENT_RUNS as i64 {
                        return Err(error::Error::QuotaExceeded(format!(
                            "User {email} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
                        )));
                    }
                }

                if workspace_id != "demo" {
                    let workspace_usage = if let Some(workspace_usage) = workspace_usage {
                        workspace_usage
                    } else {
                        sqlx::query_scalar!(
                            "SELECT usage.usage + 1 FROM usage
                            WHERE is_workspace IS TRUE AND
                            month_ = EXTRACT(YEAR FROM current_date) * 12 + EXTRACT(MONTH FROM current_date)
                            AND id = $1",
                            billing_w_id
                        )
                        .fetch_optional(db)
                        .await?
                        .flatten()
                        .unwrap_or(1)
                    };
                    if team_plan_status.premium {
                        // team plan is premium but past due, we check if the workspace has exceeded the max tolerated executions
                        if team_plan_status.max_tolerated_executions.is_none()
                            || workspace_usage > team_plan_status.max_tolerated_executions.unwrap()
                        {
                            // A canceled plan reuses the past-due cap for its last month. Only read
                            // here, on the refusal path, to word it.
                            let plan_canceled = sqlx::query_scalar::<_, bool>(
                                "SELECT COALESCE(plan = 'team_canceled', false) FROM workspace_settings WHERE workspace_id = $1",
                            )
                            .bind(&billing_w_id)
                            .fetch_optional(db)
                            .await?
                            .unwrap_or(false);
                            return Err(error::Error::QuotaExceeded(if plan_canceled {
                                format!("Workspace {workspace_id} team plan was canceled and has used the executions paid for this month. Jobs can run again on the free plan from the 1st, or right away by subscribing again in the workspace settings.")
                            } else {
                                format!("Workspace {workspace_id} team plan is past due and isn't allowed to run any more jobs. Please fix your payment method in the workspace settings.")
                            }));
                        }
                    } else {
                        if workspace_usage > MAX_FREE_EXECS
                            && !matches!(job_payload, JobPayload::Dependencies { .. })
                            && !matches!(job_payload, JobPayload::FlowDependencies { .. })
                            && !matches!(job_payload, JobPayload::AppDependencies { .. })
                        {
                            return Err(error::Error::QuotaExceeded(format!(
                                "Workspace {workspace_id} has exceeded the free usage limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                            )));
                        }

                        // These two burst guards intentionally stay keyed to `workspace_id`, not the
                        // billing root: the shared caps are the monthly usage above (metered to the
                        // root) and the per-user in-queue/concurrent guards further up (keyed by email,
                        // global across the whole family). Keying these to the root would count the
                        // root's own queue rather than this workspace's load or the family's; a true
                        // family-shared burst cap would need a family-wide subquery, not worth the
                        // hot-path cost for this downgrade-only soft guard.
                        let in_queue_workspace = sqlx::query_scalar!(
                            "SELECT COUNT(id) FROM v2_job_queue WHERE workspace_id = $1",
                            workspace_id
                        )
                        .fetch_one(db)
                        .await?
                        .unwrap_or(0);

                        if in_queue_workspace > MAX_FREE_EXECS as i64 {
                            return Err(error::Error::QuotaExceeded(format!(
                                "Workspace {workspace_id} has exceeded the jobs in queue limit of {MAX_FREE_EXECS} that applies outside of premium workspaces."
                            )));
                        }

                        let concurrent_runs_workspace = sqlx::query_scalar!(
                        "SELECT COUNT(id) FROM v2_job_queue WHERE running = true AND workspace_id = $1",
                            workspace_id
                        )
                        .fetch_one(db)
                        .await?
                        .unwrap_or(0);

                        if concurrent_runs_workspace > MAX_FREE_CONCURRENT_RUNS as i64 {
                            return Err(error::Error::QuotaExceeded(format!(
                                "Workspace {workspace_id} has exceeded the concurrent runs limit of {MAX_FREE_CONCURRENT_RUNS} that applies outside of premium workspaces."
                            )));
                        }
                    }
                }
            }
        }
    }

    #[derive(Default)]
    struct JobPayloadUntagged {
        runnable_id: Option<i64>,
        runnable_path: Option<String>,
        raw_code_tuple: Option<(String, Option<String>)>, // (content, lock)
        job_kind: JobKind,
        raw_flow: Option<FlowValue>,
        flow_status: Option<FlowStatus>,
        language: Option<ScriptLang>,
        cache_ttl: Option<i32>,
        cache_ignore_s3_path: Option<bool>,
        dedicated_worker: Option<bool>,
        _low_level_priority: Option<i16>,
        concurrency_settings: ConcurrencySettings,
        debouncing_settings: DebouncingSettings,
        retry_settings: RetrySettings,
        labels: Option<Vec<String>>,
        /// The target's own `job_token_scopes` setting.
        job_token_scopes: Option<Vec<String>>,
        /// A `dependencies` job that only compiles an already-deployed script's binary.
        /// It shares the job kind, but not the queue policy lock generation needs.
        build_binary_only: bool,
    }
    let mut preprocessed = None;
    #[allow(unused)]
    let JobPayloadUntagged {
        runnable_id,
        runnable_path,
        raw_code_tuple,
        mut job_kind,
        raw_flow,
        flow_status,
        language,
        cache_ttl,
        cache_ignore_s3_path,
        dedicated_worker,
        _low_level_priority,
        mut concurrency_settings,
        debouncing_settings,
        retry_settings,
        labels,
        job_token_scopes,
        build_binary_only,
    } = match job_payload {
        JobPayload::ScriptHash {
            hash,
            path,
            cache_ttl,
            cache_ignore_s3_path,
            language,
            dedicated_worker,
            priority,
            apply_preprocessor,
            concurrency_settings,
            debouncing_settings,
            labels,
            job_token_scopes,
        } => {
            if apply_preprocessor {
                preprocessed = Some(false);
            }

            JobPayloadUntagged {
                runnable_id: Some(hash.0),
                runnable_path: Some(path),
                job_kind: JobKind::Script,
                language: Some(language),
                concurrency_settings,
                debouncing_settings,
                cache_ttl,
                cache_ignore_s3_path,
                dedicated_worker,
                _low_level_priority: priority,
                labels,
                job_token_scopes,
                ..Default::default()
            }
        }
        JobPayload::FlowScript {
            id, // flow_node(id).
            language,
            cache_ttl,
            cache_ignore_s3_path,
            dedicated_worker,
            path,
            concurrency_settings,
        } => JobPayloadUntagged {
            runnable_id: Some(id.0),
            runnable_path: Some(path),
            job_kind: JobKind::FlowScript,
            language: Some(language),
            concurrency_settings,
            cache_ttl,
            cache_ignore_s3_path,
            dedicated_worker,
            ..Default::default()
        },
        JobPayload::FlowNode { id, path, no_inherited_flow_env } => {
            let data = cache::flow::fetch_flow(db, id).await?;
            let value = data.value();
            let status = Some(FlowStatus { no_inherited_flow_env, ..FlowStatus::new(value) });
            // Keep inserting `value` if not all workers are updated.
            // Starting at `v1.440`, the value is fetched on pull from the flow node id.
            let value_o = if !MIN_VERSION_IS_AT_LEAST_1_440.met().await {
                Some(value.clone())
            } else {
                // `raw_flow` is fetched on pull.
                None
            };
            JobPayloadUntagged {
                runnable_id: Some(id.0),
                runnable_path: Some(path),
                job_kind: JobKind::FlowNode,
                raw_flow: value_o,
                flow_status: status,
                ..Default::default()
            }
        }
        JobPayload::AppScript {
            id, // app_script(id).
            path,
            language,
            cache_ttl,
        } => JobPayloadUntagged {
            runnable_id: Some(id.0),
            runnable_path: path,
            job_kind: JobKind::AppScript,
            language: Some(language),
            cache_ttl,
            ..Default::default()
        },
        JobPayload::ScriptHub { path, apply_preprocessor } => {
            if path == "hub/7771/slack" || path == "hub/7836/slack" || path == "hub/9084/slack" {
                // these scripts send app reports to slack
                // they use the slack bot token and should therefore be run with permissions to access it
                permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
                email = SUPERADMIN_NOTIFICATION_EMAIL;
            }

            if apply_preprocessor {
                preprocessed = Some(false);
            }

            let hub_script =
                get_full_hub_script_by_path(StripPath(path.clone()), &HTTP_CLIENT, Some(db))
                    .await?;

            JobPayloadUntagged {
                runnable_path: Some(path),
                job_kind: JobKind::Script_Hub,
                language: Some(hub_script.language),
                ..Default::default()
            }
        }
        JobPayload::Code(RawCode {
            content,
            path,
            hash,
            mut language,
            lock,
            cache_ttl,
            cache_ignore_s3_path,
            dedicated_worker,
            tag: _,
            concurrency_settings,
            debouncing_settings,
            modules,
        }) => {
            // Reconcile the preview language with the `//native` annotation, mirroring the
            // deploy-time logic in `worker_lockfiles`. The editor sends `bun` for a TypeScript
            // script even when it carries `//native`, which would otherwise tag the preview as
            // `bun` and route it to a regular bun worker. A native-mode worker neither matches
            // the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native`
            // script on a native-only worker setup fails. Normalizing to `bunnative` (tag
            // `nativets`) makes the preview run exactly like the deployed script would.
            if language == ScriptLang::Bun || language == ScriptLang::Bunnative {
                let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
                if anns.native && language == ScriptLang::Bun {
                    language = ScriptLang::Bunnative;
                } else if !anns.native && language == ScriptLang::Bunnative {
                    language = ScriptLang::Bun;
                }
            }
            if let Some(ref modules) = modules {
                match serde_json::to_string(modules).and_then(|s| RawValue::from_string(s)) {
                    Ok(raw) => {
                        let extra = args.extra.get_or_insert_with(HashMap::new);
                        extra.insert(MODULES_ARG.to_string(), raw);
                    }
                    Err(e) => {
                        tracing::warn!("Failed to serialize modules for preview job: {e}");
                    }
                }
            }
            JobPayloadUntagged {
                runnable_id: hash,
                runnable_path: path,
                raw_code_tuple: Some((content, lock)),
                job_kind: JobKind::Preview,
                language: Some(language),
                concurrency_settings: concurrency_settings.into(),
                debouncing_settings,
                cache_ttl,
                cache_ignore_s3_path,
                dedicated_worker,
                ..Default::default()
            }
        }
        JobPayload::Dependencies {
            hash,
            language,
            path,
            dedicated_worker,
            debouncing_settings,
        } => JobPayloadUntagged {
            runnable_id: Some(hash.0),
            runnable_path: Some(path),
            job_kind: JobKind::Dependencies,
            language: Some(language),
            dedicated_worker,
            debouncing_settings,
            ..Default::default()
        },

        JobPayload::BuildBinary { path, hash, language } => JobPayloadUntagged {
            runnable_id: Some(hash.0),
            runnable_path: Some(path),
            job_kind: JobKind::Dependencies,
            language: Some(language),
            build_binary_only: true,
            ..Default::default()
        },

        // CLI usage, is not modifying db, no need for debouncing.
        JobPayload::RawScriptDependencies { script_path, content, language } => {
            JobPayloadUntagged {
                runnable_path: Some(script_path),
                raw_code_tuple: Some((content, None)),
                job_kind: JobKind::Dependencies,
                language: Some(language),
                ..Default::default()
            }
        }

        // CLI usage, is not modifying db, no need for debouncing.
        JobPayload::RawFlowDependencies { path, flow_value } => JobPayloadUntagged {
            runnable_path: Some(path),
            job_kind: JobKind::FlowDependencies,
            raw_flow: Some(flow_value),
            ..Default::default()
        },
        JobPayload::FlowDependencies { path, dedicated_worker, version, debouncing_settings } => {
            #[cfg(test)]
            let skip_compat = args
                .args
                .contains_key("dbg_create_job_for_unexistant_flow_version");

            #[cfg(not(test))]
            let skip_compat = false;

            // Keep inserting `value` if not all workers are updated.
            // Starting at `v1.440`, the value is fetched on pull from the version id.
            let value_o = if !MIN_VERSION_IS_AT_LEAST_1_440.met().await && !skip_compat {
                let mut ntx = tx.into_tx().await?;
                // The version has been inserted only within the transaction.
                let data = cache::flow::fetch_version(&mut *ntx, version).await?;
                tx = PushIsolationLevel::Transaction(ntx);
                Some(data.value().clone())
            } else {
                // `raw_flow` is fetched on pull.
                None
            };
            JobPayloadUntagged {
                runnable_id: Some(version),
                runnable_path: Some(path.clone()),
                job_kind: JobKind::FlowDependencies,
                raw_flow: value_o,
                dedicated_worker,
                debouncing_settings,
                ..Default::default()
            }
        }
        JobPayload::AppDependencies { path, version, debouncing_settings } => JobPayloadUntagged {
            runnable_id: Some(version),
            runnable_path: Some(path.clone()),
            job_kind: JobKind::AppDependencies,
            debouncing_settings,
            ..Default::default()
        },
        JobPayload::RawFlow { mut value, path, restarted_from } => {
            refuse_step_scopes_on_outdated_workers(&value).await?;
            add_virtual_items_if_necessary(&mut value.modules);

            let flow_status: FlowStatus = match restarted_from {
                Some(restarted_from_val) => {
                    let (_, _, _, step_n, truncated_modules, user_states, cleanup_module, _) =
                        restarted_flows_resolution(
                            db,
                            workspace_id,
                            restarted_from_val.flow_job_id,
                            restarted_from_val.step_id.as_str(),
                            restarted_from_val.branch_or_iteration_n,
                            restarted_from_val.flow_version,
                            restarted_from_val.nested.is_some(),
                            // RawFlow queues the request's (possibly edited) definition, not the
                            // stored one, so zombie reuse of the stored step is unsafe here.
                            false,
                        )
                        .await?;
                    FlowStatus {
                        step: step_n,
                        modules: truncated_modules,
                        // failure_module is reset
                        failure_module: Box::new(FlowStatusModuleWParent {
                            parent_module: None,
                            module_status: FlowStatusModule::WaitingForPriorSteps {
                                id: "failure".to_string(),
                            },
                        }),
                        cleanup_module,
                        // retry status is reset
                        retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
                        // TODO: for now, flows with approval conditions aren't supported for restart
                        approval_conditions: None,
                        restarted_from: Some(RestartedFrom {
                            flow_job_id: restarted_from_val.flow_job_id,
                            step_id: restarted_from_val.step_id,
                            branch_or_iteration_n: restarted_from_val.branch_or_iteration_n,
                            flow_version: restarted_from_val.flow_version,
                            branch_chosen: restarted_from_val.branch_chosen,
                            nested: restarted_from_val.nested,
                        }),
                        user_states,
                        preprocessor_module: None,
                        stream_job: None,
                        chat_input_enabled: None,
                        memory_id: None,
                        no_inherited_flow_env: false,
                    }
                }
                _ => {
                    value.preprocessor_module = None;
                    FlowStatus::new(&value)
                } // this is a new flow being pushed, flow_status is set to flow_value
            };
            let cache_ttl = value.cache_ttl.map(|x| x as i32);
            let priority = value.priority;
            JobPayloadUntagged {
                runnable_path: path,
                job_kind: JobKind::FlowPreview,
                flow_status: Some(flow_status),
                cache_ttl,
                cache_ignore_s3_path: value.cache_ignore_s3_path,
                _low_level_priority: priority,
                concurrency_settings: value.concurrency_settings.clone(),
                debouncing_settings: value.debouncing_settings.clone(),
                raw_flow: Some(value),
                ..Default::default()
            }
        }
        JobPayload::SingleStepFlow {
            path,
            hash,
            flow_version,
            language,
            retry,
            error_handler_path,
            error_handler_args,
            skip_handler,
            args,
            cache_ttl,
            cache_ignore_s3_path,
            priority,
            tag_override,
            trigger_path,
            apply_preprocessor,
            debouncing_settings,
            concurrency_settings,
        } => 'ssf: {
            // Determine if this is a flow or a script
            let is_flow = flow_version.is_some();

            // Native retry: a bare script wrapped only to gain a retry (no
            // skip/error-handler modules) is pushed as a real `Script` job that
            // carries the policy in `runnable_settings`. This avoids spawning a
            // one-step flow — and its extra job rows, flow_status, and UI
            // projection — for the common schedule/pipeline retry case. The flow
            // path below is kept only for handler-bearing or flow-wrapping cases.
            //
            // Gated on the runnable-settings min version: on a mixed-version
            // fleet the policy can't be persisted (`insert_rs` would drop it), so
            // we fall back to the flow wrapper to preserve retry semantics.
            // `retry_if` is always materialized natively and evaluated on the
            // failure path (see `eval_retry_if`). On a worker built without the
            // `quickjs` feature it cannot be evaluated and fails closed (no retry);
            // the flow path is not a fallback, since the flow runtime needs quickjs
            // too.
            // A hub script has no hash and runs as a `Script_Hub` job.
            let is_hub = hash.is_none() && path.starts_with("hub/");
            let native_retry = !is_flow
                && skip_handler.is_none()
                && error_handler_path.is_none()
                && (hash.is_some() || is_hub)
                && language.is_some()
                && windmill_common::runnable_settings::min_version_supports_runnable_settings_v0()
                    .await;
            if native_retry {
                if apply_preprocessor {
                    preprocessed = Some(false);
                }
                // Preserve worker affinity: normal ScriptHash pushes carry the
                // script's `dedicated_worker` (it drives the dedicated tag below),
                // but the SingleStepFlow payload doesn't — resolve it from the
                // script row so a dedicated-worker script keeps its dedicated pool.
                // The script's `job_token_scopes` is resolved the same way.
                let (dedicated_worker, job_token_scopes) = if let Some(h) = &hash {
                    // Read on the non-RLS pool: push_inner is also entered with RLS
                    // isolation variants under which the script row may be invisible,
                    // which would mis-resolve dedicated_worker routing.
                    sqlx::query_as::<_, (Option<bool>, Option<Vec<String>>)>(
                        "SELECT dedicated_worker, job_token_scopes FROM script WHERE hash = $1 AND workspace_id = $2",
                    )
                    .bind(h.0)
                    .bind(workspace_id)
                    .fetch_optional(db)
                    .await?
                    .unwrap_or((None, None))
                } else {
                    (None, None)
                };
                break 'ssf JobPayloadUntagged {
                    runnable_id: hash.map(|h| h.0),
                    runnable_path: Some(path),
                    job_kind: if is_hub {
                        JobKind::Script_Hub
                    } else {
                        JobKind::Script
                    },
                    language,
                    dedicated_worker,
                    job_token_scopes,
                    concurrency_settings,
                    debouncing_settings,
                    retry_settings: retry.as_ref().map(RetrySettings::from).unwrap_or_default(),
                    cache_ttl,
                    cache_ignore_s3_path,
                    _low_level_priority: priority,
                    ..Default::default()
                };
            }

            // Build modules list
            let mut modules = vec![];

            // Add skip validation module if provided
            if let Some(skip_handler) = skip_handler {
                let mut skip_input_transforms = HashMap::<String, InputTransform>::new();
                for (arg_name, arg_value) in skip_handler.args {
                    skip_input_transforms
                        .insert(arg_name, InputTransform::Static { value: arg_value });
                }

                modules.push(FlowModule {
                    id: "skip_validation".to_string(),
                    value: to_raw_value(&FlowModuleValue::Script {
                        input_transforms: skip_input_transforms,
                        path: skip_handler.path,
                        hash: None,
                        tag_override: None,
                        is_trigger: None,
                        pass_flow_input_directly: None,
                    }),
                    stop_after_if: Some(StopAfterIf {
                        expr: skip_handler.stop_condition,
                        skip_if_stopped: true,
                        error_message: Some(skip_handler.stop_message),
                        error_include_result: false,
                    }),
                    ..Default::default()
                });
            }

            // Add main module (script or flow)
            let mut main_input_transforms = HashMap::<String, InputTransform>::new();
            for (arg_name, arg_value) in args {
                main_input_transforms.insert(arg_name, InputTransform::Static { value: arg_value });
            }

            let main_module = if is_flow {
                FlowModule {
                    id: "a".to_string(),
                    value: to_raw_value(&FlowModuleValue::Flow {
                        path: path.clone(),
                        input_transforms: main_input_transforms,
                        pass_flow_input_directly: None,
                    }),
                    retry,
                    pass_flow_input_directly: Some(true),
                    ..Default::default()
                }
            } else {
                FlowModule {
                    id: "a".to_string(),
                    value: to_raw_value(&FlowModuleValue::Script {
                        input_transforms: main_input_transforms,
                        path: path.clone(),
                        hash,
                        tag_override,
                        is_trigger: None,
                        pass_flow_input_directly: None,
                    }),
                    retry,
                    apply_preprocessor: Some(apply_preprocessor),
                    ..Default::default()
                }
            };
            modules.push(main_module);

            // Build failure module if error handler is provided
            let failure_module = if let Some(error_handler_path) = error_handler_path {
                let mut input_transforms = HashMap::<String, InputTransform>::new();
                input_transforms.insert(
                    "error".to_string(),
                    InputTransform::Javascript { expr: "error".to_string() },
                );
                input_transforms.insert(
                    "path".to_string(),
                    InputTransform::Static { value: to_raw_value(&path) },
                );
                input_transforms.insert(
                    "is_flow".to_string(),
                    InputTransform::Static { value: to_raw_value(&is_flow) },
                );
                input_transforms.insert(
                    "trigger_path".to_string(),
                    InputTransform::Static { value: to_raw_value(&trigger_path) },
                );
                input_transforms.insert(
                    "workspace_id".to_string(),
                    InputTransform::Static { value: to_raw_value(&workspace_id) },
                );
                input_transforms.insert(
                    "email".to_string(),
                    InputTransform::Static { value: to_raw_value(&email) },
                );
                // for the below transforms to work, make sure that flow_job_id and started_at are added to the eval context when pusing the error handler job
                input_transforms.insert(
                    "job_id".to_string(),
                    InputTransform::Javascript { expr: "flow_job_id".to_string() },
                );
                input_transforms.insert(
                    "started_at".to_string(),
                    InputTransform::Javascript { expr: "started_at".to_string() },
                );

                if let Some(error_handler_args) = error_handler_args {
                    for (arg_name, arg_value) in error_handler_args {
                        input_transforms
                            .insert(arg_name, InputTransform::Static { value: arg_value });
                    }
                }

                Some(Box::new(FlowModule {
                    id: "failure".to_string(),
                    value: to_raw_value(&FlowModuleValue::Script {
                        input_transforms,
                        path: error_handler_path,
                        hash: None,
                        tag_override: None,
                        is_trigger: None,
                        pass_flow_input_directly: None,
                    }),
                    ..Default::default()
                }))
            } else {
                None
            };

            let flow_value = FlowValue {
                modules,
                failure_module,
                concurrency_settings: concurrency_settings.clone(),
                debouncing_settings: debouncing_settings.clone(),
                priority,
                cache_ttl: cache_ttl.map(|val| val as u32),
                cache_ignore_s3_path: cache_ignore_s3_path,
                same_worker: false,
                preserve_step_tags: false,
                early_return: None,
                skip_expr: None,
                preprocessor_module: None,
                chat_input_enabled: None,
                flow_env: None,
                delete_after_use: None,
                delete_after_secs: None,
            };
            // this is a new flow being pushed, flow_status is set to flow_value:
            let flow_status: FlowStatus = FlowStatus::new(&flow_value);
            JobPayloadUntagged {
                runnable_path: Some(path),
                job_kind: JobKind::SingleStepFlow,
                raw_flow: Some(flow_value),
                flow_status: Some(flow_status),
                cache_ttl,
                cache_ignore_s3_path,
                _low_level_priority: priority,
                concurrency_settings,
                debouncing_settings,
                ..Default::default()
            }
        }
        JobPayload::Flow {
            path,
            dedicated_worker,
            apply_preprocessor,
            version,
            labels,
            job_token_scopes,
        } => {
            let mut ntx = tx.into_tx().await?;
            // Do not use the lite version unless all workers are updated.
            let data = if *DISABLE_FLOW_SCRIPT
                || (!MIN_VERSION_IS_AT_LEAST_1_432.met().await && !*CLOUD_HOSTED)
            {
                cache::flow::fetch_version(&mut *ntx, version).await
            } else {
                // Fallback to the original version if the lite version is not found.
                // This also prevent a race condition where the flow is run just after deploy and
                // the lite version is still being created.
                match cache::flow::fetch_version_lite(&mut *ntx, version).await {
                    Ok(data) => Ok(data),
                    Err(_) => cache::flow::fetch_version(&mut *ntx, version).await,
                }
            }?;
            tx = PushIsolationLevel::Transaction(ntx);

            let mut value = data.value().clone();
            refuse_step_scopes_on_outdated_workers(&value).await?;
            let priority = value.priority;
            let cache_ttl = value.cache_ttl.map(|x| x as i32);
            let cache_ignore_s3_path = value.cache_ignore_s3_path;
            let mut concurrency_settings = value.concurrency_settings.clone();
            let mut debouncing_settings = value.debouncing_settings.clone();

            if !apply_preprocessor {
                value.preprocessor_module = None;
            } else {
                tag = None;

                concurrency_settings.concurrent_limit = None;
                // TODO: May be re-enable?
                debouncing_settings.debounce_delay_s = None;

                preprocessed = Some(false);
            }

            // this is a new flow being pushed, status is set to `value`.
            let status = FlowStatus::new(&value);

            // Keep inserting `value` if not all workers are updated.
            // Starting at `v1.440`, the value is fetched on pull from the version id.
            let value_o = if !MIN_VERSION_IS_AT_LEAST_1_440.met().await {
                let mut value = value;
                add_virtual_items_if_necessary(&mut value.modules);
                if same_worker {
                    value.same_worker = true;
                }
                Some(value)
            } else {
                // `raw_flow` is fetched on pull, the mutations from the other branch are replaced
                // by additional checks when handling the flow.
                None
            };
            JobPayloadUntagged {
                runnable_id: Some(version), // Starting from `v1.436`, the version id is used to fetch the value on pull.
                runnable_path: Some(path),
                job_kind: JobKind::Flow,
                raw_flow: value_o,
                flow_status: Some(status),
                cache_ttl,
                cache_ignore_s3_path,
                dedicated_worker,
                _low_level_priority: priority,
                concurrency_settings,
                debouncing_settings,
                labels,
                job_token_scopes,
                ..Default::default()
            }
        }
        JobPayload::RestartedFlow {
            completed_job_id,
            step_id,
            branch_or_iteration_n,
            flow_version,
            branch_chosen,
            nested,
        } => {
            let (
                version,
                flow_path,
                flow_data,
                step_n,
                truncated_modules,
                user_states,
                cleanup_module,
                original_kind,
            ) = restarted_flows_resolution(
                db,
                workspace_id,
                completed_job_id,
                step_id.as_str(),
                branch_or_iteration_n,
                flow_version,
                nested.is_some(),
                // RestartedFlow resolves and queues the completed job's stored definition, so the
                // step validated for reuse is the one that will run.
                true,
            )
            .await?;

            let restarted_flow_status = FlowStatus {
                step: step_n,
                modules: truncated_modules,
                // failure_module is reset
                failure_module: Box::new(FlowStatusModuleWParent {
                    parent_module: None,
                    module_status: FlowStatusModule::WaitingForPriorSteps {
                        id: "failure".to_string(),
                    },
                }),
                cleanup_module,
                // retry status is reset
                retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
                // TODO: for now, flows with approval conditions aren't supported for restart
                approval_conditions: None,
                restarted_from: Some(RestartedFrom {
                    flow_job_id: completed_job_id,
                    step_id,
                    branch_or_iteration_n,
                    flow_version,
                    branch_chosen,
                    nested,
                }),
                user_states,
                preprocessor_module: None,
                stream_job: None,
                chat_input_enabled: None,
                memory_id: None,
                no_inherited_flow_env: false,
            };
            // The completed job's own scopes are gone with its `job_perms` row, so the restart
            // takes the flow's current setting (and the restarting caller's ceiling).
            let job_token_scopes = match &flow_path {
                Some(flow_path) => sqlx::query_scalar::<_, Option<Vec<String>>>(
                    "SELECT job_token_scopes FROM flow WHERE path = $1 AND workspace_id = $2",
                )
                .bind(flow_path)
                .bind(workspace_id)
                .fetch_optional(db)
                .await?
                .flatten(),
                None => None,
            };
            let value = flow_data.value();
            refuse_step_scopes_on_outdated_workers(value).await?;
            let priority = value.priority;
            let concurrency_settings = value.concurrency_settings.clone();
            let debouncing_settings = value.debouncing_settings.clone();
            let cache_ttl = value.cache_ttl.map(|x| x as i32);
            // Keep inserting `value` if not all workers are updated.
            // Starting at `v1.440`, the value is fetched on pull from the version id.
            let value_o = if version.is_none() || !MIN_VERSION_IS_AT_LEAST_1_440.met().await {
                Some(value.clone())
            } else {
                // `raw_flow` is fetched on pull.
                None
            };
            JobPayloadUntagged {
                runnable_id: version,
                runnable_path: flow_path,
                job_kind: original_kind,
                raw_flow: value_o,
                flow_status: Some(restarted_flow_status),
                cache_ttl,
                cache_ignore_s3_path: value.cache_ignore_s3_path,
                _low_level_priority: priority,
                concurrency_settings,
                debouncing_settings,
                job_token_scopes,
                ..Default::default()
            }
        }
        JobPayload::DeploymentCallback { path, debouncing_settings, concurrency_key_append } => {
            // `resolve_concurrency_key` prepends `{workspace_id}/` on cloud builds
            // (compiled into every EE build), so that is what the key must leave room
            // for here.
            #[cfg(feature = "cloud")]
            let reserved_prefix_len = workspace_id.len() + 1;
            #[cfg(not(feature = "cloud"))]
            let reserved_prefix_len = 0;
            let concurrency_key =
                git_sync_concurrency_key(workspace_id, concurrency_key_append, reserved_prefix_len);
            JobPayloadUntagged {
                runnable_path: Some(path.clone()),
                job_kind: JobKind::DeploymentCallback,
                concurrency_settings: ConcurrencySettings {
                    concurrency_key: Some(concurrency_key),
                    concurrent_limit: Some(1),
                    concurrency_time_window_s: Some(0),
                },
                debouncing_settings,
                ..Default::default()
            }
        }
        JobPayload::Identity => {
            JobPayloadUntagged { job_kind: JobKind::Identity, ..Default::default() }
        }
        JobPayload::Noop => JobPayloadUntagged { job_kind: JobKind::Noop, ..Default::default() },
        JobPayload::AIAgent { path } => JobPayloadUntagged {
            runnable_path: Some(path),
            job_kind: JobKind::AIAgent,
            ..Default::default()
        },
    };

    // Guard against an already-stored `concurrent_limit <= 0` reaching the queue: it would
    // register a zero-slot concurrency key and permanently block the job. Coerce it to
    // disabled before it is persisted onto the job row / concurrency key here.
    concurrency_settings = concurrency_settings.normalized();

    // Enforce concurrency limit on all dependency jobs.
    // TODO: We can ignore this for scripts djobs. The main reason we need all djobs to be sequential is because we have
    // nodes_to_relock and we need all locks whose corresponding steps aren't in nodes_to_relock be already present.
    //
    // This is not the case for scripts, so we can potentially have multiple djobs for scripts at the same time.
    if let (Some(path), true) = (
        &runnable_path,
        cfg!(feature = "private")
            && job_kind.is_dependency()
            && !*WMDEBUG_NO_DEBOUNCING
            && MIN_VERSION_SUPPORTS_DEBOUNCING.met().await,
    ) {
        // A binary build takes minutes and writes no lock, so it gets its own key: sharing
        // the lock-generation slot would park the next deploy of that path behind a compile.
        // Still limit 1, to collapse redundant builds of the same script.
        concurrency_settings.concurrency_key = Some(if build_binary_only {
            format!("build_binary:{workspace_id}/{path}")
        } else {
            format!("dependency:{workspace_id}/{path}")
        });
        concurrency_settings.concurrent_limit = Some(1);
    }

    let final_priority: Option<i16>;
    #[cfg(not(feature = "enterprise"))]
    {
        // priority is only available on EE. Do not compute it on CE
        final_priority = None;
    }
    #[cfg(feature = "enterprise")]
    {
        final_priority = if *CLOUD_HOSTED {
            // for cloud hosted instance, priority queues is disabled
            None
        } else if _priority_override.is_some() {
            _priority_override
        } else {
            // else it takes the priority defined at the script/flow level, if it's a script or flow
            _low_level_priority
        }; // else it remains empty, i.e. no priority
    }
    // prioritize flow steps to drain the queue faster
    let final_priority = if flow_step_id.is_some() && final_priority.is_none() {
        Some(0)
    } else if job_kind == JobKind::Dependencies && !build_binary_only {
        // A binary build is a background optimization, not a deploy the user waits on, so
        // it does not get the dependency-job jump ahead of ordinary jobs on its tag.
        Some(0)
    } else {
        final_priority
    };

    let is_running = same_worker || running;
    if let Some(flow) = raw_flow.as_ref() {
        same_worker = same_worker || flow.same_worker;

        for module in flow.modules.iter() {
            if let Some(retry) = &module.retry {
                if retry.max_attempts() > MAX_RETRY_ATTEMPTS {
                    Err(Error::BadRequest(format!(
                        "retry attempts exceeds the maximum of {MAX_RETRY_ATTEMPTS}"
                    )))?
                }

                if matches!(retry.max_interval(), Some(interval) if interval > MAX_RETRY_INTERVAL) {
                    let max = MAX_RETRY_INTERVAL.as_secs();
                    Err(Error::BadRequest(format!(
                        "retry interval exceeds the maximum of {max} seconds"
                    )))?
                }
            }
        }
    }

    let (raw_code, raw_lock) = raw_code_tuple
        .map(|e| (Some(e.0), e.1))
        .unwrap_or_else(|| (None, None));

    let job_token_scopes = windmill_common::scopes::intersect_job_token_scopes(
        scope_ceiling,
        job_token_scopes.as_deref(),
    );

    // A dedicated worker runs every job it serves with its own unscoped worker token, so a
    // job with a restricted token runs on the regular workers of its language instead.
    let tag = if dedicated_worker.is_some_and(|x| x) && job_token_scopes.is_none() {
        let flow_prefix = if job_kind == JobKind::Flow || job_kind == JobKind::FlowDependencies {
            "flow/"
        } else {
            ""
        };
        let full_path = format!(
            "{}{}",
            flow_prefix,
            runnable_path.clone().expect("dedicated script has a path")
        );
        windmill_common::worker::dedicated_worker_tag(workspace_id, &full_path)
    } else if let Some(tag) = configured_dependency_job_tag(
        &job_kind,
        language.as_ref(),
        build_binary_only && tag.is_some(),
        workspace_id,
        db,
    )
    .await
    {
        tag
    } else {
        let interpolated_tag = match tag {
            Some(x) => resolve_push_tag(&x, &args, workspace_id, db).await,
            None => None,
        };
        let effective_ws = per_workspace_tag(&workspace_id, db).await;

        let default = || {
            let ntag = if job_kind.is_flow()
                || job_kind == JobKind::Identity
                || job_kind == JobKind::AIAgent
            {
                "flow".to_string()
            } else if job_kind == JobKind::Dependencies
                || job_kind == JobKind::FlowDependencies
                || job_kind == JobKind::DeploymentCallback
                || job_kind == JobKind::AppDependencies
            {
                // using the dependency tag for deployment callback for now. We can create a separate tag when we need
                "dependency".to_string()
            } else {
                "deno".to_string()
            };
            match &effective_ws {
                Some(ws) => format!("{}-{}", ntag, ws),
                None => ntag,
            }
        };

        interpolated_tag.unwrap_or_else(|| {
            if job_kind.is_preview()
                && PREVIEW_TAGS_OVERRIDE.load(std::sync::atomic::Ordering::Relaxed)
            {
                match &effective_ws {
                    Some(ws) => format!("preview-{}", ws),
                    None => "preview".to_string(),
                }
            } else {
                language
                    .as_ref()
                    .map(|x| {
                        let tag_lang = if x == &ScriptLang::Bunnative {
                            if job_kind == JobKind::Dependencies {
                                ScriptLang::Bun.as_str()
                            } else {
                                ScriptLang::Nativets.as_str()
                            }
                        } else {
                            x.as_str()
                        };
                        match &effective_ws {
                            Some(ws) => format!("{}-{}", tag_lang, ws),
                            None => tag_lang.to_string(),
                        }
                    })
                    .unwrap_or_else(default)
            }
        })
    };

    let mut tx = tx.into_tx().await?;

    let job_id: Uuid = if let Some(job_id) = job_id {
        let conflicting_id = sqlx::query_scalar!("SELECT 1 FROM v2_job WHERE id = $1", job_id)
            .fetch_optional(&mut *tx)
            .await?;

        if conflicting_id.is_some() {
            return Err(Error::BadRequest(format!(
                "Job with id {job_id} already exists"
            )));
        }

        job_id
    } else {
        Ulid::new().into()
    };

    #[cfg(feature = "private")]
    if schedule_path.is_none() && flow_step_id.is_none() {
        crate::jobs_ee::maybe_debounce(
            &debouncing_settings,
            &mut scheduled_for_o,
            &runnable_path,
            workspace_id,
            job_kind,
            job_id,
            &args,
            &mut tx,
        )
        .await?
    }

    let (trigger_path, trigger_kind) = trigger
        .map_or_else(
            || schedule_path.map(|path| (Some(path), JobTriggerKind::Schedule)),
            |trigger| Some((trigger.trigger_path, trigger.trigger_kind)),
        )
        .unzip();

    // Which trigger kinds an instance actually fires. Counted here rather than
    // aggregated from `v2_job` later: that table's only usable index is
    // (workspace_id, created_at), so a windowed GROUP BY over it is a full scan.
    //
    // Root jobs only. A scheduled flow hands every step push its own
    // `schedule_path` (see `FlowJob::schedule_path`), so counting per push would
    // score one run as a fire per step job — a loop pushes two of those per
    // iteration — burying every other kind, and would sit on the per-step path.
    //
    // A job a suspended trigger parks is not a fire: it counts as `fired` only
    // when `resume_suspended_trigger_jobs` releases it, or never if discarded.
    // Both are counted before the queue caps below and the caller's commit, so a
    // rejected push still counts; accepted for a telemetry counter.
    if flow_step_id.is_none() {
        if let Some(kind) = trigger_kind.as_ref() {
            let action = if suspended_mode.unwrap_or(false) {
                "suspended"
            } else {
                "fired"
            };
            windmill_common::feature_usage::log_feature_usage("trigger", action, kind.as_str());
        }
    }

    #[cfg(feature = "cloud")]
    if *CLOUD_HOSTED {
        check_workspace_queue_cap(&mut *tx, workspace_id).await?;
    }

    if has_active_concurrency_limit(concurrency_settings.concurrent_limit) {
        let concurrency_key = resolve_concurrency_key(
            workspace_id,
            &args,
            &runnable_path,
            job_kind,
            concurrency_settings.concurrency_key.clone(),
        );
        #[cfg(feature = "cloud")]
        if *CLOUD_HOSTED {
            check_concurrency_key_queue_cap(&mut *tx, &concurrency_key).await?;
        }
        insert_resolved_concurrency_key(
            &concurrency_key,
            &mut *tx,
            job_id,
            &runnable_path,
            workspace_id,
        )
        .await?;
    }
    let stringified_args = if *JOB_ARGS_AUDIT_LOGS {
        Some(serde_json::to_string(&args).map_err(|e| {
            Error::internal_err(format!(
                "Could not serialize args for audit log of job {job_id}: {e:#}"
            ))
        })?)
    } else {
        None
    };

    let raw_flow = raw_flow.map(Json);
    let preprocessed = preprocessed.or_else(|| match flow_step_id.as_deref() {
        Some("preprocessor") => Some(false),
        _ => None,
    });

    let job_authed = match authed {
        Some(authed)
            if authed.email == email
                && authed.username == permissioned_as_to_username(&permissioned_as) =>
        {
            authed.clone()
        }
        _ => {
            if authed.is_some() {
                tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching direclty permissions for job {job_id}...")
            }
            windmill_common::auth::fetch_authed_from_permissioned_as(
                &permissioned_as,
                email,
                workspace_id,
                &mut *tx,
            )
            .await
            .map_err(|e| match e {
                Error::NotFound(_) => e,
                _ => Error::internal_err(format!(
                    "Could not get permissions directly for job {job_id}: {e:#}"
                )),
            })?
        }
    };

    let folders = job_authed
        .folders
        .iter()
        .filter_map(|x| serde_json::to_value(x).ok())
        .collect::<Vec<_>>();

    // if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)
    //     values ($1, $2, $3, $4, $5, $6, $7, $8)
    //     ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
    //     job_id,
    //     job_authed.email,
    //     job_authed.username,
    //     job_authed.is_admin,
    //     job_authed.is_operator,
    //     folders.as_slice(),
    //     job_authed.groups.as_slice(),
    //     workspace_id,
    // ).execute(&mut *tx).await {
    //     tracing::error!("Could not insert job_perms for job {job_id}: {err:#}");
    // }

    let root_job = if root_job.is_some()
        && (root_job == flow_innermost_root_job.or(parent_job).or(Some(job_id)))
    {
        // We only save the root job if it's not the innermost root job, parent job, or the job itself as an optimization
        // Reference: see [`windmill_worker::common::get_root_job_id`] for logic on determining the root job.
        None
    } else {
        root_job
    };

    let (job_kind, scheduled_for_o) = if suspended_mode.unwrap_or(false) {
        let unassigned_job_kind = match job_kind {
            JobKind::Script => JobKind::UnassignedScript,
            JobKind::Flow => JobKind::UnassignedFlow,
            JobKind::SingleStepFlow => JobKind::UnassignedSinglestepFlow,
            _ => return Err(Error::internal_err(format!("Cannot suspend job of kind {job_kind:?} as it is not a script, flow or single step flow"))),
        };
        (
            unassigned_job_kind,
            Some(Utc::now() + chrono::Duration::days(30)),
        )
    } else {
        (job_kind, scheduled_for_o)
    };

    let runnable_settings_handle = windmill_common::runnable_settings::insert_rs(
        RunnableSettings {
            debouncing_settings: debouncing_settings.insert_cached(db).await?,
            concurrency_settings: concurrency_settings.insert_cached(db).await?,
            retry_settings: retry_settings.insert_cached(db).await?,
        },
        db,
    )
    .await?;

    let (guarded_concurrent_limit, guarded_concurrency_time_window_s) =
        if windmill_common::runnable_settings::min_version_supports_runnable_settings_v0().await {
            (None, None)
        } else {
            (
                concurrency_settings.concurrent_limit,
                concurrency_settings.concurrency_time_window_s,
            )
        };
    // jsonb rejects the `\u0000` escape (22P05), which would fail the whole push, and args are
    // not always what a caller typed: a flow step's are evaluated from the previous step's
    // in-memory result, an agent tool's are written by the model.
    let serialized_args = serde_json::value::to_raw_value(&args).map_err(|e| {
        Error::internal_err(format!("Could not serialize args of job {job_id}: {e:#}"))
    })?;
    let sanitized_args = match strip_json_nul(serialized_args.get()) {
        Cow::Owned(stripped) => RawValue::from_string(stripped).map_err(|e| {
            Error::internal_err(format!("Could not sanitize args of job {job_id}: {e:#}"))
        })?,
        Cow::Borrowed(_) => serialized_args,
    };
    sqlx::query!(
        "WITH inserted_job AS (
            INSERT INTO v2_job (
                id, -- 1
                workspace_id, -- 2
                raw_code, -- 3
                raw_lock, -- 4
                raw_flow, -- 5
                tag, -- 6
                parent_job, -- 7
                created_by, -- 8
                permissioned_as, -- 9
                runnable_id, -- 10
                runnable_path, -- 11
                args, -- 12
                kind, -- 13
                trigger, -- 14
                script_lang, -- 15
                same_worker, -- 16
                pre_run_error, -- 17
                permissioned_as_email, -- 18
                visible_to_owner, -- 19
                flow_innermost_root_job, -- 20
                root_job, -- 38
                concurrent_limit, -- 21
                concurrency_time_window_s, -- 22
                timeout, -- 23
                flow_step_id, -- 24
                cache_ttl, -- 25
                priority, -- 26
                trigger_kind, -- 39
                script_entrypoint_override, -- 12
                preprocessed, -- 27,
                labels -- 44
            ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18,
            $19, $20, $38, $21, $22, $23, $24, $25, $26, $39::job_trigger_kind,
            ($12::JSONB)->>'_ENTRYPOINT_OVERRIDE', $27,
            -- $44 (payload labels) merged with the labels of the runnable's folder, if any
            -- ($45/$46 are runnable_path/workspace_id again, kept separate to avoid parameter type conflicts)
            (SELECT CASE WHEN fl.labels IS NULL THEN $44
                ELSE (SELECT array_agg(DISTINCT lbl) FROM unnest(COALESCE($44, ARRAY[]::TEXT[]) || fl.labels) lbl)
            END
            FROM folder_labels($46, $45) AS fl(labels)))
        ),
        inserted_runtime AS (
            INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)
        ),
        inserted_job_perms AS (
            INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email, job_token_scopes)
            values ($1, $32, $33, $34, $35, $36, $37, $2, $41, $47)
            ON CONFLICT (job_id) DO UPDATE SET email = EXCLUDED.email, username = EXCLUDED.username, is_admin = EXCLUDED.is_admin, is_operator = EXCLUDED.is_operator, folders = EXCLUDED.folders, groups = EXCLUDED.groups, workspace_id = EXCLUDED.workspace_id, end_user_email = EXCLUDED.end_user_email, job_token_scopes = EXCLUDED.job_token_scopes
        )
        INSERT INTO v2_job_queue
            (workspace_id, id, running, scheduled_for, started_at, tag, priority, cache_ignore_s3_path, runnable_settings_handle)
            VALUES ($2, $1, $28, COALESCE($29, now()), CASE WHEN $27 OR $40 THEN now() END, $30, $31, $42, $43)",
        job_id,
        workspace_id,
        raw_code,
        raw_lock,
        raw_flow as Option<Json<FlowValue>>,
        tag,
        parent_job,
        user,
        permissioned_as,
        runnable_id,
        runnable_path.clone(),
        Json(sanitized_args) as Json<Box<RawValue>>,
        job_kind.clone() as JobKind,
        trigger_path.flatten(),
        language as Option<ScriptLang>,
        same_worker,
        pre_run_error.map(|e| e.to_string()),
        // `job_authed`'s, not the handed-in `email`: unless the caller's own authed already names
        // this identity, it came through `fetch_authed_from_permissioned_as`, which re-resolves the
        // address from the principal's live binding. The same statement writes it to
        // `job_perms.email`, and the two columns naming different accounts is what this prevents.
        job_authed.email,
        visible_to_owner,
        flow_innermost_root_job,
        guarded_concurrent_limit,
        guarded_concurrency_time_window_s,
        custom_timeout,
        flow_step_id,
        cache_ttl,
        final_priority,
        preprocessed,
        is_running,
        scheduled_for_o,
        tag,
        final_priority,
        job_authed.email,
        job_authed.username,
        job_authed.is_admin,
        job_authed.is_operator,
        folders.as_slice(),
        job_authed.groups.as_slice(),
        root_job,
        trigger_kind as Option<JobTriggerKind>,
        running,
        end_user_email,
        cache_ignore_s3_path,
        runnable_settings_handle,
        labels.as_deref() as Option<&[String]>,
        runnable_path,
        workspace_id,
        job_token_scopes.as_deref() as Option<&[String]>,
    )
    .execute(&mut *tx)
    .warn_after_seconds(1)
    .await?;

    // RunnableSettings::insert(RunnableType::Job)

    //     tracing::debug!("Pushing job {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}");
    //     let uuid = sqlx::query_scalar!(
    //         "INSERT INTO v2_job_queue
    //             (workspace_id, id, running, scheduled_for, started_at, tag, priority)
    //             VALUES ($1, $2, $3, COALESCE($4, now()), CASE WHEN $3 THEN now() END, $5, $6) \
    //          RETURNING id AS \"id!\"",
    //         workspace_id,
    //         job_id,
    // ,
    //     )
    //     .fetch_one(&mut *tx)
    //     .warn_after_seconds(1)
    //     .await
    //     .map_err(|e| Error::internal_err(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?;

    // sqlx::query!(
    //     "INSERT INTO v2_job_runtime (id, ping) VALUES ($1, null)",
    //     job_id
    // )
    // .execute(&mut *tx)
    // .await?;
    if let Some(flow_status) = flow_status {
        sqlx::query!(
            "INSERT INTO v2_job_status (id, flow_status) VALUES ($1, $2)",
            job_id,
            Json(flow_status) as Json<FlowStatus>,
        )
        .execute(&mut *tx)
        .warn_after_seconds(1)
        .await?;
    }

    tracing::debug!("Pushed {job_id}");
    // TODO: technically the job isn't queued yet, as the transaction can be rolled back. Should be solved when moving these metrics to the queue abstraction.
    #[cfg(feature = "prometheus")]
    if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
        QUEUE_PUSH_COUNT.inc();
    }
    otel_incr_queue_push_count();

    {
        let uuid_string = job_id.to_string();
        let uuid_str = uuid_string.as_str();

        let mut hm = HashMap::from([("uuid", uuid_str), ("permissioned_as", &permissioned_as)]);

        let s: String;
        let operation_name = match job_kind {
            JobKind::Preview => "jobs.run.preview",
            JobKind::Script => {
                s = ScriptHash(runnable_id.unwrap()).to_string();
                hm.insert("hash", s.as_str());
                "jobs.run.script"
            }
            JobKind::Flow => "jobs.run.flow",
            JobKind::FlowPreview => "jobs.run.flow_preview",
            JobKind::SingleStepFlow => "jobs.run.single_step_flow",
            JobKind::Script_Hub => "jobs.run.script_hub",
            JobKind::Dependencies => "jobs.run.dependencies",
            JobKind::Identity => "jobs.run.identity",
            JobKind::Noop => "jobs.run.noop",
            JobKind::FlowDependencies => "jobs.run.flow_dependencies",
            JobKind::AppDependencies => "jobs.run.app_dependencies",
            JobKind::DeploymentCallback => "jobs.run.deployment_callback",
            JobKind::FlowScript => "jobs.run.flow_script",
            JobKind::FlowNode => "jobs.run.flow_node",
            JobKind::AppScript => "jobs.run.app_script",
            JobKind::AIAgent => "jobs.run.ai_agent",
            JobKind::UnassignedScript => "jobs.run.unassigned_script",
            JobKind::UnassignedFlow => "jobs.run.unassigned_flow",
            JobKind::UnassignedSinglestepFlow => "jobs.run.unassigned_singlestepflow",
        };

        let runs_on_behalf = format!("u/{user}") != permissioned_as && user != permissioned_as;
        // `user` is the end user of `permissioned_as` when the job runs as someone else, and
        // `audit_end_user` takes the one `end_user` slot ahead of it. Record it here so it stays
        // searchable: `username` holds the run-as identity in that case, not the caller.
        if runs_on_behalf && audit_end_user.is_some_and(|e| e != user) {
            hm.insert("created_by", user);
        }
        let audit_author = AuditAuthor {
            // `job_authed`'s address, matching `v2_job` and `job_perms` above.
            email: job_authed.email.clone(),
            username: if runs_on_behalf {
                windmill_common::auth::permissioned_as_to_username(&permissioned_as)
            } else {
                user.to_string()
            },
            // `username` is the identity the job runs as; `end_user` identifies the caller
            // behind it, as it does for an app run. An explicitly passed one therefore wins
            // over `user`, which only stands in for a caller when the job runs as someone else.
            username_override: audit_end_user
                .map(|s| s.to_string())
                .or_else(|| runs_on_behalf.then(|| user.to_string())),
            token_prefix: token_prefix.map(|s| s.to_string()),
        };

        if let Some(ref stringified_args) = stringified_args {
            hm.insert("args", stringified_args);
        }

        audit_log(
            &mut *tx,
            &audit_author,
            operation_name,
            ActionKind::Execute,
            workspace_id,
            runnable_path.as_ref().map(|x| x.as_str()),
            Some(hm),
        )
        .warn_after_seconds(1)
        .await?;
    }

    Ok((job_id, tx))
}

fn resolve_concurrency_key<'d>(
    workspace_id: &str,
    args: &PushArgs<'d>,
    script_path: &Option<String>,
    job_kind: JobKind,
    custom_concurrency_key: Option<String>,
) -> String {
    custom_concurrency_key
        .map(|x| {
            let interpolated = interpolate_args(x.clone(), args, workspace_id);
            // In cloud mode, enforce workspace isolation by prefixing with workspace
            // if the custom key doesn't already specify $workspace
            #[cfg(feature = "cloud")]
            {
                if !x.contains("$workspace") {
                    format!("{}/{}", workspace_id, interpolated)
                } else {
                    interpolated
                }
            }
            #[cfg(not(feature = "cloud"))]
            {
                interpolated
            }
        })
        .unwrap_or(fullpath_with_workspace(
            workspace_id,
            script_path.as_ref(),
            &job_kind,
        ))
}

/// Resolves the concurrency key, applies the cloud-only queue-depth cap, then registers the key.
///
/// `concurrent_limit: None` still registers the key but skips the cap: without a limit nothing
/// serializes the key, so there is no backlog to bound. Callers register keys for tag
/// interpolation alone, so the two are not interchangeable.
///
/// Takes a `Copy` executor because the cap runs a query before the insert on the same one. `push`
/// cannot use this — it holds a `&mut Transaction` — so it performs the same three steps inline.
pub async fn insert_concurrency_key_capped<'d, 'c, E: PgExecutor<'c> + Copy>(
    workspace_id: &str,
    args: &PushArgs<'d>,
    script_path: &Option<String>,
    job_kind: JobKind,
    custom_concurrency_key: Option<String>,
    concurrent_limit: Option<i32>,
    db: E,
    job_id: Uuid,
) -> Result<(), Error> {
    let concurrency_key = resolve_concurrency_key(
        workspace_id,
        args,
        script_path,
        job_kind,
        custom_concurrency_key,
    );
    #[cfg(feature = "cloud")]
    if *CLOUD_HOSTED && has_active_concurrency_limit(concurrent_limit) {
        check_concurrency_key_queue_cap(db, &concurrency_key).await?;
    }
    #[cfg(not(feature = "cloud"))]
    let _ = concurrent_limit;
    insert_resolved_concurrency_key(&concurrency_key, db, job_id, script_path, workspace_id).await
}

async fn insert_resolved_concurrency_key<'c>(
    concurrency_key: &str,
    db: impl PgExecutor<'c>,
    job_id: Uuid,
    script_path: &Option<String>,
    workspace_id: &str,
) -> Result<(), Error> {
    sqlx::query!(
        "WITH inserted_concurrency_counter AS (
                INSERT INTO concurrency_counter (concurrency_id, job_uuids)
                VALUES ($1, '{}'::jsonb)
                ON CONFLICT DO NOTHING
            )
            INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
        concurrency_key,
        job_id,
    )
    .execute(db)
    .warn_after_seconds(3)
    .await
    .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
    Ok(())
}

/// Counts jobs *waiting* behind `concurrency_key`, scanning at most `limit` rows.
///
/// Three constraints on the query below, each pinned by a test in
/// `tests/concurrency_key_queue_depth_test.rs`:
/// - Any `scheduled_for`: the limiter parks blocked jobs in the future, so a gated backlog is
///   almost entirely future-dated and `scheduled_for <= now()` would never see it.
/// - `running = false`: running jobs keep `ended_at` NULL, and counting them would charge a key
///   for the concurrency it is licensed to use.
/// - `LIMIT` inside the `EXISTS`, not outside: rows whose job left the queue without
///   `add_completed_job` keep `ended_at` NULL forever (the retention sweep only matches
///   `ended_at <= …`), so an outside `LIMIT` would let them be rescanned on every push.
///
/// Takes an already-resolved key and performs no authorization; `pub` only so the integration
/// test can reach it. Never call it with a caller-supplied key — it would leak queue depth
/// across workspaces.
pub async fn concurrency_key_queue_depth<'c>(
    db: impl PgExecutor<'c>,
    concurrency_key: &str,
    limit: i64,
) -> Result<i64, Error> {
    sqlx::query_scalar!(
        "SELECT count(*) FROM (
            SELECT ck.job_id FROM concurrency_key ck
            WHERE ck.key = $1 AND ck.ended_at IS NULL
            LIMIT $2
        ) s WHERE EXISTS (
            SELECT 1 FROM v2_job_queue q WHERE q.id = s.job_id AND q.running = false
        )",
        concurrency_key,
        limit,
    )
    .fetch_one(db)
    .warn_after_seconds(3)
    .await
    .map_err(|e| {
        Error::internal_err(format!(
            "Could not count queued jobs for concurrency_key={concurrency_key}: {e:#}"
        ))
    })
    .map(|c| c.unwrap_or(0))
}

/// Rejects the push when `concurrency_key` already has `CONCURRENCY_KEY_MAX_QUEUED` jobs queued.
///
/// Caller must runtime-gate this on `*CLOUD_HOSTED`, and must call it from *every push path*
/// that registers a concurrency key: a flow with a preprocessor is pushed with `concurrent_limit`
/// cleared and only registers its key later from `worker_flow`, so gating `push` alone leaves
/// trigger-driven flows uncapped.
///
/// `add_batch_jobs` writes keys directly and is deliberately exempt: it is superadmin-only, and a
/// superadmin can set the cap to `0` anyway. `import_queued_jobs` is likewise exempt because it
/// is rejected outright on cloud.
#[cfg(feature = "cloud")]
async fn check_concurrency_key_queue_cap<'c>(
    db: impl PgExecutor<'c>,
    concurrency_key: &str,
) -> Result<(), Error> {
    let cap = windmill_common::worker::CONCURRENCY_KEY_MAX_QUEUED
        .load(std::sync::atomic::Ordering::Relaxed);
    if cap == 0 {
        return Ok(());
    }
    let cap = cap as i64;
    let depth = concurrency_key_queue_depth(db, concurrency_key, cap).await?;
    if depth >= cap {
        return Err(Error::QuotaExceeded(format!(
            "Too many jobs queued behind concurrency key '{concurrency_key}': at least {depth} \
             jobs are already waiting and the limit is {cap}. Jobs sharing a concurrency key run \
             at most `concurrent_limit` at a time, so this queue is growing faster than it can \
             drain. Cancel the backlog, slow down the caller, or raise the concurrency limit."
        )));
    }
    Ok(())
}

/// Bounded count of a workspace's non-running queued jobs, capped at `limit` so the scan
/// stops once the ceiling is reached rather than counting an entire runaway backlog.
///
/// Internal helper for `check_workspace_queue_cap`, `pub` only so the integration test can call
/// it (like `concurrency_key_queue_depth`). Returns a count, not job contents; the caller is
/// responsible for any authorization — it takes the workspace id as given.
pub async fn workspace_queue_depth<'c>(
    db: impl PgExecutor<'c>,
    workspace_id: &str,
    limit: i64,
) -> Result<i64, Error> {
    sqlx::query_scalar!(
        "SELECT count(*) FROM (
            SELECT 1 FROM v2_job_queue
            WHERE workspace_id = $1 AND running = false
            LIMIT $2
        ) s",
        workspace_id,
        limit,
    )
    .fetch_one(db)
    .warn_after_seconds(3)
    .await
    .map_err(|e| {
        Error::internal_err(format!(
            "Could not count queued jobs for workspace={workspace_id}: {e:#}"
        ))
    })
    .map(|c| c.unwrap_or(0))
}

/// Rejects the push when the workspace already has `WORKSPACE_MAX_QUEUED_JOBS` jobs queued.
///
/// Caller must runtime-gate this on `*CLOUD_HOSTED`. It runs on every push (not only
/// concurrency-limited ones) because a workspace can flood the queue across many keys or with
/// keyless jobs. It caps *new* pushes past the ceiling; jobs already queued still drain, so an
/// in-flight flow only ever fails to push further work while the workspace is at the ceiling.
///
/// This is a soft ceiling, like the per-key cap: the count and the insert are not serialized, so
/// a burst of concurrent pushes can land a handful over the limit. That is fine and intentional
/// — the cap exists to stop an unbounded runaway, not to enforce an exact quota, and a
/// per-workspace lock on every push would add hot-path contention for no practical gain.
#[cfg(feature = "cloud")]
async fn check_workspace_queue_cap<'c>(
    db: impl PgExecutor<'c>,
    workspace_id: &str,
) -> Result<(), Error> {
    let cap = windmill_common::worker::WORKSPACE_MAX_QUEUED_JOBS
        .load(std::sync::atomic::Ordering::Relaxed);
    if cap == 0 {
        return Ok(());
    }
    let cap = cap as i64;
    let depth = workspace_queue_depth(db, workspace_id, cap).await?;
    if depth >= cap {
        return Err(Error::QuotaExceeded(format!(
            "Too many jobs queued in workspace '{workspace_id}': at least {depth} jobs are \
             already waiting and the instance limit is {cap}. Cancel the backlog or slow down \
             whatever is creating jobs before pushing more."
        )));
    }
    Ok(())
}

// pub async fn insert_debounce_key<'d, 'c>(
//     workspace_id: &str,
//     args: &PushArgs<'d>,
//     script_path: &Option<String>,
//     job_kind: JobKind,
//     custom_concurrency_key: Option<String>,
//     tx: &mut Transaction<'c, Postgres>,
//     job_id: Uuid,
// ) -> Result<(), Error> {
//     let concurrency_key = custom_concurrency_key
//         .map(|x| interpolate_args(x, args, workspace_id))
//         .unwrap_or(fullpath_with_workspace(
//             workspace_id,
//             script_path.as_ref(),
//             &job_kind,
//         ));
//     sqlx::query!(
//         "WITH inserted_concurrency_counter AS (
//                 INSERT INTO concurrency_counter (concurrency_id, job_uuids)
//                 VALUES ($1, '{}'::jsonb)
//                 ON CONFLICT DO NOTHING
//             )
//             INSERT INTO concurrency_key(key, job_id) VALUES ($1, $2)",
//         concurrency_key,
//         job_id,
//     )
//     .execute(&mut **tx)
//     .warn_after_seconds(3)
//     .await
//     .map_err(|e| Error::internal_err(format!("Could not insert concurrency_key={concurrency_key} for job_id={job_id} script_path={script_path:?} workspace_id={workspace_id}: {e:#}")))?;
//     Ok(())
// }

/// The result payload a job cancelled anywhere carries. Callers that hold the cancel
/// outside a `MiniPulledJob` — a row read after the pull, say — go through this rather
/// than rebuilding the shape.
pub fn canceled_result(reason: Option<&str>, canceler: Option<&str>) -> serde_json::Value {
    let reason = reason.unwrap_or("no reason given");
    let canceler = canceler.unwrap_or("unknown");
    serde_json::json!({"message": format!("Job canceled: {reason} by {canceler}"), "name": "Canceled", "reason": reason, "canceler": canceler})
}

pub fn canceled_job_to_result(job: &MiniPulledJob) -> serde_json::Value {
    canceled_result(job.canceled_reason.as_deref(), job.canceled_by.as_deref())
}

/// Helper function to create a restarted module for branch/iteration restart
fn create_restarted_module(
    module: &FlowStatusModule,
    module_definition: &FlowModule,
    branch_or_iteration_n: usize,
    restart_step_id: &str,
) -> Result<FlowStatusModule, Error> {
    match module_definition.get_value() {
        Ok(FlowModuleValue::BranchAll { branches, parallel, .. }) => {
            if parallel {
                return Err(Error::internal_err(format!(
                    "Module {} is a parallel branchall. It can only be restarted at a given branch if it's sequential",
                    restart_step_id,
                )));
            }
            let total_branch_number = module.flow_jobs().map(|v| v.len()).unwrap_or(0);
            if total_branch_number <= branch_or_iteration_n {
                return Err(Error::internal_err(format!(
                    "Branch-all module {} has only {} branches. It can't be restarted on branch {}",
                    restart_step_id, total_branch_number, branch_or_iteration_n,
                )));
            }
            let mut new_flow_jobs = module.flow_jobs().unwrap_or_default();
            new_flow_jobs.truncate(branch_or_iteration_n);
            let mut new_flow_jobs_success = module.flow_jobs_success();
            if let Some(new_flow_jobs_success) = new_flow_jobs_success.as_mut() {
                new_flow_jobs_success.truncate(branch_or_iteration_n);
            }
            let mut new_flow_jobs_timeline = module.flow_jobs_duration();
            if let Some(new_flow_jobs_timeline) = new_flow_jobs_timeline.as_mut() {
                new_flow_jobs_timeline.truncate(branch_or_iteration_n);
            }
            Ok(FlowStatusModule::InProgress {
                id: module.id(),
                job: new_flow_jobs[new_flow_jobs.len() - 1],
                iterator: None,
                flow_jobs: Some(new_flow_jobs),
                flow_jobs_success: new_flow_jobs_success,
                flow_jobs_duration: new_flow_jobs_timeline,
                branch_chosen: None,
                branchall: Some(BranchAllStatus {
                    branch: branch_or_iteration_n - 1,
                    len: branches.len(),
                }),
                parallel,
                while_loop: false,
                progress: None,
                agent_actions: None,
                agent_actions_success: None,
            })
        }
        Ok(FlowModuleValue::ForloopFlow { parallel, .. }) => {
            if parallel {
                return Err(Error::internal_err(format!(
                    "Module {} is not parallel loop. It can only be restarted at a given iteration if it's sequential",
                    restart_step_id,
                )));
            }
            let total_iterations = module.flow_jobs().map(|v| v.len()).unwrap_or(0);
            if total_iterations <= branch_or_iteration_n {
                return Err(Error::internal_err(format!(
                    "For-loop module {} doesn't cannot be restarted on iteration number {} as it has only {} iterations",
                    restart_step_id,
                    branch_or_iteration_n,
                    total_iterations,
                )));
            }
            let mut new_flow_jobs = module.flow_jobs().unwrap_or_default();
            new_flow_jobs.truncate(branch_or_iteration_n);
            let mut new_flow_jobs_success = module.flow_jobs_success();
            if let Some(new_flow_jobs_success) = new_flow_jobs_success.as_mut() {
                new_flow_jobs_success.truncate(branch_or_iteration_n);
            }
            let mut new_flow_jobs_timeline = module.flow_jobs_duration();
            if let Some(new_flow_jobs_timeline) = new_flow_jobs_timeline.as_mut() {
                new_flow_jobs_timeline.truncate(branch_or_iteration_n);
            }
            Ok(FlowStatusModule::InProgress {
                id: module.id(),
                job: new_flow_jobs[new_flow_jobs.len() - 1],
                iterator: Some(FlowIterator {
                    index: branch_or_iteration_n - 1,
                    itered: None,
                    itered_len: None,
                }),
                flow_jobs: Some(new_flow_jobs),
                flow_jobs_success: new_flow_jobs_success,
                flow_jobs_duration: new_flow_jobs_timeline,
                branch_chosen: None,
                branchall: None,
                parallel,
                while_loop: false,
                progress: None,
                agent_actions: None,
                agent_actions_success: None,
            })
        }
        _ => Err(Error::internal_err(format!(
            "Module {} is not a branchall or forloop, unable to restart it at step {:?}",
            restart_step_id, branch_or_iteration_n
        ))),
    }
}

/// A between-steps-zombie step: an `InProgress` module (in an otherwise terminal,
/// reaped flow) whose every child is recorded as a `success` completion. Only the
/// module's final state transition was lost, so the whole step is derivable and
/// safe to reuse on restart. Children incomplete/failed/cancelled ⟹ not a zombie.
async fn is_derivable_between_steps_zombie(
    db: &Pool<Postgres>,
    workspace_id: &str,
    module: &FlowStatusModule,
) -> Result<bool, Error> {
    // The module's own cursor must prove it reached the end (a serial loop/branch-all reaped
    // mid-fan-out has an all-success prefix but unrun remaining iterations); while-loops are
    // never derivable. Children-success is verified below.
    if !module.is_between_steps_complete() {
        return Ok(false);
    }
    let child_ids: Vec<Uuid> = module
        .flow_jobs()
        .filter(|v| !v.is_empty())
        .or_else(|| module.job().map(|j| vec![j]))
        .unwrap_or_default();
    if child_ids.is_empty() {
        return Ok(false);
    }
    let success_children = sqlx::query_scalar!(
        "SELECT count(*) FROM v2_job_completed
         WHERE workspace_id = $1 AND id = ANY($2) AND status = 'success'",
        workspace_id,
        &child_ids,
    )
    .fetch_one(db)
    .await?
    .unwrap_or(0);
    Ok(success_children == child_ids.len() as i64)
}

/// Convert a between-steps-zombie `InProgress` module (validated by
/// [`is_derivable_between_steps_zombie`]) into the `Success` it would have become
/// had its dropped transition landed, reusing all completed children. Downstream
/// steps re-derive this step's result from `flow_jobs`/`job` on demand
/// (`get_previous_job_result`), so no aggregate needs recomputing here.
fn reuse_completed_zombie_module(module: FlowStatusModule) -> FlowStatusModule {
    match module {
        FlowStatusModule::InProgress {
            id,
            job,
            flow_jobs,
            flow_jobs_success,
            flow_jobs_duration,
            branch_chosen,
            agent_actions,
            agent_actions_success,
            ..
        } => FlowStatusModule::Success {
            id,
            job,
            // Every child was verified successful, so normalise the success
            // vector (the dropped transition may have left the last entry unset).
            flow_jobs_success: flow_jobs_success
                .map(|v| v.into_iter().map(|_| Some(true)).collect()),
            flow_jobs,
            flow_jobs_duration,
            branch_chosen,
            approvers: vec![],
            failed_retries: vec![],
            skipped: false,
            agent_actions,
            agent_actions_success,
        },
        other => other,
    }
}

/// Loads the flow version a restart switches to. The version id is caller-supplied and the
/// restarted job keeps the original job's path, so it must be a version of that same flow in
/// that same workspace: any other id would run foreign code under the original path.
/// This only ties the version to the flow; it does not authorize the caller. `workspace_id`
/// and `flow_path` must come from the original job, which the caller is already allowed to
/// restart.
pub async fn fetch_restart_flow_version(
    db: &Pool<Postgres>,
    workspace_id: &str,
    flow_path: &str,
    version: i64,
) -> Result<Arc<FlowData>, Error> {
    let belongs = sqlx::query_scalar!(
        "SELECT EXISTS(SELECT 1 FROM flow_version WHERE id = $1 AND workspace_id = $2 AND path = $3) AS \"exists!\"",
        version,
        workspace_id,
        flow_path,
    )
    .fetch_one(db)
    .await?;
    if !belongs {
        return Err(Error::BadRequest(format!(
            "flow version {version} is not a version of flow {flow_path} in workspace {workspace_id}"
        )));
    }
    cache::flow::fetch_version(db, version).await
}

async fn restarted_flows_resolution(
    db: &Pool<Postgres>,
    workspace_id: &str,
    completed_flow_id: Uuid,
    restart_step_id: &str,
    branch_or_iteration_n: Option<usize>,
    flow_version: Option<i64>,
    // A nested restart chain (RestartedFrom.nested) descends into the restart step's child to
    // re-run an inner step; zombie reuse would skip the whole container and ignore it.
    nested_restart: bool,
    // Zombie reuse validates the restart step against the completed job's STORED definition and
    // synthesizes Success from its recorded children. That is only sound when the run being queued
    // uses that same definition (JobPayload::RestartedFlow). A JobPayload::RawFlow restart queues
    // the editor's current, possibly EDITED, definition instead, so reuse would skip the edited
    // step and reuse the old child result; disable it there.
    allow_zombie_reuse: bool,
) -> Result<
    (
        Option<i64>,
        Option<String>,
        Arc<FlowData>,
        i32,
        Vec<FlowStatusModule>,
        HashMap<String, serde_json::Value>,
        FlowCleanupModule,
        JobKind,
    ),
    Error,
> {
    let row = sqlx::query!(
        "SELECT
            j.runnable_path as script_path, j.runnable_id AS \"script_hash: ScriptHash\",
            j.kind AS \"job_kind!: JobKind\", c.canceled_by,
            COALESCE(c.flow_status, c.workflow_as_code_status) AS \"flow_status: Json<Box<RawValue>>\",
            j.raw_flow AS \"raw_flow: Json<Box<RawValue>>\"
        FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.id = $1 and j.workspace_id = $2",
        completed_flow_id,
        workspace_id,
    )
    .fetch_one(db) // TODO: should we try to use the passed-in `tx` here?
    .await
    .map_err(|err| {
        Error::internal_err(format!(
            "completed job not found for UUID {} in workspace {}: {}",
            completed_flow_id, workspace_id, err
        ))
    })?;

    // Zombie reuse must only apply to flows the zombie monitor reaped (canceled_by = 'monitor').
    // An ordinary force-cancel copies the same live flow_status, so a user canceling after a child
    // succeeds but before the parent transition lands produces the identical InProgress/all-success
    // shape; those must retain restart-from-step semantics (the step re-runs).
    let reaped_by_monitor = row.canceled_by.as_deref() == Some("monitor");

    let current_flow_version = row.script_hash.map(|x| x.0);
    let is_version_change = flow_version.is_some()
        && current_flow_version.is_some()
        && flow_version != current_flow_version
        && row.job_kind == JobKind::Flow;

    let flow_data = if is_version_change {
        let flow_path = row.script_path.as_deref().ok_or_else(|| {
            Error::BadRequest(format!(
                "completed flow {completed_flow_id} has no path to restart a version of"
            ))
        })?;
        fetch_restart_flow_version(db, workspace_id, flow_path, flow_version.unwrap()).await?
    } else {
        cache::job::fetch_flow(db, &row.job_kind, row.script_hash)
            .or_else(|_| {
                cache::job::fetch_preview_flow(db.into(), &completed_flow_id, row.raw_flow)
            })
            .await?
    };

    let flow_value = flow_data.value();
    let flow_status = row
        .flow_status
        .as_ref()
        .and_then(|v| serde_json::from_str::<FlowStatus>(v.get()).ok())
        .ok_or(Error::internal_err(format!(
            "Unable to parse flow status for job {} in workspace {}",
            completed_flow_id, workspace_id,
        )))?;

    let mut step_n = 0;
    let mut dependent_module = false;
    let mut truncated_modules: Vec<FlowStatusModule> = vec![];

    if is_version_change {
        // When the flow version has changed, create flow status from scratch
        // based on the new flow, but match modules from the old flow status where possible
        for module_definition in &flow_value.modules {
            let module_id = &module_definition.id;

            if module_id == restart_step_id {
                // Mark this and all following modules as WaitingForPriorSteps
                if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 {
                    truncated_modules
                        .push(FlowStatusModule::WaitingForPriorSteps { id: module_id.clone() });
                } else {
                    // Handle branch/iteration restart for version changes
                    let branch_n = branch_or_iteration_n.unwrap();
                    // Try to find matching module in old flow status
                    if let Some(old_module) =
                        flow_status.modules.iter().find(|m| &m.id() == module_id)
                    {
                        truncated_modules.push(create_restarted_module(
                            old_module,
                            module_definition,
                            branch_n,
                            restart_step_id,
                        )?);
                    } else {
                        // Module not found in old flow, mark as waiting
                        truncated_modules
                            .push(FlowStatusModule::WaitingForPriorSteps { id: module_id.clone() });
                    }
                }
                dependent_module = true;
            } else if dependent_module {
                truncated_modules
                    .push(FlowStatusModule::WaitingForPriorSteps { id: module_id.clone() });
            } else {
                // Before the restart step, try to match with old flow status
                if let Some(old_module) = flow_status.modules.iter().find(|m| &m.id() == module_id)
                {
                    truncated_modules.push(old_module.clone());
                } else {
                    truncated_modules
                        .push(FlowStatusModule::WaitingForPriorSteps { id: module_id.clone() });
                }
                step_n += 1;
            }
        }
    } else {
        // Original logic for same version
        for module in flow_status.modules {
            let Some(module_definition) = flow_value
                .modules
                .iter()
                .find(|flow_value_module| flow_value_module.id == module.id())
            else {
                // skip module as it doesn't appear in the flow_value anymore
                continue;
            };
            if module.id() == restart_step_id {
                // Reuse is only safe when there is a NEXT step to advance into (advancing past the
                // last module lands on the failure step) and the step's definition carries no
                // completion/arming semantics that reuse would skip (stop predicates, skip_if,
                // suspend, sleep); such a step must re-run, not be synthesized as Success.
                let has_next_step = flow_value
                    .modules
                    .last()
                    .is_none_or(|m| m.id != restart_step_id);
                // A whole-step restart is `None` (restart API with the field omitted) or `Some(0)`
                // (the run page's "Re-start from" button always sends 0); both mean "redo this step",
                // which for a monitor-reaped zombie means reuse it. `Some(n>=1)` is an explicit
                // partial container restart and keeps its existing reuse-0..n-1 / rerun-from-n path.
                if allow_zombie_reuse
                    && reaped_by_monitor
                    && branch_or_iteration_n.unwrap_or(0) == 0
                    && !nested_restart
                    && has_next_step
                    && module_definition.allows_zombie_reuse()
                    && is_derivable_between_steps_zombie(db, workspace_id, &module).await?
                {
                    // Between-steps-zombie recovery: this step's children all
                    // completed but its final state transition was dropped (the
                    // flow was reaped by the zombie monitor). Reuse the completed
                    // step verbatim and restart from the NEXT step, so no child
                    // re-runs and only the dropped transition is replayed onward.
                    step_n += 1;
                    truncated_modules.push(reuse_completed_zombie_module(module));
                } else if branch_or_iteration_n.is_none() || branch_or_iteration_n.unwrap() == 0 {
                    // if the module ID is the one we want to restart the flow at, or if it's past it in the flow,
                    // set the module as WaitingForPriorSteps as it needs to be re-run
                    // The module as WaitingForPriorSteps as the entire module (i.e. all the branches) need to be re-run
                    truncated_modules
                        .push(FlowStatusModule::WaitingForPriorSteps { id: module.id() });
                } else {
                    // expect a module to be either a branchall (resp. loop), and resume the flow from this branch (resp. iteration)
                    truncated_modules.push(create_restarted_module(
                        &module,
                        module_definition,
                        branch_or_iteration_n.unwrap(),
                        restart_step_id,
                    )?);
                }
                dependent_module = true;
            } else if dependent_module {
                truncated_modules.push(FlowStatusModule::WaitingForPriorSteps { id: module.id() });
            } else {
                // else we simply "transfer" the module from the completed flow to the new one if it's a success
                step_n = step_n + 1;
                match module.clone() {
                    FlowStatusModule::Success { .. } => Ok(truncated_modules.push(module)),
                    _ => Err(Error::internal_err(format!(
                        "Flow cannot be restarted from a non successful module",
                    ))),
                }?;
            }
        }
    }

    if !dependent_module {
        // step not found in flow.
        return Err(Error::internal_err(format!(
            "Flow cannot be restarted from step {} as it could not be found.",
            restart_step_id
        )));
    }

    Ok((
        if is_version_change {
            flow_version
        } else {
            row.script_hash.map(|x| x.0)
        },
        row.script_path,
        flow_data,
        step_n,
        truncated_modules,
        flow_status.user_states,
        flow_status.cleanup_module,
        // Preserve the original job kind (e.g. FlowNode for BranchOne/loop wrapped
        // children) so the new run uses the same lookup path. Setting kind=Flow when
        // the original was FlowNode would cause `cache::job::fetch_flow` to query
        // `flow_version` with a `flow_node` id and fail at runtime.
        if is_version_change {
            JobKind::Flow
        } else {
            row.job_kind
        },
    ))
}

// Wrapper struct to send both job and optional flow_runners to dedicated workers
pub struct DedicatedWorkerJob {
    pub job: Arc<MiniPulledJob>,
    pub flow_runners: Option<Arc<FlowRunners>>,
    pub done_tx: Option<oneshot::Sender<()>>,
}

#[derive(Debug)]
pub struct FlowRunners {
    pub runners: HashMap<String, Sender<DedicatedWorkerJob>>,
    pub handles: Vec<JoinHandle<()>>,
    pub job_id: Uuid,
}

impl Drop for FlowRunners {
    fn drop(&mut self) {
        let total_runners = self.handles.len();
        tracing::info!(
            "dropping {} flow runners for job {}",
            total_runners,
            self.job_id
        );

        // First, drop all senders to signal workers to stop gracefully
        self.runners.clear();

        // Spawn a background task to wait with timeout and abort if needed
        let handles = std::mem::take(&mut self.handles);
        let job_id = self.job_id;

        tokio::spawn(async move {
            // Extract abort handles before consuming the join handles
            let abort_handles: Vec<_> = handles.iter().map(|h| h.abort_handle()).collect();

            // Wait up to 5 seconds for natural termination
            let timeout_result = tokio::time::timeout(
                tokio::time::Duration::from_secs(5),
                futures::future::join_all(handles),
            )
            .await;

            match timeout_result {
                Ok(_) => {
                    tracing::info!(
                        "all {} flow runners for job {} terminated gracefully",
                        total_runners,
                        job_id
                    );
                }
                Err(_) => {
                    // Timeout reached, abort only the handles that haven't finished
                    let mut aborted = 0;
                    for abort_handle in abort_handles {
                        if !abort_handle.is_finished() {
                            abort_handle.abort();
                            aborted += 1;
                        }
                    }
                    let graceful = total_runners - aborted;
                    tracing::warn!(
                        "flow runners for job {}: {} terminated gracefully, {} aborted after 5s timeout",
                        job_id, graceful, aborted
                    );
                }
            }
        });
    }
}

#[derive(Debug, Serialize, Deserialize)]
pub struct SameWorkerPayload {
    pub job_id: Uuid,
    pub recoverable: bool,
    #[serde(skip)]
    pub flow_runners: Option<Arc<FlowRunners>>,
}

pub async fn get_same_worker_job(
    db: &DB,
    same_worker_job: &SameWorkerPayload,
) -> windmill_common::error::Result<Option<PulledJob>> {
    sqlx::query_as::<_, PulledJob>(
        "WITH ping AS (
                        UPDATE v2_job_runtime SET ping = NOW() WHERE id = $1
                    ),
                    started_at AS (
                        UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1
                    )
                    SELECT
                    v2_job_queue.workspace_id,
                    v2_job_queue.id,
                    v2_job.args,
                    v2_job.parent_job,
                    v2_job.created_by,
                    v2_job_queue.started_at,
                    scheduled_for,
                    v2_job.runnable_path,
                    v2_job.kind,
                    v2_job.runnable_id,
                    v2_job_queue.canceled_reason,
                    v2_job_queue.canceled_by,
                    v2_job.permissioned_as,
                    v2_job.permissioned_as_email,
                    v2_job_status.flow_status,
                    v2_job.tag,
                    v2_job.script_lang,
                    v2_job.same_worker,
                    v2_job.pre_run_error,
                    v2_job.concurrent_limit,
                    v2_job.concurrency_time_window_s,
                    v2_job.flow_innermost_root_job,
                    v2_job.root_job,
                    v2_job.timeout,
                    v2_job.flow_step_id,
                    v2_job.cache_ttl,
                    v2_job_queue.cache_ignore_s3_path,
                    v2_job_queue.runnable_settings_handle,
                    v2_job_queue.priority,
                    v2_job.preprocessed,
                    v2_job.script_entrypoint_override,
                    v2_job.trigger,
                    v2_job.trigger_kind,
                    v2_job.visible_to_owner,
                    p.job_token_scopes,
                    v2_job.raw_code,
                    v2_job.raw_lock,
                    v2_job.raw_flow,
                    COALESCE(pj.runnable_path, v2_job.args->>'_FLOW_PATH') as parent_runnable_path,
                    p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
                    p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders, p.end_user_email as permissioned_as_end_user_email
                    FROM v2_job_queue
                    INNER JOIN v2_job ON v2_job.id = v2_job_queue.id
                    LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id
                    LEFT JOIN job_perms p ON p.job_id = v2_job.id
                    LEFT JOIN v2_job pj ON v2_job.parent_job = pj.id
                    WHERE v2_job_queue.id = $1
",
    )
    .bind(same_worker_job.job_id)
    .fetch_optional(db)
    .await
    .map_err(|e| {
        Error::internal_err(format!(
            "Impossible to fetch same_worker job {}: {}",
            same_worker_job.job_id, e
        ))
    })
}

#[cfg(test)]
mod git_sync_concurrency_key_tests {
    use super::git_sync_concurrency_key;

    /// The reservation is what keeps `{workspace_id}/` + the key inside the
    /// VARCHAR(255) column on cloud builds. Without it, a suffix that fits the
    /// bare budget is emitted verbatim and the prefixed insert fails.
    #[test]
    fn hashes_a_suffix_that_only_fits_before_the_prefix() {
        let ws = "some-workspace";
        let suffix: String = format!("u/user/{}", "x".repeat(220));
        let bare = git_sync_concurrency_key(ws, Some(suffix.clone()), 0);
        assert!(bare.len() <= 255 && bare.ends_with(&suffix));

        let reserved = git_sync_concurrency_key(ws, Some(suffix), ws.len() + 1);
        assert!(
            ws.len() + 1 + reserved.len() <= 255,
            "prefixed key must fit the column, got {}",
            ws.len() + 1 + reserved.len()
        );
    }

    #[test]
    fn keeps_distinct_repos_distinct_when_hashed() {
        let ws = "w";
        let long = "x".repeat(300);
        let a = git_sync_concurrency_key(ws, Some(format!("u/user/a{long}")), 0);
        let b = git_sync_concurrency_key(ws, Some(format!("u/user/b{long}")), 0);
        assert_ne!(a, b);
        assert!(a.len() <= 255 && b.len() <= 255);
    }
}

#[cfg(test)]
mod preset_handler_path_tests {
    use super::is_preset_handler_path;

    #[test]
    fn only_pinned_hub_scripts_are_presets() {
        assert!(is_preset_handler_path(
            "script/hub/28791/slack/schedule-recovery-handler-slack"
        ));
        // The hub resolves by id: any other id is arbitrary hub code whatever its name.
        assert!(!is_preset_handler_path(
            "hub/13968/workspace-or-schedule-error-handler-slack"
        ));
        assert!(!is_preset_handler_path(
            "script/u/me/workspace-or-schedule-error-handler-slack"
        ));
    }

    /// A preset version bumped in the frontend without being pinned here would silently run as
    /// the schedule's owner and lose access to the Slack bot token.
    #[test]
    fn frontend_presets_are_pinned() {
        let hub_paths: std::collections::HashMap<String, String> =
            serde_json::from_str(include_str!("../../../frontend/src/lib/hubPaths.json")).unwrap();
        for (name, path) in hub_paths {
            if path.contains("-handler-") {
                assert!(
                    is_preset_handler_path(&path),
                    "{name}: {path} is not pinned"
                );
            }
        }
    }
}

#[cfg(test)]
mod result_metadata_tests {
    use super::{ResultMetadata, ValidableJson};
    use serde_json::value::RawValue;

    fn from_raw(json: &str) -> ResultMetadata {
        RawValue::from_string(json.to_string())
            .unwrap()
            .result_metadata()
    }

    fn from_value(json: &str) -> ResultMetadata {
        serde_json::from_str::<serde_json::Value>(json)
            .unwrap()
            .result_metadata()
    }

    #[test]
    fn array_result_carries_no_markers() {
        for json in [r#"[["label"], "boom"]"#, r#"[null, "boom"]"#] {
            for meta in [from_raw(json), from_value(json)] {
                assert!(
                    meta.wm_labels.is_none() && meta.wm_failure.is_none(),
                    "{json}"
                );
            }
        }
        let meta = from_raw(r#"{"wm_labels": ["label"], "wm_failure": "boom"}"#);
        assert_eq!(meta.wm_labels, Some(vec!["label".to_string()]));
        assert_eq!(meta.wm_failure.as_deref(), Some("boom"));
    }
}

#[cfg(test)]
mod render_tag_path_tests {
    use super::{interpolate_args, render_tag_path, PushArgs};
    use serde_json::value::RawValue;
    use std::collections::HashMap;

    fn render(root: &str, path: &str) -> String {
        render_tag_path(
            Some(&RawValue::from_string(root.to_string()).unwrap()),
            path,
        )
    }

    // Existing `$args[...]` tags route on exactly these renderings.
    #[test]
    fn renders_like_args_tags() {
        assert_eq!(render(r#""eu""#, ""), "eu");
        assert_eq!(render(r#"{"a": {"b": "eu"}}"#, "a.b"), "eu");
        assert_eq!(render(r#"{"n": 4}"#, "n"), "4");
        assert_eq!(render("null", ""), "null");
        assert_eq!(render(r#"{"a": 1}"#, "b.c"), "");
        assert_eq!(render(r#"{"a": ["eu"]}"#, "a.0"), "");
        assert_eq!(render_tag_path(None, "a"), "");

        let args = HashMap::from([("cfg".to_string(), raw(r#"{"lang": "eu"}"#))]);
        let push_args = PushArgs {
            args: &args,
            extra: Some(HashMap::from([("e".to_string(), raw(r#""x""#))])),
        };
        assert_eq!(
            interpolate_args("w-$args[cfg.lang]-$args[e]".to_string(), &push_args, "ws"),
            "w-eu-x"
        );

        let args = HashMap::from([("_MODULES".to_string(), raw(r#""allowed-""#))]);
        let push_args = PushArgs { args: &args, extra: None };
        assert_eq!(
            interpolate_args("$args[_MODULES]private".to_string(), &push_args, "ws"),
            "private"
        );
    }

    fn raw(json: &str) -> Box<RawValue> {
        RawValue::from_string(json.to_string()).unwrap()
    }
}