/*
 * Author: Ruben Fiszel
 * Copyright: Windmill Labs, Inc 2022
 * This file and its contents are licensed under the AGPLv3 License.
 * Please see the included NOTICE for copyright information and
 * LICENSE-AGPL for a copy of the license.
 */

use anyhow::Context;
use monitor::{
    load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
    reload_instance_python_version_setting, reload_nuget_config_setting,
    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
    send_logs_to_object_store,
};
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
use std::{
    collections::HashMap,
    fs::{create_dir_all, DirBuilder},
    net::{IpAddr, Ipv4Addr, SocketAddr},
    time::{Duration, Instant},
};
use tokio::{fs::File, io::AsyncReadExt, task::JoinHandle};
use uuid::Uuid;
use windmill_api::HTTP_CLIENT;

#[cfg(feature = "enterprise")]
use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICENSE_KEY_VALID};

use windmill_common::{
    get_database_url,
    global_settings::{
        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
        DEFAULT_TAGS_WORKSPACES_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
        EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING,
        HUB_BASE_URL_SETTING, INDEXER_SETTING, INSTANCE_PYTHON_VERSION_SETTING,
        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
        NUGET_CONFIG_SETTING, OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING,
        REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
        RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
        TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
    },
    scripts::ScriptLang,
    stats_ee::schedule_stats,
    utils::{hostname, rd_string, Mode, GIT_VERSION, MODE_AND_ADDONS},
    worker::{reload_custom_tags_setting, HUB_CACHE_DIR, TMP_DIR, TMP_LOGS_DIR, WORKER_GROUP},
    KillpillSender, METRICS_ENABLED,
};

#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
use monitor::monitor_mem;

#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
use tikv_jemallocator::Jemalloc;

#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;

#[cfg(feature = "parquet")]
use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;

use windmill_worker::{
    get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR,
    DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR,
    NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR,
    PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR,
    TAR_PY313_CACHE_DIR, UV_CACHE_DIR,
};

use crate::monitor::{
    initial_load, load_keep_job_dir, load_metrics_debug_enabled, load_require_preexisting_user,
    load_tag_per_workspace_enabled, load_tag_per_workspace_workspaces, monitor_db,
    reload_base_url_setting, reload_bunfig_install_scopes_setting,
    reload_critical_alert_mute_ui_setting, reload_critical_error_channels_setting,
    reload_extra_pip_index_url_setting, reload_hub_base_url_setting,
    reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key,
    reload_npm_config_registry_setting, reload_pip_index_url_setting,
    reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
    reload_worker_config,
};

#[cfg(feature = "parquet")]
use crate::monitor::reload_s3_cache_setting;

const DEFAULT_NUM_WORKERS: usize = 1;
const DEFAULT_PORT: u16 = 8000;
const DEFAULT_SERVER_BIND_ADDR: Ipv4Addr = Ipv4Addr::new(0, 0, 0, 0);

mod ee;
mod monitor;

#[inline(always)]
fn create_and_run_current_thread_inner<F, R>(future: F) -> R
where
    F: std::future::Future<Output = R> + 'static,
    R: Send + 'static,
{
    let rt = tokio::runtime::Builder::new_multi_thread()
        .enable_all()
        .worker_threads(32)
        .build()
        .unwrap();

    // Since this is the main future, we want to box it in debug mode because it tends to be fairly
    // large and the compiler won't optimize repeated copies. We also make this runtime factory
    // function #[inline(always)] to avoid holding the unboxed, unused future on the stack.
    #[cfg(debug_assertions)]
    // SAFETY: this this is guaranteed to be running on a current-thread executor
    let future = Box::pin(future);

    rt.block_on(future)
}

lazy_static::lazy_static! {
    static ref PG_LISTENER_REFRESH_PERIOD_SECS: u64 = std::env::var("PG_LISTENER_REFRESH_PERIOD_SECS")
        .ok()
        .and_then(|x| x.parse::<u64>().ok())
        .unwrap_or(3600 * 12);
}

pub fn main() -> anyhow::Result<()> {
    // https://github.com/denoland/deno/blob/main/cli/main.rs#L477
    #[cfg(feature = "deno_core")]
    let unrecognized_v8_flags = deno_core::v8_set_flags(vec![
        "--stack-size=1024".to_string(),
        // TODO(bartlomieju): I think this can be removed as it's handled by `deno_core`
        // and its settings.
        // deno_ast removes TypeScript `assert` keywords, so this flag only affects JavaScript
        // TODO(petamoriken): Need to check TypeScript `assert` keywords in deno_ast
        "--no-harmony-import-assertions".to_string(),
    ])
    .into_iter()
    .skip(1)
    .collect::<Vec<_>>();

    #[cfg(feature = "deno_core")]
    if !unrecognized_v8_flags.is_empty() {
        println!("Unrecognized V8 flags: {:?}", unrecognized_v8_flags);
    }

    #[cfg(feature = "deno_core")]
    deno_core::JsRuntime::init_platform(None, false);
    create_and_run_current_thread_inner(windmill_main())
}

async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
    let file_path = file_path.unwrap_or("./hubPaths.json".to_string());
    let mut file = File::open(&file_path)
        .await
        .with_context(|| format!("Could not open {}, make sure it exists", &file_path))?;
    let mut contents = String::new();
    file.read_to_string(&mut contents).await?;
    let paths = serde_json::from_str::<HashMap<String, String>>(&contents).with_context(|| {
        format!(
            "Could not parse {}, make sure it is a valid JSON object with string keys and values",
            &file_path
        )
    })?;

    create_dir_all(HUB_CACHE_DIR)?;
    create_dir_all(BUN_BUNDLE_CACHE_DIR)?;

    for path in paths.values() {
        tracing::info!("Caching hub script at {path}");
        let res = get_hub_script_content_and_requirements(Some(path), None).await?;
        if res
            .language
            .as_ref()
            .is_some_and(|x| x == &ScriptLang::Deno)
        {
            let job_dir = format!("{}/cache_init/{}", TMP_DIR, Uuid::new_v4());
            create_dir_all(&job_dir)?;
            let _ = windmill_worker::generate_deno_lock(
                &Uuid::nil(),
                &res.content,
                &mut 0,
                &mut None,
                &job_dir,
                None,
                "global",
                "global",
                "",
                &mut None,
            )
            .await?;
            tokio::fs::remove_dir_all(job_dir).await?;
        } else if res.language.as_ref().is_some_and(|x| x == &ScriptLang::Bun) {
            let job_id = Uuid::new_v4();
            let job_dir = format!("{}/cache_init/{}", TMP_DIR, job_id);
            create_dir_all(&job_dir)?;
            if let Some(lockfile) = res.lockfile {
                let _ = windmill_worker::prepare_job_dir(&lockfile, &job_dir).await?;
                let envs = windmill_worker::get_common_bun_proc_envs(None).await;
                let _ = windmill_worker::install_bun_lockfile(
                    &mut 0,
                    &mut None,
                    &job_id,
                    "admins",
                    None,
                    &job_dir,
                    "cache_init",
                    envs.clone(),
                    false,
                    &mut None,
                )
                .await?;

                let _ = windmill_common::worker::write_file(&job_dir, "main.js", &res.content)?;

                if let Err(e) = windmill_worker::prebundle_bun_script(
                    &res.content,
                    Some(&lockfile),
                    &path,
                    &job_id,
                    "admins",
                    None,
                    &job_dir,
                    "",
                    "cache_init",
                    "",
                    &mut None,
                )
                .await
                {
                    panic!("Error prebundling bun script: {e:#}");
                }
            } else {
                tracing::warn!("No lockfile found for bun script {path}, skipping...");
            }
            tokio::fs::remove_dir_all(job_dir).await?;
        }
    }
    Ok(())
}

async fn windmill_main() -> anyhow::Result<()> {
    dotenv::dotenv().ok();

    if std::env::var("RUST_LOG").is_err() {
        std::env::set_var("RUST_LOG", "info")
    }

    if let Err(_e) = rustls::crypto::ring::default_provider().install_default() {
        tracing::error!("Failed to install rustls crypto provider");
    }

    let hostname = hostname();

    let mode_and_addons = MODE_AND_ADDONS.clone();
    let mode = mode_and_addons.mode;

    if mode == Mode::Standalone {
        println!("Running in standalone mode");
    }

    #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
    println!("jemalloc enabled");

    let cli_arg = std::env::args().nth(1).unwrap_or_default();

    match cli_arg.as_str() {
        "cache" => {
            #[cfg(feature = "embedding")]
            {
                println!("Caching embedding model...");
                windmill_api::embeddings::ModelInstance::load_model_files().await?;
                println!("Cached embedding model");
            }
            #[cfg(not(feature = "embedding"))]
            {
                println!("Embeddings are not enabled, ignoring...");
            }

            cache_hub_scripts(std::env::args().nth(2)).await?;

            return Ok(());
        }
        "-v" | "--version" | "version" => {
            println!("Windmill {}", GIT_VERSION);
            return Ok(());
        }
        _ => {}
    }

    #[allow(unused_mut)]
    let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
        0
    } else {
        std::env::var("NUM_WORKERS")
            .ok()
            .and_then(|x| x.parse::<i32>().ok())
            .unwrap_or(DEFAULT_NUM_WORKERS as i32)
    };

    if num_workers > 1 && !std::env::var("WORKER_GROUP").is_ok_and(|x| x == "native") {
        println!(
            "We STRONGLY recommend using at most 1 worker per container, use at your own risks"
        );
    }

    let server_mode = !std::env::var("DISABLE_SERVER")
        .ok()
        .and_then(|x| x.parse::<bool>().ok())
        .unwrap_or(false)
        && (mode == Mode::Server || mode == Mode::Standalone);

    let indexer_mode = mode == Mode::Indexer;

    let server_bind_address: IpAddr = if server_mode || indexer_mode {
        std::env::var("SERVER_BIND_ADDR")
            .ok()
            .and_then(|x| x.parse().ok())
            .unwrap_or(IpAddr::from(DEFAULT_SERVER_BIND_ADDR))
    } else {
        IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
    };

    println!("Connecting to database...");
    let db = windmill_common::initial_connection().await?;

    let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;

    tracing::info!(
        "PostgreSQL version: {} (windmill require PG >= 14)",
        num_version
            .ok()
            .flatten()
            .unwrap_or_else(|| "UNKNOWN".to_string())
    );
    load_otel(&db).await;

    tracing::info!("Database connected");

    let environment = load_base_url(&db)
        .await
        .unwrap_or_else(|_| "local".to_string())
        .trim_start_matches("https://")
        .trim_start_matches("http://")
        .split(".")
        .next()
        .unwrap_or_else(|| "local")
        .to_string();

    let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);

    let is_agent = mode == Mode::Agent;

    let mut migration_handle: Option<JoinHandle<()>> = None;
    #[cfg(feature = "parquet")]
    let disable_s3_store = std::env::var("DISABLE_S3_STORE")
        .ok()
        .is_some_and(|x| x == "1" || x == "true");

    if !is_agent && !indexer_mode {
        let skip_migration = std::env::var("SKIP_MIGRATION")
            .map(|val| val == "true")
            .unwrap_or(false);

        if !skip_migration {
            // migration code to avoid break
            migration_handle = windmill_api::migrate_db(&db).await?;
        } else {
            tracing::info!("SKIP_MIGRATION set, skipping db migration...")
        }
    }

    drop(db);
    let worker_mode = num_workers > 0;

    let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?;

    let (killpill_tx, mut killpill_rx) = KillpillSender::new(2);
    let mut monitor_killpill_rx = killpill_tx.subscribe();
    let (killpill_phase2_tx, _killpill_phase2_rx) = tokio::sync::broadcast::channel::<()>(2);
    let server_killpill_rx = killpill_phase2_tx.subscribe();

    let shutdown_signal =
        windmill_common::shutdown_signal(killpill_tx.clone(), killpill_tx.subscribe());

    #[cfg(feature = "enterprise")]
    tracing::info!(
        "
##############################
Windmill Enterprise Edition {GIT_VERSION}
##############################"
    );

    #[cfg(not(feature = "enterprise"))]
    tracing::info!(
        "
##############################
Windmill Community Edition {GIT_VERSION}
##############################"
    );

    display_config(&ENV_SETTINGS);

    if let Err(e) = reload_base_url_setting(&db).await {
        tracing::error!("Error loading base url: {:?}", e)
    }

    if let Err(e) = reload_critical_error_channels_setting(&db).await {
        tracing::error!("Could loading critical error emails setting: {:?}", e);
    }

    #[cfg(feature = "enterprise")]
    {
        // load the license key and check if it's valid
        // if not valid and not server mode just quit
        // if not expired and server mode then force renewal
        // if key still invalid and num_workers > 0, set to 0
        if let Err(err) = reload_license_key(&db).await {
            tracing::error!("Failed to reload license key: {err:#}");
        }
        let valid_key = *LICENSE_KEY_VALID.read().await;
        if !valid_key && !server_mode {
            tracing::error!("Invalid license key, workers require a valid license key");
        }
        if server_mode {
            // only force renewal if invalid but not empty (= expired)
            let renewed_now = maybe_renew_license_key_on_start(
                &HTTP_CLIENT,
                &db,
                !valid_key && !LICENSE_KEY_ID.read().await.is_empty(),
            )
            .await;
            if renewed_now {
                if let Err(err) = reload_license_key(&db).await {
                    tracing::error!("Failed to reload license key: {err:#}");
                }
            }
        }
    }

    if server_mode || worker_mode || indexer_mode {
        let port_var = std::env::var("PORT").ok().and_then(|x| x.parse().ok());

        let port = if server_mode || indexer_mode {
            port_var.unwrap_or(DEFAULT_PORT as u16)
        } else {
            port_var.unwrap_or(0)
        };

        let default_base_internal_url = format!("http://localhost:{}", port.to_string());
        // since it's only on server mode, the port is statically defined
        let base_internal_url: String = if let Ok(base_url) = std::env::var("BASE_INTERNAL_URL") {
            if !is_agent {
                tracing::warn!("BASE_INTERNAL_URL is now unecessary and ignored unless the mode is 'agent', you can remove it.");
                default_base_internal_url.clone()
            } else {
                base_url
            }
        } else {
            default_base_internal_url.clone()
        };

        initial_load(
            &db,
            killpill_tx.clone(),
            worker_mode,
            server_mode,
            #[cfg(feature = "parquet")]
            disable_s3_store,
        )
        .await;

        monitor_db(
            &db,
            &base_internal_url,
            server_mode,
            worker_mode,
            true,
            killpill_tx.clone(),
        )
        .await;

        #[cfg(feature = "prometheus")]
        crate::monitor::monitor_pool(&db).await;

        send_logs_to_object_store(&db, &hostname, &mode);

        #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
        if !worker_mode {
            monitor_mem().await;
        }

        let addr = SocketAddr::from((server_bind_address, port));

        let (base_internal_tx, base_internal_rx) = tokio::sync::oneshot::channel::<String>();

        DirBuilder::new()
            .recursive(true)
            .create("/tmp/windmill")
            .expect("could not create initial server dir");

        #[cfg(feature = "tantivy")]
        let should_index_jobs = mode == Mode::Indexer || mode_and_addons.indexer;

        reload_indexer_config(&db).await;

        #[cfg(feature = "tantivy")]
        let (index_reader, index_writer) = if should_index_jobs {
            let mut indexer_rx = killpill_rx.resubscribe();

            let (mut reader, mut writer) = (None, None);
            tokio::select! {
                _ = indexer_rx.recv() => {
                    tracing::info!("Received killpill, aborting index initialization");
                },
                res = windmill_indexer::completed_runs_ee::init_index(&db) => {
                        let res = res?;
                        reader = Some(res.0);
                        writer = Some(res.1);
                }

            }
            (reader, writer)
        } else {
            (None, None)
        };

        #[cfg(feature = "tantivy")]
        let indexer_f = {
            let indexer_rx = killpill_rx.resubscribe();
            let index_writer2 = index_writer.clone();
            async {
                if let Some(index_writer) = index_writer2 {
                    windmill_indexer::completed_runs_ee::run_indexer(
                        db.clone(),
                        index_writer,
                        indexer_rx,
                    )
                    .await?;
                }
                Ok(())
            }
        };

        #[cfg(all(feature = "tantivy", feature = "parquet"))]
        let (log_index_reader, log_index_writer) = if should_index_jobs {
            let mut indexer_rx = killpill_rx.resubscribe();

            let (mut reader, mut writer) = (None, None);
            tokio::select! {
                _ = indexer_rx.recv() => {
                    tracing::info!("Received killpill, aborting index initialization");
                },
                res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => {
                        let res = res?;
                        reader = Some(res.0);
                        writer = Some(res.1);
                }

            }
            (reader, writer)
        } else {
            (None, None)
        };

        #[cfg(all(feature = "tantivy", feature = "parquet"))]
        let log_indexer_f = {
            let log_indexer_rx = killpill_rx.resubscribe();
            let log_index_writer2 = log_index_writer.clone();
            async {
                if let Some(log_index_writer) = log_index_writer2 {
                    windmill_indexer::service_logs_ee::run_indexer(
                        db.clone(),
                        log_index_writer,
                        log_indexer_rx,
                    )
                    .await?;
                }
                Ok(())
            }
        };

        #[cfg(not(feature = "tantivy"))]
        let index_reader = None;

        #[cfg(not(feature = "tantivy"))]
        let indexer_f = async { Ok(()) as anyhow::Result<()> };

        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
        let log_index_reader = None;

        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
        let log_indexer_f = async { Ok(()) as anyhow::Result<()> };

        let server_f = async {
            if !is_agent {
                windmill_api::run_server(
                    db.clone(),
                    index_reader,
                    log_index_reader,
                    addr,
                    server_killpill_rx,
                    base_internal_tx,
                    server_mode,
                    #[cfg(feature = "smtp")]
                    base_internal_url.clone(),
                )
                .await?;
            } else {
                base_internal_tx
                    .send(base_internal_url.clone())
                    .map_err(|e| {
                        anyhow::anyhow!("Could not send base_internal_url to agent: {e:#}")
                    })?;
            }
            Ok(()) as anyhow::Result<()>
        };

        let workers_f = async {
            let mut rx = killpill_rx.resubscribe();

            if !killpill_rx.try_recv().is_ok() {
                let base_internal_url = base_internal_rx.await?;
                if worker_mode {
                    run_workers(
                        db.clone(),
                        rx,
                        killpill_tx.clone(),
                        num_workers,
                        base_internal_url.clone(),
                        is_agent,
                        hostname.clone(),
                    )
                    .await?;
                    tracing::info!("All workers exited.");
                    killpill_tx.send();
                } else {
                    rx.recv().await?;
                }
            }
            if killpill_phase2_tx.receiver_count() > 0 {
                if worker_mode {
                    tracing::info!("Starting phase 2 of shutdown");
                }
                killpill_phase2_tx.send(())?;
                if worker_mode {
                    tracing::info!("Phase 2 of shutdown completed");
                }
            }
            Ok(())
        };

        let monitor_f = async {
            let db = db.clone();
            let tx = killpill_tx.clone();

            let base_internal_url = base_internal_url.to_string();
            let db_url: String = get_database_url().await?;

            let h = tokio::spawn(async move {
                let mut listener = retry_listen_pg(&db_url).await;
                let mut last_listener_refresh = Instant::now();
                loop {
                    tokio::select! {
                        biased;
                        Some(_) = async { if let Some(jh) = migration_handle.take() {
                            tracing::info!("migration job finished");
                            Some(jh.await)
                        } else {
                            None
                        }} => {
                           continue;
                        },
                        _ = monitor_killpill_rx.recv() => {
                            tracing::info!("received killpill for monitor job");
                            break;
                        },
                        notification = listener.try_recv() => {
                            match notification {
                                Ok(n) => {
                                    if n.is_none() {
                                        tracing::error!("Could not receive notification, attempting to reconnect to pg listener");
                                        continue;
                                    }
                                    let n = n.unwrap();
                                    tracing::info!("Received new pg notification: {n:?}");
                                    match n.channel() {
                                        "notify_config_change" => {
                                            match n.payload() {
                                                "server" if server_mode => {
                                                    tracing::error!("Server config change detected but server config is obsolete: {}", n.payload());
                                                },
                                                a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
                                                    tracing::info!("Worker config change detected: {}", n.payload());
                                                    reload_worker_config(&db, tx.clone(), true).await;
                                                },
                                                _ => {
                                                    tracing::debug!("config changed but did not target this server/worker");
                                                }
                                            }
                                        },
                                        "notify_webhook_change" => {
                                            let workspace_id = n.payload();
                                            tracing::info!("Webhook change detected, invalidating webhook cache: {}", workspace_id);
                                            windmill_api::webhook_util::WEBHOOK_CACHE.remove(workspace_id);
                                        },
                                        "notify_global_setting_change" => {
                                            tracing::info!("Global setting change detected: {}", n.payload());
                                            match n.payload() {
                                                BASE_URL_SETTING => {
                                                    if let Err(e) = reload_base_url_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload base url setting");
                                                    }
                                                },
                                                OAUTH_SETTING => {
                                                    if let Err(e) = reload_base_url_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload oauth setting");
                                                    }
                                                },
                                                CUSTOM_TAGS_SETTING => {
                                                    if let Err(e) = reload_custom_tags_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload custom tags setting");
                                                    }
                                                },
                                                LICENSE_KEY_SETTING => {
                                                    if let Err(e) = reload_license_key(&db).await {
                                                        tracing::error!("Failed to reload license key: {e:#}");
                                                    }
                                                },
                                                DEFAULT_TAGS_PER_WORKSPACE_SETTING => {
                                                    if let Err(e) = load_tag_per_workspace_enabled(&db).await {
                                                        tracing::error!("Error loading default tag per workspace: {e:#}");
                                                    }
                                                },
                                                DEFAULT_TAGS_WORKSPACES_SETTING => {
                                                    if let Err(e) = load_tag_per_workspace_workspaces(&db).await {
                                                        tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
                                                    }
                                                }
                                                SMTP_SETTING => {
                                                    reload_smtp_config(&db).await;
                                                },
                                                TEAMS_SETTING => {
                                                    tracing::info!("Teams setting changed.");
                                                },
                                                INDEXER_SETTING => {
                                                    reload_indexer_config(&db).await;
                                                },
                                                TIMEOUT_WAIT_RESULT_SETTING => {
                                                    reload_timeout_wait_result_setting(&db).await
                                                },
                                                RETENTION_PERIOD_SECS_SETTING => {
                                                    reload_retention_period_setting(&db).await
                                                },
                                                MONITOR_LOGS_ON_OBJECT_STORE_SETTING => {
                                                    reload_delete_logs_periodically_setting(&db).await
                                                },
                                                JOB_DEFAULT_TIMEOUT_SECS_SETTING => {
                                                    reload_job_default_timeout_setting(&db).await
                                                },
                                                #[cfg(feature = "parquet")]
                                                OBJECT_STORE_CACHE_CONFIG_SETTING => {
                                                    if !disable_s3_store {
                                                        reload_s3_cache_setting(&db).await
                                                    }
                                                },
                                                SCIM_TOKEN_SETTING => {
                                                    reload_scim_token_setting(&db).await
                                                },
                                                EXTRA_PIP_INDEX_URL_SETTING => {
                                                    reload_extra_pip_index_url_setting(&db).await
                                                },
                                                PIP_INDEX_URL_SETTING => {
                                                    reload_pip_index_url_setting(&db).await
                                                },
                                                INSTANCE_PYTHON_VERSION_SETTING => {
                                                    reload_instance_python_version_setting(&db).await
                                                },
                                                NPM_CONFIG_REGISTRY_SETTING => {
                                                    reload_npm_config_registry_setting(&db).await
                                                },
                                                BUNFIG_INSTALL_SCOPES_SETTING => {
                                                    reload_bunfig_install_scopes_setting(&db).await
                                                },
                                                NUGET_CONFIG_SETTING => {
                                                    reload_nuget_config_setting(&db).await
                                                },
                                                KEEP_JOB_DIR_SETTING => {
                                                    load_keep_job_dir(&db).await;
                                                },
                                                REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING => {
                                                    load_require_preexisting_user(&db).await;
                                                },
                                                EXPOSE_METRICS_SETTING  => {
                                                    tracing::info!("Metrics setting changed, restarting");
                                                    send_delayed_killpill(&tx, 40, "metrics setting change").await;
                                                },
                                                EMAIL_DOMAIN_SETTING => {
                                                    tracing::info!("Email domain setting changed");
                                                    if server_mode {
                                                        send_delayed_killpill(&tx, 4, "email domain setting change").await;
                                                    }
                                                },
                                                EXPOSE_DEBUG_METRICS_SETTING => {
                                                    if let Err(e) = load_metrics_debug_enabled(&db).await {
                                                        tracing::error!(error = %e, "Could not reload debug metrics setting");
                                                    }
                                                },
                                                OTEL_SETTING => {
                                                    tracing::info!("OTEL setting changed, restarting");
                                                    send_delayed_killpill(&tx, 4, "OTEL setting change").await;
                                                },
                                                REQUEST_SIZE_LIMIT_SETTING => {
                                                    if server_mode {
                                                        tracing::info!("Request limit size change detected, killing server expecting to be restarted");
                                                        send_delayed_killpill(&tx, 4, "request size limit change").await;
                                                    }
                                                },
                                                SAML_METADATA_SETTING => {
                                                    tracing::info!("SAML metadata change detected, killing server expecting to be restarted");
                                                    send_delayed_killpill(&tx, 0, "SAML metadata change").await;
                                                },
                                                HUB_BASE_URL_SETTING => {
                                                    if let Err(e) = reload_hub_base_url_setting(&db, server_mode).await {
                                                        tracing::error!(error = %e, "Could not reload hub base url setting");
                                                    }
                                                },
                                                CRITICAL_ERROR_CHANNELS_SETTING => {
                                                    if let Err(e) = reload_critical_error_channels_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload critical error emails setting");
                                                    }
                                                },
                                                JWT_SECRET_SETTING => {
                                                    if let Err(e) = reload_jwt_secret_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload jwt secret setting");
                                                    }
                                                },
                                                CRITICAL_ALERT_MUTE_UI_SETTING => {
                                                    tracing::info!("Critical alert UI setting changed");
                                                    if let Err(e) = reload_critical_alert_mute_ui_setting(&db).await {
                                                        tracing::error!(error = %e, "Could not reload critical alert UI setting");
                                                    }
                                                },
                                                a @_ => {
                                                    tracing::info!("Unrecognized Global Setting Change Payload: {:?}", a);
                                                }
                                            }
                                        },
                                        _ => {
                                            tracing::warn!("Unknown notification received");
                                            continue;
                                        }
                                    }
                                },
                                Err(e) => {
                                    tracing::error!(error = %e, "Could not receive notification, attempting to reconnect listener");
                                    tokio::select! {
                                        biased;
                                        _ = monitor_killpill_rx.recv() => {
                                            tracing::info!("received killpill for monitor job");
                                            break;
                                        },
                                        new_listener = retry_listen_pg(&db_url) => {
                                            listener = new_listener;
                                            continue;
                                        }
                                    }
                                }
                            };
                        },
                        _ = tokio::time::sleep(Duration::from_secs(30))    => {
                            if last_listener_refresh.elapsed() > Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS) {
                                tracing::info!("Refreshing pg listeners, settings and license key after {}s", Duration::from_secs(*PG_LISTENER_REFRESH_PERIOD_SECS).as_secs());
                                if let Err(e) = listener.unlisten_all().await {
                                    tracing::error!(error = %e, "Could not unlisten to database");
                                }
                                listener = retry_listen_pg(&db_url).await;
                                initial_load(
                                    &db,
                                    tx.clone(),
                                    worker_mode,
                                    server_mode,
                                    #[cfg(feature = "parquet")]
                                    disable_s3_store,
                                )
                                .await;
                                #[cfg(feature = "enterprise")]
                                if let Err(err) = reload_license_key(&db).await {
                                    tracing::error!("Failed to reload license key: {err:#}");
                                }
                                last_listener_refresh = Instant::now();
                            }

                            tracing::info!("monitor task started");
                            monitor_db(
                                &db,
                                &base_internal_url,
                                server_mode,
                                worker_mode,
                                false,
                                tx.clone(),
                            )
                            .await;
                            tracing::info!("monitor task finished");
                        },
                    }
                }
            });

            if let Err(e) = h.await {
                tracing::error!("Error waiting for monitor handle: {e:#}")
            }
            tracing::info!("Monitor exited");
            killpill_tx.send();
            Ok(()) as anyhow::Result<()>
        };

        let metrics_f = async {
            let enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);

            #[cfg(not(all(feature = "enterprise", feature = "prometheus")))]
            if enabled {
                tracing::error!("Metrics are only available in the EE, ignoring...");
            }

            #[cfg(all(feature = "enterprise", feature = "prometheus"))]
            if let Err(e) = windmill_common::serve_metrics(
                *windmill_common::METRICS_ADDR,
                _killpill_phase2_rx,
                num_workers > 0,
                enabled,
            )
            .await
            {
                tracing::error!("Error serving metrics: {e:#}");
            }

            Ok(()) as anyhow::Result<()>
        };

        if server_mode {
            schedule_stats(&db, &HTTP_CLIENT).await;
        }

        futures::try_join!(
            shutdown_signal,
            workers_f,
            monitor_f,
            server_f,
            metrics_f,
            indexer_f,
            log_indexer_f
        )?;
    } else {
        tracing::info!("Nothing to do, exiting.");
    }
    send_current_log_file_to_object_store(&db, &hostname, &mode).await;

    tracing::info!("Exiting connection pool");
    tokio::select! {
        _ = db.close() => {
            tracing::info!("Database connection pool closed");
        },
        _ = tokio::time::sleep(Duration::from_secs(15)) => {
            tracing::warn!("Could not close database connection pool in time (15s). Exiting anyway.");
        }
    }
    Ok(())
}

async fn listen_pg(url: &str) -> Option<PgListener> {
    let mut listener = match PgListener::connect(url).await {
        Ok(l) => l,
        Err(e) => {
            tracing::error!(error = %e, "Could not connect to database");
            return None;
        }
    };

    if let Err(e) = listener
        .listen_all(vec![
            "notify_config_change",
            "notify_global_setting_change",
            "notify_webhook_change",
        ])
        .await
    {
        tracing::error!(error = %e, "Could not listen to database");
        return None;
    }

    return Some(listener);
}

async fn retry_listen_pg(url: &str) -> PgListener {
    let mut listener = listen_pg(url).await;
    loop {
        if listener.is_none() {
            tracing::info!("Retrying listening to pg listen in 5 seconds");
            tokio::time::sleep(Duration::from_secs(5)).await;
            listener = listen_pg(url).await;
        } else {
            tracing::info!("Successfully connected to pg listen");
            return listener.unwrap();
        }
    }
}

fn display_config(envs: &[&str]) {
    tracing::info!(
        "config: {}",
        envs.iter()
            .filter(|env| std::env::var(env).is_ok())
            .map(|env| {
                format!(
                    "{}: {}",
                    env,
                    std::env::var(env).unwrap_or_else(|_| "not set".to_string())
                )
            })
            .collect::<Vec<String>>()
            .join(", ")
    )
}

pub async fn run_workers(
    db: Pool<Postgres>,
    mut rx: tokio::sync::broadcast::Receiver<()>,
    tx: KillpillSender,
    num_workers: i32,
    base_internal_url: String,
    agent_mode: bool,
    hostname: String,
) -> anyhow::Result<()> {
    let mut killpill_rxs = vec![];
    for _ in 0..num_workers {
        killpill_rxs.push(rx.resubscribe());
    }

    if rx.try_recv().is_ok() {
        tracing::info!("Received killpill, exiting");
        return Ok(());
    }
    let instance_name = hostname
        .clone()
        .replace(" ", "")
        .split("-")
        .last()
        .unwrap()
        .to_ascii_lowercase()
        .to_string();

    // #[cfg(tokio_unstable)]
    // let monitor = tokio_metrics::TaskMonitor::new();

    let ip = windmill_common::external_ip::get_ip()
        .await
        .unwrap_or_else(|e| {
            tracing::warn!(error = e.to_string(), "failed to get external IP");
            "unretrievable IP".to_string()
        });

    let mut handles = Vec::with_capacity(num_workers as usize);

    for x in [
        TMP_LOGS_DIR,
        UV_CACHE_DIR,
        DENO_CACHE_DIR,
        DENO_CACHE_DIR_DEPS,
        DENO_CACHE_DIR_NPM,
        BUN_CACHE_DIR,
        PY310_CACHE_DIR,
        PY311_CACHE_DIR,
        PY312_CACHE_DIR,
        PY313_CACHE_DIR,
        TAR_PY310_CACHE_DIR,
        TAR_PY311_CACHE_DIR,
        TAR_PY312_CACHE_DIR,
        TAR_PY313_CACHE_DIR,
        BUN_BUNDLE_CACHE_DIR,
        GO_CACHE_DIR,
        GO_BIN_CACHE_DIR,
        RUST_CACHE_DIR,
        CSHARP_CACHE_DIR,
        NU_CACHE_DIR,
        HUB_CACHE_DIR,
        POWERSHELL_CACHE_DIR,
    ] {
        DirBuilder::new()
            .recursive(true)
            .create(x)
            .expect("could not create initial worker dir");
    }

    tracing::info!(
        "Starting {num_workers} workers and SLEEP_QUEUE={}ms",
        *windmill_worker::SLEEP_QUEUE
    );
    for i in 1..(num_workers + 1) {
        let db1 = db.clone();
        let instance_name = instance_name.clone();
        let worker_name = format!("wk-{}-{}-{}", *WORKER_GROUP, &instance_name, rd_string(5));
        let ip = ip.clone();
        let rx = killpill_rxs.pop().unwrap();
        let tx = tx.clone();
        let base_internal_url = base_internal_url.clone();
        let hostname = hostname.clone();

        handles.push(tokio::spawn(async move {
            if num_workers > 1 {
                tracing::info!(worker = %worker_name, "starting worker {i}");
            }

            let f = windmill_worker::run_worker(
                &db1,
                &hostname,
                worker_name,
                i as u64,
                num_workers as u32,
                &ip,
                rx,
                tx,
                &base_internal_url,
                agent_mode,
            );

            // #[cfg(tokio_unstable)]
            // {
            //     monitor.monitor(f, "worker").await
            // }

            // #[cfg(not(tokio_unstable))]
            // {
            f.await
            // }
        }));
    }

    futures::future::try_join_all(handles).await?;
    Ok(())
}

async fn send_delayed_killpill(tx: &KillpillSender, mut max_delay_secs: u64, context: &str) {
    if max_delay_secs == 0 {
        max_delay_secs = 1;
    }
    // Random delay to avoid all servers/workers shutting down simultaneously
    let rd_delay = rand::rng().random_range(0..max_delay_secs);
    tracing::info!("Scheduling {context} shutdown in {rd_delay}s");
    tokio::time::sleep(Duration::from_secs(rd_delay)).await;

    tx.send();
}