From 8f198ba68ceaa1f503cddfc246ecbbfc7f90f11d Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Wed, 20 Nov 2024 22:40:36 +0100 Subject: [PATCH] feat: Indexer improvements: s3 backup logic reworked, settings on the frontend (#4763) * Add settings page + common code for indexer * Prepare sqlx --- ...43061932dc5b4b17cf13d5cf8d833aedbe6d5.json | 2 +- ...9177a87e1accd192402e21db5ae09c3498ab0.json | 3 +- ...8c47886545796424f8612135b711e2b9ddb6c.json | 3 +- ...98a03f751b246c40daf056fced0fd91f6dd73.json | 3 +- ...1218ac19ab2a4bc5f8b42deb2476f85d0deac.json | 3 +- ...7f5033b9c9afc344d9c3e385ba20a3ad2197a.json | 2 +- ...62c3c267ca336a8b6bec5b29d4409030ed561.json | 3 +- ...94f91df7e588d4d2431bc85f4d8734920c8bf.json | 3 +- ...8b3ca6d7fa05600067f379659b59b6f2bf418.json | 3 +- ...eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json | 3 +- ...36e65991b1d21a3306390fb1c7ca38babafe3.json | 3 +- ...53ff66b37ed8110a0b67aedb9f25e6a7383e1.json | 3 +- ...160ad1962668fc3305d8e80ae91ef73614a80.json | 2 +- ...371fe28ec8233fa80413324ce0194ce909e3c.json | 20 +++ ...1e273d25543401639a68f868e1e4fd8b0640a.json | 3 +- backend/Cargo.lock | 3 +- backend/ee-repo-ref.txt | 2 +- backend/src/main.rs | 22 ++- backend/src/monitor.rs | 16 +- .../windmill-common/src/global_settings.rs | 1 + backend/windmill-common/src/indexer.rs | 114 ++++++++++++++ backend/windmill-common/src/lib.rs | 1 + backend/windmill-common/src/worker.rs | 3 +- backend/windmill-indexer/Cargo.toml | 1 + .../windmill-indexer/src/completed_runs_ee.rs | 2 +- .../windmill-indexer/src/service_logs_ee.rs | 8 +- .../lib/components/InstanceSettings.svelte | 140 ++++++++++++++++++ .../src/lib/components/instanceSettings.ts | 10 ++ .../search/GlobalSearchModal.svelte | 4 +- .../(root)/(logged)/service_logs/+page.svelte | 2 +- 30 files changed, 348 insertions(+), 40 deletions(-) create mode 100644 backend/.sqlx/query-e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c.json create mode 100644 backend/windmill-common/src/indexer.rs diff --git a/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json b/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json index c8b5e3086f..043d236a6b 100644 --- a/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json +++ b/backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "bool", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json index fc94db0309..109f357f74 100644 --- a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json +++ b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json @@ -72,7 +72,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json b/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json index e74be644c7..a82d32768e 100644 --- a/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json +++ b/backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json @@ -52,7 +52,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json index a23bb3827f..8a12aca84a 100644 --- a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json +++ b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json @@ -58,7 +58,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json index 6bc0fb8a10..b35a48638a 100644 --- a/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json +++ b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json @@ -57,7 +57,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 75b8108281..1fa370e682 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "bool", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json index 5605518c9c..36579b3062 100644 --- a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json +++ b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json @@ -52,7 +52,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json index c4a2ff0a10..7cbd85f23b 100644 --- a/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json +++ b/backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json @@ -30,7 +30,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json b/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json index 4f8a78ca66..b955a01fb6 100644 --- a/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json +++ b/backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json @@ -57,7 +57,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index 2393bc83df..3019cb0068 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -52,7 +52,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json index cc825ff56f..8b3824a2a2 100644 --- a/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json +++ b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json @@ -38,7 +38,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json b/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json index 164157b02f..fafd943f04 100644 --- a/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json +++ b/backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json @@ -78,7 +78,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json b/backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json index 271c395f9a..103219fe05 100644 --- a/backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json +++ b/backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "bool", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c.json b/backend/.sqlx/query-e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c.json new file mode 100644 index 0000000000..1c108e19a2 --- /dev/null +++ b/backend/.sqlx/query-e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM global_settings WHERE name = 'indexer_settings'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c" +} diff --git a/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json index 5e4f522147..8f459f96fd 100644 --- a/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json +++ b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json @@ -57,7 +57,8 @@ "php", "bunnative", "rust", - "ansible" + "ansible", + "csharp" ] } } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d58f9470ae..1faa25d419 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -10908,6 +10908,7 @@ dependencies = [ "anyhow", "bytes", "chrono", + "const_format", "futures", "lazy_static", "object_store", diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0042b552e0..ff96bbc548 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5bb4a7ce8660effc14b4ab96b17ff11f4048bfe9 \ No newline at end of file +0a64c5754a5e7c5b5e12ab7dc80d252f3f301e96 diff --git a/backend/src/main.rs b/backend/src/main.rs index d94537110a..eb6c423e0e 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,8 +8,7 @@ use anyhow::Context; use monitor::{ - reload_timeout_wait_result_setting, send_current_log_file_to_object_store, - send_logs_to_object_store, + reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -30,15 +29,7 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE use windmill_common::{ global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, - CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, - EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, - JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, - LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, - TIMEOUT_WAIT_RESULT_SETTING, + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -529,6 +520,8 @@ Windmill Community Edition {GIT_VERSION} let should_index_jobs = mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone); + reload_indexer_config(&db).await; + #[cfg(feature = "tantivy")] let (index_reader, index_writer) = if should_index_jobs { let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?; @@ -548,7 +541,7 @@ Windmill Community Edition {GIT_VERSION} index_writer, indexer_rx, ) - .await; + .await?; } Ok(()) } @@ -573,7 +566,7 @@ Windmill Community Edition {GIT_VERSION} log_index_writer, log_indexer_rx, ) - .await; + .await?; } Ok(()) } @@ -737,6 +730,9 @@ Windmill Community Edition {GIT_VERSION} SMTP_SETTING => { reload_smtp_config(&db).await; }, + INDEXER_SETTING => { + reload_indexer_config(&db).await; + }, TIMEOUT_WAIT_RESULT_SETTING => { reload_timeout_wait_result_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 43c5867dce..a623d76d39 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -43,6 +43,7 @@ use windmill_common::{ REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, + indexer::load_indexer_config, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, @@ -51,8 +52,8 @@ use windmill_common::{ utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{ load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting, - update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, - WORKER_CONFIG, WORKER_GROUP, + update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, + SMTP_CONFIG, WORKER_CONFIG, WORKER_GROUP, }, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, @@ -1186,6 +1187,17 @@ pub async fn reload_smtp_config(db: &Pool) { } } +pub async fn reload_indexer_config(db: &Pool) { + let indexer_config = load_indexer_config(&db).await; + if let Err(e) = indexer_config { + tracing::error!("Error reloading indexer config: {:?}", e) + } else { + let mut wc = INDEXER_CONFIG.write().await; + tracing::info!("Reloading smtp config..."); + *wc = indexer_config.unwrap() + } +} + pub async fn reload_worker_config( db: &DB, tx: tokio::sync::broadcast::Sender<()>, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index a48db22b1f..f24153ec38 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -15,6 +15,7 @@ pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; pub const SCIM_TOKEN_SETTING: &str = "scim_token"; pub const SAML_METADATA_SETTING: &str = "saml_metadata"; pub const SMTP_SETTING: &str = "smtp_settings"; +pub const INDEXER_SETTING: &str = "indexer_settings"; pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result"; pub const UNIQUE_ID_SETTING: &str = "uid"; diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs new file mode 100644 index 0000000000..31ad8778b6 --- /dev/null +++ b/backend/windmill-common/src/indexer.rs @@ -0,0 +1,114 @@ +use serde::Deserialize; + +use crate::{error, DB}; + +pub const BYTES_PER_MB: u64 = 1_048_576; +pub const BYTES_PER_KB: u64 = 1024; + +#[derive(Clone, Debug)] +pub struct TantivyIndexerSettings { + pub writer_memory_budget: u64, + pub commit_job_max_batch_size: u64, + pub commit_log_max_batch_size: u64, + pub refresh_index_period: u64, + pub refresh_log_index_period: u64, + pub max_indexed_job_log_size: usize, +} + +impl Default for TantivyIndexerSettings { + fn default() -> Self { + TantivyIndexerSettings { + writer_memory_budget: 300_000_000, + commit_job_max_batch_size: 100_000, + commit_log_max_batch_size: 10_000, + refresh_index_period: 300, + refresh_log_index_period: 300, + max_indexed_job_log_size: 1_000_000, + } + } +} +#[derive(Deserialize, Default)] +pub struct TantivyIndexerSettingsOpt { + pub writer_memory_budget: Option, + pub commit_job_max_batch_size: Option, + pub commit_log_max_batch_size: Option, + pub refresh_index_period: Option, + pub refresh_log_index_period: Option, + pub max_indexed_job_log_size: Option, +} + +pub async fn load_indexer_config(db: &DB) -> error::Result { + let config: TantivyIndexerSettingsOpt = + sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'indexer_settings'",) + .fetch_optional(db) + .await? + .map(|x| serde_json::from_value(x).ok()) + .flatten() + .unwrap_or_default(); + + let TantivyIndexerSettings { + commit_job_max_batch_size, + commit_log_max_batch_size, + refresh_index_period, + refresh_log_index_period, + max_indexed_job_log_size, + writer_memory_budget, + } = get_indexer_rates_from_env(); + + Ok(TantivyIndexerSettings { + writer_memory_budget: config.writer_memory_budget.unwrap_or(writer_memory_budget), + commit_job_max_batch_size: config + .commit_job_max_batch_size + .unwrap_or(commit_job_max_batch_size), + commit_log_max_batch_size: config + .commit_log_max_batch_size + .unwrap_or(commit_log_max_batch_size), + refresh_index_period: config.refresh_index_period.unwrap_or(refresh_index_period), + refresh_log_index_period: config + .refresh_log_index_period + .unwrap_or(refresh_log_index_period), + max_indexed_job_log_size: config + .max_indexed_job_log_size + .unwrap_or(max_indexed_job_log_size), + }) +} + +pub fn get_env_var(env_var: &str) -> Option { + match std::env::var(env_var).map(|x| x.parse()) { + Ok(Ok(i)) => Some(i), + Err(_) => None, + + Ok(Err(e)) => { + tracing::error!("Failed to parse env var {}: {}", env_var, e); + None + } + } +} + +pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings { + let mut settings = TantivyIndexerSettings::default(); + + if let Some(b) = get_env_var("TANTIVY_INDEX_WRITER_MEMORY_BUDGET__MB") { + settings.writer_memory_budget = b * BYTES_PER_MB as u64; + } + if let Some(b) = get_env_var("TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE") { + settings.commit_job_max_batch_size = b; + } + if let Some(b) = get_env_var("TANTIVY_SERVICE_LOG_COMMIT_MAX_BATCH_SIZE") { + settings.commit_log_max_batch_size = b; + } + if let Some(b) = get_env_var("TANTIVY_REFRESH_INDEX_PERIOD__S") { + settings.refresh_index_period = b; + } + if let Some(b) = get_env_var("TANTIVY_REFRESH_LOG_INDEX_PERIOD__S") { + settings.refresh_log_index_period = b; + } + if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__MB") { + settings.max_indexed_job_log_size = (b * BYTES_PER_MB) as usize; + } + if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") { + settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize; + } + + settings +} diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 8eb57cb236..91f49da937 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -28,6 +28,7 @@ pub mod external_ip; pub mod flow_status; pub mod flows; pub mod global_settings; +pub mod indexer; pub mod job_metrics; #[cfg(feature = "parquet")] pub mod job_s3_helpers_ee; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 2f7f4b18e0..ee6b95a19b 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -17,7 +17,7 @@ use std::{ use tokio::sync::RwLock; use windmill_macros::annotations; -use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB}; +use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB}; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); @@ -70,6 +70,7 @@ lazy_static::lazy_static! { pub static ref SMTP_CONFIG: Arc>> = Arc::new(RwLock::new(None)); + pub static ref INDEXER_CONFIG: Arc> = Arc::new(RwLock::new(TantivyIndexerSettings::default())); pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok(); diff --git a/backend/windmill-indexer/Cargo.toml b/backend/windmill-indexer/Cargo.toml index b345e02917..14ab53beef 100644 --- a/backend/windmill-indexer/Cargo.toml +++ b/backend/windmill-indexer/Cargo.toml @@ -30,3 +30,4 @@ bytes.workspace = true object_store = { workspace = true, optional = true} tokio-tar.workspace = true lazy_static.workspace = true +const_format.workspace = true diff --git a/backend/windmill-indexer/src/completed_runs_ee.rs b/backend/windmill-indexer/src/completed_runs_ee.rs index 79bcbcff77..46887ee62b 100644 --- a/backend/windmill-indexer/src/completed_runs_ee.rs +++ b/backend/windmill-indexer/src/completed_runs_ee.rs @@ -8,7 +8,7 @@ pub struct IndexReader; #[derive(Clone)] pub struct IndexWriter; -pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> { +pub async fn init_index(_db: Pool) -> Result<(IndexReader, IndexWriter), Error> { Err(anyhow!("Cannot initialize index: not in EE").into()) } diff --git a/backend/windmill-indexer/src/service_logs_ee.rs b/backend/windmill-indexer/src/service_logs_ee.rs index 79bcbcff77..8ef070dcea 100644 --- a/backend/windmill-indexer/src/service_logs_ee.rs +++ b/backend/windmill-indexer/src/service_logs_ee.rs @@ -3,18 +3,18 @@ use windmill_common::error::Error; use anyhow::anyhow; #[derive(Clone)] -pub struct IndexReader; +pub struct ServiceLogIndexReader; #[derive(Clone)] -pub struct IndexWriter; +pub struct ServiceLogIndexWriter; -pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> { +pub async fn init_index(_db: Pool) -> Result<(ServiceLogIndexReader, ServiceLogIndexWriter), Error> { Err(anyhow!("Cannot initialize index: not in EE").into()) } pub async fn run_indexer( _db: Pool, - mut _index_writer: IndexWriter, + mut _index_writer: ServiceLogIndexWriter, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, ) { tracing::error!("Cannot run indexer: not in EE"); diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 213318e6fe..acfa8f6cb6 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -106,6 +106,9 @@ if (values['smtp_settings'] == undefined) { values['smtp_settings'] = {} } + if (values['indexer_settings'] == undefined) { + values['indexer_settings'] = {} + } loading = false latestKeyRenewalAttempt = await SettingService.getLatestKeyRenewalAttempt() @@ -324,6 +327,12 @@ Learn more + {:else if category == "Indexer/Search"} +
The indexer service unlocks full text search across jobs and service logs. It requires spinning up its own separate container + Learn how to
{:else if category == 'Registries'}
Add private registries for Pip, Bun and npm. {/if}
+ {:else if setting.fieldType == 'indexer_rates'} +
+ {#if values[setting.key]} +
+ + { + if (e.target instanceof HTMLInputElement) { + if (e.target.valueAsNumber) { + values[setting.key].writer_memory_budget = + e.target.valueAsNumber * (1024 * 1024) + } + } + }} + value={values[setting.key].writer_memory_budget / (1024 * 1024)} + /> +
+

Completed Job Index

+
+ + +
+
+ + +
+
+ + { + if (e.target instanceof HTMLInputElement) { + if (e.target.valueAsNumber) { + values[setting.key].max_indexed_job_log_size = + e.target.valueAsNumber * 1024 + } + } + }} + value={values[setting.key].max_indexed_job_log_size / 1024} + /> +
+

Service Logs Index

+
+ + +
+
+ + +
+ {/if} +
{:else if setting.fieldType == 'smtp_connect'}
{#if values[setting.key]} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 13fe33bb72..2a8dd95aad 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -20,6 +20,7 @@ export interface Setting { | 'critical_error_channels' | 'slack_connect' | 'smtp_connect' + | 'indexer_rates' storage: SettingStorage advancedToggle?: { label: string @@ -249,6 +250,15 @@ export const settings: Record = { ee_only: '' } ], + 'Indexer/Search': [ + { + label: '', + key: 'indexer_settings', + fieldType: 'indexer_rates', + storage: 'setting', + ee_only: 'Full text search across jobs and service logs is an EE feature' + } + ], Slack: [ { label: 'Slack', diff --git a/frontend/src/lib/components/search/GlobalSearchModal.svelte b/frontend/src/lib/components/search/GlobalSearchModal.svelte index 021d82acfc..599e8853cb 100644 --- a/frontend/src/lib/components/search/GlobalSearchModal.svelte +++ b/frontend/src/lib/components/search/GlobalSearchModal.svelte @@ -701,7 +701,7 @@
{#if indexMetadata.indexed_until} - Most recently indexed job was created @@ -745,7 +745,7 @@
{#if indexMetadata.indexed_until} - Most recently indexed job was created diff --git a/frontend/src/routes/(root)/(logged)/service_logs/+page.svelte b/frontend/src/routes/(root)/(logged)/service_logs/+page.svelte index 7dc9cb4580..9989f8c861 100644 --- a/frontend/src/routes/(root)/(logged)/service_logs/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/service_logs/+page.svelte @@ -11,7 +11,7 @@ let queryParseErrors: string[] | undefined = undefined -
+