feat: Indexer improvements: s3 backup logic reworked, settings on the frontend (#4763)

* Add settings page + common code for indexer

* Prepare sqlx

backend/.sqlx/query-02424907504848e983bfa89eec343061932dc5b4b17cf13d5cf8d833aedbe6d5.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "?column?",
+        "name": "bool",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json

@@ -72,7 +72,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-1a612eb0b64eddd2c5657ef73598c47886545796424f8612135b711e2b9ddb6c.json

@@ -52,7 +52,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json

@@ -58,7 +58,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "?column?",
+        "name": "bool",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json

@@ -52,7 +52,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json

@@ -30,7 +30,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-9d25fbd21a63e6e9ccfdbf0460c8b3ca6d7fa05600067f379659b59b6f2bf418.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json

@@ -52,7 +52,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json

@@ -38,7 +38,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-d5a8614286c170e0d175903cd1b53ff66b37ed8110a0b67aedb9f25e6a7383e1.json

@@ -78,7 +78,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/.sqlx/query-d6615719bf8db4b333ed55c9a3c160ad1962668fc3305d8e80ae91ef73614a80.json

@@ -5,7 +5,7 @@
     "columns": [
       {
         "ordinal": 0,
-        "name": "?column?",
+        "name": "bool",
         "type_info": "Bool"
       }
     ],

backend/.sqlx/query-e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT value FROM global_settings WHERE name = 'indexer_settings'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "value",
+        "type_info": "Jsonb"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e2d8028163c014f4cdff0d85550371fe28ec8233fa80413324ce0194ce909e3c"
+}

backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json

@@ -57,7 +57,8 @@
                 "php",
                 "bunnative",
                 "rust",
-                "ansible"
+                "ansible",
+                "csharp"
               ]
             }
           }

backend/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "Inflector"
@@ -10908,6 +10908,7 @@ dependencies = [
  "anyhow",
  "bytes",
  "chrono",
+ "const_format",
  "futures",
  "lazy_static",
  "object_store",

backend/ee-repo-ref.txt

@@ -1 +1 @@
-5bb4a7ce8660effc14b4ab96b17ff11f4048bfe9
+0a64c5754a5e7c5b5e12ab7dc80d252f3f301e96

backend/src/main.rs

@@ -8,8 +8,7 @@
 
 use anyhow::Context;
 use monitor::{
-    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
-    send_logs_to_object_store,
+    reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store
 };
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -30,15 +29,7 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE
 
 use windmill_common::{
     global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
-        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
-        DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
-        EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING,
-        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
-        LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING,
-        REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
-        RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
-        TIMEOUT_WAIT_RESULT_SETTING,
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING,CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING
     },
     scripts::ScriptLang,
     stats_ee::schedule_stats,
@@ -529,6 +520,8 @@ Windmill Community Edition {GIT_VERSION}
         let should_index_jobs =
             mode == Mode::Indexer || (enable_standalone_indexer && mode == Mode::Standalone);
 
+        reload_indexer_config(&db).await;
+
         #[cfg(feature = "tantivy")]
         let (index_reader, index_writer) = if should_index_jobs {
             let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?;
@@ -548,7 +541,7 @@ Windmill Community Edition {GIT_VERSION}
                         index_writer,
                         indexer_rx,
                     )
-                    .await;
+                    .await?;
                 }
                 Ok(())
             }
@@ -573,7 +566,7 @@ Windmill Community Edition {GIT_VERSION}
                         log_index_writer,
                         log_indexer_rx,
                     )
-                    .await;
+                    .await?;
                 }
                 Ok(())
             }
@@ -737,6 +730,9 @@ Windmill Community Edition {GIT_VERSION}
                                                 SMTP_SETTING => {
                                                     reload_smtp_config(&db).await;
                                                 },
+                                                INDEXER_SETTING => {
+                                                    reload_indexer_config(&db).await;
+                                                },
                                                 TIMEOUT_WAIT_RESULT_SETTING => {
                                                     reload_timeout_wait_result_setting(&db).await
                                                 },

backend/src/monitor.rs

@@ -43,6 +43,7 @@ use windmill_common::{
         REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
         SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
     },
+    indexer::load_indexer_config,
     jobs::QueuedJob,
     oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
     server::load_smtp_config,
@@ -51,8 +52,8 @@ use windmill_common::{
     utils::{now_from_db, rd_string, report_critical_error, Mode},
     worker::{
         load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
-        update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG,
-        WORKER_CONFIG, WORKER_GROUP,
+        update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG,
+        SMTP_CONFIG, WORKER_CONFIG, WORKER_GROUP,
     },
     BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
     HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
@@ -1186,6 +1187,17 @@ pub async fn reload_smtp_config(db: &Pool<Postgres>) {
     }
 }
 
+pub async fn reload_indexer_config(db: &Pool<Postgres>) {
+    let indexer_config = load_indexer_config(&db).await;
+    if let Err(e) = indexer_config {
+        tracing::error!("Error reloading indexer config: {:?}", e)
+    } else {
+        let mut wc = INDEXER_CONFIG.write().await;
+        tracing::info!("Reloading smtp config...");
+        *wc = indexer_config.unwrap()
+    }
+}
+
 pub async fn reload_worker_config(
     db: &DB,
     tx: tokio::sync::broadcast::Sender<()>,

backend/windmill-common/src/global_settings.rs

@@ -15,6 +15,7 @@ pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
 pub const SCIM_TOKEN_SETTING: &str = "scim_token";
 pub const SAML_METADATA_SETTING: &str = "saml_metadata";
 pub const SMTP_SETTING: &str = "smtp_settings";
+pub const INDEXER_SETTING: &str = "indexer_settings";
 pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
 
 pub const UNIQUE_ID_SETTING: &str = "uid";

backend/windmill-common/src/indexer.rs

@@ -0,0 +1,114 @@
+use serde::Deserialize;
+
+use crate::{error, DB};
+
+pub const BYTES_PER_MB: u64 = 1_048_576;
+pub const BYTES_PER_KB: u64 = 1024;
+
+#[derive(Clone, Debug)]
+pub struct TantivyIndexerSettings {
+    pub writer_memory_budget: u64,
+    pub commit_job_max_batch_size: u64,
+    pub commit_log_max_batch_size: u64,
+    pub refresh_index_period: u64,
+    pub refresh_log_index_period: u64,
+    pub max_indexed_job_log_size: usize,
+}
+
+impl Default for TantivyIndexerSettings {
+    fn default() -> Self {
+        TantivyIndexerSettings {
+            writer_memory_budget: 300_000_000,
+            commit_job_max_batch_size: 100_000,
+            commit_log_max_batch_size: 10_000,
+            refresh_index_period: 300,
+            refresh_log_index_period: 300,
+            max_indexed_job_log_size: 1_000_000,
+        }
+    }
+}
+#[derive(Deserialize, Default)]
+pub struct TantivyIndexerSettingsOpt {
+    pub writer_memory_budget: Option<u64>,
+    pub commit_job_max_batch_size: Option<u64>,
+    pub commit_log_max_batch_size: Option<u64>,
+    pub refresh_index_period: Option<u64>,
+    pub refresh_log_index_period: Option<u64>,
+    pub max_indexed_job_log_size: Option<usize>,
+}
+
+pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSettings> {
+    let config: TantivyIndexerSettingsOpt =
+        sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'indexer_settings'",)
+            .fetch_optional(db)
+            .await?
+            .map(|x| serde_json::from_value(x).ok())
+            .flatten()
+            .unwrap_or_default();
+
+    let TantivyIndexerSettings {
+        commit_job_max_batch_size,
+        commit_log_max_batch_size,
+        refresh_index_period,
+        refresh_log_index_period,
+        max_indexed_job_log_size,
+        writer_memory_budget,
+    } = get_indexer_rates_from_env();
+
+    Ok(TantivyIndexerSettings {
+        writer_memory_budget: config.writer_memory_budget.unwrap_or(writer_memory_budget),
+        commit_job_max_batch_size: config
+            .commit_job_max_batch_size
+            .unwrap_or(commit_job_max_batch_size),
+        commit_log_max_batch_size: config
+            .commit_log_max_batch_size
+            .unwrap_or(commit_log_max_batch_size),
+        refresh_index_period: config.refresh_index_period.unwrap_or(refresh_index_period),
+        refresh_log_index_period: config
+            .refresh_log_index_period
+            .unwrap_or(refresh_log_index_period),
+        max_indexed_job_log_size: config
+            .max_indexed_job_log_size
+            .unwrap_or(max_indexed_job_log_size),
+    })
+}
+
+pub fn get_env_var(env_var: &str) -> Option<u64> {
+    match std::env::var(env_var).map(|x| x.parse()) {
+        Ok(Ok(i)) => Some(i),
+        Err(_) => None,
+
+        Ok(Err(e)) => {
+            tracing::error!("Failed to parse env var {}: {}", env_var, e);
+            None
+        }
+    }
+}
+
+pub fn get_indexer_rates_from_env() -> TantivyIndexerSettings {
+    let mut settings = TantivyIndexerSettings::default();
+
+    if let Some(b) = get_env_var("TANTIVY_INDEX_WRITER_MEMORY_BUDGET__MB") {
+        settings.writer_memory_budget = b * BYTES_PER_MB as u64;
+    }
+    if let Some(b) = get_env_var("TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE") {
+        settings.commit_job_max_batch_size = b;
+    }
+    if let Some(b) = get_env_var("TANTIVY_SERVICE_LOG_COMMIT_MAX_BATCH_SIZE") {
+        settings.commit_log_max_batch_size = b;
+    }
+    if let Some(b) = get_env_var("TANTIVY_REFRESH_INDEX_PERIOD__S") {
+        settings.refresh_index_period = b;
+    }
+    if let Some(b) = get_env_var("TANTIVY_REFRESH_LOG_INDEX_PERIOD__S") {
+        settings.refresh_log_index_period = b;
+    }
+    if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__MB") {
+        settings.max_indexed_job_log_size = (b * BYTES_PER_MB) as usize;
+    }
+    if let Some(b) = get_env_var("TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__KB") {
+        settings.max_indexed_job_log_size = (b * BYTES_PER_KB) as usize;
+    }
+
+    settings
+}

backend/windmill-common/src/lib.rs

@@ -28,6 +28,7 @@ pub mod external_ip;
 pub mod flow_status;
 pub mod flows;
 pub mod global_settings;
+pub mod indexer;
 pub mod job_metrics;
 #[cfg(feature = "parquet")]
 pub mod job_s3_helpers_ee;

backend/windmill-common/src/worker.rs

@@ -17,7 +17,7 @@ use std::{
 use tokio::sync::RwLock;
 use windmill_macros::annotations;
 
-use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB};
+use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB};
 
 lazy_static::lazy_static! {
     pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -70,6 +70,7 @@ lazy_static::lazy_static! {
 
 
     pub static ref SMTP_CONFIG: Arc<RwLock<Option<Smtp>>> = Arc::new(RwLock::new(None));
+    pub static ref INDEXER_CONFIG: Arc<RwLock<TantivyIndexerSettings>> = Arc::new(RwLock::new(TantivyIndexerSettings::default()));
 
 
     pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();

backend/windmill-indexer/Cargo.toml

@@ -30,3 +30,4 @@ bytes.workspace = true
 object_store = { workspace = true, optional = true}
 tokio-tar.workspace = true
 lazy_static.workspace = true
+const_format.workspace = true

backend/windmill-indexer/src/completed_runs_ee.rs

@@ -8,7 +8,7 @@ pub struct IndexReader;
 #[derive(Clone)]
 pub struct IndexWriter;
 
-pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
+pub async fn init_index(_db: Pool<Postgres>) -> Result<(IndexReader, IndexWriter), Error> {
     Err(anyhow!("Cannot initialize index: not in EE").into())
 }
 

backend/windmill-indexer/src/service_logs_ee.rs

@@ -3,18 +3,18 @@ use windmill_common::error::Error;
 use anyhow::anyhow;
 
 #[derive(Clone)]
-pub struct IndexReader;
+pub struct ServiceLogIndexReader;
 
 #[derive(Clone)]
-pub struct IndexWriter;
+pub struct ServiceLogIndexWriter;
 
-pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> {
+pub async fn init_index(_db: Pool<Postgres>) -> Result<(ServiceLogIndexReader, ServiceLogIndexWriter), Error> {
     Err(anyhow!("Cannot initialize index: not in EE").into())
 }
 
 pub async fn run_indexer(
     _db: Pool<Postgres>,
-    mut _index_writer: IndexWriter,
+    mut _index_writer: ServiceLogIndexWriter,
     mut _killpill_rx: tokio::sync::broadcast::Receiver<()>,
 ) {
     tracing::error!("Cannot run indexer: not in EE");

frontend/src/lib/components/InstanceSettings.svelte

@@ -106,6 +106,9 @@
 		if (values['smtp_settings'] == undefined) {
 			values['smtp_settings'] = {}
 		}
+		if (values['indexer_settings'] == undefined) {
+			values['indexer_settings'] = {}
+		}
 		loading = false
 
 		latestKeyRenewalAttempt = await SettingService.getLatestKeyRenewalAttempt()
@@ -324,6 +327,12 @@
 							<a target="_blank" href="https://www.windmill.dev/docs/misc/setup_smtp">Learn more</a
 							></div
 						>
+					{:else if category == "Indexer/Search"}
+						<div class="text-secondary pb-4 text-xs"
+							>The indexer service unlocks full text search across jobs and service logs. It requires spinning up its own separate container
+							<a target="_blank" href="https://www.windmill.dev/docs/core_concepts/search_bar#setup">Learn how to</a
+							></div
+						>
 					{:else if category == 'Registries'}
 						<div class="text-secondary pb-4 text-xs">
 							Add private registries for Pip, Bun and npm. <a
@@ -984,6 +993,137 @@
 														</Button>
 													{/if}
 												</div>
+											{:else if setting.fieldType == 'indexer_rates'}
+												<div class="flex flex-col gap-4 mt-4">
+													{#if values[setting.key]}
+														<div>
+															<label for="writer_memory_budget" class="block text-sm font-medium">
+																Index writer memory budget (MB)
+																<Tooltip>
+																	The allocated memory arena for the indexer. A bigger value means
+																	less writing to disk and potentially higher indexing throughput
+																</Tooltip>
+															</label>
+															<input
+																type="number"
+																id="writer_memory_budget"
+																placeholder="300"
+																on:input={(e) => {
+																	if (e.target instanceof HTMLInputElement) {
+																		if (e.target.valueAsNumber) {
+																			values[setting.key].writer_memory_budget =
+																				e.target.valueAsNumber * (1024 * 1024)
+																		}
+																	}
+																}}
+																value={values[setting.key].writer_memory_budget / (1024 * 1024)}
+															/>
+														</div>
+														<h3>Completed Job Index</h3>
+														<div>
+															<label
+																for="commit_job_max_batch_size"
+																class="block text-sm font-medium"
+															>
+																Commit max batch size <Tooltip>
+																	The max amount of documents (here jobs) per commit. To optimize
+																	indexing throughput, it is best to keep this as high as possible.
+																	However, especially when reindexing the whole instance, it can be
+																	useful to have a limit on how many jobs can be written without
+																	being commited. A commit will make the jobs available for search,
+																	constitute a "checkpoint" state in the indexing and will be
+																	logged.
+																</Tooltip>
+															</label>
+															<input
+																type="number"
+																id="commit_job_max_batch_size"
+																placeholder="100000"
+																bind:value={values[setting.key].commit_job_max_batch_size}
+															/>
+														</div>
+														<div>
+															<label for="refresh_index_period" class="block text-sm font-medium">
+																Refresh index period (s) <Tooltip>
+																	The index will query new jobs peridically and write them on the
+																	index. This setting sets that period.
+																</Tooltip></label
+															>
+															<input
+																type="number"
+																id="refresh_index_period"
+																placeholder="300"
+																bind:value={values[setting.key].refresh_index_period}
+															/>
+														</div>
+														<div>
+															<label
+																for="max_indexed_job_log_size"
+																class="block text-sm font-medium"
+															>
+																Max indexed job log size (KB) <Tooltip>
+																	Job logs are included when indexing, but to avoid the index size
+																	growing artificially, the logs will be truncated after a size has
+																	been reached.
+																</Tooltip>
+															</label>
+															<input
+																type="number"
+																id="max_indexed_job_log_size"
+																placeholder="1024"
+																on:input={(e) => {
+																	if (e.target instanceof HTMLInputElement) {
+																		if (e.target.valueAsNumber) {
+																			values[setting.key].max_indexed_job_log_size =
+																				e.target.valueAsNumber * 1024
+																		}
+																	}
+																}}
+																value={values[setting.key].max_indexed_job_log_size / 1024}
+															/>
+														</div>
+														<h3>Service Logs Index</h3>
+														<div>
+															<label
+																for="commit_log_max_batch_size"
+																class="block text-sm font-medium"
+																>Commit max batch size Commit max batch size <Tooltip>
+																	The max amount of documents per commit. In this case 1 document is
+																	one log file representing all logs during 1 minute for a specific
+																	host. To optimize indexing throughput, it is best to keep this as
+																	high as possible. However, especially when reindexing the whole
+																	instance, it can be useful to have a limit on how many logs can be
+																	written without being commited. A commit will make the logs
+																	available for search, appear as a log line, and be a "checkpoint"
+																	of the indexing progress.
+																</Tooltip>
+															</label>
+															<input
+																type="number"
+																id="commit_log_max_batch_size"
+																placeholder="10000"
+																bind:value={values[setting.key].commit_log_max_batch_size}
+															/>
+														</div>
+														<div>
+															<label
+																for="refresh_log_index_period"
+																class="block text-sm font-medium"
+															>
+																Refresh index period (s) <Tooltip>
+																	The index will query new service logs peridically and write them
+																	on the index. This setting sets that period.
+																</Tooltip></label
+															>
+															<input
+																type="number"
+																id="refresh_log_index_period"
+																placeholder="300"
+																bind:value={values[setting.key].refresh_log_index_period}
+															/>
+														</div>
+													{/if}
+												</div>
 											{:else if setting.fieldType == 'smtp_connect'}
 												<div class="flex flex-col gap-4 mt-4">
 													{#if values[setting.key]}

frontend/src/lib/components/instanceSettings.ts

@@ -20,6 +20,7 @@ export interface Setting {
 		| 'critical_error_channels'
 		| 'slack_connect'
 		| 'smtp_connect'
+		| 'indexer_rates'
 	storage: SettingStorage
 	advancedToggle?: {
 		label: string
@@ -249,6 +250,15 @@ export const settings: Record<string, Setting[]> = {
 			ee_only: ''
 		}
 	],
+	'Indexer/Search': [
+		{
+			label: '',
+			key: 'indexer_settings',
+			fieldType: 'indexer_rates',
+			storage: 'setting',
+			ee_only: 'Full text search across jobs and service logs is an EE feature'
+		}
+	],
 	Slack: [
 		{
 			label: 'Slack',

frontend/src/lib/components/search/GlobalSearchModal.svelte

@@ -701,7 +701,7 @@
 									<div class="flex flex-row pt-3 pl-4 items-center text-xs text-secondary">
 										{#if indexMetadata.indexed_until}
 											<span class="px-2">
-												Most recently indexed job was created <TimeAgo
+												Most recently indexed job was created at <TimeAgo
 													agoOnlyIfRecent
 													date={indexMetadata.indexed_until || ''}
 												/>
@@ -745,7 +745,7 @@
 									<div class="flex flex-row pt-10 text-xs text-secondary">
 										{#if indexMetadata.indexed_until}
 											<span class="px-2">
-												Most recently indexed job was created <TimeAgo
+												Most recently indexed job was created at <TimeAgo
 													agoOnlyIfRecent
 													date={indexMetadata.indexed_until}
 												/>

frontend/src/routes/(root)/(logged)/service_logs/+page.svelte

@@ -11,7 +11,7 @@
 	let queryParseErrors: string[] | undefined = undefined
 </script>
 
-<div class="w-full h-screen">
+<div class="w-full h-screen px-2">
 	<div class="px-2">
 		<div class="flex items-center space-x-2 flex-row justify-between">
 			<div class="flex flex-row flex-wrap justify-between py-2 my-4 px-4 gap-1 items-center">