From 6987a368467d06f48001d8279735d9e00a59120f Mon Sep 17 00:00:00 2001 From: wendrul <53628737+wendrul@users.noreply.github.com> Date: Fri, 29 Nov 2024 19:14:05 +0100 Subject: [PATCH] feat: indexer extra settings + parallel downloads + many improvements (#4822) * Nits on the service logs page * Show all the hosts returned by query + sumOtherDocCount warning * Remove from index endpoint * Fix tests * Prepare sqlx --- ...74b9a1035ec21176468f147998ac89fbed465.json | 14 + backend/src/main.rs | 16 +- backend/windmill-api/openapi.yaml | 23 + backend/windmill-common/src/indexer.rs | 18 +- backend/windmill-common/src/lib.rs | 2 +- backend/windmill-indexer/src/lib.rs | 3 - .../src/lib/components/ConfirmButton.svelte | 33 + .../lib/components/InstanceSettings.svelte | 21 +- .../lib/components/ServiceLogsInner.svelte | 726 +++++++++--------- .../SplitPanesOrColumnOnMobile.svelte | 37 + 10 files changed, 536 insertions(+), 357 deletions(-) create mode 100644 backend/.sqlx/query-134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465.json create mode 100644 frontend/src/lib/components/ConfirmButton.svelte create mode 100644 frontend/src/lib/components/splitPanes/SplitPanesOrColumnOnMobile.svelte diff --git a/backend/.sqlx/query-134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465.json b/backend/.sqlx/query-134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465.json new file mode 100644 index 0000000000..c8aa02d387 --- /dev/null +++ b/backend/.sqlx/query-134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE global_settings SET value = $1 WHERE name = 'indexer_settings'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465" +} diff --git a/backend/src/main.rs b/backend/src/main.rs index 1a9fdbaebe..d403ad212a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,9 @@ use anyhow::Context; use monitor::{ - reload_delete_logs_periodically_setting, reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store + reload_delete_logs_periodically_setting, reload_indexer_config, + reload_timeout_wait_result_setting, send_current_log_file_to_object_store, + send_logs_to_object_store, }; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; @@ -29,7 +31,15 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE use windmill_common::{ global_settings::{ - BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING + BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, + DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, + EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, + LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, + OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -537,7 +547,7 @@ Windmill Community Edition {GIT_VERSION} _ = indexer_rx.recv() => { tracing::info!("Received killpill, aborting index initialization"); }, - res = windmill_indexer::service_logs_ee::init_index(&db) => { + res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => { let res = res?; reader = Some(res.0); writer = Some(res.1); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c075518a60..3d11ec421d 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -10127,6 +10127,29 @@ paths: description: count of log lines that matched the query per hostname type: object + /srch/index/delete/{idx_name}: + delete: + summary: Restart container and delete the index to recreate it. + operationId: clearIndex + tags: + - indexSearch + parameters: + - name: idx_name + in: path + required: true + schema: + type: string + enum: + - JobIndex + - ServiceLogIndex + responses: + "200": + description: idx to be deleted and container restarting + content: + text/plain: + schema: + type: string + components: securitySchemes: bearerAuth: diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index 31ad8778b6..1a24028fc3 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -1,4 +1,4 @@ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::{error, DB}; @@ -13,6 +13,8 @@ pub struct TantivyIndexerSettings { pub refresh_index_period: u64, pub refresh_log_index_period: u64, pub max_indexed_job_log_size: usize, + pub should_clear_job_index: bool, + pub should_clear_log_index: bool, } impl Default for TantivyIndexerSettings { @@ -24,10 +26,12 @@ impl Default for TantivyIndexerSettings { refresh_index_period: 300, refresh_log_index_period: 300, max_indexed_job_log_size: 1_000_000, + should_clear_job_index: false, + should_clear_log_index: false, } } } -#[derive(Deserialize, Default)] +#[derive(Deserialize, Serialize, Default, sqlx::FromRow, Clone)] pub struct TantivyIndexerSettingsOpt { pub writer_memory_budget: Option, pub commit_job_max_batch_size: Option, @@ -35,6 +39,8 @@ pub struct TantivyIndexerSettingsOpt { pub refresh_index_period: Option, pub refresh_log_index_period: Option, pub max_indexed_job_log_size: Option, + pub should_clear_job_index: Option, + pub should_clear_log_index: Option, } pub async fn load_indexer_config(db: &DB) -> error::Result { @@ -53,6 +59,8 @@ pub async fn load_indexer_config(db: &DB) -> error::Result error::Result + import { Button } from './common' + import { Check, X } from 'lucide-svelte' + import { createEventDispatcher } from 'svelte' + + export let confirmation: string = 'Are you sure?' + let firstClick = false + const dispatch = createEventDispatcher() + + +
+{#if !firstClick} + +{:else} + {confirmation} + + +{/if} +
diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index acfa8f6cb6..0b509b0814 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1,7 +1,7 @@ @@ -393,366 +420,371 @@ -