feat: indexer extra settings + parallel downloads + many improvements (#4822)

* Nits on the service logs page * Show all the hosts returned by query + sumOtherDocCount warning * Remove from index endpoint * Fix tests * Prepare sqlx
2026-09-12 00:06:14 +00:00 · 2024-11-29 19:14:05 +01:00
parent 9abed3a30a
commit 6987a36846
10 changed files with 536 additions and 357 deletions
@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE global_settings SET value = $1 WHERE name = 'indexer_settings'",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Jsonb"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "134fe14dd47e80a9ef9b245a59e74b9a1035ec21176468f147998ac89fbed465"
+}
@@ -8,7 +8,9 @@

 use anyhow::Context;
 use monitor::{
-    reload_delete_logs_periodically_setting, reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store
+    reload_delete_logs_periodically_setting, reload_indexer_config,
+    reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
+    send_logs_to_object_store,
 };
 use rand::Rng;
 use sqlx::{postgres::PgListener, Pool, Postgres};
@@ -29,7 +31,15 @@ use windmill_common::ee::{maybe_renew_license_key_on_start, LICENSE_KEY_ID, LICE

 use windmill_common::{
    global_settings::{
-        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING
+        BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
+        CRITICAL_ERROR_CHANNELS_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
+        DEFAULT_TAGS_WORKSPACES_SETTING, ENV_SETTINGS, EXPOSE_DEBUG_METRICS_SETTING,
+        EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
+        JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
+        LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
+        OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
+        REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
+        SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
    },
    scripts::ScriptLang,
    stats_ee::schedule_stats,
@@ -537,7 +547,7 @@ Windmill Community Edition {GIT_VERSION}
                _ = indexer_rx.recv() => {
                    tracing::info!("Received killpill, aborting index initialization");
                },
-                res = windmill_indexer::service_logs_ee::init_index(&db) => {
+                res = windmill_indexer::service_logs_ee::init_index(&db, killpill_tx.clone()) => {
                        let res = res?;
                        reader = Some(res.0);
                        writer = Some(res.1);
@@ -10127,6 +10127,29 @@ paths:
                    description: count of log lines that matched the query per hostname
                    type: object

+  /srch/index/delete/{idx_name}:
+    delete:
+      summary: Restart container and delete the index to recreate it.
+      operationId: clearIndex
+      tags:
+        - indexSearch
+      parameters:
+        - name: idx_name
+          in: path
+          required: true
+          schema:
+            type: string
+            enum:
+              - JobIndex
+              - ServiceLogIndex
+      responses:
+        "200":
+          description: idx to be deleted and container restarting
+          content:
+            text/plain:
+              schema:
+                type: string
+
 components:
  securitySchemes:
    bearerAuth:
@@ -1,4 +1,4 @@
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};

 use crate::{error, DB};

@@ -13,6 +13,8 @@ pub struct TantivyIndexerSettings {
    pub refresh_index_period: u64,
    pub refresh_log_index_period: u64,
    pub max_indexed_job_log_size: usize,
+    pub should_clear_job_index: bool,
+    pub should_clear_log_index: bool,
 }

 impl Default for TantivyIndexerSettings {
@@ -24,10 +26,12 @@ impl Default for TantivyIndexerSettings {
            refresh_index_period: 300,
            refresh_log_index_period: 300,
            max_indexed_job_log_size: 1_000_000,
+            should_clear_job_index: false,
+            should_clear_log_index: false,
        }
    }
 }
-#[derive(Deserialize, Default)]
+#[derive(Deserialize, Serialize, Default, sqlx::FromRow, Clone)]
 pub struct TantivyIndexerSettingsOpt {
    pub writer_memory_budget: Option<u64>,
    pub commit_job_max_batch_size: Option<u64>,
@@ -35,6 +39,8 @@ pub struct TantivyIndexerSettingsOpt {
    pub refresh_index_period: Option<u64>,
    pub refresh_log_index_period: Option<u64>,
    pub max_indexed_job_log_size: Option<usize>,
+    pub should_clear_job_index: Option<bool>,
+    pub should_clear_log_index: Option<bool>,
 }

 pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSettings> {
@@ -53,6 +59,8 @@ pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSetting
        refresh_log_index_period,
        max_indexed_job_log_size,
        writer_memory_budget,
+        should_clear_job_index,
+        should_clear_log_index,
    } = get_indexer_rates_from_env();

    Ok(TantivyIndexerSettings {
@@ -70,6 +78,12 @@ pub async fn load_indexer_config(db: &DB) -> error::Result<TantivyIndexerSetting
        max_indexed_job_log_size: config
            .max_indexed_job_log_size
            .unwrap_or(max_indexed_job_log_size),
+        should_clear_job_index: config
+            .should_clear_job_index
+            .unwrap_or(should_clear_job_index),
+        should_clear_log_index: config
+            .should_clear_log_index
+            .unwrap_or(should_clear_log_index),
    })
 }

@@ -55,7 +55,7 @@ pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
 pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;

 pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
-pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 24 * 14; // 2 weeks retention period for logs
+pub const SERVICE_LOG_RETENTION_SECS: i64 = 60 * 60 * 24 * 14; // 2 weeks retention period for logs

 #[macro_export]
 macro_rules! add_time {
@@ -1,6 +1,3 @@
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
 pub mod completed_runs_ee;
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
 pub mod service_logs_ee;
-#[cfg(all(feature = "enterprise", feature = "parquet"))]
 pub mod indexer_ee;
@@ -0,0 +1,33 @@
+<script lang="ts">
+	import { Button } from './common'
+	import { Check, X } from 'lucide-svelte'
+	import { createEventDispatcher } from 'svelte'
+
+	export let confirmation: string = 'Are you sure?'
+	let firstClick = false
+	const dispatch = createEventDispatcher()
+</script>
+
+	<div class="p-2 flex flex-row w-full gap-2">
+{#if !firstClick}
+	<Button
+		on:click={() => {
+			firstClick = true
+		}}><slot /></Button
+	>
+{:else}
+		{confirmation}
+		<Button
+			color="red"
+			on:click={() => {
+				firstClick = false
+				dispatch('click')
+			}}><Check /></Button
+		>
+		<Button
+			on:click={() => {
+				firstClick = false
+			}}><X /></Button
+		>
+{/if}
+	</div>
@@ -1,7 +1,7 @@
 <script lang="ts">
 	import { settings, settingsKeys, type SettingStorage } from './instanceSettings'
 	import { Button, Skeleton, Tab, TabContent, Tabs } from '$lib/components/common'
-	import { SettingService, SettingsService } from '$lib/gen'
+	import { IndexSearchService, SettingService, SettingsService } from '$lib/gen'
 	import Toggle from '$lib/components/Toggle.svelte'
 	import SecondsInput from '$lib/components/common/seconds/SecondsInput.svelte'
 	import Tooltip from '$lib/components/Tooltip.svelte'
@@ -39,6 +39,7 @@
 	import { base } from '$lib/base'
 	import { createEventDispatcher } from 'svelte'
 	import { setLicense } from '$lib/enterpriseUtils'
+	import ConfirmButton from './ConfirmButton.svelte'

 	export let tab: string = 'Core'
 	export let hideTabs: boolean = false
@@ -1122,6 +1123,24 @@
 																bind:value={values[setting.key].refresh_log_index_period}
 															/>
 														</div>
+														<h3>Reset Index</h3>
+															This buttons will clear the whole index, and the service will start reindexing from scratch. Full text search might be down during this time.
+															<div>
+																<ConfirmButton
+																	on:click={async () => {
+																		let r = await IndexSearchService.clearIndex({idxName: "JobIndex"})
+																		console.log("asasd")
+																		sendUserToast(r)
+																	}}>Clear <b>Jobs</b> Index</ConfirmButton
+																>
+																<ConfirmButton
+																	on:click={async () => {
+																		let r = await IndexSearchService.clearIndex({idxName: "ServiceLogIndex"})
+																		console.log("asasd")
+																		sendUserToast(r)
+																	}}>Clear <b>Service Logs</b> Index</ConfirmButton
+																>
+															</div>
 													{/if}
 												</div>
 											{:else if setting.fieldType == 'smtp_connect'}
@@ -1,6 +1,5 @@
 <script lang="ts">
 	import { IndexSearchService, ServiceLogsService } from '$lib/gen'
-	import { Pane, Splitpanes } from 'svelte-splitpanes'

 	import ManuelDatePicker from './runs/ManuelDatePicker.svelte'
 	import CalendarPicker from './common/calendarPicker/CalendarPicker.svelte'
@@ -15,7 +14,7 @@
 	import ClipboardCopy from 'lucide-svelte/icons/clipboard-copy'
 	import AnsiUp from 'ansi_up'
 	import { scroll_into_view_if_needed_polyfill } from './multiselect/utils'
-	import SplitPanesWrapper from './splitPanes/SplitPanesWrapper.svelte'
+	import SplitPanesOrColumnOnMobile from './splitPanes/SplitPanesOrColumnOnMobile.svelte'

 	export let searchTerm: string
 	export let queryParseErrors: string[] = []
@@ -286,6 +285,7 @@
 	let loadingLogCounts = false

 	let countsPerHost: any
+	let sumOtherDocCount: number = 0

 	async function searchLogs(
 		searchTerm: string,
@@ -298,6 +298,7 @@
 			debounceTimeout && clearTimeout(debounceTimeout)
 			logs = undefined
 			countsPerHost = undefined
+			sumOtherDocCount = 0
 			loadingLogs = false
 			loadingLogCounts = false
 			return
@@ -314,12 +315,14 @@
 					minTs,
 					maxTs
 				})
-				const buckets = (countLogsResponse.count_per_host as any)['count_per_host']['buckets']
-				countsPerHost = new Map(buckets.map(({ key, doc_count }) => [key, doc_count]));
+				const res = (countLogsResponse.count_per_host as any)['count_per_host']
+				const buckets = res['buckets']
+				sumOtherDocCount = res['sum_other_doc_count']
+				countsPerHost = new Map(buckets.map(({ key, doc_count }) => [key, doc_count]))
 				countsPerHost = buckets.reduce((acc: any, { key, doc_count }) => {
-					  acc[key] = {doc_count};
-					  return acc;
-					}, {} as Record<string, number>)
+					acc[key] = { doc_count }
+					return acc
+				}, {} as Record<string, number>)
 				queryParseErrors = countLogsResponse.query_parse_errors ?? []
 				loadingLogCounts = false
 			}
@@ -365,6 +368,30 @@
 	}

 	$: searchLogs(searchTerm, selected, minTsManual, maxTsManual, allLogs)
+
+	function allLogsOrQueryResults(allLogs: ByMode, countsPerHost: any): ByMode {
+		if (countsPerHost == undefined) {
+			return allLogs
+		}
+		let ret = {}
+
+		for (const hk of Object.keys(countsPerHost)) {
+			let u = hk.split(",")
+			let [mode, wg, hn] = [u[0], u[1], u[2]]
+
+			if (!ret[mode]) {
+				ret[mode] = {}
+			}
+			if (!ret[mode][wg]) {
+				ret[mode][wg] = {}
+			}
+			if (!ret[mode][wg][hn]) {
+				ret[mode][wg][hn] = []
+			}
+		}
+
+		return ret
+	}
 </script>

 <Drawer bind:this={logDrawer} bind:open={logDrawerOpen} size="1400px">
@@ -393,366 +420,371 @@
 	</DrawerContent>
 </Drawer>

-<SplitPanesWrapper class="hidden md:block">
-	<Splitpanes>
-		<Pane size={30} minSize={25}>
-			<div class="p-1">
-				<div
-					class="flex flex-col lg:flex-row gap-y-1 justify-between w-full relative pb-4 gap-x-0.5"
-					id="service-logs-date-pickers"
-				>
-					<div class="flex relative">
-						<input
-							type="text"
-							value={minTsManual
-								? new Date(minTsManual).toLocaleTimeString([], {
-										day: '2-digit',
-										month: '2-digit',
-										hour: '2-digit',
-										minute: '2-digit'
-								  })
-								: 'min datetime'}
-							disabled
-						/>
-						<CalendarPicker
-							label="min datetime"
-							date={minTsManual}
-							on:change={({ detail }) => {
-								minTs = undefined
-								maxTs = undefined
-								allLogs = undefined
-								minTsManual = detail
-								getAllLogs(minTsManual, maxTsManual)
-							}}
-						/>
-					</div>
-					<ManuelDatePicker
-						bind:minTs={minTsManual}
-						bind:maxTs={maxTsManual}
-						bind:this={manualPicker}
-						{loading}
-						on:loadJobs={() => {
+<SplitPanesOrColumnOnMobile>
+	<svelte:fragment slot="left-pane">
+		<div class="p-1">
+			<div
+				class="flex flex-col lg:flex-row gap-y-1 justify-between w-full relative pb-4 gap-x-0.5"
+				id="service-logs-date-pickers"
+			>
+				<div class="flex relative">
+					<input
+						type="text"
+						value={minTsManual
+							? new Date(minTsManual).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+							  })
+							: 'min datetime'}
+						disabled
+					/>
+					<CalendarPicker
+						label="min datetime"
+						date={minTsManual}
+						on:change={({ detail }) => {
 							minTs = undefined
 							maxTs = undefined
 							allLogs = undefined
+							minTsManual = detail
 							getAllLogs(minTsManual, maxTsManual)
 						}}
-						serviceLogsChoices
-						loadText={searchTerm === '' ? 'Last 1000 logfiles' : 'All time'}
+						placement="top-start"
 					/>
-					<div class="flex relative">
-						<input
-							type="text"
-							value={maxTsManual
-								? new Date(maxTsManual).toLocaleTimeString([], {
-										day: '2-digit',
-										month: '2-digit',
-										hour: '2-digit',
-										minute: '2-digit'
-								  })
-								: 'max datetime'}
-							disabled
-						/>
-						<CalendarPicker
-							label="max datetime"
-							date={maxTsManual}
-							on:change={({ detail }) => {
-								minTs = undefined
-								maxTs = undefined
-								allLogs = undefined
-								maxTsManual = detail
-								getAllLogs(minTsManual, maxTsManual)
-							}}
-						/>
-					</div>
 				</div>
-				<div class="flex w-full flex-row-reverse pb-4 -mt-2 gap-2"
-					><Toggle
-						size="xs"
-						bind:checked={withError}
-						options={{ right: 'errors > 0' }}
-						on:change={() => {
+				<ManuelDatePicker
+					bind:minTs={minTsManual}
+					bind:maxTs={maxTsManual}
+					bind:this={manualPicker}
+					{loading}
+					on:loadJobs={() => {
+						minTs = undefined
+						maxTs = undefined
+						allLogs = undefined
+						getAllLogs(minTsManual, maxTsManual)
+					}}
+					serviceLogsChoices
+					loadText={searchTerm === '' ? 'Last 1000 logfiles' : 'All time'}
+				/>
+				<div class="flex relative">
+					<input
+						type="text"
+						value={maxTsManual
+							? new Date(maxTsManual).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+							  })
+							: 'max datetime'}
+						disabled
+					/>
+					<CalendarPicker
+						label="max datetime"
+						date={maxTsManual}
+						on:change={({ detail }) => {
+							minTs = undefined
+							maxTs = undefined
 							allLogs = undefined
-							getAllLogs(minTs, maxTs)
+							maxTsManual = detail
+							getAllLogs(minTsManual, maxTsManual)
 						}}
 					/>
-					<Toggle
-						size="xs"
-						bind:checked={autoRefresh}
-						disabled={searchTerm != ''}
-						on:change={(e) => {
-							if (e.detail) {
-								getAllLogs(maxTs, undefined)
-							} else {
-								timeout && clearTimeout(timeout)
-							}
-						}}
-						options={{ right: 'auto-refresh' }}
-					/></div
-				>
-				{#if allLogs == undefined}
-					<div class="text-center pb-2"><Loader2 class="animate-spin" /></div>
-				{:else if Object.keys(allLogs).length == 0}
-					<div class="flex justify-center items-center h-full">No logs</div>
-				{:else if minTs && maxTs}
-					{@const minTsN = new Date(minTs).getTime()}
-					{@const maxTsN = new Date(maxTs).getTime()}
-					{@const diff = maxTsN - minTsN}
-					{#if searchTerm === ''}
-						<div class="flex w-full text-2xs text-tertiary pb-6">
-							<div style="width: 60px;" />
-
-							<div class="flex justify-between w-full"
-								><div
-									>{new Date(minTs).toLocaleTimeString([], {
-										day: '2-digit',
-										month: '2-digit',
-										hour: '2-digit',
-										minute: '2-digit'
-									})}</div
-								><div
-									>{new Date(maxTs).toLocaleTimeString([], {
-										day: '2-digit',
-										month: '2-digit',
-										hour: '2-digit',
-										minute: '2-digit'
-									})}</div
-								></div
-							>
-						</div>
-					{/if}
-					{#each Object.entries(allLogs) as [mode, o1]}
-						<div class="w-full pb-8">
-							<h2 class="pb-2 text-2xl">{mode}s</h2>
-							{#each Object.entries(o1) as [wg, o2]}
-								<div class="w-full px-1">
-									{#if wg && wg != ''}
-										<h4 class="pt-4">{wg}</h4>
-									{/if}
-									<div class="divide-y flex flex-col">
-										{#each Object.entries(o2).filter(([hn, files]) => {
-											if (selected && selected[0] === mode && selected[1] === wg && selected[2] === hn) {
-												return true
-											}
-											const hostKey = `${mode},${wg},${hn}`
-											if (countsPerHost && (countsPerHost[hostKey] == undefined || countsPerHost[hostKey].doc_count === 0)) {
-												return false
-											}
-											return true
-										}) as [hn, files]}
-											{@const hostKey = `${mode},${wg},${hn}`}
-											<!-- svelte-ignore a11y-click-events-have-key-events -->
-											<!-- svelte-ignore a11y-no-static-element-interactions -->
-											<div
-												class="w-full flex items-baseline justify-between rounded px-1 hover:bg-surface-hover cursor-pointer {selected &&
-												selected[0] == mode &&
-												selected[1] == wg &&
-												selected[2] == hn
-													? 'bg-surface-secondary'
-													: ''}"
-												on:click={() => {
-													selected = [mode, wg, hn]
-													upToIsLatest = true
-													upTo = getLatestUpTo(selected)
-													scrollToBottom()
-												}}
-											>
-												<div
-													class="text-sm pt-2 pl-0.5 whitespace-nowrap"
-													title={hn}
-													style="width: 90px;">{truncateRev(hn, 8)}</div
-												>
-												{#if loadingLogCounts}
-													<Loader2 size={15} class="animate-spin" />
-												{:else if countsPerHost}
-													<div class="text-tertiary text-xs">
-														{countsPerHost[hostKey]?.doc_count ?? 0} matches
-													</div>
-												{:else}
-													<div class="relative grow h-8 mr-2">
-														{#each files as file}
-															{@const okHeight = 100.0 * ((file.ok_lines * 1.0) / (max_lines ?? 1))}
-															{@const errHeight =
-																100.0 * ((file.err_lines * 1.0) / (max_lines ?? 1))}
-															<div
-																class=" w-2 bg-red-400 absolute"
-																style="left: {((file.ts - minTsN) / diff) *
-																	100}%; height: {errHeight}%; bottom: {okHeight}%;"
-															/>
-															<div
-																class="w-2 bg-surface-secondary-inverse absolute bottom-0"
-																style="left: {((file.ts - minTsN) / diff) *
-																	100}%; height: {okHeight}%"
-															/>
-														{/each}
-													</div>
-												{/if}
-											</div>
-										{/each}
-									</div>
-								</div>
-							{/each}
-						</div>
-					{/each}
-				{/if}
+				</div>
 			</div>
-		</Pane>
-		<Pane size={70} minSize={25}
-			><div class="relative h-full flex flex-col gap-1"
-				><div class="w-full bg-surface-primary-inverse text-tertiary text-xs text-center"
-					>1 min delay: logs are compacted before being available</div
-				>
-				{#if selected}
-					<div class="grow overflow-auto" id="logviewer">
-						{#if loadingLogs}
-							<div class="flex w-full justify-center items-center h-48">
-								<div class="text-tertiary text-center">
-									<Loader2 size={34} class="animate-spin" />
+			<div class="flex w-full flex-row-reverse pb-4 -mt-2 gap-2"
+				><Toggle
+					size="xs"
+					bind:checked={withError}
+					options={{ right: 'errors > 0' }}
+					on:change={() => {
+						allLogs = undefined
+						getAllLogs(minTs, maxTs)
+					}}
+				/>
+				<Toggle
+					size="xs"
+					bind:checked={autoRefresh}
+					disabled={searchTerm != ''}
+					on:change={(e) => {
+						if (e.detail) {
+							getAllLogs(maxTs, undefined)
+						} else {
+							timeout && clearTimeout(timeout)
+						}
+					}}
+					options={{ right: 'auto-refresh' }}
+				/></div
+			>
+			{#if allLogs == undefined}
+				<div class="text-center pb-2"><Loader2 class="animate-spin" /></div>
+			{:else if Object.keys(allLogs).length == 0}
+				<div class="flex justify-center items-center h-full">No logs</div>
+			{:else if minTs && maxTs}
+				{@const minTsN = new Date(minTs).getTime()}
+				{@const maxTsN = new Date(maxTs).getTime()}
+				{@const diff = maxTsN - minTsN}
+				{#if searchTerm === ''}
+					<div class="flex w-full text-2xs text-tertiary pb-6">
+						<div style="width: 60px;" />
+
+						<div class="flex justify-between w-full"
+							><div
+								>{new Date(minTs).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+								})}</div
+							><div
+								>{new Date(maxTs).toLocaleTimeString([], {
+									day: '2-digit',
+									month: '2-digit',
+									hour: '2-digit',
+									minute: '2-digit'
+								})}</div
+							></div
+						>
+					</div>
+				{/if}
+				{#each Object.entries(allLogsOrQueryResults(allLogs, countsPerHost)) as [mode, o1]}
+					<div class="w-full pb-8">
+						<h2 class="pb-2 text-2xl">{mode}s</h2>
+						{#each Object.entries(o1) as [wg, o2]}
+							<div class="w-full px-1">
+								{#if wg && wg != ''}
+									<h4 class="pt-4">{wg}</h4>
+								{/if}
+								<div class="divide-y flex flex-col">
+									{#each Object.entries(o2).filter(([hn, files]) => {
+										if (selected && selected[0] === mode && selected[1] === wg && selected[2] === hn) {
+											return true
+										}
+										const hostKey = `${mode},${wg},${hn}`
+										if (countsPerHost && (countsPerHost[hostKey] == undefined || countsPerHost[hostKey].doc_count === 0)) {
+											return false
+										}
+										return true
+									}) as [hn, files]}
+										{@const hostKey = `${mode},${wg},${hn}`}
+										<!-- svelte-ignore a11y-click-events-have-key-events -->
+										<!-- svelte-ignore a11y-no-static-element-interactions -->
+										<div
+											class="w-full flex items-baseline justify-between rounded px-1 hover:bg-surface-hover cursor-pointer {selected &&
+											selected[0] == mode &&
+											selected[1] == wg &&
+											selected[2] == hn
+												? 'bg-surface-secondary'
+												: ''}"
+											on:click={() => {
+												selected = [mode, wg, hn]
+												upToIsLatest = true
+												upTo = getLatestUpTo(selected)
+												scrollToBottom()
+											}}
+										>
+											<div
+												class="text-sm pt-2 pl-0.5 whitespace-nowrap"
+												title={hn}
+												style="width: 90px;">{truncateRev(hn, 8)}</div
+											>
+											{#if loadingLogCounts}
+												<Loader2 size={15} class="animate-spin" />
+											{:else if countsPerHost}
+												<div class="text-tertiary text-xs">
+													{countsPerHost[hostKey]?.doc_count ?? 0} matches
+												</div>
+											{:else}
+												<div class="relative grow h-8 mr-2">
+													{#each files as file}
+														{@const okHeight = 100.0 * ((file.ok_lines * 1.0) / (max_lines ?? 1))}
+														{@const errHeight = 100.0 * ((file.err_lines * 1.0) / (max_lines ?? 1))}
+														<div
+															class=" w-2 bg-red-400 absolute"
+															style="left: {((file.ts - minTsN) / diff) *
+																100}%; height: {errHeight}%; bottom: {okHeight}%;"
+														/>
+														<div
+															class="w-2 bg-surface-secondary-inverse absolute bottom-0"
+															style="left: {((file.ts - minTsN) / diff) *
+																100}%; height: {okHeight}%"
+														/>
+													{/each}
+												</div>
+											{/if}
+										</div>
+									{/each}
 								</div>
 							</div>
-						{:else if logs != undefined}
-							<div class="flex flex-col min-w-full w-fit">
-								{#each logs.hits as { snippet_fragment, snippet_highlighted, document }}
-									<LogSnippetViewer
-										content={snippet_fragment || document.logs[0]}
-										highlighted={snippet_highlighted}
-										on:click={() => {
-											let logLineNumber = document.line_number[0]
-											let logFile = document.file_name[0]
-											let host = document.host[0]
-											let jsonFmt = document.json_fmt[0]
-											seeLogContext(logLineNumber, logFile, host, jsonFmt)
-										}}
-									/>
-								{/each}
-								{#if logs.hits.length === 0}
-									<div class="text-center py-20 text-bold text-xl text-tertiary"> No logs </div>
-								{/if}
-								{#if logs.hits.length === 30}
-									<div class="pl-6 py-6 text-sm text-secondary">
-										Older matches were truncated from this search, try refining your filters to get
-										more precise results.
-									</div>
-								{/if}
-								<div class="py-20" />
+						{/each}
+					</div>
+				{/each}
+				{#if !loadingLogCounts && sumOtherDocCount != 0}
+					<div class="text-tertiary italic text-sm">
+						Note: {sumOtherDocCount} additional matches weren't grouped into any of the above hosts.
+					</div>
+				{/if}
+			{/if}
+		</div>
+	</svelte:fragment>
+	<svelte:fragment slot="right-pane">
+		<div class="relative h-full flex flex-col gap-1 pb-2">
+			{#if selected}
+				{#if !loadingLogs && logs == undefined}
+					<div class="w-full bg-surface-primary-inverse text-tertiary text-xs text-center">
+						1 min delay: logs are compacted before being available
+					</div>
+				{/if}
+				<div class="grow overflow-auto" id="logviewer">
+					{#if loadingLogs}
+						<div class="flex w-full justify-center items-center h-48">
+							<div class="text-tertiary text-center">
+								<Loader2 size={34} class="animate-spin" />
 							</div>
-						{:else}
-							{#each getLogs(selected, upTo) as file}
-								<div
-									style="min-height: {logsContent[file.file_path]
-										? 10
-										: (file.ok_lines + file.err_lines) / 20}px;"
+						</div>
+					{:else if logs != undefined}
+						<div class="flex flex-col min-w-full w-fit">
+							{#each logs.hits as { snippet_fragment, snippet_highlighted, document }}
+								<LogSnippetViewer
+									content={snippet_fragment || document.logs[0]}
+									highlighted={snippet_highlighted}
+									on:click={() => {
+										let logLineNumber = document.line_number[0]
+										let logFile = document.file_name[0]
+										let host = document.host[0]
+										let jsonFmt = document.json_fmt[0]
+										seeLogContext(logLineNumber, logFile, host, jsonFmt)
+									}}
+								/>
+							{/each}
+							{#if logs.hits.length === 0}
+								<div class="text-center py-20 text-bold text-xl text-tertiary"> No logs </div>
+							{/if}
+							{#if logs.hits.length === 1000}
+								<div class="pl-6 py-6 text-sm text-secondary">
+									Older matches were truncated from this search, try refining your filters to get
+									more precise results.
+								</div>
+							{/if}
+							<div class="py-20" />
+						</div>
+					{:else}
+						{#each getLogs(selected, upTo) as file}
+							<div
+								style="min-height: {logsContent[file.file_path]
+									? 10
+									: Math.min(file.ok_lines + file.err_lines, 30) * 16}px;"
+							>
+								<div class="bg-surface-primary-inverse text-sm font-semibold px-1"
+									>{new Date(file.ts).toLocaleTimeString([], {
+										day: '2-digit',
+										month: '2-digit',
+										hour: '2-digit',
+										minute: '2-digit'
+									})}</div
 								>
-									<div class="bg-surface-primary-inverse text-sm font-semibold px-1"
-										>{new Date(file.ts).toLocaleTimeString([], {
-											day: '2-digit',
-											month: '2-digit',
-											hour: '2-digit',
-											minute: '2-digit'
-										})}</div
-									>
-									{#if logsContent[file.file_path] == undefined}
-										<div
-											class="animate-skeleton dark:bg-frost-900/50 [animation-delay:1000ms] h-full w-full"
-										/>
-									{:else if logsContent[file.file_path]}
-										{#if logsContent[file.file_path].error}
-											{#if logsContent[file.file_path].error?.startsWith('Not Found')}
-												<div class="text-xs pb-4 pt-2 text-secondary"
-													>Log file is missing. Log files require a shared log volume to be mounted
-													across servers and workers or to use the EE S3/object storage integration
-													for logs. To avoid mounting a shared volume, set the EE object store logs
-													in the instance settings</div
-												>
-											{:else}
-												<div class="text-xs text-red-400 pb-4"
-													>{logsContent[file.file_path].error}</div
-												>
-											{/if}
-										{:else if logsContent[file.file_path].content}
-											<!-- svelte-ignore a11y-click-events-have-key-events -->
-											<!-- svelte-ignore a11y-no-static-element-interactions -->
-											<div on:click|preventDefault class="pr-2"
-												><LogViewer
-													noAutoScroll
-													noMaxH
-													isLoading={false}
-													tag={undefined}
-													content={processLogWithJsonFmt(
-														logsContent[file.file_path].content,
-														file.json_fmt
-													)}
-												/></div
+								{#if logsContent[file.file_path] == undefined}
+									<div
+										class="animate-skeleton dark:bg-frost-900/50 [animation-delay:1000ms] h-full w-full"
+									/>
+								{:else if logsContent[file.file_path]}
+									{#if logsContent[file.file_path].error}
+										{#if logsContent[file.file_path].error?.startsWith('Not Found')}
+											<div class="text-xs pb-4 pt-2 text-secondary"
+												>Log file is missing. Log files require a shared log volume to be mounted
+												across servers and workers or to use the EE S3/object storage integration
+												for logs. To avoid mounting a shared volume, set the EE object store logs in
+												the instance settings</div
 											>
 										{:else}
-											<div>No logs</div>
+											<div class="text-xs text-red-400 pb-4"
+												>{logsContent[file.file_path].error}</div
+											>
 										{/if}
+									{:else if logsContent[file.file_path].content}
+										<!-- svelte-ignore a11y-click-events-have-key-events -->
+										<!-- svelte-ignore a11y-no-static-element-interactions -->
+										<div on:click|preventDefault class="pr-2"
+											><LogViewer
+												noAutoScroll
+												noMaxH
+												isLoading={false}
+												tag={undefined}
+												content={processLogWithJsonFmt(
+													logsContent[file.file_path].content,
+													file.json_fmt
+												)}
+											/></div
+										>
+									{:else}
+										<div>No logs</div>
 									{/if}
-								</div>
-							{/each}
-						{/if}
-					</div>
-					{#if searchTerm == ''}
-						<div class="flex w-full items-center gap-4">
-							<div class="text-tertiary px-1 text-2xs">Last 5 log files up to:</div>
-							<div class="flex grow text-xs justify-center px-2 items-center gap-2">
-								{#if upTo}
-									<button
-										on:click={() => {
-											if (upTo) {
-												upToIsLatest = false
-												upTo = new Date(new Date(upTo).getTime() - 5 * 60 * 1000).toISOString()
-											}
-										}}>{'<'} 5m</button
-									>
-								{:else}
-									<div />
-								{/if}
-
-								<div class="flex gap-1 relative items-center"
-									><div class="flex gap-1 relative">
-										<input
-											type="text"
-											value={upTo
-												? new Date(upTo).toLocaleTimeString([], {
-														day: '2-digit',
-														month: '2-digit',
-														hour: '2-digit',
-														minute: '2-digit'
-												  })
-												: ''}
-											disabled
-										/><CalendarPicker bind:date={upTo} label="Logs up to" /></div
-									></div
-								>
-								{#if upTo}
-									<button
-										on:click={() => {
-											if (upTo) {
-												upToIsLatest = false
-												upTo = new Date(new Date(upTo).getTime() + 5 * 60 * 1000).toISOString()
-											}
-										}}>5m {'>'}</button
-									>
-								{:else}
-									<div />
 								{/if}
 							</div>
-							<div>
-								<button
-									class="text-xs"
-									on:click={() => {
-										upTo = new Date().toISOString()
-										upToIsLatest = true
-									}}>now</button
-								>
-							</div>
-						</div>
+						{/each}
 					{/if}
-				{:else}
-					<div class="flex justify-center items-center pt-8">Select a host to see its logs</div>
-				{/if}</div
-			></Pane
+				</div>
+				{#if searchTerm == ''}
+					<div class="flex w-full items-center gap-4">
+						<div class="text-tertiary px-1 text-2xs">Last 5 log files up to:</div>
+						<div class="flex grow text-xs justify-center px-2 items-center gap-2">
+							{#if upTo}
+								<button
+									on:click={() => {
+										if (upTo) {
+											upToIsLatest = false
+											upTo = new Date(new Date(upTo).getTime() - 5 * 60 * 1000).toISOString()
+										}
+									}}>{'<'} 5m</button
+								>
+							{:else}
+								<div />
+							{/if}
+
+							<div class="flex gap-1 relative items-center"
+								><div class="flex gap-1 relative">
+									<input
+										type="text"
+										value={upTo
+											? new Date(upTo).toLocaleTimeString([], {
+													day: '2-digit',
+													month: '2-digit',
+													hour: '2-digit',
+													minute: '2-digit'
+											  })
+											: ''}
+										disabled
+									/><CalendarPicker bind:date={upTo} label="Logs up to" /></div
+								></div
+							>
+							{#if upTo}
+								<button
+									on:click={() => {
+										if (upTo) {
+											upToIsLatest = false
+											upTo = new Date(new Date(upTo).getTime() + 5 * 60 * 1000).toISOString()
+										}
+									}}>5m {'>'}</button
+								>
+							{:else}
+								<div />
+							{/if}
+						</div>
+						<div>
+							<button
+								class="text-xs"
+								on:click={() => {
+									upTo = new Date().toISOString()
+									upToIsLatest = true
+								}}>now</button
+							>
+						</div>
+					</div>
+				{/if}
+			{:else}
+				<div class="flex justify-center items-center pt-8">Select a host to see its logs</div>
+			{/if}</div
 		>
-	</Splitpanes>
-</SplitPanesWrapper>
+	</svelte:fragment>
+</SplitPanesOrColumnOnMobile>
@@ -0,0 +1,37 @@
+<script lang="ts">
+	import { Pane, Splitpanes } from 'svelte-splitpanes'
+	import SplitPanesWrapper from './SplitPanesWrapper.svelte'
+
+	export let leftPaneSize = 30
+	export let leftPaneMinSize = 25
+	export let rightPaneSize = 70
+	export let rightPaneMinSize = 25
+	export let rightPaneIsFirstInCol = false
+
+	let clientWidth = window.innerWidth
+</script>
+
+<main class="h-screen w-full" bind:clientWidth>
+	{#if clientWidth >= 768}
+		<SplitPanesWrapper class="hidden md:block">
+			<Splitpanes>
+				<Pane size={30} minSize={25}>
+					<slot name="left-pane" />
+				</Pane>
+				<Pane size={70} minSize={25}>
+					<slot name="right-pane" />
+				</Pane>
+			</Splitpanes>
+		</SplitPanesWrapper>
+	{:else}
+		<div class="flex flex-col">
+			{#if rightPaneIsFirstInCol}
+				<slot name="right-pane" />
+				<slot name="left-pane" />
+			{:else}
+				<slot name="left-pane" />
+				<slot name="right-pane" />
+			{/if}
+		</div>
+	{/if}
+</main>