From 229dcba554d68e0259a8aabb9e495f8ddb24e09f Mon Sep 17 00:00:00 2001 From: Sai Nikhil Date: Thu, 3 Oct 2024 15:17:59 +0530 Subject: [PATCH] fix: Updated Search History (#4694) 1.New Fields --> executed at introduced --> this helps in user to know when this search got executed. 2. when expands Function defination got introduced if user searched with function then it will be displayed. 3. Pagination got introduced to show all the results by default per page user can able to see 100 rows. 4. ## Summary by CodeRabbit ## Release Notes - **New Features** - Enhanced search history component with pagination and improved data handling. - Added new fields in usage tracking to capture function-specific data. - New method for dynamic pagination control in the search history component. - **Improvements** - Streamlined search result processing by removing unnecessary filtering. - Improved timestamp handling in usage reporting. - Flexible handling of the size parameter in search history retrieval. - Refined partitioning logic and filtering capabilities in search functionality. - Enhanced user interface responsiveness in the search history component. - **Bug Fixes** - Adjusted sorting functionality and pagination logic in the search history component. These updates enhance the user experience and improve data accuracy throughout the application. --------- Co-authored-by: Subhradeep Chakraborty --- src/config/src/meta/usage.rs | 6 + src/handler/http/request/search/mod.rs | 19 +- src/service/search/cache/mod.rs | 3 +- src/service/search/mod.rs | 5 + src/service/usage/mod.rs | 6 +- web/src/plugins/logs/Index.vue | 1 + web/src/plugins/logs/SearchHistory.vue | 307 ++++++++++++++++++++----- web/src/services/search.ts | 1 - 8 files changed, 272 insertions(+), 76 deletions(-) diff --git a/src/config/src/meta/usage.rs b/src/config/src/meta/usage.rs index a7d1a1aa45..0c93867bc4 100644 --- a/src/config/src/meta/usage.rs +++ b/src/config/src/meta/usage.rs @@ -98,6 +98,8 @@ pub struct UsageData { pub took_wait_in_queue: Option, #[serde(skip_serializing_if = "Option::is_none")] pub result_cache_ratio: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, } #[derive(Hash, PartialEq, Eq)] @@ -241,6 +243,8 @@ pub struct RequestStats { #[serde(default)] pub request_body: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub cached_ratio: Option, #[serde(skip_serializing_if = "Option::is_none")] pub compressed_size: Option, @@ -266,6 +270,7 @@ impl Default for RequestStats { records: 0, response_time: 0.0, request_body: None, + function: None, cached_ratio: None, compressed_size: None, min_ts: None, @@ -285,6 +290,7 @@ impl From for RequestStats { size: meta.original_size as f64 / SIZE_IN_MB, records: meta.records, response_time: 0.0, + function: None, request_body: None, cached_ratio: None, compressed_size: Some(meta.compressed_size as f64 / SIZE_IN_MB), diff --git a/src/handler/http/request/search/mod.rs b/src/handler/http/request/search/mod.rs index 1380383f75..cc4f9ac1d3 100644 --- a/src/handler/http/request/search/mod.rs +++ b/src/handler/http/request/search/mod.rs @@ -21,7 +21,7 @@ use chrono::{Duration, Utc}; use config::{ get_config, meta::{ - search::{SearchEventType, SearchHistoryHitResponse}, + search::SearchEventType, sql::resolve_stream_names, stream::StreamType, usage::{RequestStats, UsageType, USAGE_STREAM}, @@ -1558,23 +1558,6 @@ pub async fn search_history( } }; - search_res.hits = search_res - .hits - .into_iter() - .filter_map(|hit| match SearchHistoryHitResponse::try_from(hit) { - Ok(response) => match serde_json::to_value(response) { - Ok(json_value) => Some(json_value), - Err(e) => { - log::error!("[trace_id {}] Serialization error: {:?}", trace_id, e); - None - } - }, - Err(e) => { - log::error!("[trace_id {}] Deserialization error: {:?}", trace_id, e); - None - } - }) - .collect::>(); search_res.trace_id = trace_id.clone(); // report http metrics diff --git a/src/service/search/cache/mod.rs b/src/service/search/cache/mod.rs index 105f3650d5..65270f8186 100644 --- a/src/service/search/cache/mod.rs +++ b/src/service/search/cache/mod.rs @@ -279,11 +279,13 @@ pub async fn search( res.histogram_interval = Some(c_resp.histogram_interval); } + let num_fn = req.query.query_fn.is_some() as u16; let req_stats = RequestStats { records: res.hits.len() as i64, response_time: time, size: res.scan_size as f64, request_body: Some(req.query.sql), + function: req.query.query_fn, user_email: user_id, min_ts: Some(req.query.start_time), max_ts: Some(req.query.end_time), @@ -300,7 +302,6 @@ pub async fn search( result_cache_ratio: Some(res.result_cache_ratio), ..Default::default() }; - let num_fn = req.query.query_fn.is_some() as u16; report_request_usage_stats( req_stats, org_id, diff --git a/src/service/search/mod.rs b/src/service/search/mod.rs index de41a0482a..b3ead53bf0 100644 --- a/src/service/search/mod.rs +++ b/src/service/search/mod.rs @@ -216,6 +216,11 @@ pub async fn search( response_time: time, size: res.scan_size as f64, request_body: Some(req_query.sql.clone()), + function: if req_query.query_fn.is_empty() { + None + } else { + Some(req_query.query_fn.clone()) + }, user_email: user_id, min_ts: Some(req_query.start_time), max_ts: Some(req_query.end_time), diff --git a/src/service/usage/mod.rs b/src/service/usage/mod.rs index 6bc8c7e74f..2630ed70f3 100644 --- a/src/service/usage/mod.rs +++ b/src/service/usage/mod.rs @@ -15,7 +15,7 @@ use std::sync::Arc; -use chrono::{Datelike, Timelike, Utc}; +use chrono::{DateTime, Datelike, Timelike}; use config::{ get_config, meta::{ @@ -61,6 +61,7 @@ pub async fn report_request_usage_stats( .with_label_values(&[org_id, stream_name, stream_type.to_string().as_str()]) .inc_by((stats.size * SIZE_IN_MB) as u64); let event: UsageEvent = usage_type.into(); + let now = DateTime::from_timestamp_micros(timestamp).unwrap(); if !get_config().common.usage_enabled { return; @@ -68,7 +69,6 @@ pub async fn report_request_usage_stats( let request_body = stats.request_body.unwrap_or(usage_type.to_string()); let user_email = stats.user_email.unwrap_or("".to_owned()); - let now = Utc::now(); let mut usage = vec![]; @@ -89,6 +89,7 @@ pub async fn report_request_usage_stats( ), org_id: org_id.to_owned(), request_body: request_body.to_owned(), + function: None, size: stats.size, unit: "MB".to_owned(), user_email: user_email.to_owned(), @@ -127,6 +128,7 @@ pub async fn report_request_usage_stats( unit: "MB".to_owned(), user_email, response_time: stats.response_time, + function: stats.function, num_records: stats.records, stream_type, stream_name: stream_name.to_owned(), diff --git a/web/src/plugins/logs/Index.vue b/web/src/plugins/logs/Index.vue index 9880f03293..069a58cd18 100644 --- a/web/src/plugins/logs/Index.vue +++ b/web/src/plugins/logs/Index.vue @@ -856,6 +856,7 @@ export default defineComponent({ query: { action: "history", org_identifier: store.state.selectedOrganization.identifier, + type: "search_history", }, }); showSearchHistory.value = true; diff --git a/web/src/plugins/logs/SearchHistory.vue b/web/src/plugins/logs/SearchHistory.vue index c71457d4c9..0c0e469424 100644 --- a/web/src/plugins/logs/SearchHistory.vue +++ b/web/src/plugins/logs/SearchHistory.vue @@ -44,14 +44,14 @@ -
+
- + + + + + + + {{ props.row[col.field] }} + + + - + -
- SQL Query: - -
+
+
+ SQL Query : + +
+
-
+ 
{{ props.row?.sql }}
-
+ +
+
+
+ Function Defination : + +
+ +
+ +
+ 
{{ props.row?.function }}
+ +
+ + + +
+ + + +
+ + + -
- -
+
(0); + + const pagination = ref({ page: 1, - rowsPerPage: 50 - } + rowsPerPage: 100 + }) + const selectedPerPage = ref(pagination.value.rowsPerPage); + const generateColumns = (data: any) => { if (data.length === 0) return []; // Define the desired column order and names const desiredColumns = [ + {key : '#' , label : '#',align: 'center',sortable: false}, { key: 'trace_id', label: 'Trace ID' }, + { key: 'executed_time', label: 'Executed At' }, + { key: 'start_time', label: 'Start Time' }, { key: 'end_time', label: 'End Time' }, { key: 'duration', label: 'Duration' }, @@ -195,6 +317,7 @@ { key: 'scan_records', label: 'Scan Records' }, { key: 'cached_ratio', label: 'Cached Ratio' }, { key: 'sql', label: 'SQL Query' }, + ]; return desiredColumns.map(({ key, label }) => { @@ -207,35 +330,39 @@ } if(key == "scan_size"){ - columnWidth = 80 + columnWidth = 100 } if(key == "duration"){ + align='left' columnWidth = 100 } if( key == 'start_time' || key == 'end_time'){ columnWidth = 200 } if(key == 'sql'){ - columnWidth = 350 + columnWidth = 300 sortable = false; } if(key == "trace_id"){ columnWidth = 250 - align = "left" sortable = false; } + if(key == "#"){ + columnWidth = 100 + sortable = false; + align = "left" + } // Custom width for each column return { name: key, // Field name label: label, // Column label field: key, // Field accessor - align: 'center', + align: align, sortable: sortable, style: `max-width: ${columnWidth}px; width: ${columnWidth}px;`, }; }); }; - const fetchSearchHistory = async ( ) => { columnsToBeRendered.value = []; @@ -249,24 +376,42 @@ const response = await searchService.get_history( org_identifier,startTime,endTime, ); const limitedHits = response.data.hits; - columnsToBeRendered.value = generateColumns(limitedHits); - limitedHits.forEach((hit:any)=>{ - const {formatted, raw} = calculateDuration(hit.start_time, hit.end_time); + const filteredHits = limitedHits.filter((hit) => hit.event === "Search"); + if(filteredHits.length > 0){ + resultTotal.value = filteredHits.length; + } + columnsToBeRendered.value = generateColumns(filteredHits); + filteredHits.forEach((hit:any)=>{ + + const {formatted, raw} = calculateDuration(hit.min_ts, hit.max_ts); hit.duration = formatted; hit.rawDuration = raw; - hit.toBeStoredStartTime = hit.start_time; - hit.toBeStoredEndTime = hit.end_time; - hit.start_time = timestampToTimezoneDate(hit.start_time / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS"); - hit.end_time = timestampToTimezoneDate(hit.end_time / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS"); - hit.rawTook = hit.took; - hit.took = formatTime(hit.took); - hit.rawScanRecords = hit.scan_records; - hit.rawScanSize = hit.scan_size; + hit.toBeStoredStartTime = hit.min_ts; + hit.toBeStoredEndTime = hit.max_ts; + hit.start_time = timestampToTimezoneDate(hit.min_ts / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS"); + hit.end_time = timestampToTimezoneDate(hit.max_ts / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS"); + hit.rawTook = hit.response_time; + hit.took = formatTime(hit.response_time); + hit.scan_records = hit.num_records; + hit.rawScanRecords = hit.num_records; + hit.scan_size = hit.size + hit.unit; + hit.rawScanSize = hit.size; + hit.cached_ratio = hit.cached_ratio; hit.rawCachedRatio = hit.cached_ratio; + hit.sql = hit.request_body; + hit.function = hit.function; + hit.rawExecutedTime = hit._timestamp; + hit.executed_time = timestampToTimezoneDate(hit._timestamp / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS"); + }) - dataToBeLoaded.value = response.data.hits; + dataToBeLoaded.value = filteredHits; isLoading.value = false; } catch (error) { + $q.notify({ + type: "negative", + message: "Failed to fetch search history. Please try again later.", + timeout: 5000, + }); console.log(error, "error") isLoading.value = false; } finally { @@ -334,6 +479,13 @@ return data.sort((a, b) => a.toBeStoredEndTime - b.toBeStoredEndTime); } + if(sortBy == "executed_time"){ + if (descending) { + return data.sort((a, b) => b.rawExecutedTime - a.rawExecutedTime); + } + return data.sort((a, b) => a.rawExecutedTime - b.rawExecutedTime); + + } @@ -341,11 +493,11 @@ // return a.rawDuration - b.rawDuration; } - const copyToClipboard = (text) => { + const copyToClipboard = (text,type) => { navigator.clipboard.writeText(text).then(() => { $q.notify({ type: "positive", - message: "Content Copied Successfully!", + message: `${type} Copied Successfully!`, timeout: 5000, }); }).catch(() => { @@ -451,25 +603,44 @@ const query = b64EncodeUnicode(row.sql); + const queryObject = { + stream_type: "logs", + stream, + period: '15m', + refresh, + sql_mode: "true", + query, + defined_schemas: "user_defined_schema", + org_identifier: row.org_id, + quick_mode: "false", + show_histogram: "true", + type: "search_history_re_apply" + }; + + if(row.hasOwnProperty('function') && row.function){ + const functionContent = b64EncodeUnicode(row.function); + queryObject['functionContent'] = functionContent; + } + + router.push({ path: "/logs", - query: { - stream_type: "logs", - stream, - period: '15m', - refresh, - sql_mode: "true", - query, - defined_schemas:"user_defined_schema", - org_identifier: row.org_id, - quick_mode: "false", - show_histogram: "true", - type: "search_history_re_apply" - - }, + query: queryObject, }); }; + const changePagination = (val: { label: string; value: any }) => { + if(val.label == "All"){ + val.value = dataToBeLoaded.value.length; + val.label = 'All'; + } + selectedPerPage.value = val.value; + pagination.value.rowsPerPage = val.value; + qTable.value.setPagination(pagination.value); + + // pagination.value.page = 1; + + }; watch(() => props.isClicked, (value) => { if(value == true && !isLoading.value){ @@ -497,6 +668,10 @@ formatTime, delayMessage, sortMethod, + resultTotal, + perPageOptions, + changePagination, + selectedPerPage, }; // Watch the searchObj for changes @@ -505,6 +680,7 @@ \ No newline at end of file diff --git a/web/src/services/search.ts b/web/src/services/search.ts index f11d963a50..7458665094 100644 --- a/web/src/services/search.ts +++ b/web/src/services/search.ts @@ -199,7 +199,6 @@ const search = { stream_type : "logs", org_identifier, user_email : null, - size:50, }; // Add startTime and endTime to the payload if provided if (startTime) {