fix: Updated Search History (#4694)

1.New Fields --> executed at introduced --> this helps in user to know
when this search got executed.
2. when expands Function defination got introduced if user searched with
function then it will be displayed.
3. Pagination got introduced to show all the results by default per page
user can able to see 100 rows.
4. 

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Release Notes

- **New Features**
- Enhanced search history component with pagination and improved data
handling.
	- Added new fields in usage tracking to capture function-specific data.
- New method for dynamic pagination control in the search history
component.

- **Improvements**
- Streamlined search result processing by removing unnecessary
filtering.
	- Improved timestamp handling in usage reporting.
	- Flexible handling of the size parameter in search history retrieval.
- Refined partitioning logic and filtering capabilities in search
functionality.
- Enhanced user interface responsiveness in the search history
component.

- **Bug Fixes**
- Adjusted sorting functionality and pagination logic in the search
history component.

These updates enhance the user experience and improve data accuracy
throughout the application.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Subhradeep Chakraborty <chakrabortysubhradeep556@gmail.com>
This commit is contained in:
Sai Nikhil
2024-10-03 15:17:59 +05:30
committed by GitHub
co-authored by Subhradeep Chakraborty
parent 63dc8ddcce
commit 229dcba554
8 changed files with 272 additions and 76 deletions
+6
View File
@@ -98,6 +98,8 @@ pub struct UsageData {
pub took_wait_in_queue: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub result_cache_ratio: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub function: Option<String>,
}
#[derive(Hash, PartialEq, Eq)]
@@ -241,6 +243,8 @@ pub struct RequestStats {
#[serde(default)]
pub request_body: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub function: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cached_ratio: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub compressed_size: Option<f64>,
@@ -266,6 +270,7 @@ impl Default for RequestStats {
records: 0,
response_time: 0.0,
request_body: None,
function: None,
cached_ratio: None,
compressed_size: None,
min_ts: None,
@@ -285,6 +290,7 @@ impl From<FileMeta> for RequestStats {
size: meta.original_size as f64 / SIZE_IN_MB,
records: meta.records,
response_time: 0.0,
function: None,
request_body: None,
cached_ratio: None,
compressed_size: Some(meta.compressed_size as f64 / SIZE_IN_MB),
+1 -18
View File
@@ -21,7 +21,7 @@ use chrono::{Duration, Utc};
use config::{
get_config,
meta::{
search::{SearchEventType, SearchHistoryHitResponse},
search::SearchEventType,
sql::resolve_stream_names,
stream::StreamType,
usage::{RequestStats, UsageType, USAGE_STREAM},
@@ -1558,23 +1558,6 @@ pub async fn search_history(
}
};
search_res.hits = search_res
.hits
.into_iter()
.filter_map(|hit| match SearchHistoryHitResponse::try_from(hit) {
Ok(response) => match serde_json::to_value(response) {
Ok(json_value) => Some(json_value),
Err(e) => {
log::error!("[trace_id {}] Serialization error: {:?}", trace_id, e);
None
}
},
Err(e) => {
log::error!("[trace_id {}] Deserialization error: {:?}", trace_id, e);
None
}
})
.collect::<Vec<_>>();
search_res.trace_id = trace_id.clone();
// report http metrics
+2 -1
View File
@@ -279,11 +279,13 @@ pub async fn search(
res.histogram_interval = Some(c_resp.histogram_interval);
}
let num_fn = req.query.query_fn.is_some() as u16;
let req_stats = RequestStats {
records: res.hits.len() as i64,
response_time: time,
size: res.scan_size as f64,
request_body: Some(req.query.sql),
function: req.query.query_fn,
user_email: user_id,
min_ts: Some(req.query.start_time),
max_ts: Some(req.query.end_time),
@@ -300,7 +302,6 @@ pub async fn search(
result_cache_ratio: Some(res.result_cache_ratio),
..Default::default()
};
let num_fn = req.query.query_fn.is_some() as u16;
report_request_usage_stats(
req_stats,
org_id,
+5
View File
@@ -216,6 +216,11 @@ pub async fn search(
response_time: time,
size: res.scan_size as f64,
request_body: Some(req_query.sql.clone()),
function: if req_query.query_fn.is_empty() {
None
} else {
Some(req_query.query_fn.clone())
},
user_email: user_id,
min_ts: Some(req_query.start_time),
max_ts: Some(req_query.end_time),
+4 -2
View File
@@ -15,7 +15,7 @@
use std::sync::Arc;
use chrono::{Datelike, Timelike, Utc};
use chrono::{DateTime, Datelike, Timelike};
use config::{
get_config,
meta::{
@@ -61,6 +61,7 @@ pub async fn report_request_usage_stats(
.with_label_values(&[org_id, stream_name, stream_type.to_string().as_str()])
.inc_by((stats.size * SIZE_IN_MB) as u64);
let event: UsageEvent = usage_type.into();
let now = DateTime::from_timestamp_micros(timestamp).unwrap();
if !get_config().common.usage_enabled {
return;
@@ -68,7 +69,6 @@ pub async fn report_request_usage_stats(
let request_body = stats.request_body.unwrap_or(usage_type.to_string());
let user_email = stats.user_email.unwrap_or("".to_owned());
let now = Utc::now();
let mut usage = vec![];
@@ -89,6 +89,7 @@ pub async fn report_request_usage_stats(
),
org_id: org_id.to_owned(),
request_body: request_body.to_owned(),
function: None,
size: stats.size,
unit: "MB".to_owned(),
user_email: user_email.to_owned(),
@@ -127,6 +128,7 @@ pub async fn report_request_usage_stats(
unit: "MB".to_owned(),
user_email,
response_time: stats.response_time,
function: stats.function,
num_records: stats.records,
stream_type,
stream_name: stream_name.to_owned(),
+1
View File
@@ -856,6 +856,7 @@ export default defineComponent({
query: {
action: "history",
org_identifier: store.state.selectedOrganization.identifier,
type: "search_history",
},
});
showSearchHistory.value = true;
+253 -54
View File
@@ -44,14 +44,14 @@
</div>
</div>
<div class="full-width">
<div class="">
<q-page>
<q-table
ref="qTable"
dense
:rows="dataToBeLoaded"
:columns="columnsToBeRendered"
:pagination="pagination"
hide-bottom
:pagination.sync="pagination"
row-key="trace_id"
:rows-per-page-options="[]"
class="custom-table"
@@ -61,7 +61,30 @@
<template v-slot:body="props">
<q-tr :props="props" @click="triggerExpand(props)">
<q-tr
:data-test="`stream-association-table-${props.row.trace_id}-row`"
:props="props"
style="cursor: pointer"
@click="triggerExpand(props)"
>
<q-td >
<q-btn
dense
flat
size="xs"
:icon="
expandedRow != props.row.trace_id
? 'expand_more'
: 'expand_less'
"
/>
</q-td>
<q-td v-for="col in columnsToBeRendered.slice(1)" :key="col.name" :props="props">
{{ props.row[col.field] }}
</q-td>
</q-tr>
<!-- <q-tr :props="props" @click="triggerExpand(props)">
<q-td
:class="`column-${col.name}`"
v-for="col in props.cols"
@@ -71,21 +94,40 @@
{{ col.value }}
</q-td>
</q-tr>
</q-tr> -->
<q-tr v-show="expandedRow === props.row.trace_id" :props="props" >
<q-td colspan="100%">
<q-td colspan="100%">
<div class="text-left tw-px-2 expanded-content">
<strong >SQL Query:</strong>
<div class="tw-flex tw-items-start tw-justify-center" >
<div class="text-left tw-px-2 q-mb-sm expanded-content">
<div class="tw-flex tw-items-center q-py-sm ">
<strong >SQL Query : <span> <q-btn
@click.stop="copyToClipboard(props.row.sql, 'SQL Query')"
size="xs"
dense
flat
icon="content_copy"
class="copy-btn-sql tw-ml-2 tw-py-2 tw-px-2 "
/></span></strong>
<q-btn
@click.stop="goToLogs(props.row)"
size="xs"
label="Logs"
dense
class="copy-btn tw-py-2 tw-mx-2 tw-px-2"
icon="search"
flat
style="color: #F2452F; border: #F2452F 1px solid;font-weight:bolder;"
/>
</div>
<div class="tw-flex tw-items-start tw-justify-center" >
<div class="scrollable-content ">
<div class="scrollable-content expanded-sql ">
<pre style="text-wrap: wrap;">{{ props.row?.sql }}</pre>
</div>
<div class="tw-pl-2">
<!-- <div class="tw-pl-2">
<q-btn
@click.stop="copyToClipboard(props.row.sql)"
size="xs"
@@ -98,21 +140,81 @@
label="Go To Logs"
class="copy-btn tw-py-3 tw-mx-2"
/>
</div>
</div> -->
</div>
</div>
<div v-if="props.row?.function" class="text-left q-mb-sm tw-px-2 expanded-content">
<div class="tw-flex tw-items-center q-py-sm ">
<strong >Function Defination : <span> <q-btn
@click.stop="copyToClipboard(props.row.function, 'Function Defination')"
size="xs"
dense
flat
icon="content_copy"
class="copy-btn-function tw-ml-2 tw-py-2 tw-px-2 "
/></span></strong>
</div>
<div class="tw-flex tw-items-start tw-justify-center" >
<div class="scrollable-content expanded-function ">
<pre style="text-wrap: wrap;">{{ props.row?.function }}</pre>
</div>
<!--
<div class="tw-pl-2 tw-flex tw-my-auto">
<q-btn
@click.stop="copyToClipboard(props.row.function)"
size="xs"
icon="content_copy"
class="copy-btn tw-py-3"
/>
<q-btn
@click.stop="goToLogs(props.row)"
size="xs"
label="Go To Logs"
class="copy-btn tw-py-3 tw-mx-2"
/>
</div> -->
</div>
</div>
</q-td>
</q-tr>
</template>
<template #bottom="scope">
<div class="tw-ml-auto tw-mr-2">
Max Limit : <b>1000</b>
</div>
<q-separator style="height: 1.5rem; margin: auto 0; " vertical inset class="q-mr-md" />
<div class="q-pl-md" >
<QTablePagination
:scope="scope"
:position="'bottom'"
:resultTotal="resultTotal"
:perPageOptions="perPageOptions"
@update:changeRecordPerPage="changePagination"
/>
</div>
</template>
<template #no-data>
<div v-if="!isLoading" class="tw-flex tw-mx-auto">
<NoData />
</div>
</template>
</q-table>
<div v-if="!isLoading && dataToBeLoaded.length === 0">
<NoData />
</div>
<div
v-if="isLoading"
class="text-center full-width full-height q-mt-lg tw-flex tw-justify-center"
@@ -144,11 +246,14 @@
import { useI18n } from 'vue-i18n';
import { date , QTable , useQuasar } from 'quasar';
import type { Ref } from "vue";
import QTablePagination from "@/components/shared/grid/Pagination.vue";
export default defineComponent({
name: "SearchHistoryComponent",
components: {
DateTime,
NoData,
QTablePagination,
},
props: {
isClicked: {
@@ -177,16 +282,33 @@
const expandedRow = ref( []); // Array to track expanded rows
const isLoading = ref(false);
const pagination = {
const perPageOptions: any = [
{ label: "5", value: 5 },
{ label: "10", value: 10 },
{ label: "20", value: 20 },
{ label: "50", value: 50 },
{ label: "100", value: 100 },
{ label: "All", value: 0 },
];
const resultTotal = ref<number>(0);
const pagination = ref({
page: 1,
rowsPerPage: 50
}
rowsPerPage: 100
})
const selectedPerPage = ref(pagination.value.rowsPerPage);
const generateColumns = (data: any) => {
if (data.length === 0) return [];
// Define the desired column order and names
const desiredColumns = [
{key : '#' , label : '#',align: 'center',sortable: false},
{ key: 'trace_id', label: 'Trace ID' },
{ key: 'executed_time', label: 'Executed At' },
{ key: 'start_time', label: 'Start Time' },
{ key: 'end_time', label: 'End Time' },
{ key: 'duration', label: 'Duration' },
@@ -195,6 +317,7 @@
{ key: 'scan_records', label: 'Scan Records' },
{ key: 'cached_ratio', label: 'Cached Ratio' },
{ key: 'sql', label: 'SQL Query' },
];
return desiredColumns.map(({ key, label }) => {
@@ -207,35 +330,39 @@
}
if(key == "scan_size"){
columnWidth = 80
columnWidth = 100
}
if(key == "duration"){
align='left'
columnWidth = 100
}
if( key == 'start_time' || key == 'end_time'){
columnWidth = 200
}
if(key == 'sql'){
columnWidth = 350
columnWidth = 300
sortable = false;
}
if(key == "trace_id"){
columnWidth = 250
align = "left"
sortable = false;
}
if(key == "#"){
columnWidth = 100
sortable = false;
align = "left"
}
// Custom width for each column
return {
name: key, // Field name
label: label, // Column label
field: key, // Field accessor
align: 'center',
align: align,
sortable: sortable,
style: `max-width: ${columnWidth}px; width: ${columnWidth}px;`,
};
});
};
const fetchSearchHistory = async ( ) => {
columnsToBeRendered.value = [];
@@ -249,24 +376,42 @@
const response = await searchService.get_history( org_identifier,startTime,endTime,
);
const limitedHits = response.data.hits;
columnsToBeRendered.value = generateColumns(limitedHits);
limitedHits.forEach((hit:any)=>{
const {formatted, raw} = calculateDuration(hit.start_time, hit.end_time);
const filteredHits = limitedHits.filter((hit) => hit.event === "Search");
if(filteredHits.length > 0){
resultTotal.value = filteredHits.length;
}
columnsToBeRendered.value = generateColumns(filteredHits);
filteredHits.forEach((hit:any)=>{
const {formatted, raw} = calculateDuration(hit.min_ts, hit.max_ts);
hit.duration = formatted;
hit.rawDuration = raw;
hit.toBeStoredStartTime = hit.start_time;
hit.toBeStoredEndTime = hit.end_time;
hit.start_time = timestampToTimezoneDate(hit.start_time / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS");
hit.end_time = timestampToTimezoneDate(hit.end_time / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS");
hit.rawTook = hit.took;
hit.took = formatTime(hit.took);
hit.rawScanRecords = hit.scan_records;
hit.rawScanSize = hit.scan_size;
hit.toBeStoredStartTime = hit.min_ts;
hit.toBeStoredEndTime = hit.max_ts;
hit.start_time = timestampToTimezoneDate(hit.min_ts / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS");
hit.end_time = timestampToTimezoneDate(hit.max_ts / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS");
hit.rawTook = hit.response_time;
hit.took = formatTime(hit.response_time);
hit.scan_records = hit.num_records;
hit.rawScanRecords = hit.num_records;
hit.scan_size = hit.size + hit.unit;
hit.rawScanSize = hit.size;
hit.cached_ratio = hit.cached_ratio;
hit.rawCachedRatio = hit.cached_ratio;
hit.sql = hit.request_body;
hit.function = hit.function;
hit.rawExecutedTime = hit._timestamp;
hit.executed_time = timestampToTimezoneDate(hit._timestamp / 1000, store.state.timezone, "yyyy-MM-dd HH:mm:ss.SSS");
})
dataToBeLoaded.value = response.data.hits;
dataToBeLoaded.value = filteredHits;
isLoading.value = false;
} catch (error) {
$q.notify({
type: "negative",
message: "Failed to fetch search history. Please try again later.",
timeout: 5000,
});
console.log(error, "error")
isLoading.value = false;
} finally {
@@ -334,6 +479,13 @@
return data.sort((a, b) => a.toBeStoredEndTime - b.toBeStoredEndTime);
}
if(sortBy == "executed_time"){
if (descending) {
return data.sort((a, b) => b.rawExecutedTime - a.rawExecutedTime);
}
return data.sort((a, b) => a.rawExecutedTime - b.rawExecutedTime);
}
@@ -341,11 +493,11 @@
// return a.rawDuration - b.rawDuration;
}
const copyToClipboard = (text) => {
const copyToClipboard = (text,type) => {
navigator.clipboard.writeText(text).then(() => {
$q.notify({
type: "positive",
message: "Content Copied Successfully!",
message: `${type} Copied Successfully!`,
timeout: 5000,
});
}).catch(() => {
@@ -451,25 +603,44 @@
const query = b64EncodeUnicode(row.sql);
const queryObject = {
stream_type: "logs",
stream,
period: '15m',
refresh,
sql_mode: "true",
query,
defined_schemas: "user_defined_schema",
org_identifier: row.org_id,
quick_mode: "false",
show_histogram: "true",
type: "search_history_re_apply"
};
if(row.hasOwnProperty('function') && row.function){
const functionContent = b64EncodeUnicode(row.function);
queryObject['functionContent'] = functionContent;
}
router.push({
path: "/logs",
query: {
stream_type: "logs",
stream,
period: '15m',
refresh,
sql_mode: "true",
query,
defined_schemas:"user_defined_schema",
org_identifier: row.org_id,
quick_mode: "false",
show_histogram: "true",
type: "search_history_re_apply"
},
query: queryObject,
});
};
const changePagination = (val: { label: string; value: any }) => {
if(val.label == "All"){
val.value = dataToBeLoaded.value.length;
val.label = 'All';
}
selectedPerPage.value = val.value;
pagination.value.rowsPerPage = val.value;
qTable.value.setPagination(pagination.value);
// pagination.value.page = 1;
};
watch(() => props.isClicked, (value) => {
if(value == true && !isLoading.value){
@@ -497,6 +668,10 @@
formatTime,
delayMessage,
sortMethod,
resultTotal,
perPageOptions,
changePagination,
selectedPerPage,
};
// Watch the searchObj for changes
@@ -505,6 +680,7 @@
</script>
<style lang="scss" scoped >
.expanded-content {
padding: 0 3rem;
min-width: 100vh;
max-height: 100vh; /* Set a fixed height for the container */
overflow: hidden; /* Hide overflow by default */
@@ -519,20 +695,43 @@
max-height: 200px;
/* Use the full height of the parent */
text-wrap: normal;
background-color: #e8e8e8;
color: black;
}
.q-td {
overflow: hidden;
white-space: nowrap;
text-overflow: ellipsis;
}
.custom-table .q-tr > .q-td:first-child {
.custom-table .q-tr > .q-td:nth-child(2){
text-align: left;
}
.copy-btn-sql{
border: #7A54A2 1px solid;
color: #7A54A2;
}
.copy-btn-function{
border: #0A7EBC 1px solid;
color: #0A7EBC;
}
.warning-text {
color: #F5A623;
border: 1px solid #F5A623;
border-radius: 2px ;
}
.expanded-sql{
border-left: #7A54A2 3px solid;
}
.expanded-function{
border-left: #0A7EBC 3px solid;
}
</style>
-1
View File
@@ -199,7 +199,6 @@ const search = {
stream_type : "logs",
org_identifier,
user_email : null,
size:50,
};
// Add startTime and endTime to the payload if provided
if (startTime) {