From 2bea89307c089fb267753f7e1d3ced13fe174419 Mon Sep 17 00:00:00 2001 From: Sai Nikhil Date: Thu, 27 Feb 2025 16:07:45 +0530 Subject: [PATCH] fix: search jobs UI (#5706) ## Summary by CodeRabbit ## Release Notes - **New Features** - Introduced search job scheduling functionality. - Added ability to create, list, cancel, retry, and delete scheduled searches. - Enhanced user interface for managing scheduled search jobs, including a new component for displaying scheduled jobs. - Implemented conditional rendering for search history and search scheduler components. - Added new methods for handling scheduled searches within the service layer. - **Improvements** - Refined search bar interface with new scheduling options. - Updated pagination handling for job-related data. - Enhanced overall functionality related to job management and pagination. - Improved loading indicators and user experience during data retrieval. - **Bug Fixes** - Corrected conditional rendering of search history components. - Optimized job data retrieval and pagination logic. - Simplified sorting logic within the search history component. --- .../playwright-tests/sanity2.spec.js | 2 +- web/src/composables/useLogs.ts | 433 +++++-- web/src/plugins/logs/Index.vue | 103 +- web/src/plugins/logs/SearchBar.vue | 282 +++- web/src/plugins/logs/SearchHistory.vue | 6 - web/src/plugins/logs/SearchResult.vue | 54 +- web/src/plugins/logs/SearchSchedulersList.vue | 1151 +++++++++++++++++ web/src/services/search.ts | 129 ++ 8 files changed, 2022 insertions(+), 138 deletions(-) create mode 100644 web/src/plugins/logs/SearchSchedulersList.vue diff --git a/tests/ui-testing/playwright-tests/sanity2.spec.js b/tests/ui-testing/playwright-tests/sanity2.spec.js index 2dcdf00fdd..be6fa291df 100644 --- a/tests/ui-testing/playwright-tests/sanity2.spec.js +++ b/tests/ui-testing/playwright-tests/sanity2.spec.js @@ -497,7 +497,7 @@ test.skip("should display error if timestamp past the ingestion time limit", asy await page.locator('[data-test="logs-search-bar-refresh-btn"]').click(); await page.getByRole('button', { name: 'Search History' }).click(); await page.locator('[data-test="add-alert-title"]').click(); - await page.getByText('arrow_back_ios_new').click(); + await page.getByText('arrow_back_ios_new').first().click() await page.waitForTimeout(1000); // Use a more specific locator for 'e2e_automate' by targeting its unique container or parent element diff --git a/web/src/composables/useLogs.ts b/web/src/composables/useLogs.ts index a8ca4675b0..9ffba364eb 100644 --- a/web/src/composables/useLogs.ts +++ b/web/src/composables/useLogs.ts @@ -81,6 +81,7 @@ const defaultObject = { runQuery: false, loading: false, loadingHistogram: false, + loadingCounter: false, loadingStream: false, loadingSavedView: false, shouldIgnoreWatcher: false, @@ -144,6 +145,8 @@ const defaultObject = { }, showPagination: true, }, + jobId: "", + jobRecords: "100", scrollInfo: {}, pageType: "logs", // 'logs' or 'stream regions: [], @@ -151,6 +154,7 @@ const defaultObject = { useUserDefinedSchemas: "user_defined_schema", hasUserDefinedSchemas: false, selectedTraceStream: "", + showSearchScheduler: false, }, data: { query: "", @@ -599,7 +603,7 @@ const useLogs = () => { const query = generateURLQuery(false); if ( Object.hasOwn(query, "type") && - query.type == "search_history_re_apply" + query.type == "search_history_re_apply" || query.type == "search_scheduler" ) { delete query.type; } @@ -1332,6 +1336,9 @@ const useLogs = () => { }; const refreshPartitionPagination = (regenrateFlag: boolean = false) => { + if (searchObj.meta.jobId != "") { + return; + } try { const { rowsPerPage } = searchObj.meta.resultGrid; const { currentPage } = searchObj.data.resultGrid; @@ -1502,6 +1509,7 @@ const useLogs = () => { // window will have more priority // if window has use_web_socket property then use that // else use organization settings + searchObj.meta.jobId = ""; const shouldUseWebSocket = isWebSocketEnabled(); const isMultiStreamSearch = @@ -1618,6 +1626,7 @@ const useLogs = () => { JSON.stringify(queryReq), ); + // get the current page detail and set it into query request queryReq.query.start_time = searchObj.data.queryResults.partitionDetail.paginations[ @@ -1688,6 +1697,7 @@ const useLogs = () => { searchObj.meta.refreshHistogram = false; if (searchObj.data.queryResults.hits.length > 0) { if (searchObj.data.stream.selectedStream.length > 1) { + searchObj.data.histogram = { xData: [], yData: [], @@ -1797,6 +1807,7 @@ const useLogs = () => { } if (searchObj.data.stream.selectedStream.length > 1) { + searchObj.data.histogram = { xData: [], yData: [], @@ -1826,8 +1837,104 @@ const useLogs = () => { notificationMsg.value = ""; } }; + const getJobData = async (isPagination = false) => { + try { + // get websocket enable config from store + // window will have more priority + // if window has use_web_socket property then use that + // else use organization settings + const queryReq: any = buildSearch(); + if (queryReq == false) { + throw new Error(notificationMsg.value || "Something went wrong."); + } + if (searchObj.meta.jobId == "") { + queryReq.query.size = parseInt(searchObj.meta.jobRecords); + } + // reset query data and get partition detail for given query. + + if (queryReq != null) { + // in case of live refresh, reset from to 0 + if ( + searchObj.meta.refreshInterval > 0 && + router.currentRoute.value.name == "logs" + ) { + queryReq.query.from = 0; + searchObj.meta.refreshHistogram = true; + } + + // get function definition + if ( + searchObj.data.tempFunctionContent != "" && + searchObj.meta.toggleFunction + ) { + queryReq.query["query_fn"] = + b64EncodeUnicode(searchObj.data.tempFunctionContent) || ""; + } + + // in case of relative time, set start_time and end_time to query + // it will be used in pagination request + if (searchObj.data.datetime.type === "relative") { + if (!isPagination) initialQueryPayload.value = cloneDeep(queryReq); + else { + if ( + searchObj.meta.refreshInterval == 0 && + router.currentRoute.value.name == "logs" && + searchObj.data.queryResults.hasOwnProperty("hits") + ) { + const start_time: number = + initialQueryPayload.value?.query?.start_time || 0; + const end_time: number = + initialQueryPayload.value?.query?.end_time || 0; + queryReq.query.start_time = start_time; + queryReq.query.end_time = end_time; + } + } + } + delete queryReq.aggs; + } + searchObj.data.queryResults.subpage = 1; + if (searchObj.meta.jobId == "" ) { + searchService + .schedule_search( + { + org_identifier: searchObj.organizationIdentifier, + query: queryReq, + page_type: searchObj.data.stream.streamType, + }, + "UI", + ).then((res: any) => { + $q.notify({ + type: "positive", + message: "Job Added Succesfully", + timeout: 2000, + actions: [ + { + label: "Go To Job Scheduler", + color: "white", + handler: () => routeToSearchSchedule(), + }, + ], + }); + }) + } + else { + await getPaginatedData(queryReq); + } + if (searchObj.meta.jobId == ""){ + searchObj.data.histogram.chartParams.title = getHistogramTitle(); + } + } catch (e: any) { + searchObj.loading = false; + showErrorNotification( + notificationMsg.value || "Error occurred during the search operation.", + ); + throw e; + // notificationMsg.value = ""; + } + }; function resetHistogramWithError(errorMsg: string, errorCode: number = 0) { + searchObj.data.histogram = { xData: [], yData: [], @@ -1859,6 +1966,7 @@ const useLogs = () => { } function generateHistogramSkeleton() { + if ( searchObj.data.queryResults.hasOwnProperty("aggs") && searchObj.data.queryResults.aggs @@ -1962,101 +2070,114 @@ const useLogs = () => { const getPageCount = async (queryReq: any) => { return new Promise((resolve, reject) => { - searchObj.data.countErrorMsg = ""; - queryReq.query.size = 0; - delete queryReq.query.from; - delete queryReq.query.quick_mode; - queryReq.query["sql_mode"] = "full"; - - queryReq.query.track_total_hits = true; - - const { traceparent, traceId } = generateTraceContext(); - addTraceId(traceId); - - searchService - .search( - { - org_identifier: searchObj.organizationIdentifier, - query: queryReq, - page_type: searchObj.data.stream.streamType, - traceparent, - }, - "UI", - ) - .then(async (res) => { - // check for total records update for the partition and update pagination accordingly - // searchObj.data.queryResults.partitionDetail.partitions.forEach( - // (item: any, index: number) => { - searchObj.data.queryResults.scan_size += res.data.scan_size; - searchObj.data.queryResults.took += res.data.took; - for (const [ - index, - item, - ] of searchObj.data.queryResults.partitionDetail.partitions.entries()) { - if ( - (searchObj.data.queryResults.partitionDetail.partitionTotal[ - index - ] == -1 || - searchObj.data.queryResults.partitionDetail.partitionTotal[ - index - ] < res.data.total) && - queryReq.query.start_time == item[0] - ) { - searchObj.data.queryResults.partitionDetail.partitionTotal[ - index - ] = res.data.total; + try { + searchObj.loadingCounter = true; + searchObj.data.countErrorMsg = ""; + queryReq.query.size = 0; + delete queryReq.query.from; + delete queryReq.query.quick_mode; + queryReq.query["sql_mode"] = "full"; + + queryReq.query.track_total_hits = true; + + const { traceparent, traceId } = generateTraceContext(); + addTraceId(traceId); + + searchService + .search( + { + org_identifier: searchObj.organizationIdentifier, + query: queryReq, + page_type: searchObj.data.stream.streamType, + traceparent, + }, + "UI", + ) + .then(async (res) => { + // check for total records update for the partition and update pagination accordingly + // searchObj.data.queryResults.partitionDetail.partitions.forEach( + // (item: any, index: number) => { + searchObj.data.queryResults.scan_size += res.data.scan_size; + searchObj.data.queryResults.took += res.data.took; + if (searchObj.meta.jobId == "") { + for (const [ + index, + item, + ] of searchObj.data.queryResults.partitionDetail.partitions.entries()) { + if ( + (searchObj.data.queryResults.partitionDetail.partitionTotal[ + index + ] == -1 || + searchObj.data.queryResults.partitionDetail.partitionTotal[ + index + ] < res.data.total) && + queryReq.query.start_time == item[0] + ) { + searchObj.data.queryResults.partitionDetail.partitionTotal[ + index + ] = res.data.total; + } + } } - } - - let regeratePaginationFlag = false; - if (res.data.hits.length != searchObj.meta.resultGrid.rowsPerPage) { - regeratePaginationFlag = true; - } - // if total records in partition is greater than recordsPerPage then we need to update pagination - // setting up forceFlag to true to update pagination as we have check for pagination already created more than currentPage + 3 pages. - refreshPartitionPagination(regeratePaginationFlag); - searchObj.data.histogram.chartParams.title = getHistogramTitle(); - resolve(true); - }) - .catch((err) => { - searchObj.loading = false; + + + let regeratePaginationFlag = false; + if (res.data.hits.length != searchObj.meta.resultGrid.rowsPerPage) { + regeratePaginationFlag = true; + } + // if total records in partition is greater than recordsPerPage then we need to update pagination + // setting up forceFlag to true to update pagination as we have check for pagination already created more than currentPage + 3 pages. + refreshPartitionPagination(regeratePaginationFlag); + searchObj.data.histogram.chartParams.title = getHistogramTitle(); + searchObj.loadingCounter = false; + resolve(true); + }) + .catch((err) => { + searchObj.loading = false; + searchObj.loadingCounter = false; + // Reset cancel query on search error searchObj.data.isOperationCancelled = false; let trace_id = ""; - searchObj.data.countErrorMsg = - "Error while retrieving total events: "; - if (err.response != undefined) { - if (err.response.data.hasOwnProperty("trace_id")) { - trace_id = err.response.data?.trace_id; + searchObj.data.countErrorMsg = + "Error while retrieving total events: "; + if (err.response != undefined) { + if (err.response.data.hasOwnProperty("trace_id")) { + trace_id = err.response.data?.trace_id; + } + } else { + if (err.hasOwnProperty("trace_id")) { + trace_id = err?.trace_id; + } } - } else { - if (err.hasOwnProperty("trace_id")) { - trace_id = err?.trace_id; + + const customMessage = logsErrorMessage(err?.response?.data.code); + searchObj.data.errorCode = err?.response?.data.code; + + notificationMsg.value = searchObj.data.countErrorMsg; + + if (err?.request?.status >= 429) { + notificationMsg.value = err?.response?.data?.message; + searchObj.data.countErrorMsg += err?.response?.data?.message; } - } + + if (trace_id) { + searchObj.data.countErrorMsg += " TraceID:" + trace_id; + notificationMsg.value += " TraceID:" + trace_id; + trace_id = ""; + } + reject(false); + }) + .finally(() => { + removeTraceId(traceId); + }); + } catch (e) { + searchObj.loadingCounter = false; + reject(false); + } - const customMessage = logsErrorMessage(err?.response?.data.code); - searchObj.data.errorCode = err?.response?.data.code; - - notificationMsg.value = searchObj.data.countErrorMsg; - - if (err?.request?.status >= 429) { - notificationMsg.value = err?.response?.data?.message; - searchObj.data.countErrorMsg += err?.response?.data?.message; - } - - if (trace_id) { - searchObj.data.countErrorMsg += " TraceID:" + trace_id; - notificationMsg.value += " TraceID:" + trace_id; - trace_id = ""; - } - reject(false); - }) - .finally(() => { - removeTraceId(traceId); - }); }); }; @@ -2070,8 +2191,8 @@ const useLogs = () => { updateGridColumns(); filterHitsColumns(); + searchObj.data.histogram.chartParams.title = getHistogramTitle(); - searchObj.data.histogram.chartParams.title = getHistogramTitle(); }; const getPaginatedData = async ( @@ -2099,7 +2220,9 @@ const useLogs = () => { showCancelSearchNotification(); return; } - + if (searchObj.meta.jobId != ""){ + searchObj.meta.resultGrid.rowsPerPage = queryReq.query.size; + } const parsedSQL: any = fnParsedSQL(); searchObj.meta.resultGrid.showPagination = true; if (searchObj.meta.sqlMode == true) { @@ -2127,12 +2250,14 @@ const useLogs = () => { const { traceparent, traceId } = generateTraceContext(); addTraceId(traceId); - - searchService - .search( + const decideSearch = searchObj.meta.jobId + ? "get_scheduled_search_result" + : "search"; + searchService[decideSearch]( { org_identifier: searchObj.organizationIdentifier, query: queryReq, + jobId: searchObj.meta.jobId ? searchObj.meta.jobId : "", page_type: searchObj.data.stream.streamType, traceparent, }, @@ -2179,19 +2304,21 @@ const useLogs = () => { // check for total records update for the partition and update pagination accordingly // searchObj.data.queryResults.partitionDetail.partitions.forEach( // (item: any, index: number) => { - for (const [ - index, - item, - ] of searchObj.data.queryResults.partitionDetail.partitions.entries()) { - if ( - searchObj.data.queryResults.partitionDetail.partitionTotal[ - index - ] == -1 && - queryReq.query.start_time == item[0] - ) { - searchObj.data.queryResults.partitionDetail.partitionTotal[ - index - ] = res.data.total; + if (searchObj.meta.jobId == ""){ + for (const [ + index, + item, + ] of searchObj.data.queryResults.partitionDetail.partitions.entries()) { + if ( + searchObj.data.queryResults.partitionDetail.partitionTotal[ + index + ] == -1 && + queryReq.query.start_time == item[0] + ) { + searchObj.data.queryResults.partitionDetail.partitionTotal[ + index + ] = res.data.total; + } } } @@ -2214,8 +2341,9 @@ const useLogs = () => { } // if total records in partition is greater than recordsPerPage then we need to update pagination // setting up forceFlag to true to update pagination as we have check for pagination already created more than currentPage + 3 pages. - refreshPartitionPagination(regeratePaginationFlag); - + if (searchObj.meta.jobId == "") { + refreshPartitionPagination(regeratePaginationFlag); + } // Scan-size and took time in histogram title // For the initial request, we get histogram and logs data. So, we need to sum the scan_size and took time of both the requests. // For the pagination request, we only get logs data. So, we need to consider scan_size and took time of only logs request. @@ -2251,6 +2379,10 @@ const useLogs = () => { ); searchObj.data.queryResults.hits = res.data.hits; } else { + if (searchObj.meta.jobId != ""){ + searchObj.data.queryResults.total = res.data.total; + + } if (!queryReq.query.hasOwnProperty("track_total_hits")) { delete res.data.total; } @@ -2264,6 +2396,7 @@ const useLogs = () => { // check for pagination request for the partition and check for subpage if we have to pull data from multiple partitions // it will check for subpage and if subpage is present then it will send pagination request for next partition if ( + searchObj.meta.jobId == "" && searchObj.data.queryResults.partitionDetail.paginations[ searchObj.data.resultGrid.currentPage - 1 ].length > searchObj.data.queryResults.subpage && @@ -2291,7 +2424,7 @@ const useLogs = () => { searchObj.data.queryResults.subpage++; setTimeout(async () => { - processPostPaginationData(); + processPostPaginationData(); // searchObj.data.functionError = ""; // if ( @@ -2303,6 +2436,11 @@ const useLogs = () => { }, 0); await getPaginatedData(queryReq, true); } + if (searchObj.meta.jobId != ""){ + searchObj.meta.resultGrid.rowsPerPage = queryReq.query.size; + searchObj.data.queryResults.pagination = []; + refreshJobPagination(true); + } await processPostPaginationData(); @@ -2406,6 +2544,16 @@ const useLogs = () => { } }; + const routeToSearchSchedule = () => { + router.push({ + query:{ + action: "search_scheduler", + org_identifier: store.state.selectedOrganization.identifier, + type: "search_scheduler_list" + } + }); + } + const getHistogramQueryData = (queryReq: any) => { return new Promise((resolve, reject) => { if (searchObj.data.isOperationCancelled) { @@ -2427,7 +2575,6 @@ const useLogs = () => { const { traceparent, traceId } = generateTraceContext(); addTraceId(traceId); queryReq.query.size = -1; - searchService .search( { @@ -3351,7 +3498,7 @@ const useLogs = () => { endCount = searchObj.meta.resultGrid.rowsPerPage * (currentPage + 1); if ( currentPage >= - (searchObj.communicationMethod === "ws" + (searchObj.communicationMethod === "ws" || searchObj.meta.jobId != "" ? searchObj.data.queryResults?.pagination?.length : searchObj.data.queryResults.partitionDetail?.paginations ?.length || 0) - @@ -3477,7 +3624,6 @@ const useLogs = () => { searchObj.data.queryResults.total = num_records; } - const chartParams = { title: getHistogramTitle(), unparsed_x_data: unparsed_x_data, @@ -3751,7 +3897,27 @@ const useLogs = () => { // await getSavedViews(); await getFunctions(); await extractFields(); - await getQueryData(); + if (searchObj.meta.jobId == ""){ + await getQueryData(); + } + else{ + + await getJobData(); + + } + refreshData(); + } catch (e: any) { + searchObj.loading = false; + + } + }; + const loadJobData = async () => { + try { + resetFunctions(); + await getStreamList(); + await getFunctions(); + await extractFields(); + await getJobData(); refreshData(); } catch (e: any) { searchObj.loading = false; @@ -3784,7 +3950,13 @@ const useLogs = () => { ) { delete router.currentRoute.value.query.type; } + const queryTimeout = setTimeout(() => { + if (searchObj.loading) { + searchObj.meta.showSearchScheduler = true; + } + }, 120000); await getQueryData(); + clearTimeout(queryTimeout); } catch (e: any) { console.log("Error while loading logs data"); } @@ -5336,6 +5508,8 @@ const useLogs = () => { const { rowsPerPage } = searchObj.meta.resultGrid; const { currentPage } = searchObj.data.resultGrid; + if(searchObj.meta.jobId != "") searchObj.meta.resultGrid.rowsPerPage = 100; + let total = 0; let totalPages = 0; @@ -5371,6 +5545,37 @@ const useLogs = () => { return false; } }; + const refreshJobPagination = (regenrateFlag: boolean = false) => { + try { + const { rowsPerPage } = searchObj.meta.resultGrid; + const { currentPage } = searchObj.data.resultGrid; + + if (searchObj.meta.jobId != "") { + // searchObj.meta.resultGrid.rowsPerPage = 100; + } + + let totalPages = 0; + + searchObj.data.queryResults.pagination = []; + + totalPages = Math.ceil(searchObj.data.queryResults.total / rowsPerPage); + + for (let i = 0; i < totalPages; i++) { + if (i + 1 > currentPage + 10) { + break; + } + searchObj.data.queryResults.pagination.push({ + from: i * rowsPerPage + 1, + size: rowsPerPage, + }); + } + // console.log("searchObj.data.queryResults.pagination", searchObj.data.queryResults.pagination); + } catch (e: any) { + console.log("Error while refreshing pagination", e); + notificationMsg.value = "Error while refreshing pagination."; + return false; + } + }; const getPageCountThroughSocket = async (queryReq: any) => { if ( @@ -5498,6 +5703,7 @@ const useLogs = () => { fieldValues, extractFields, getQueryData, + getJobData, searchAroundData, updateGridColumns, refreshData, @@ -5532,10 +5738,13 @@ const useLogs = () => { extractValueQuery, initialQueryPayload, refreshPagination, + loadJobData, + refreshJobPagination, enableRefreshInterval, buildWebSocketPayload, initializeWebSocketConnection, addRequestId, + routeToSearchSchedule, }; }; diff --git a/web/src/plugins/logs/Index.vue b/web/src/plugins/logs/Index.vue index 37c5f4bd8c..245ead72c8 100644 --- a/web/src/plugins/logs/Index.vue +++ b/web/src/plugins/logs/Index.vue @@ -18,7 +18,7 @@ along with this program. If not, see . @@ -386,6 +395,7 @@ import { buildSqlQuery, getFieldsFromQuery } from "@/utils/query/sqlUtils"; import useNotifications from "@/composables/useNotifications"; import SearchBar from "@/plugins/logs/SearchBar.vue"; import SearchHistory from "@/plugins/logs/SearchHistory.vue"; +import SearchSchedulersList from "@/plugins/logs/SearchSchedulersList.vue"; import { type ActivationState, PageType } from "@/ts/interfaces/logs.ts"; import { isWebSocketEnabled } from "@/utils/zincutils"; @@ -393,6 +403,7 @@ export default defineComponent({ name: "PageSearch", components: { SearchBar, + SearchSchedulersList, IndexList: defineAsyncComponent( () => import("@/plugins/logs/IndexList.vue"), ), @@ -446,9 +457,14 @@ export default defineComponent({ // so added this condition to avoid that this.searchObj.meta.refreshHistogram = true; this.searchObj.data.queryResults.aggs = null; + if (this.searchObj.meta.jobId == "") { + await this.getQueryData(false); + this.refreshHistogramChart(); + } + else{ + await this.getJobData(false); + } - await this.getQueryData(false); - this.refreshHistogramChart(); if (config.isCloud == "true") { segment.track("Button Click", { @@ -471,8 +487,14 @@ export default defineComponent({ // this.searchObj.data.resultGrid.currentPage = // this.searchObj.data.resultGrid.currentPage + 1; this.searchObj.loading = true; - await this.getQueryData(true); - this.refreshHistogramChart(); + if(this.searchObj.meta.jobId == ""){ + await this.getQueryData(true); + this.refreshHistogramChart(); + + } + else{ + await this.getJobData(false); + } if (config.isCloud == "true") { segment.track("Button Click", { @@ -533,12 +555,14 @@ export default defineComponent({ let { searchObj, getQueryData, + getJobData, fieldValues, updateGridColumns, refreshData, updateUrlQueryParams, loadLogsData, updateStreams, + loadJobData, restoreUrlQueryParams, handleRunQuery, generateHistogramData, @@ -561,6 +585,8 @@ export default defineComponent({ const searchResultRef = ref(null); const searchBarRef = ref(null); const showSearchHistory = ref(false); + const showSearchScheduler = ref(false); + const showJobScheduler = ref(false); let parser: any; const isLogsMounted = ref(false); @@ -630,6 +656,17 @@ export default defineComponent({ ) { showSearchHistory.value = true; } + if ( + router.currentRoute.value.query.hasOwnProperty("action") && + router.currentRoute.value.query.action == "search_scheduler" + ) { + if(config.isEnterprise == 'true'){ + showSearchScheduler.value = true; + } + else{ + router.back(); + } + } }); onActivated(() => { @@ -662,6 +699,7 @@ export default defineComponent({ () => { if (!router.currentRoute.value.query.hasOwnProperty("action")) { showSearchHistory.value = false; + showSearchScheduler.value = false; } if ( router.currentRoute.value.query.hasOwnProperty("action") && @@ -669,6 +707,17 @@ export default defineComponent({ ) { showSearchHistory.value = true; } + if ( + router.currentRoute.value.query.hasOwnProperty("action") && + router.currentRoute.value.query.action == "search_scheduler" + ) { + if(config.isEnterprise == 'true'){ + showSearchScheduler.value = true; + } + else{ + router.back(); + } + } }, // (action) => { // if (action === "history") { @@ -680,6 +729,8 @@ export default defineComponent({ () => router.currentRoute.value.query.type, async (type) => { if (type == "search_history_re_apply") { + searchObj.meta.jobId = ""; + searchObj.organizationIdetifier = router.currentRoute.value.query.org_identifier; searchObj.data.stream.selectedStream.value = @@ -709,6 +760,31 @@ export default defineComponent({ } }, ); + watch( + () => router.currentRoute.value.query.type, + async (type) => { + if (type == "search_scheduler") { + searchObj.organizationIdetifier = + router.currentRoute.value.query.org_identifier; + searchObj.data.stream.selectedStream.value = + router.currentRoute.value.query.stream; + searchObj.data.stream.streamType = + router.currentRoute.value.query.stream_type; + resetSearchObj(); + + // As when redirecting from search history to logs page, date type was getting set as absolute, so forcefully keeping it relative. + searchBarRef.value.dateTimeRef.setAbsoluteTime( + router.currentRoute.value.query.from, + router.currentRoute.value.query.to, + ); + searchObj.data.datetime.type = "absolute"; + searchObj.meta.searchApplied = false; + resetStreamData(); + await restoreUrlQueryParams(); + await loadLogsData(); + } + }, + ); const importSqlParser = async () => { const useSqlParser: any = await import("@/composables/useParser"); @@ -722,6 +798,8 @@ export default defineComponent({ try { await getQueryData(); refreshHistogramChart(); + showJobScheduler.value = true; + } catch (e) { console.log(e); } @@ -1359,6 +1437,10 @@ export default defineComponent({ showSearchHistory.value = false; refreshHistogramChart(); }; + const closeSearchSchedulerFn = () => { + router.back(); + showSearchScheduler.value = false; + }; // watch for changes in the visualize toggle // if it is in visualize mode, then set the query and stream name in the dashboard panel @@ -1472,6 +1554,7 @@ export default defineComponent({ splitterModel, // loadPageData, getQueryData, + getJobData, searchResultRef, runQueryFn, refreshData, @@ -1512,6 +1595,9 @@ export default defineComponent({ initializeWebSocketConnection, addRequestId, isWebSocketEnabled, + showJobScheduler, + showSearchScheduler, + closeSearchSchedulerFn, }; }, computed: { @@ -1596,7 +1682,10 @@ export default defineComponent({ this.resetHistogramWithError( "Histogram is not available for multi stream search.", ); - } else if (this.searchObj.meta.histogramDirtyFlag == true) { + } else if ( + this.searchObj.meta.histogramDirtyFlag == true && + this.searchObj.meta.jobId == "" + ) { this.searchObj.meta.histogramDirtyFlag = false; // this.handleRunQuery(); diff --git a/web/src/plugins/logs/SearchBar.vue b/web/src/plugins/logs/SearchBar.vue index 320ed5320e..f8b250bf4e 100644 --- a/web/src/plugins/logs/SearchBar.vue +++ b/web/src/plugins/logs/SearchBar.vue @@ -399,6 +399,51 @@ along with this program. If not, see . + + + + + + + + Create Scheduled Search + + + + + + + + + List Scheduled Search + + + + + . + + + +
Schedule Search Job
+ + + +
+
+ No of Records: + + + Number of records can be specified eg: if the no. of records is 1000 then user can get maximum of 1000 records + + +
+ + +
+
+ Maximum 100000 events can be returned in schedule job +
+
+ + Histogram will be disabled for the schedule job +
+ + + + + + + + + { return { name: "", @@ -1043,8 +1170,7 @@ export default defineComponent({ QueryEditor, SyntaxGuide, AutoRefreshInterval, - ConfirmDialog, - }, + ConfirmDialog, }, emits: [ "searchdata", "onChangeInterval", @@ -1191,6 +1317,8 @@ export default defineComponent({ extractFields, cancelQuery, setSelectedStreams, + getJobData, + routeToSearchSchedule, } = useLogs(); const queryEditorRef = ref(null); @@ -1212,6 +1340,8 @@ export default defineComponent({ const confirmDialogVisible: boolean = ref(false); const confirmSavedViewDialogVisible: boolean = ref(false); + const searchSchedulerJob = ref(false); + const autoSearchSchedulerJob = ref(false); let confirmCallback; let streamName = ""; @@ -1270,6 +1400,56 @@ export default defineComponent({ }, { immediate: true, deep: true }, ); + watch( + () => searchObj.meta.showSearchScheduler, + (showSearchScheduler) => { + if(showSearchScheduler){ + searchSchedulerJob.value = true; + } + }, + { immediate: true, deep: true }, + ); + watch( + () => searchObj.meta.functionEditorPlaceholderFlag, + (val) => { + if ( + searchObj.meta.jobId != "" && + val == true && + (router.currentRoute.value.query.functionContent || searchObj.data.tempFunctionContent != "") + ) { + if (!checkFnQuery(searchObj.data.tempFunctionContent)) { + $q.notify({ + message: "Job Context have been removed", + color: "warning", + position: "bottom", + timeout: 2000, + }); + searchObj.meta.jobId = ""; + searchObj.data.queryResults.hits = []; + // getQueryData(false); + } + } + }, + { immediate: true, deep: true }, + ); + watch( + () => searchObj.meta.showHistogram, + (val) => { + + if(val == true && searchObj.meta.jobId != "" ){ + $q.notify({ + message: "Histogram is not available for scheduled search", + color: "negative", + position: "bottom", + timeout: 2000, + }); + searchObj.meta.showHistogram = false; + searchObj.loadingHistogram = false; + } + + }, + { immediate: true, deep: true }, + ); watch( () => searchObj.data.stream.functions, (funs) => { @@ -1278,6 +1458,7 @@ export default defineComponent({ { immediate: true, deep: true }, ); + onBeforeMount(async () => { await importSqlParser(); }); @@ -1326,6 +1507,12 @@ export default defineComponent({ }; const updateQueryValue = (value: string) => { + // if (searchObj.meta.jobId != "") { + // searchObj.meta.jobId = ""; + // getQueryData(false); + // } + + searchObj.data.editorValue = value; if (searchObj.meta.quickMode === true) { const parsedSQL = fnParsedSQL(); @@ -1434,9 +1621,27 @@ export default defineComponent({ } } } + //here we reset the job id if user change the query and move outside of the editor + if ( + searchObj.meta.jobId != "" && + searchObj.meta.queryEditorPlaceholderFlag == true + ) { + if (!checkQuery(value)) { + $q.notify({ + message: "Job Context have been removed", + color: "warning", + position: "bottom", + timeout: 2000, + }); + searchObj.meta.jobId = ""; + searchObj.data.queryResults.hits = []; + // getQueryData(false); + } + } } catch (e) { console.log(e, "Logs: Error while updating query value"); } + }; const handleEscKey = (event: KeyboardEvent) => { if (event.key === "Escape") { @@ -2817,6 +3022,70 @@ export default defineComponent({ ) ); }); + const addJobScheduler = async () => { + + try{ + // if(searchObj.meta.jobId != ""){ + // searchObj.meta.jobId = ""; + // } + if (searchObj.meta.jobId != "") { + $q.notify({ + type: "negative", + message: "Job Already Scheduled , please change some parameters to schedule new job", + timeout: 3000, + }); + return; + } + if ( + searchObj.meta.jobRecords > 100000 || + searchObj.meta.jobRecords == 0 || + searchObj.meta.jobRecords < 0 + ) { + $q.notify({ + type: "negative", + message: "Job Scheduler should be between 1 and 100000", + timeout: 3000, + }); + return; + } + + searchSchedulerJob.value = false; + searchObj.meta.showSearchScheduler = false; + await getJobData(); + + } + catch (e){ + if(e.response.status != 403){ + $q.notify({ + type: "negative", + message: "Error while adding job", + timeout: 3000, + }); + return; + } + } + + }; + + const createScheduleJob = () => { + searchSchedulerJob.value = true; + searchObj.meta.jobRecords = 100; + } + + const checkQuery = (query) => { + const jobQuery = router.currentRoute.value.query.query; + if (jobQuery == b64EncodeUnicode(query)) { + return true; + } + return false; + } + const checkFnQuery = (fnQuery) => { + const jobFnQuery = router.currentRoute.value.query.functionContent; + if (jobFnQuery == b64EncodeUnicode(fnQuery)) { + return true; + } + return false; + } // [END] cancel running queries @@ -2909,11 +3178,18 @@ export default defineComponent({ fnParsedSQL, iconRight, functionToggleIcon, + searchSchedulerJob, + autoSearchSchedulerJob, + addJobScheduler, + routeToSearchSchedule, + createScheduleJob, searchTerm, filteredFunctionOptions, confirmUpdate, updateViewObj, updateSavedViews, + checkQuery, + checkFnQuery, }; }, computed: { diff --git a/web/src/plugins/logs/SearchHistory.vue b/web/src/plugins/logs/SearchHistory.vue index 641727bf74..e426e13db1 100644 --- a/web/src/plugins/logs/SearchHistory.vue +++ b/web/src/plugins/logs/SearchHistory.vue @@ -491,12 +491,6 @@ return data.sort((a, b) => a.rawExecutedTime - b.rawExecutedTime); } - - - - - - // return a.rawDuration - b.rawDuration; } const copyToClipboard = (text,type) => { navigator.clipboard.writeText(text).then(() => { diff --git a/web/src/plugins/logs/SearchResult.vue b/web/src/plugins/logs/SearchResult.vue index 28880b3b2f..418ebd3835 100644 --- a/web/src/plugins/logs/SearchResult.vue +++ b/web/src/plugins/logs/SearchResult.vue @@ -33,10 +33,29 @@ along with this program. If not, see . :htmlContent="searchObj.data.countErrorMsg" /> -
+
{{ noOfRecordsTitle }} + + + + Fetching the search events + + +
+
+ . :max=" Math.max( 1, - (searchObj.communicationMethod === 'ws' + (searchObj.communicationMethod === 'ws' || + searchObj.meta.jobId != '' ? searchObj.data.queryResults?.pagination?.length : searchObj.data.queryResults?.partitionDetail?.paginations ?.length) || 0, @@ -310,19 +330,33 @@ export default defineComponent({ } else if (actionType == "recordsPerPage") { this.searchObj.data.resultGrid.currentPage = 1; this.pageNumberInput = this.searchObj.data.resultGrid.currentPage; - this.searchObj.communicationMethod === "ws" - ? this.refreshPagination() - : this.refreshPartitionPagination(true); + if (this.searchObj.communicationMethod === "ws") { + if (this.searchObj.meta.jobId == "") { + this.refreshPagination(); + } else { + this.refreshJobPagination(); + } + } else { + if (this.searchObj.meta.jobId !== "") { + this.refreshJobPagination(); + } else { + this.refreshPartitionPagination(true); + } + } this.$emit("update:recordsPerPage"); this.scrollTableToTop(0); } else if (actionType == "pageChange") { + //here at first the queryResults is undefined so we are checking if it is undefined then we are setting it to empty array + if(this.searchObj.meta.jobId != "" && this.searchObj.data.queryResults.paginations == undefined){ + this.searchObj.data.queryResults.pagination = []; + } const maxPages = - this.searchObj.communicationMethod === "ws" + this.searchObj.communicationMethod === "ws" || + this.searchObj.meta.jobId != "" ? this.searchObj.data.queryResults.pagination.length - : this.searchObj.data.queryResults.partitionDetail.paginations + : this.searchObj.data.queryResults?.partitionDetail?.paginations .length; - - if (this.pageNumberInput > Math.ceil(maxPages)) { + if (this.pageNumberInput > Math.ceil(maxPages) && this.searchObj.meta.jobId == "") { this.$q.notify({ type: "negative", message: @@ -391,6 +425,7 @@ export default defineComponent({ reorderSelectedFields, getFilterExpressionByFieldType, refreshPagination, + refreshJobPagination, } = useLogs(); const pageNumberInput = ref(1); const totalHeight = ref(0); @@ -609,6 +644,7 @@ export default defineComponent({ reorderSelectedFields, getPaginations, refreshPagination, + refreshJobPagination, }; }, computed: { diff --git a/web/src/plugins/logs/SearchSchedulersList.vue b/web/src/plugins/logs/SearchSchedulersList.vue new file mode 100644 index 0000000000..f2b792d1db --- /dev/null +++ b/web/src/plugins/logs/SearchSchedulersList.vue @@ -0,0 +1,1151 @@ + + + diff --git a/web/src/services/search.ts b/web/src/services/search.ts index 63c9193555..5801c56539 100644 --- a/web/src/services/search.ts +++ b/web/src/services/search.ts @@ -225,6 +225,135 @@ const search = { payload, // Send the payload as the request body ); }, + schedule_search: ( + { + org_identifier, + query, + page_type = "logs", + traceparent, + }: { + org_identifier: string; + query: any; + page_type: string; + traceparent?: string; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + const url = `/api/${org_identifier}/search_jobs?type=${page_type}&search_type=${search_type}&use_cache=${use_cache}`; + return http({ headers: { traceparent } }).post(url, query); + }, + cancel_scheduled_search: ( + { + org_identifier, + jobId, + traceparent, + }: { + org_identifier: string; + jobId: string; + traceparent?: string; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + const url = `/api/${org_identifier}/search_jobs/${jobId}/cancel`; + return http({ headers: { traceparent } }).post(url); + }, + retry_scheduled_search: ( + { + org_identifier, + jobId, + traceparent, + }: { + org_identifier: string; + jobId: string; + traceparent?: string; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + const url = `/api/${org_identifier}/search_jobs/${jobId}/retry`; + return http({ headers: { traceparent } }).post(url); + }, + delete_scheduled_search: ( + { + org_identifier, + jobId, + traceparent, + }: { + org_identifier: string; + jobId: string; + traceparent?: string; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + const url = `/api/${org_identifier}/search_jobs/${jobId}`; + return http({ headers: { traceparent } }).delete(url); + }, + get_scheduled_search_list: ( + { + org_identifier, + page_type = "logs", + traceparent, + }: { + org_identifier: string; + page_type: string; + traceparent?: string; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + const url = `/api/${org_identifier}/search_jobs?type=${page_type}&search_type=${search_type}&use_cache=${use_cache}`; + return http({ headers: { traceparent } }).get(url); + }, + get_scheduled_search_result: ( + { + org_identifier, + page_type = "logs", + jobId, + traceparent, + query, + }: { + org_identifier: string; + jobId: string; + page_type: string; + traceparent?: string; + query: any; + }, + search_type: string = "UI", + ) => { + if (!traceparent) traceparent = generateTraceContext()?.traceparent; + const { size, from } = query.query; + const use_cache: boolean = + (window as any).use_cache !== undefined + ? (window as any).use_cache + : true; + let url = `/api/${org_identifier}/search_jobs/${jobId}/result?type=${page_type}&search_type=${search_type}&use_cache=${use_cache}`; + url += `&size=${size}&from=${from}`; + + return http({ headers: { traceparent } }).get(url); + }, }; export default search;