feat: Dashboard top N results (#4258)

- #4302
<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

## Summary by CodeRabbit

- **New Features**
- Introduced new configuration options for dashboard panels, including
settings for "Top Results" and an "Others" series to enhance data
visualization.
- Added an input field in the dashboard configuration panel for users to
specify the number of top values displayed.

- **Improvements**
- Enhanced data processing capabilities for SQL data, allowing better
aggregation and sorting of results.
- Streamlined the dashboard configuration flow for an improved user
experience.
- Improved error handling and logging in data processing to enhance
debugging and visibility.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

src/common/meta/dashboards/v4/mod.rs

@@ -177,6 +177,10 @@ pub struct PanelConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     decimals: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    top_results: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    top_results_others: Option<bool>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     axis_width: Option<f64>,
     #[serde(skip_serializing_if = "Option::is_none")]
     axis_border_show: Option<bool>,

web/src/components/dashboards/addPanel/ConfigPanel.vue

@@ -15,7 +15,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 -->
 
 <template>
-  <div style="padding-bottom: 30px;">
+  <div style="padding-bottom: 30px">
     <div class="" style="max-width: 300px">
       <div class="q-mb-sm" style="font-weight: 600">
         {{ t("dashboard.description") }}
@@ -172,7 +172,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         :display-value="`${
           dashboardPanelData.data.config.unit
             ? unitOptions.find(
-                (it) => it.value == dashboardPanelData.data.config.unit
+                (it) => it.value == dashboardPanelData.data.config.unit,
               )?.label
             : 'Default'
         }`"
@@ -201,8 +201,10 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         min="0"
         max="100"
         @update:model-value="
-        (value: any) => (dashboardPanelData.data.config.decimals = ( typeof value == 'number' && value >= 0) ? value : 2)
-      "
+          (value: any) =>
+            (dashboardPanelData.data.config.decimals =
+              typeof value == 'number' && value >= 0 ? value : 2)
+        "
         :rules="[
           (val: any) =>
             (val >= 0 && val <= 100) || 'Decimals must be between 0 and 100',
@@ -328,7 +330,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
               handleBlur(
                 dashboardPanelData.data.config.map_symbol_style.size_by_value,
                 1,
-                'min'
+                'min',
               )
             "
             :label="t('dashboard.minimum')"
@@ -359,7 +361,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
               handleBlur(
                 dashboardPanelData.data.config.map_symbol_style.size_by_value,
                 100,
-                'max'
+                'max',
               )
             "
             :label="t('dashboard.maximum')"
@@ -389,7 +391,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
             handleBlur(
               dashboardPanelData.data.config.map_symbol_style,
               2,
-              'size_fixed'
+              'size_fixed',
             )
           "
           :label="t('dashboard.fixedValue')"
@@ -497,6 +499,118 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
         </template>
       </q-input>
 
+      <div class="space"></div>
+      <q-input
+        v-if="
+          [
+            'area',
+            'bar',
+            'line',
+            'h-bar',
+            'h-stacked',
+            'scatter',
+            'area-stacked',
+            'stacked',
+          ].includes(dashboardPanelData.data.type) && !promqlMode
+        "
+        v-model.number="dashboardPanelData.data.config.top_results"
+        :min="0"
+        @update:model-value="
+          (value: any) =>
+            (dashboardPanelData.data.config.top_results = value ? value : null)
+        "
+        label="Top Results"
+        color="input-border"
+        bg-color="input-bg"
+        class="q-py-sm showLabelOnTop"
+        stack-label
+        outlined
+        filled
+        dense
+        label-slot
+        placeholder="ALL"
+        :type="'number'"
+        :disable="
+          dashboardPanelData.data.queries[
+            dashboardPanelData.layout.currentQueryIndex
+          ].fields.breakdown.length == 0
+        "
+        data-test="dashboard-config-top_results"
+        ><template v-slot:label>
+          <div class="row items-center all-pointer-events">
+            Show top n values
+            <div>
+              <q-icon
+                class="q-ml-xs"
+                size="20px"
+                name="info"
+                data-test="dashboard-config-top_results-info"
+              />
+              <q-tooltip
+                class="bg-grey-8"
+                anchor="top middle"
+                self="bottom middle"
+                max-width="250px"
+              >
+                <b
+                  >This is only applicable when breakdown field is available</b
+                >
+                <br />
+                <br />
+                Specify the number of top N values to show when breakdown field
+                is available.
+              </q-tooltip>
+            </div>
+          </div>
+        </template></q-input
+      >
+
+      <div
+        class="row items-center"
+        v-if="
+          [
+            'area',
+            'bar',
+            'line',
+            'h-bar',
+            'h-stacked',
+            'scatter',
+            'area-stacked',
+            'stacked',
+          ].includes(dashboardPanelData.data.type) && !promqlMode
+        "
+      >
+        <q-toggle
+          v-model="dashboardPanelData.data.config.top_results_others"
+          label="Add 'others' series"
+          data-test="dashboard-config-top_results_others"
+          :disable="
+            dashboardPanelData.data.queries[
+              dashboardPanelData.layout.currentQueryIndex
+            ].fields.breakdown.length == 0
+          "
+        />
+
+        <q-icon
+          class="q-ml-xs"
+          size="20px"
+          name="info"
+          data-test="dashboard-config-top_results-others-info"
+        >
+          <q-tooltip
+            class="bg-grey-8"
+            anchor="top middle"
+            self="bottom middle"
+            max-width="250px"
+          >
+            Include an 'others' series for values outside the top results when
+            using breakdown fields.
+          </q-tooltip>
+        </q-icon>
+      </div>
+
+      <div class="space"></div>
+
       <CommonAutoComplete
         v-if="promqlMode"
         :label="t('common.legend')"
@@ -548,7 +662,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
       <q-toggle
         v-if="
           ['area', 'line', 'area-stacked'].includes(
-            dashboardPanelData.data.type
+            dashboardPanelData.data.type,
           )
         "
         v-model="dashboardPanelData.data.config.connect_nulls"
@@ -561,7 +675,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
       <q-input
         v-if="
           ['area', 'line', 'area-stacked'].includes(
-            dashboardPanelData.data.type
+            dashboardPanelData.data.type,
           )
         "
         v-model="dashboardPanelData.data.config.no_value_replacement"
@@ -637,7 +751,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
               dashboardPanelData.layout.currentQueryIndex
             ].config,
             1,
-            'weight_fixed'
+            'weight_fixed',
           )
         "
         :label="t('common.weight')"
@@ -768,7 +882,7 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
       <Drilldown
         v-if="
           !['html', 'markdown', 'geomap', 'maps'].includes(
-            dashboardPanelData.data.type
+            dashboardPanelData.data.type,
           )
         "
         :variablesData="variablesData"
@@ -834,6 +948,11 @@ export default defineComponent({
         dashboardPanelData.data.config.axis_border_show = false;
       }
 
+      // by default, use top_results_others as false
+      if (!dashboardPanelData.data.config.top_results_others) {
+        dashboardPanelData.data.config.top_results_others = false;
+      }
+
       // Ensure that the nested structure is initialized
       if (!dashboardPanelData.data.config.map_symbol_style) {
         dashboardPanelData.data.config.map_symbol_style = {
@@ -1049,8 +1168,8 @@ export default defineComponent({
             label: it.name,
             value: it.name,
           };
-        }
-      )
+        },
+      ),
     );
 
     return {

web/src/composables/useDashboardPanel.ts

@@ -47,6 +47,8 @@ const getDefaultDashboardPanelData: any = () => ({
       unit: null,
       unit_custom: null,
       decimals: 2,
+      top_results: null,
+      top_results_others: false,
       axis_width: null,
       axis_border_show: false,
       legend_width: {

web/src/utils/dashboard/convertSQLData.ts

@@ -83,6 +83,121 @@ export const convertSQLData = async (
       : [];
   };
 
+  // Step 1: Get the X-Axis key
+  const xAxisKeys = getXAxisKeys();
+
+  // Step 2: Get the Y-Axis key
+  const yAxisKeys = getYAxisKeys();
+
+  const zAxisKeys = getZAxisKeys();
+
+  const breakDownKeys = getBreakDownKeys();
+
+  /**
+   * Process the SQL data and convert it into a format suitable for the echarts
+   * library.
+   *
+   * This function takes in the raw data returned from the SQL query and the
+   * panel schema, and returns a processed data array that can be fed into the
+   * echarts library.
+   *
+   * Here's a step-by-step breakdown of what this function does:
+   *
+   * 1. It checks if the data is empty or if the first item in the array is not
+   * an object. If either condition is true, it returns an empty array.
+   *
+   * 2. It extracts the top_results and top_results_others values from the
+   * panel schema config.
+   *
+   * 3. If top_results is not enabled, it simply returns the inner data array
+   * without any modifications.
+   *
+   * 4. It extracts the breakdown key, y-axis key, and x-axis key from the
+   * panel schema.
+   *
+   * 5. It aggregates the y-axis values by breakdown, ignoring items without a
+   * breakdown key. This is done using the reduce() method.
+   *
+   * 6. It sorts the breakdown object by value in descending order and extracts
+   * the top keys based on the configured number of top results. This is done
+   * using the Object.entries() and sort() methods.
+   *
+   * 7. It initializes an empty result array and an empty others object.
+   *
+   * 8. It loops through the inner data array and checks if the breakdown value
+   * is in the top keys. If it is, it adds the item to the result array. If it's
+   * not, and top_results_others is enabled, it adds the item to the others
+   * object.
+   *
+   * 9. If top_results_others is enabled, it loops through the others object and
+   * adds the aggregated values to the result array.
+   *
+   * 10. Finally, it returns the result array.
+   *
+   * @param {any[]} data - The data returned from the SQL query.
+   * @param {any} panelSchema - The schema of the panel.
+   * @returns {any[]} - The processed data array.
+   */
+  const processData = (data: any[], panelSchema: any): any[] => {
+    if (!data.length || !Array.isArray(data[0])) {
+      return [];
+    }
+
+    const { top_results, top_results_others } = panelSchema.config;
+    const innerDataArray = data[0];
+    if (!top_results || !breakDownKeys.length) {
+      return innerDataArray;
+    }
+
+    const breakdownKey = breakDownKeys[0];
+    const yAxisKey = yAxisKeys[0];
+    const xAxisKey = xAxisKeys[0];
+
+    // Step 1: Aggregate y_axis values by breakdown, ignoring items without a breakdown key
+    const breakdown = innerDataArray.reduce((acc, item) => {
+      const breakdownValue = item[breakdownKey];
+      const yAxisValue = item[yAxisKey];
+      if (breakdownValue) {
+        acc[breakdownValue] = (acc[breakdownValue] || 0) + (+yAxisValue || 0);
+      }
+      return acc;
+    }, {});
+
+    // Step 2: Sort and extract the top keys based on the configured number of top results
+    const topKeys = Object.entries(breakdown)
+      .sort(([, a]: any, [, b]: any) => b - a)
+      .slice(0, top_results)
+      .map(([key]) => key);
+
+    // Step 3: Initialize result array and others object for aggregation
+    const resultArray: any[] = [];
+    const othersObj: any = {};
+
+    innerDataArray.forEach((item) => {
+      const breakdownValue = item[breakdownKey];
+      if (topKeys.includes(breakdownValue)) {
+        resultArray.push(item);
+      } else if (top_results_others) {
+        const xAxisValue = String(item[xAxisKey]);
+        othersObj[xAxisValue] =
+          (othersObj[xAxisValue] || 0) + (+item[yAxisKey] || 0);
+      }
+    });
+
+    // Step 4: Add 'others' aggregation to the result array if enabled
+    if (top_results_others) {
+      Object.entries(othersObj).forEach(([xAxisValue, yAxisValue]) => {
+        resultArray.push({
+          [breakdownKey]: "others",
+          [xAxisKey]: xAxisValue,
+          [yAxisKey]: yAxisValue,
+        });
+      });
+    }
+
+    return resultArray;
+  };
+
   const getMarkLineData = (panelSchema: any) => {
     return (
       panelSchema?.config?.mark_line?.map((markLine: any) => {
@@ -101,6 +216,9 @@ export const convertSQLData = async (
   };
 
   const noValueConfigOption = panelSchema.config?.no_value_replacement;
+
+  const processedData = processData(searchQueryData, panelSchema);
+
   const missingValue = () => {
     // Get the interval in minutes
     const interval = resultMetaData.value.map(
@@ -112,7 +230,7 @@ export const convertSQLData = async (
       !metadata.queries ||
       !["area-stacked", "line", "area"].includes(panelSchema.type)
     ) {
-      return JSON.parse(JSON.stringify(searchQueryData[0]));
+      return JSON.parse(JSON.stringify(processedData));
     }
 
     // Extract and process metaDataStartTime
@@ -127,7 +245,7 @@ export const convertSQLData = async (
     const intervalMillis = interval * 1000;
 
     // Identify the time-based key
-    const searchQueryDataFirstEntry = searchQueryData[0][0];
+    const searchQueryDataFirstEntry = processedData[0];
 
     const keys = [
       ...getXAxisKeys(),
@@ -140,7 +258,7 @@ export const convertSQLData = async (
     );
 
     if (!timeBasedKey) {
-      return JSON.parse(JSON.stringify(searchQueryData[0]));
+      return JSON.parse(JSON.stringify(processedData));
     }
 
     // Extract and process metaDataEndTime
@@ -162,14 +280,14 @@ export const convertSQLData = async (
 
     // Create a set of unique xAxis values
     const uniqueXAxisValues = new Set(
-      searchQueryData[0].map((d: any) => d[uniqueKey]),
+      processedData.map((d: any) => d[uniqueKey]),
     );
 
     const filledData: any = [];
     let currentTime = binnedDate;
     // Create a map of existing data
     const searchDataMap = new Map();
-    searchQueryData[0]?.forEach((d: any) => {
+    processedData?.forEach((d: any) => {
       const key =
         xAxisKeysWithoutTimeStamp.length > 0 ||
         breakdownAxisKeysWithoutTimeStamp.length > 0
@@ -265,16 +383,6 @@ export const convertSQLData = async (
     return result;
   };
 
-  // Step 1: Get the X-Axis key
-  const xAxisKeys = getXAxisKeys();
-
-  // Step 2: Get the Y-Axis key
-  const yAxisKeys = getYAxisKeys();
-
-  const zAxisKeys = getZAxisKeys();
-
-  const breakDownKeys = getBreakDownKeys();
-
   const legendPosition = getLegendPosition(
     panelSchema.config?.legends_position,
   );
@@ -1015,12 +1123,12 @@ export const convertSQLData = async (
       const key1 = breakDownKeys[0];
       // get the unique value of the second xAxis's key
       const stackedXAxisUniqueValue = [
-        ...new Set(searchQueryData[0].map((obj: any) => obj[key1])),
+        ...new Set(processedData.map((obj: any) => obj[key1])),
       ].filter((it) => it);
 
       options.series = stackedXAxisUniqueValue?.map((key: any) => {
         // queryData who has the xaxis[1] key as well from xAxisUniqueValue.
-        const data = searchQueryData[0].filter((it: any) => it[key1] == key);
+        const data = processedData.filter((it: any) => it[key1] == key);
         const seriesObj = {
           name: key,
           ...defaultSeriesProps,
@@ -1224,12 +1332,12 @@ export const convertSQLData = async (
       const key1 = breakDownKeys[0];
       // get the unique value of the second xAxis's key
       const stackedXAxisUniqueValue = [
-        ...new Set(searchQueryData[0].map((obj: any) => obj[key1])),
+        ...new Set(processedData.map((obj: any) => obj[key1])),
       ].filter((it) => it);
 
       options.series = stackedXAxisUniqueValue?.map((key: any) => {
         // queryData who has the xaxis[1] key as well from xAxisUniqueValue.
-        const data = searchQueryData[0].filter((it: any) => it[key1] == key);
+        const data = processedData.filter((it: any) => it[key1] == key);
         const seriesObj = {
           name: key,
           ...defaultSeriesProps,