perf(monitor): improve monitor query and chart performance for large ranges (#13874)

* perf(monitor): improve monitor query and chart performance for large ranges

- enable LTTB sampling in the shared line chart to bound rendered points
- empty io/network in monitor search now means no name filter, returning
  all device rows in one query instead of fanning out per device
- merge device names recorded in the monitor db into io/network/gpu
  options so removed devices stay selectable; move logic to service layer
- add composite indexes (name, created_at) on monitor_ios/monitor_networks
  and (product_name, created_at) on monitor_gpus
- stop migrating the unused MonitorGPU model into MonitorDB
- fix the memory param which was compared against "mem" and never matched

* chore(monitor): remove monitor test fixtures

---------

Co-authored-by: ssongliu <sloooop1x@gmail.com>

agent/app/api/v2/monitor.go

@@ -1,13 +1,9 @@
 package v2
 
 import (
-	"sort"
-
 	"github.com/1Panel-dev/1Panel/agent/app/api/v2/helper"
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/gin-gonic/gin"
-	"github.com/shirou/gopsutil/v4/disk"
-	"github.com/shirou/gopsutil/v4/net"
 )
 
 // @Tags Monitor
@@ -96,14 +92,7 @@ func (b *BaseApi) UpdateMonitorSetting(c *gin.Context) {
 // @Security Timestamp
 // @Router /hosts/monitor/netoptions [get]
 func (b *BaseApi) GetNetworkOptions(c *gin.Context) {
-	netStat, _ := net.IOCounters(true)
-	var options []string
-	options = append(options, "all")
-	for _, net := range netStat {
-		options = append(options, net.Name)
-	}
-	sort.Strings(options)
-	helper.SuccessWithData(c, options)
+	helper.SuccessWithData(c, monitorService.LoadNetworkOptions())
 }
 
 // @Tags Monitor
@@ -113,12 +102,5 @@ func (b *BaseApi) GetNetworkOptions(c *gin.Context) {
 // @Security Timestamp
 // @Router /hosts/monitor/iooptions [get]
 func (b *BaseApi) GetIOOptions(c *gin.Context) {
-	diskStat, _ := disk.IOCounters()
-	var options []string
-	options = append(options, "all")
-	for _, net := range diskStat {
-		options = append(options, net.Name)
-	}
-	sort.Strings(options)
-	helper.SuccessWithData(c, options)
+	helper.SuccessWithData(c, monitorService.LoadIOOptions())
 }

agent/app/repo/monitor.go

@@ -29,6 +29,8 @@ type IMonitorRepo interface {
 	GetGPUDevices() ([]model.MonitorGPU, error)
 	GetIO(opts ...DBOption) ([]model.MonitorIO, error)
 	GetNetwork(opts ...DBOption) ([]model.MonitorNetwork, error)
+	GetIONames() ([]string, error)
+	GetNetworkNames() ([]string, error)
 
 	CreateMonitorBase(model model.MonitorBase) error
 	BatchCreateMonitorGPU(list []model.MonitorGPU) error
@@ -86,6 +88,18 @@ func (u *MonitorRepo) GetNetwork(opts ...DBOption) ([]model.MonitorNetwork, erro
 	err := db.Find(&data).Error
 	return data, err
 }
+func (u *MonitorRepo) GetIONames() ([]string, error) {
+	var names []string
+	err := global.MonitorDB.Model(&model.MonitorIO{}).Distinct().Pluck("name", &names).Error
+	return names, err
+}
+
+func (u *MonitorRepo) GetNetworkNames() ([]string, error) {
+	var names []string
+	err := global.MonitorDB.Model(&model.MonitorNetwork{}).Distinct().Pluck("name", &names).Error
+	return names, err
+}
+
 func (u *MonitorRepo) CreateMonitorBase(model model.MonitorBase) error {
 	return global.MonitorDB.Create(&model).Error
 }

agent/app/service/monitor.go

@@ -52,6 +52,8 @@ type IMonitorService interface {
 	UpdateSetting(key, value string) error
 	CleanData(monitorType string) error
 
+	LoadIOOptions() []string
+	LoadNetworkOptions() []string
 	LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error)
 	LoadVLLMCurrent(ctx context.Context, req dto.MonitorVLLMCurrent) (model.MonitorVLLM, error)
 	CleanVLLMMonitor(req dto.MonitorVLLMClean) error
@@ -92,7 +94,7 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa
 				base.TopCPUItems = processes
 				base.TopCPU = ""
 			}
-			if req.Param == "all" || req.Param == "mem" {
+			if req.Param == "all" || req.Param == "memory" {
 				var processes []dto.Process
 				_ = json.Unmarshal([]byte(base.TopMem), &processes)
 				base.TopMemItems = processes
@@ -103,7 +105,11 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa
 		data = append(data, itemData)
 	}
 	if req.Param == "all" || req.Param == "io" {
-		bases, err := monitorRepo.GetIO(repo.WithByName(req.IO), repo.WithByCreatedAt(req.StartTime, req.EndTime))
+		ioOpts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime)}
+		if len(req.IO) != 0 {
+			ioOpts = append(ioOpts, repo.WithByName(req.IO))
+		}
+		bases, err := monitorRepo.GetIO(ioOpts...)
 		if err != nil {
 			return nil, err
 		}
@@ -117,7 +123,11 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa
 		data = append(data, itemData)
 	}
 	if req.Param == "all" || req.Param == "network" {
-		bases, err := monitorRepo.GetNetwork(repo.WithByName(req.Network), repo.WithByCreatedAt(req.StartTime, req.EndTime))
+		netOpts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime)}
+		if len(req.Network) != 0 {
+			netOpts = append(netOpts, repo.WithByName(req.Network))
+		}
+		bases, err := monitorRepo.GetNetwork(netOpts...)
 		if err != nil {
 			return nil, err
 		}
@@ -133,6 +143,37 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa
 	return data, nil
 }
 
+func (m *MonitorService) LoadIOOptions() []string {
+	optionSet := make(map[string]struct{})
+	if diskStat, err := disk.IOCounters(); err == nil {
+		for _, item := range diskStat {
+			optionSet[item.Name] = struct{}{}
+		}
+	}
+	// union with names recorded in the monitor db so removed devices stay selectable
+	if names, err := monitorRepo.GetIONames(); err == nil {
+		for _, name := range names {
+			optionSet[name] = struct{}{}
+		}
+	}
+	return sortedMonitorOptions(optionSet)
+}
+
+func (m *MonitorService) LoadNetworkOptions() []string {
+	optionSet := make(map[string]struct{})
+	if netStat, err := net.IOCounters(true); err == nil {
+		for _, item := range netStat {
+			optionSet[item.Name] = struct{}{}
+		}
+	}
+	if names, err := monitorRepo.GetNetworkNames(); err == nil {
+		for _, name := range names {
+			optionSet[name] = struct{}{}
+		}
+	}
+	return sortedMonitorOptions(optionSet)
+}
+
 func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
 	var data dto.MonitorGPUOptions
 	exist, client := accelerator.New()
@@ -942,3 +983,14 @@ func loadHostSysPath() string {
 	}
 	return hostSys
 }
+
+func sortedMonitorOptions(optionSet map[string]struct{}) []string {
+	options := make([]string, 0, len(optionSet))
+	for name := range optionSet {
+		if len(name) != 0 && name != "all" {
+			options = append(options, name)
+		}
+	}
+	sort.Strings(options)
+	return append([]string{"all"}, options...)
+}

agent/init/hook/hook.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/1Panel-dev/1Panel/agent/app/dto"
 	"github.com/1Panel-dev/1Panel/agent/app/model"
@@ -13,6 +14,7 @@ import (
 	"github.com/1Panel-dev/1Panel/agent/global"
 	"github.com/1Panel-dev/1Panel/agent/utils/alert_push"
 	"github.com/1Panel-dev/1Panel/agent/utils/xpack"
+	"gorm.io/gorm"
 )
 
 func Init() {
@@ -174,6 +176,36 @@ func initAlertTask() {
 }
 
 func initMonitorDB() {
-	_ = global.MonitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorGPU{}, &model.MonitorIO{})
+	_ = global.MonitorDB.AutoMigrate(&model.MonitorBase{}, &model.MonitorNetwork{}, &model.MonitorIO{})
+	_ = global.GPUMonitorDB.AutoMigrate(&model.MonitorGPU{})
 	_ = global.TaskDB.AutoMigrate(&model.Task{})
+	// building indexes on large monitor tables can take seconds, keep it off the startup path;
+	// WAL mode leaves readers unblocked and busy_timeout covers the collector's inserts meanwhile
+	go ensureMonitorIndexes()
+}
+
+func ensureMonitorIndexes() {
+	indexes := []struct {
+		db   *gorm.DB
+		stmt string
+	}{
+		// created_at alone serves unfiltered range queries and retention cleanup
+		{global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_bases_created ON monitor_bases(created_at)"},
+		{global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_ios_created ON monitor_ios(created_at)"},
+		{global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_networks_created ON monitor_networks(created_at)"},
+		{global.GPUMonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_gpus_created ON monitor_gpus(created_at)"},
+		// (name, created_at) serves per-device range queries and distinct name lookups
+		{global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_ios_name_created ON monitor_ios(name, created_at)"},
+		{global.MonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_networks_name_created ON monitor_networks(name, created_at)"},
+		{global.GPUMonitorDB, "CREATE INDEX IF NOT EXISTS idx_monitor_gpus_product_created ON monitor_gpus(product_name, created_at)"},
+	}
+	start := time.Now()
+	for _, index := range indexes {
+		if err := index.db.Exec(index.stmt).Error; err != nil {
+			global.LOG.Warnf("create monitor index failed, stmt: %s, err: %v", index.stmt, err)
+		}
+	}
+	if elapsed := time.Since(start); elapsed > time.Second {
+		global.LOG.Infof("monitor indexes ready, took %s", elapsed)
+	}
 }

frontend/src/components/v-charts/components/Line.vue

@@ -139,6 +139,7 @@ function initChart() {
                 data: item?.data,
                 showSymbol: item.showSymbol ?? false,
                 symbolSize: item.symbolSize,
+                sampling: 'lttb',
                 yAxisIndex: item.yAxisIndex ? 1 : null,
             });
         });
@@ -227,6 +228,7 @@ function initChart() {
     };
     const tooltip = props.option.tooltip || defaultTooltip;
     const option = {
+        animation: (props.option.xData?.length ?? 0) <= 3000,
         title: [
             {
                 left: props.option.titleLeft ?? 'center',

frontend/src/views/host/monitor/monitor/index.vue

@@ -327,11 +327,7 @@ const searchGlobal = () => {
     timeRangeMemory.value = timeRangeGlobal.value;
     timeRangeIO.value = timeRangeGlobal.value;
     timeRangeNetwork.value = timeRangeGlobal.value;
-    search('load');
-    search('cpu');
-    search('memory');
-    search('io');
-    search('network');
+    search('all');
 };
 
 const search = async (param: string) => {
@@ -355,7 +351,7 @@ const search = async (param: string) => {
             searchInfo.network = networkChoose.value || 'all';
             break;
         case 'all':
-            searchTime.value = timeRangeNetwork.value;
+            searchTime.value = timeRangeGlobal.value;
             searchInfo.io = ioChoose.value || 'all';
             searchInfo.network = networkChoose.value || 'all';
             break;