From d27d6db1eac02503a31dafc11083d8a6b105a696 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Mon, 28 Sep 2026 18:32:12 +0800 Subject: [PATCH] feat: enhance native GPU NPU and XPU monitoring (#13939) --- agent/app/dto/monitor.go | 55 +- agent/app/model/monitor.go | 49 +- agent/app/repo/monitor.go | 79 ++ agent/app/service/dashboard.go | 9 +- agent/app/service/monitor.go | 320 ++++--- agent/init/migration/migrate.go | 1 + agent/init/migration/migrations/init.go | 7 + .../utils/ai_tools/accelerator/accelerator.go | 27 +- agent/utils/ai_tools/accelerator/metric.go | 5 +- agent/utils/ai_tools/accelerator/normalize.go | 142 +-- agent/utils/ai_tools/accelerator/provider.go | 18 +- agent/utils/ai_tools/accelerator/types.go | 88 +- agent/utils/ai_tools/gpu/amd.go | 75 +- agent/utils/ai_tools/gpu/client.go | 4 + agent/utils/ai_tools/gpu/nvidia.go | 145 +-- agent/utils/ai_tools/gpu/nvidia_types.go | 43 +- agent/utils/ai_tools/gpu/types.go | 37 + agent/utils/ai_tools/npu/npu.go | 62 +- agent/utils/ai_tools/npu/types.go | 8 + agent/utils/ai_tools/xpu/types.go | 30 +- agent/utils/ai_tools/xpu/xpu.go | 66 +- frontend/src/api/interface/ai.ts | 101 ++- .../components/v-charts/components/Line.vue | 11 +- frontend/src/lang/modules/en.ts | 55 +- frontend/src/lang/modules/es-es.ts | 55 +- frontend/src/lang/modules/fa.ts | 54 +- frontend/src/lang/modules/ja.ts | 53 +- frontend/src/lang/modules/ko.ts | 53 +- frontend/src/lang/modules/lo.ts | 53 +- frontend/src/lang/modules/ms.ts | 55 +- frontend/src/lang/modules/pt-br.ts | 55 +- frontend/src/lang/modules/ru.ts | 55 +- frontend/src/lang/modules/tr.ts | 54 +- frontend/src/lang/modules/zh-Hant.ts | 53 +- frontend/src/lang/modules/zh.ts | 53 +- .../gpu/current/components/DeviceDetails.vue | 90 ++ frontend/src/views/ai/gpu/current/index.vue | 854 +++++++++--------- frontend/src/views/ai/gpu/history/index.vue | 783 ++++++---------- 38 files changed, 2492 insertions(+), 1265 deletions(-) create mode 100644 frontend/src/views/ai/gpu/current/components/DeviceDetails.vue diff --git a/agent/app/dto/monitor.go b/agent/app/dto/monitor.go index 1056faf29..195d3e5b2 100644 --- a/agent/app/dto/monitor.go +++ b/agent/app/dto/monitor.go @@ -44,6 +44,8 @@ type MonitorGPUOptions struct { Options []string `json:"options"` } type GPUChartHide struct { + DeviceID string `json:"deviceID"` + Legacy bool `json:"legacy"` ProductName string `json:"productName"` Type string `json:"type"` Process bool `json:"process"` @@ -55,23 +57,54 @@ type GPUChartHide struct { Speed bool `json:"speed"` } type MonitorGPUSearch struct { + Aggregation string `json:"aggregation" validate:"omitempty,oneof=avg max"` + DeviceID string `json:"deviceID"` + Legacy bool `json:"legacy"` ProductName string `json:"productName"` StartTime time.Time `json:"startTime"` EndTime time.Time `json:"endTime"` } type MonitorGPUData struct { - Date []time.Time `json:"date"` - GPUValue []float64 `json:"gpuValue"` - TemperatureValue []float64 `json:"temperatureValue"` - PowerTotal []float64 `json:"powerTotal"` - PowerUsed []float64 `json:"powerUsed"` - PowerPercent []float64 `json:"powerPercent"` - MemoryTotal []float64 `json:"memoryTotal"` - MemoryUsed []float64 `json:"memoryUsed"` - MemoryPercent []float64 `json:"memoryPercent"` - SpeedValue []int `json:"speedValue"` + MemoryActivity []*float64 `json:"memoryActivity"` + EncoderUtil []*float64 `json:"encoderUtil"` + DecoderUtil []*float64 `json:"decoderUtil"` + JPEGUtil []*float64 `json:"jpegUtil"` + OFAUtil []*float64 `json:"ofaUtil"` + MediaUtil []*float64 `json:"mediaUtil"` + ComputeUtil []*float64 `json:"computeUtil"` + CopyUtil []*float64 `json:"copyUtil"` + HotspotTemperature []*float64 `json:"hotspotTemperature"` + FanRPM []*float64 `json:"fanRPM"` + AICPUUtil []*float64 `json:"aiCPUUtil"` + CtrlCPUUtil []*float64 `json:"ctrlCPUUtil"` + DDRUsed []*float64 `json:"ddrUsed"` + DDRTotal []*float64 `json:"ddrTotal"` + HBMUsed []*float64 `json:"hbmUsed"` + HBMTotal []*float64 `json:"hbmTotal"` + DDRBandwidth []*float64 `json:"ddrBandwidth"` + HBMBandwidth []*float64 `json:"hbmBandwidth"` + MemoryBandwidth []*float64 `json:"memoryBandwidth"` + MediaFrequency []*float64 `json:"mediaFrequency"` + HugepagesUsed []*float64 `json:"hugepagesUsed"` + HugepagesTotal []*float64 `json:"hugepagesTotal"` - ProcessCount []int `json:"processCount"` + BucketSeconds int64 `json:"bucketSeconds"` + SampleCount int64 `json:"sampleCount"` + MemoryTemperatureValue []*float64 `json:"memoryTemperatureValue"` + FrequencyValue []*float64 `json:"frequencyValue"` + MemoryFrequencyValue []*float64 `json:"memoryFrequencyValue"` + Date []time.Time `json:"date"` + GPUValue []*float64 `json:"gpuValue"` + TemperatureValue []*float64 `json:"temperatureValue"` + PowerTotal []*float64 `json:"powerTotal"` + PowerUsed []*float64 `json:"powerUsed"` + PowerPercent []*float64 `json:"powerPercent"` + MemoryTotal []*float64 `json:"memoryTotal"` + MemoryUsed []*float64 `json:"memoryUsed"` + MemoryPercent []*float64 `json:"memoryPercent"` + SpeedValue []*float64 `json:"speedValue"` + + ProcessCount []*float64 `json:"processCount"` GPUProcesses [][]GPUProcess `json:"gpuProcesses"` } diff --git a/agent/app/model/monitor.go b/agent/app/model/monitor.go index 6ec42ab9c..6344b787e 100644 --- a/agent/app/model/monitor.go +++ b/agent/app/model/monitor.go @@ -33,14 +33,45 @@ type MonitorNetwork struct { } type MonitorGPU struct { + MemoryUtil *float64 `json:"memoryUtil"` + MemoryActivity *float64 `json:"memoryActivity"` + EncoderUtil *float64 `json:"encoderUtil"` + DecoderUtil *float64 `json:"decoderUtil"` + JPEGUtil *float64 `json:"jpegUtil"` + OFAUtil *float64 `json:"ofaUtil"` + MediaUtil *float64 `json:"mediaUtil"` + ComputeUtil *float64 `json:"computeUtil"` + CopyUtil *float64 `json:"copyUtil"` + HotspotTemperature *float64 `json:"hotspotTemperature"` + FanRPM *float64 `json:"fanRPM"` + AICPUUtil *float64 `json:"aiCPUUtil"` + CtrlCPUUtil *float64 `json:"ctrlCPUUtil"` + DDRUsed *float64 `json:"ddrUsed"` + DDRTotal *float64 `json:"ddrTotal"` + HBMUsed *float64 `json:"hbmUsed"` + HBMTotal *float64 `json:"hbmTotal"` + DDRBandwidth *float64 `json:"ddrBandwidth"` + HBMBandwidth *float64 `json:"hbmBandwidth"` + MemoryBandwidth *float64 `json:"memoryBandwidth"` + MediaFrequency *float64 `json:"mediaFrequency"` + HugepagesUsed *float64 `json:"hugepagesUsed"` + HugepagesTotal *float64 `json:"hugepagesTotal"` + + MemoryTemperature *float64 `json:"memoryTemperature"` + DeviceID string `json:"deviceID"` + DeviceType string `json:"deviceType"` + ProcessStatus string `json:"processStatus"` + Frequency *float64 `json:"frequency"` + MemoryFrequency *float64 `json:"memoryFrequency"` + IntervalSeconds int `json:"intervalSeconds"` BaseModel - ProductName string `json:"productName"` - GPUUtil float64 `json:"gpuUtil"` - Temperature float64 `json:"temperature"` - PowerDraw float64 `json:"powerDraw"` - MaxPowerLimit float64 `json:"maxPowerLimit"` - MemUsed float64 `json:"memUsed"` - MemTotal float64 `json:"memTotal"` - FanSpeed int `json:"fanSpeed"` - Processes string `json:"processes"` + ProductName string `json:"productName"` + GPUUtil *float64 `json:"gpuUtil"` + Temperature *float64 `json:"temperature"` + PowerDraw *float64 `json:"powerDraw"` + MaxPowerLimit *float64 `json:"maxPowerLimit"` + MemUsed *float64 `json:"memUsed"` + MemTotal *float64 `json:"memTotal"` + FanSpeed *float64 `json:"fanSpeed"` + Processes string `json:"processes"` } diff --git a/agent/app/repo/monitor.go b/agent/app/repo/monitor.go index 381425228..a5b2c5f4c 100644 --- a/agent/app/repo/monitor.go +++ b/agent/app/repo/monitor.go @@ -1,6 +1,8 @@ package repo import ( + "fmt" + "strings" "time" "github.com/1Panel-dev/1Panel/agent/app/model" @@ -10,9 +12,20 @@ import ( type MonitorRepo struct{} +type GPUHistoryPoint struct { + model.MonitorGPU + Bucket int64 + PowerPercent *float64 + MemoryPercent *float64 + ProcessCount *float64 +} + type IMonitorRepo interface { GetBase(opts ...DBOption) ([]model.MonitorBase, error) GetGPU(opts ...DBOption) ([]model.MonitorGPU, error) + CountGPU(opts ...DBOption) (int64, error) + GetGPUHistory(start time.Time, bucketSeconds int64, aggregation string, opts ...DBOption) ([]GPUHistoryPoint, error) + GetGPUDevices() ([]model.MonitorGPU, error) GetIO(opts ...DBOption) ([]model.MonitorIO, error) GetNetwork(opts ...DBOption) ([]model.MonitorNetwork, error) @@ -26,6 +39,7 @@ type IMonitorRepo interface { DelMonitorNet(timeForDelete time.Time) error WithByProductName(name string) DBOption + WithByGPUDevice(deviceID, name string, legacy bool) DBOption } func NewIMonitorRepo() IMonitorRepo { @@ -102,3 +116,68 @@ func (s *MonitorRepo) WithByProductName(name string) DBOption { return g.Where("product_name = ?", name) } } + +func (u *MonitorRepo) GetGPUDevices() ([]model.MonitorGPU, error) { + var data []model.MonitorGPU + err := global.GPUMonitorDB.Model(&model.MonitorGPU{}).Select("device_id, product_name, device_type").Group("device_id, product_name, device_type").Order("product_name, device_id").Find(&data).Error + return data, err +} + +func (u *MonitorRepo) WithByGPUDevice(deviceID, name string, legacy bool) DBOption { + return func(db *gorm.DB) *gorm.DB { + if deviceID != "" { + return db.Where("device_id = ?", deviceID) + } + db = db.Where("product_name = ?", name) + if legacy { + db = db.Where("device_id IS NULL OR device_id = ''") + } + return db + } +} + +func (u *MonitorRepo) CountGPU(opts ...DBOption) (int64, error) { + db := global.GPUMonitorDB.Model(&model.MonitorGPU{}) + for _, opt := range opts { + db = opt(db) + } + var count int64 + err := db.Count(&count).Error + return count, err +} + +func (u *MonitorRepo) GetGPUHistory(start time.Time, bucketSeconds int64, aggregation string, opts ...DBOption) ([]GPUHistoryPoint, error) { + db := global.GPUMonitorDB.Model(&model.MonitorGPU{}) + for _, opt := range opts { + db = opt(db) + } + expressions := []string{ + "CASE WHEN max_power_limit > 0 THEN 100.0 * power_draw / max_power_limit END", + "CASE WHEN mem_total > 0 AND mem_used IS NOT NULL THEN 100.0 * mem_used / mem_total ELSE memory_util END", + "CASE WHEN (process_status = 'ok' OR process_status IS NULL OR process_status = '') AND json_valid(processes) THEN CASE WHEN json_type(processes) = 'array' THEN json_array_length(processes) END END", + } + aliases := []string{"power_percent", "memory_percent", "process_count"} + columns := []string{"*"} + if bucketSeconds > 0 { + operation := "AVG" + if aggregation == "max" { + operation = "MAX" + } + columns = []string{fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d AS bucket", start.Unix(), bucketSeconds)} + for _, column := range []string{"memory_activity", "encoder_util", "decoder_util", "jpeg_util", "ofa_util", "media_util", "compute_util", "copy_util", "hotspot_temperature", "fan_rpm", "ai_cpu_util", "ctrl_cpu_util", "ddr_used", "ddr_total", "hbm_used", "hbm_total", "ddr_bandwidth", "hbm_bandwidth", "memory_bandwidth", "media_frequency", "hugepages_used", "hugepages_total", "gpu_util", "temperature", "memory_temperature", "power_draw", "max_power_limit", "mem_used", "mem_total", "frequency", "memory_frequency", "fan_speed"} { + columns = append(columns, operation+"("+column+") AS "+column) + } + for i := range expressions { + expressions[i] = operation + "(" + expressions[i] + ")" + } + db = db.Group("bucket").Order("bucket ASC") + } else { + db = db.Order("created_at ASC, id ASC") + } + for i, expression := range expressions { + columns = append(columns, expression+" AS "+aliases[i]) + } + var data []GPUHistoryPoint + err := db.Select(strings.Join(columns, ", ")).Scan(&data).Error + return data, err +} diff --git a/agent/app/service/dashboard.go b/agent/app/service/dashboard.go index 4458d4064..19268b1b1 100644 --- a/agent/app/service/dashboard.go +++ b/agent/app/service/dashboard.go @@ -588,6 +588,9 @@ func loadAcceleratorInfo() ([]dto.GPUInfo, []dto.NPUInfo, []dto.XPUInfo) { xpuData []dto.XPUInfo ) for _, device := range snapshot.Devices { + if device.ParentID != "" { + continue + } switch device.Kind { case accelerator.KindGPU: if device.GPU == nil { @@ -597,7 +600,11 @@ func loadAcceleratorInfo() ([]dto.GPUInfo, []dto.NPUInfo, []dto.XPUInfo) { if err := copier.Copy(&dataItem, device.GPU); err != nil { continue } - dataItem.PowerUsage = dataItem.PowerDraw + " / " + dataItem.MaxPowerLimit + dataItem.MaxPowerLimit = device.GPU.PowerLimit + dataItem.PowerUsage = dataItem.PowerDraw + if dataItem.MaxPowerLimit != "" { + dataItem.PowerUsage += " / " + dataItem.MaxPowerLimit + } dataItem.MemoryUsage = dataItem.MemUsed + " / " + dataItem.MemTotal gpuData = append(gpuData, dataItem) case accelerator.KindNPU: diff --git a/agent/app/service/monitor.go b/agent/app/service/monitor.go index 1f4ffe66e..d0f5f9156 100644 --- a/agent/app/service/monitor.go +++ b/agent/app/service/monitor.go @@ -128,115 +128,135 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions { var data dto.MonitorGPUOptions - exist, client := accelerator.New() - if !exist { - return data - } - snapshot, err := client.Collect(context.Background()) - if err != nil { - global.LOG.Errorf("Load accelerator info failed, err: %v", err) - return data - } - if warning := snapshot.Warning(); warning != nil { - global.LOG.Warnf("Load accelerator info partially failed, err: %v", warning) - } - return loadGPUOptions(snapshot) -} - -func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions { - var data dto.MonitorGPUOptions - hasGPUOrNPU := false - hasXPU := false - for _, item := range snapshot.Devices { - if item.Kind == accelerator.KindXPU { - hasXPU = true + seen := make(map[string]bool) + if exist, client := accelerator.New(); exist { + snapshot, err := client.Collect(context.Background()) + if err != nil { + global.LOG.Warnf("Load accelerator options failed: %v", err) } else { - hasGPUOrNPU = true + data = loadGPUOptions(snapshot) + for _, item := range data.ChartHide { + seen[item.DeviceID] = true + } } } - switch { - case hasGPUOrNPU && hasXPU: - data.GPUType = "mixed" - case hasXPU: - data.GPUType = "xpu" - case hasGPUOrNPU: - data.GPUType = "gpu" + devices, err := monitorRepo.GetGPUDevices() + if err != nil { + global.LOG.Warnf("Load accelerator history options failed: %v", err) + return data } - - sort.Slice(snapshot.Devices, func(i, j int) bool { - if snapshot.Devices[i].Kind != snapshot.Devices[j].Kind { - return snapshot.Devices[i].Kind < snapshot.Devices[j].Kind + for _, device := range devices { + key := device.DeviceID + if key == "" { + key = "legacy:" + device.ProductName } - if snapshot.Devices[i].Vendor != snapshot.Devices[j].Vendor { - return snapshot.Devices[i].Vendor < snapshot.Devices[j].Vendor + if seen[key] { + continue } - if snapshot.Devices[i].NPUIndex != snapshot.Devices[j].NPUIndex { - return snapshot.Devices[i].NPUIndex < snapshot.Devices[j].NPUIndex - } - if snapshot.Devices[i].ChipIndex != snapshot.Devices[j].ChipIndex { - return snapshot.Devices[i].ChipIndex < snapshot.Devices[j].ChipIndex - } - return snapshot.Devices[i].Index < snapshot.Devices[j].Index - }) - for _, item := range snapshot.Devices { - optionType := "gpu" - if item.Kind == accelerator.KindXPU { - optionType = "xpu" - } - chartHide := dto.GPUChartHide{ - ProductName: item.Label, - Type: optionType, - GPU: !item.Capabilities.Utilization, - Memory: !item.Capabilities.Memory, - Power: !item.Capabilities.Power, - PowerLimit: !item.Capabilities.PowerLimit, - Temperature: !item.Capabilities.Temperature, - Speed: !item.Capabilities.FanSpeed, - } - data.ChartHide = append(data.ChartHide, chartHide) - data.Options = append(data.Options, chartHide.ProductName) + seen[key] = true + data.ChartHide = append(data.ChartHide, dto.GPUChartHide{DeviceID: device.DeviceID, ProductName: device.ProductName, Type: device.DeviceType, Legacy: device.DeviceID == ""}) + data.Options = append(data.Options, device.ProductName) } return data } func (m *MonitorService) LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.MonitorGPUData, error) { - loc, _ := time.LoadLocation(common.LoadTimeZoneByCmd()) - req.StartTime = req.StartTime.In(loc) - req.EndTime = req.EndTime.In(loc) var data dto.MonitorGPUData - gpuList, err := monitorRepo.GetGPU(repo.WithByCreatedAt(req.StartTime, req.EndTime), monitorRepo.WithByProductName(req.ProductName)) + if req.StartTime.IsZero() || req.EndTime.IsZero() || !req.EndTime.After(req.StartTime) { + return data, fmt.Errorf("invalid GPU history time range") + } + if req.DeviceID == "" && req.ProductName == "" { + return data, fmt.Errorf("GPU history requires a device") + } + if req.Aggregation != "" && req.Aggregation != "avg" && req.Aggregation != "max" { + return data, fmt.Errorf("invalid GPU history aggregation") + } + loc, err := time.LoadLocation(common.LoadTimeZoneByCmd()) if err != nil { return data, err } - - for _, gpu := range gpuList { - data.Date = append(data.Date, gpu.CreatedAt) - data.GPUValue = append(data.GPUValue, gpu.GPUUtil) - data.TemperatureValue = append(data.TemperatureValue, gpu.Temperature) - data.PowerUsed = append(data.PowerUsed, gpu.PowerDraw) - data.PowerTotal = append(data.PowerTotal, gpu.MaxPowerLimit) - if gpu.MaxPowerLimit != 0 { - data.PowerPercent = append(data.PowerPercent, gpu.PowerDraw/gpu.MaxPowerLimit*100) - } else { - data.PowerPercent = append(data.PowerPercent, float64(0)) + req.StartTime, req.EndTime = req.StartTime.In(loc), req.EndTime.In(loc) + opts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime), monitorRepo.WithByGPUDevice(req.DeviceID, req.ProductName, req.Legacy)} + data.SampleCount, err = monitorRepo.CountGPU(opts...) + if err != nil || data.SampleCount == 0 { + return data, err + } + if data.SampleCount > 1200 { + seconds := req.EndTime.Unix() - req.StartTime.Unix() + 1 + data.BucketSeconds = (seconds + 599) / 600 + } + points, err := monitorRepo.GetGPUHistory(req.StartTime, data.BucketSeconds, req.Aggregation, opts...) + if err != nil { + return data, err + } + samples := make([]repo.GPUHistoryPoint, 0, len(points)) + if data.BucketSeconds > 0 { + next := 0 + for bucket := int64(0); bucket <= (req.EndTime.Unix()-req.StartTime.Unix())/data.BucketSeconds; bucket++ { + point := repo.GPUHistoryPoint{} + if next < len(points) && points[next].Bucket == bucket { + point = points[next] + next++ + } + point.CreatedAt = time.Unix(req.StartTime.Unix()+bucket*data.BucketSeconds, 0).In(loc) + if bucket == 0 { + point.CreatedAt = req.StartTime + } + samples = append(samples, point) } - - data.MemoryTotal = append(data.MemoryTotal, gpu.MemTotal) - data.MemoryUsed = append(data.MemoryUsed, gpu.MemUsed) - if gpu.MemTotal != 0 { - data.MemoryPercent = append(data.MemoryPercent, gpu.MemUsed/gpu.MemTotal*100) - } else { - data.MemoryPercent = append(data.MemoryPercent, float64(0)) + } else { + for i, point := range points { + if i > 0 && points[i-1].IntervalSeconds > 0 { + interval := time.Duration(points[i-1].IntervalSeconds) * time.Second + if point.CreatedAt.Sub(points[i-1].CreatedAt) > 2*interval { + samples = append(samples, repo.GPUHistoryPoint{MonitorGPU: model.MonitorGPU{BaseModel: model.BaseModel{CreatedAt: points[i-1].CreatedAt.Add(interval)}}}) + } + } + samples = append(samples, point) } - var process []dto.GPUProcess - if err := json.Unmarshal([]byte(gpu.Processes), &process); err == nil { - data.ProcessCount = append(data.ProcessCount, len(process)) - data.GPUProcesses = append(data.GPUProcesses, process) - } else { - data.ProcessCount = append(data.ProcessCount, 0) - data.GPUProcesses = append(data.GPUProcesses, []dto.GPUProcess{}) + } + for _, point := range samples { + data.Date = append(data.Date, point.CreatedAt) + data.MemoryActivity = append(data.MemoryActivity, point.MemoryActivity) + data.EncoderUtil = append(data.EncoderUtil, point.EncoderUtil) + data.DecoderUtil = append(data.DecoderUtil, point.DecoderUtil) + data.JPEGUtil = append(data.JPEGUtil, point.JPEGUtil) + data.OFAUtil = append(data.OFAUtil, point.OFAUtil) + data.MediaUtil = append(data.MediaUtil, point.MediaUtil) + data.ComputeUtil = append(data.ComputeUtil, point.ComputeUtil) + data.CopyUtil = append(data.CopyUtil, point.CopyUtil) + data.HotspotTemperature = append(data.HotspotTemperature, point.HotspotTemperature) + data.FanRPM = append(data.FanRPM, point.FanRPM) + data.AICPUUtil = append(data.AICPUUtil, point.AICPUUtil) + data.CtrlCPUUtil = append(data.CtrlCPUUtil, point.CtrlCPUUtil) + data.DDRUsed = append(data.DDRUsed, point.DDRUsed) + data.DDRTotal = append(data.DDRTotal, point.DDRTotal) + data.HBMUsed = append(data.HBMUsed, point.HBMUsed) + data.HBMTotal = append(data.HBMTotal, point.HBMTotal) + data.DDRBandwidth = append(data.DDRBandwidth, point.DDRBandwidth) + data.HBMBandwidth = append(data.HBMBandwidth, point.HBMBandwidth) + data.MemoryBandwidth = append(data.MemoryBandwidth, point.MemoryBandwidth) + data.MediaFrequency = append(data.MediaFrequency, point.MediaFrequency) + data.HugepagesUsed = append(data.HugepagesUsed, point.HugepagesUsed) + data.HugepagesTotal = append(data.HugepagesTotal, point.HugepagesTotal) + data.GPUValue = append(data.GPUValue, point.GPUUtil) + data.TemperatureValue = append(data.TemperatureValue, point.Temperature) + data.MemoryTemperatureValue = append(data.MemoryTemperatureValue, point.MemoryTemperature) + data.PowerUsed = append(data.PowerUsed, point.PowerDraw) + data.PowerTotal = append(data.PowerTotal, point.MaxPowerLimit) + data.PowerPercent = append(data.PowerPercent, point.PowerPercent) + data.MemoryPercent = append(data.MemoryPercent, point.MemoryPercent) + data.MemoryTotal = append(data.MemoryTotal, point.MemTotal) + data.MemoryUsed = append(data.MemoryUsed, point.MemUsed) + data.SpeedValue = append(data.SpeedValue, point.FanSpeed) + data.FrequencyValue = append(data.FrequencyValue, point.Frequency) + data.MemoryFrequencyValue = append(data.MemoryFrequencyValue, point.MemoryFrequency) + data.ProcessCount = append(data.ProcessCount, point.ProcessCount) + var processes []dto.GPUProcess + if data.BucketSeconds == 0 && point.ProcessCount != nil { + _ = json.Unmarshal([]byte(point.Processes), &processes) } - data.SpeedValue = append(data.SpeedValue, gpu.FanSpeed) + data.GPUProcesses = append(data.GPUProcesses, processes) } return data, nil } @@ -307,7 +327,6 @@ func (m *MonitorService) CleanData() error { } func (m *MonitorService) Run() { - saveAcceleratorDataToDB() var itemModel model.MonitorBase totalPercent, _ := cpu.Percent(3*time.Second, false) if len(totalPercent) == 1 { @@ -343,6 +362,7 @@ func (m *MonitorService) Run() { m.loadDiskIO() m.loadNetIO() + m.saveGPUData() MonitorStoreDays, err := settingRepo.Get(settingRepo.WithByKey("MonitorStoreDays")) if err != nil { @@ -602,7 +622,62 @@ func StartMonitor(removeBefore bool, interval string) error { return nil } -func saveAcceleratorDataToDB() { +func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions { + var data dto.MonitorGPUOptions + hasGPUOrNPU := false + hasXPU := false + for _, item := range snapshot.Devices { + if item.Kind == accelerator.KindXPU { + hasXPU = true + } else { + hasGPUOrNPU = true + } + } + switch { + case hasGPUOrNPU && hasXPU: + data.GPUType = "mixed" + case hasXPU: + data.GPUType = "xpu" + case hasGPUOrNPU: + data.GPUType = "gpu" + } + + sort.Slice(snapshot.Devices, func(i, j int) bool { + if snapshot.Devices[i].Kind != snapshot.Devices[j].Kind { + return snapshot.Devices[i].Kind < snapshot.Devices[j].Kind + } + if snapshot.Devices[i].Vendor != snapshot.Devices[j].Vendor { + return snapshot.Devices[i].Vendor < snapshot.Devices[j].Vendor + } + if snapshot.Devices[i].NPUIndex != snapshot.Devices[j].NPUIndex { + return snapshot.Devices[i].NPUIndex < snapshot.Devices[j].NPUIndex + } + if snapshot.Devices[i].ChipIndex != snapshot.Devices[j].ChipIndex { + return snapshot.Devices[i].ChipIndex < snapshot.Devices[j].ChipIndex + } + return snapshot.Devices[i].Index < snapshot.Devices[j].Index + }) + for _, item := range snapshot.Devices { + chartHide := dto.GPUChartHide{ + DeviceID: item.ID, + ProductName: item.Label, + Type: string(item.Kind), + } + data.ChartHide = append(data.ChartHide, chartHide) + data.Options = append(data.Options, chartHide.ProductName) + } + return data +} + +func (m *MonitorService) saveGPUData() { + status, err := settingRepo.GetValueByKey("MonitorStatus") + if err != nil { + global.LOG.Errorf("load monitor status failed: %v", err) + return + } + if status != constant.StatusEnable { + return + } exist, client := accelerator.New() if !exist { return @@ -615,27 +690,64 @@ func saveAcceleratorDataToDB() { if warning := snapshot.Warning(); warning != nil { global.LOG.Warnf("load accelerator monitor data partially failed, err: %v", warning) } + intervalSeconds := 0 + if setting, err := settingRepo.Get(settingRepo.WithByKey("MonitorInterval")); err == nil { + intervalSeconds, _ = strconv.Atoi(setting.Value) + } list := make([]model.MonitorGPU, 0, len(snapshot.Devices)) for _, device := range snapshot.Devices { - list = append(list, newMonitorGPU(device)) + item := newMonitorGPU(device) + item.CreatedAt = snapshot.Info.CollectedAt + item.IntervalSeconds = intervalSeconds + list = append(list, item) } - if err := repo.NewIMonitorRepo().BatchCreateMonitorGPU(list); err != nil { + if err := monitorRepo.BatchCreateMonitorGPU(list); err != nil { global.LOG.Errorf("batch create accelerator monitor data failed, err: %v", err) } } func newMonitorGPU(device accelerator.Device) model.MonitorGPU { item := model.MonitorGPU{ - ProductName: device.Label, - GPUUtil: device.Metrics.Utilization.ValueOrZero(), - Temperature: device.Metrics.Temperature.ValueOrZero(), - PowerDraw: device.Metrics.Power.ValueOrZero(), - MaxPowerLimit: device.Metrics.PowerLimit.ValueOrZero(), - MemUsed: device.Metrics.MemoryUsed.ValueOrZero(), - MemTotal: device.Metrics.MemoryTotal.ValueOrZero(), - FanSpeed: int(device.Metrics.FanSpeed.ValueOrZero()), + MemoryUtil: device.Metrics.MemoryUtil.Value, + MemoryActivity: device.Metrics.MemoryActivity.Value, + EncoderUtil: device.Metrics.EncoderUtil.Value, + DecoderUtil: device.Metrics.DecoderUtil.Value, + JPEGUtil: device.Metrics.JPEGUtil.Value, + OFAUtil: device.Metrics.OFAUtil.Value, + MediaUtil: device.Metrics.MediaUtil.Value, + ComputeUtil: device.Metrics.ComputeUtil.Value, + CopyUtil: device.Metrics.CopyUtil.Value, + HotspotTemperature: device.Metrics.HotspotTemperature.Value, + FanRPM: device.Metrics.FanRPM.Value, + AICPUUtil: device.Metrics.AICPUUtil.Value, + CtrlCPUUtil: device.Metrics.CtrlCPUUtil.Value, + DDRUsed: device.Metrics.DDRUsed.Value, + DDRTotal: device.Metrics.DDRTotal.Value, + HBMUsed: device.Metrics.HBMUsed.Value, + HBMTotal: device.Metrics.HBMTotal.Value, + DDRBandwidth: device.Metrics.DDRBandwidth.Value, + HBMBandwidth: device.Metrics.HBMBandwidth.Value, + MemoryBandwidth: device.Metrics.MemoryBandwidth.Value, + MediaFrequency: device.Metrics.MediaFrequency.Value, + HugepagesUsed: device.Metrics.HugepagesUsed.Value, + HugepagesTotal: device.Metrics.HugepagesTotal.Value, + + ProductName: device.Label, + DeviceID: device.ID, + DeviceType: string(device.Kind), + ProcessStatus: device.ProcessStatus, + Frequency: device.Metrics.Frequency.Value, + MemoryFrequency: device.Metrics.MemoryFrequency.Value, + MemoryTemperature: device.Metrics.MemoryTemperature.Value, + GPUUtil: device.Metrics.Utilization.Value, + Temperature: device.Metrics.Temperature.Value, + PowerDraw: device.Metrics.Power.Value, + MaxPowerLimit: device.Metrics.PowerLimit.Value, + MemUsed: device.Metrics.MemoryUsed.Value, + MemTotal: device.Metrics.MemoryTotal.Value, + FanSpeed: device.Metrics.FanSpeed.Value, } - if len(device.Processes) == 0 { + if device.ProcessStatus != "ok" { return item } processes := make([]dto.GPUProcess, 0, len(device.Processes)) diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index 2aa625bec..a74552362 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -111,6 +111,7 @@ func agentDBMigrations() []*gormigrate.Migration { migrations.SimplifyFirewallRulePolicy, migrations.AddDockerPortGuardReadOnly, migrations.MigrateFirewallPortWhitelistSources, + migrations.AddAcceleratorMetrics, } } diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 7cc0a91ed..930a3931b 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -1908,3 +1908,10 @@ var AddDockerPortGuardReadOnly = &gormigrate.Migration{ return tx.Migrator().CreateIndex(&model.DockerPortGuardPolicy{}, "idx_docker_port_guard_endpoint") }, } + +var AddAcceleratorMetrics = &gormigrate.Migration{ + ID: "20260928-accelerator-vendor-metrics", + Migrate: func(tx *gorm.DB) error { + return global.GPUMonitorDB.AutoMigrate(&model.MonitorGPU{}) + }, +} diff --git a/agent/utils/ai_tools/accelerator/accelerator.go b/agent/utils/ai_tools/accelerator/accelerator.go index 67f929c51..89f7ae24d 100644 --- a/agent/utils/ai_tools/accelerator/accelerator.go +++ b/agent/utils/ai_tools/accelerator/accelerator.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "sync" + "time" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" @@ -52,6 +53,11 @@ func (c Client) LoadInfo() (*Info, error) { } func (c Client) Collect(ctx context.Context) (*Snapshot, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() results := make([]providerResult, len(c.providers)) var wg sync.WaitGroup for index, item := range c.providers { @@ -66,8 +72,11 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) { }() } wg.Wait() + if err := ctx.Err(); err != nil { + return nil, err + } - snapshot := &Snapshot{DriverVersions: make(map[string]string)} + snapshot := &Snapshot{} var ( errs []error active []*ProviderSnapshot @@ -78,6 +87,11 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) { errs = append(errs, result.err) continue } + if result.snapshot != nil { + for _, warning := range result.snapshot.Warnings { + errs = append(errs, errors.New(warning)) + } + } if result.snapshot == nil || len(result.snapshot.Devices) == 0 { continue } @@ -90,11 +104,8 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) { if item.CudaVersion != "" { snapshot.Info.CudaVersion = item.CudaVersion } - if item.DriverVersion != "" { - snapshot.DriverVersions[item.Type] = item.DriverVersion - if item.Type == "xpu" { - xpuVersion = item.DriverVersion - } + if item.Type == "xpu" { + xpuVersion = item.DriverVersion } } snapshot.Warnings = append(snapshot.Warnings, errs...) @@ -102,6 +113,10 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) { return nil, fmt.Errorf("calling accelerator monitoring tools failed: %w", errors.Join(errs...)) } + snapshot.Info.CollectedAt = time.Now() + for _, err := range errs { + snapshot.Info.Warnings = append(snapshot.Info.Warnings, err.Error()) + } snapshot.Info.XPUDriverVersion = xpuVersion snapshot.Info.Type, snapshot.Info.DriverVersion = mergeProviderMetadata(active) return snapshot, nil diff --git a/agent/utils/ai_tools/accelerator/metric.go b/agent/utils/ai_tools/accelerator/metric.go index 4234c0931..bf8ffb78d 100644 --- a/agent/utils/ai_tools/accelerator/metric.go +++ b/agent/utils/ai_tools/accelerator/metric.go @@ -1,6 +1,7 @@ package accelerator import ( + "math" "strconv" "strings" @@ -18,11 +19,11 @@ func metric(display, unit string) Metric { return result } parsed, err := strconv.ParseFloat(matched[1], 64) - if err != nil { + if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) || (parsed < 0 && unit != "°C") { return result } normalized, ok := convertMetricUnit(parsed, matched[2], unit) - if !ok { + if !ok || math.IsNaN(normalized) || math.IsInf(normalized, 0) || (unit == "%" && normalized > 100) { return result } result.Value = &normalized diff --git a/agent/utils/ai_tools/accelerator/normalize.go b/agent/utils/ai_tools/accelerator/normalize.go index e1ca60542..e59b2af36 100644 --- a/agent/utils/ai_tools/accelerator/normalize.go +++ b/agent/utils/ai_tools/accelerator/normalize.go @@ -12,26 +12,38 @@ import ( func normalizeGPU(item *gpu.Device) Device { metrics := Metrics{ - Utilization: metric(item.GPUUtil, "%"), - Temperature: metric(item.Temperature, "°C"), - Power: metric(item.PowerDraw, "W"), - PowerLimit: metric(item.MaxPowerLimit, "W"), - MemoryUsed: memoryMetric(item.MemUsed), - MemoryTotal: memoryMetric(item.MemTotal), - FanSpeed: metric(item.FanSpeed, "%"), + MemoryActivity: metric(item.MemoryActivity, "%"), + EncoderUtil: metric(item.EncoderUtil, "%"), + DecoderUtil: metric(item.DecoderUtil, "%"), + JPEGUtil: metric(item.JPEGUtil, "%"), + OFAUtil: metric(item.OFAUtil, "%"), + MediaUtil: metric(item.MediaUtil, "%"), + HotspotTemperature: metric(item.HotspotTemperature, "°C"), + FanRPM: metric(item.FanRPM, "RPM"), + MediaFrequency: metric(item.MediaFrequency, "MHz"), + Utilization: metric(item.GPUUtil, "%"), + Temperature: metric(item.Temperature, "°C"), + MemoryTemperature: metric(item.MemoryTemperature, "°C"), + Power: metric(item.PowerDraw, "W"), + PowerLimit: metric(item.PowerLimit, "W"), + Frequency: metric(item.Frequency, "MHz"), + MemoryFrequency: metric(item.MemoryFrequency, "MHz"), + MemoryUsed: memoryMetric(item.MemUsed), + MemoryTotal: memoryMetric(item.MemTotal), + FanSpeed: metric(item.FanSpeed, "%"), } device := Device{ - ID: stableID(item.Type, item.BusID, strconv.FormatUint(uint64(item.Index), 10)), - Kind: KindGPU, - Vendor: item.Type, - Index: int(item.Index), - Name: item.ProductName, - Label: fmt.Sprintf("%d - %s", item.Index, item.ProductName), - BusID: item.BusID, - Metrics: metrics, - GPU: item, + ProcessStatus: item.ProcessStatus, + ID: stableID(item.Type, item.UUID, stableID("pci", item.BusID, strconv.FormatUint(uint64(item.Index), 10))), + Kind: KindGPU, + Vendor: item.Type, + Index: int(item.Index), + Name: item.ProductName, + Label: fmt.Sprintf("%d - %s", item.Index, item.ProductName), + BusID: item.BusID, + Metrics: metrics, + GPU: item, } - device.Capabilities = capabilities(metrics) for _, process := range item.Processes { device.Processes = append(device.Processes, Process{ PID: process.PID, @@ -45,26 +57,36 @@ func normalizeGPU(item *gpu.Device) Device { func normalizeNPU(item *npu.Device) Device { metrics := Metrics{ - Utilization: metric(item.AICore, "%"), - Temperature: metric(item.Temperature, "°C"), - Power: metric(item.PowerDraw, "W"), - MemoryUsed: memoryMetric(item.MemUsed), - MemoryTotal: memoryMetric(item.MemTotal), + AICPUUtil: metric(item.AICPUUtil, "%"), + CtrlCPUUtil: metric(item.CtrlCPUUtil, "%"), + DDRBandwidth: metric(item.DDRBandwidth, "%"), + HBMBandwidth: metric(item.HBMBandwidth, "%"), + DDRUsed: metric(item.MemoryUsed, "MiB"), + DDRTotal: metric(item.MemoryTotal, "MiB"), + HBMUsed: metric(item.HBMUsed, "MiB"), + HBMTotal: metric(item.HBMTotal, "MiB"), + HugepagesUsed: metric(item.HugepagesUsed, "pages"), + HugepagesTotal: metric(item.HugepagesTotal, "pages"), + Utilization: metric(item.AICore, "%"), + Temperature: metric(item.Temperature, "°C"), + Power: metric(item.PowerDraw, "W"), + MemoryUsed: memoryMetric(item.MemUsed), + MemoryTotal: memoryMetric(item.MemTotal), } device := Device{ - ID: fmt.Sprintf("ascend:%d:%d", item.NPUIndex, item.ChipIndex), - Kind: KindNPU, - Vendor: "ascend", - Index: int(item.Index), - NPUIndex: int(item.NPUIndex), - ChipIndex: int(item.ChipIndex), - Name: item.ProductName, - Label: fmt.Sprintf("NPU %d / Chip %d - %s", item.NPUIndex, item.ChipIndex, item.ProductName), - BusID: item.BusID, - Metrics: metrics, - NPU: item, + ProcessStatus: item.ProcessStatus, + ID: fmt.Sprintf("%s:%d", stableID("ascend", item.BusID, strconv.FormatUint(uint64(item.NPUIndex), 10)), item.ChipIndex), + Kind: KindNPU, + Vendor: "ascend", + Index: int(item.Index), + NPUIndex: int(item.NPUIndex), + ChipIndex: int(item.ChipIndex), + Name: item.ProductName, + Label: fmt.Sprintf("NPU %d / Chip %d - %s", item.NPUIndex, item.ChipIndex, item.ProductName), + BusID: item.BusID, + Metrics: metrics, + NPU: item, } - device.Capabilities = capabilities(metrics) for _, process := range item.Processes { device.Processes = append(device.Processes, Process{ PID: process.PID, @@ -78,26 +100,32 @@ func normalizeNPU(item *npu.Device) Device { func normalizeXPU(item *xpu.Device) Device { metrics := Metrics{ - Utilization: metric(item.Stats.GPUUtil, "%"), - Temperature: metric(item.Stats.Temperature, "°C"), - Power: metric(item.Stats.Power, "W"), - MemoryUsed: memoryMetric(item.Stats.MemoryUsed), - MemoryTotal: memoryMetric(item.Basic.Memory), - MemoryUtil: metric(item.Stats.MemoryUtil, "%"), - Frequency: metric(item.Stats.Frequency, "MHz"), + MediaUtil: metric(item.Stats.MediaUtil, "%"), + ComputeUtil: metric(item.Stats.ComputeUtil, "%"), + CopyUtil: metric(item.Stats.CopyUtil, "%"), + MediaFrequency: metric(item.Stats.MediaFrequency, "MHz"), + MemoryTemperature: metric(item.Stats.MemoryTemperature, "°C"), + MemoryBandwidth: metric(item.Stats.MemoryBandwidthUtil, "%"), + Utilization: metric(item.Stats.GPUUtil, "%"), + Temperature: metric(item.Stats.Temperature, "°C"), + Power: metric(item.Stats.Power, "W"), + MemoryUsed: memoryMetric(item.Stats.MemoryUsed), + MemoryTotal: memoryMetric(item.Basic.Memory), + MemoryUtil: metric(item.Stats.MemoryUtil, "%"), + Frequency: metric(item.Stats.Frequency, "MHz"), } device := Device{ - ID: stableID("xpu", item.Basic.PciBdfAddress, strconv.Itoa(item.Basic.DeviceID)), - Kind: KindXPU, - Vendor: item.Basic.VendorName, - Index: item.Basic.DeviceID, - Name: item.Basic.DeviceName, - Label: fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName), - BusID: item.Basic.PciBdfAddress, - Metrics: metrics, - XPU: item, + ProcessStatus: item.ProcessStatus, + ID: stableID("xpu", item.Basic.UUID, stableID("pci", item.Basic.PciBdfAddress, strconv.Itoa(item.Basic.DeviceID))), + Kind: KindXPU, + Vendor: item.Basic.VendorName, + Index: item.Basic.DeviceID, + Name: item.Basic.DeviceName, + Label: fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName), + BusID: item.Basic.PciBdfAddress, + Metrics: metrics, + XPU: item, } - device.Capabilities = capabilities(metrics) for _, process := range item.Processes { device.Processes = append(device.Processes, Process{ PID: strconv.Itoa(process.PID), @@ -110,18 +138,6 @@ func normalizeXPU(item *xpu.Device) Device { return device } -func capabilities(metrics Metrics) Capabilities { - return Capabilities{ - Utilization: metrics.Utilization.Available(), - Temperature: metrics.Temperature.Available(), - Power: metrics.Power.Available(), - PowerLimit: metrics.PowerLimit.Available(), - Memory: metrics.MemoryUsed.Available() || metrics.MemoryTotal.Available(), - FanSpeed: metrics.FanSpeed.Available(), - Frequency: metrics.Frequency.Available(), - } -} - func stableID(vendor, busID, fallback string) string { if busID != "" && !strings.EqualFold(busID, "N/A") { return vendor + ":" + busID diff --git a/agent/utils/ai_tools/accelerator/provider.go b/agent/utils/ai_tools/accelerator/provider.go index f62610399..d965e9686 100644 --- a/agent/utils/ai_tools/accelerator/provider.go +++ b/agent/utils/ai_tools/accelerator/provider.go @@ -2,6 +2,7 @@ package accelerator import ( "context" + "fmt" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" @@ -26,6 +27,7 @@ func (p gpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { } result := &ProviderSnapshot{ Type: info.Type, + Warnings: info.Warnings, DriverVersion: info.DriverVersion, CudaVersion: info.CudaVersion, GPUs: info.Devices, @@ -49,6 +51,7 @@ func (p npuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { } result := &ProviderSnapshot{ Type: info.Type, + Warnings: info.Warnings, DriverVersion: info.DriverVersion, NPUs: info.Devices, } @@ -71,11 +74,24 @@ func (p xpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) { } result := &ProviderSnapshot{ Type: info.Type, + Warnings: info.Warnings, DriverVersion: info.DriverVersion, XPUs: info.Devices, } for index := range result.XPUs { - result.Devices = append(result.Devices, normalizeXPU(&result.XPUs[index])) + + parent := normalizeXPU(&result.XPUs[index]) + result.Devices = append(result.Devices, parent) + for _, tile := range result.XPUs[index].Tiles { + raw := xpu.Device{Basic: result.XPUs[index].Basic, Stats: tile.Stats, ProcessStatus: "unavailable"} + raw.Basic.Memory = "" + raw.Basic.FreeMemory = "" + device := normalizeXPU(&raw) + device.ParentID = parent.ID + device.ID = fmt.Sprintf("%s:tile:%d", parent.ID, tile.TileID) + device.Label = fmt.Sprintf("%s / Tile %d", parent.Label, tile.TileID) + result.Devices = append(result.Devices, device) + } } return result, nil } diff --git a/agent/utils/ai_tools/accelerator/types.go b/agent/utils/ai_tools/accelerator/types.go index fa35a849d..a0bf24499 100644 --- a/agent/utils/ai_tools/accelerator/types.go +++ b/agent/utils/ai_tools/accelerator/types.go @@ -2,6 +2,7 @@ package accelerator import ( "errors" + "time" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu" @@ -17,6 +18,9 @@ const ( ) type Info struct { + CollectedAt time.Time `json:"collectedAt"` + Warnings []string `json:"warnings"` + Type string `json:"type"` CudaVersion string `json:"cudaVersion"` DriverVersion string `json:"driverVersion"` @@ -44,25 +48,40 @@ func (m Metric) ValueOrZero() float64 { } type Metrics struct { - Utilization Metric - Temperature Metric - Power Metric - PowerLimit Metric - MemoryUsed Metric - MemoryTotal Metric - MemoryUtil Metric - FanSpeed Metric - Frequency Metric -} + MemoryActivity Metric + EncoderUtil Metric + DecoderUtil Metric + JPEGUtil Metric + OFAUtil Metric + MediaUtil Metric + ComputeUtil Metric + CopyUtil Metric + HotspotTemperature Metric + FanRPM Metric + AICPUUtil Metric + CtrlCPUUtil Metric + DDRUsed Metric + DDRTotal Metric + HBMUsed Metric + HBMTotal Metric + DDRBandwidth Metric + HBMBandwidth Metric + MemoryBandwidth Metric + MediaFrequency Metric + HugepagesUsed Metric + HugepagesTotal Metric -type Capabilities struct { - Utilization bool - Temperature bool - Power bool - PowerLimit bool - Memory bool - FanSpeed bool - Frequency bool + MemoryTemperature Metric + Utilization Metric + Temperature Metric + Power Metric + PowerLimit Metric + MemoryUsed Metric + MemoryTotal Metric + MemoryUtil Metric + FanSpeed Metric + Frequency Metric + MemoryFrequency Metric } type Process struct { @@ -74,18 +93,19 @@ type Process struct { } type Device struct { - ID string - Kind Kind - Vendor string - Index int - NPUIndex int - ChipIndex int - Name string - Label string - BusID string - Metrics Metrics - Capabilities Capabilities - Processes []Process + ParentID string + ProcessStatus string + ID string + Kind Kind + Vendor string + Index int + NPUIndex int + ChipIndex int + Name string + Label string + BusID string + Metrics Metrics + Processes []Process GPU *gpu.Device `json:"-"` NPU *npu.Device `json:"-"` @@ -93,10 +113,9 @@ type Device struct { } type Snapshot struct { - Info Info - Devices []Device - DriverVersions map[string]string - Warnings []error + Info Info + Devices []Device + Warnings []error } func (s Snapshot) Warning() error { @@ -104,6 +123,7 @@ func (s Snapshot) Warning() error { } type ProviderSnapshot struct { + Warnings []string Type string DriverVersion string CudaVersion string diff --git a/agent/utils/ai_tools/gpu/amd.go b/agent/utils/ai_tools/gpu/amd.go index 36b4e537c..095b4d3f1 100644 --- a/agent/utils/ai_tools/gpu/amd.go +++ b/agent/utils/ai_tools/gpu/amd.go @@ -3,12 +3,13 @@ package gpu import ( "context" "fmt" + "math" "sort" + "strconv" "strings" "sync" "time" - "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" ) @@ -32,16 +33,22 @@ func findAMDSMI() (string, bool) { func (a amdSMI) LoadInfo(ctx context.Context) (*Info, error) { var ( - staticData string - metricData string - processData string - staticErr error - metricErr error - processErr error - wg sync.WaitGroup + staticData string + metricData string + extendedData string + extendedErr error + processData string + staticErr error + metricErr error + processErr error + wg sync.WaitGroup ) - wg.Add(3) + wg.Add(4) + go func() { + defer wg.Done() + extendedData, extendedErr = runAMDSMI(ctx, a.command, "metric", "--clock", "--ecc", "--json") + }() go func() { defer wg.Done() staticData, staticErr = runAMDSMI(ctx, a.command, "static", "--asic", "--bus", "--driver", "--limit", "--json") @@ -64,16 +71,25 @@ func (a amdSMI) LoadInfo(ctx context.Context) (*Info, error) { return nil, fmt.Errorf("parsing %s static output failed: %w", a.command, err) } + if metricErr == nil { + metricErr = applyAMDMetrics(info, metricData) + } if metricErr != nil { - global.LOG.Warnf("calling %s metric failed, metrics will be omitted: %v", a.command, metricErr) - } else if err := applyAMDMetrics(info, metricData); err != nil { - global.LOG.Warnf("parsing %s metric output failed, metrics will be omitted: %v", a.command, err) + info.Warnings = append(info.Warnings, fmt.Sprintf("%s metrics: %v", a.command, metricErr)) + } + if extendedErr == nil { + extendedErr = applyAMDMetrics(info, extendedData) + } + if extendedErr != nil { + info.Warnings = append(info.Warnings, fmt.Sprintf("%s extended metrics: %v", a.command, extendedErr)) + } + if processErr == nil { + processErr = applyAMDProcesses(info, processData) } if processErr != nil { - global.LOG.Warnf("calling %s process failed, process information will be omitted: %v", a.command, processErr) - } else if err := applyAMDProcesses(info, processData); err != nil { - global.LOG.Warnf("parsing %s process output failed, process information will be omitted: %v", a.command, err) + info.Warnings = append(info.Warnings, fmt.Sprintf("%s processes: %v", a.command, processErr)) } + return info, nil } @@ -96,6 +112,9 @@ func parseAMDStatic(data string) (*Info, error) { } device := Device{ Type: "amd", + UUID: amdStringAt(row, "uuid", "asic.uuid"), + DriverVersion: amdStringAt(row, "driver.version", "driver_version", "amdgpu_version"), + ProcessStatus: "unavailable", Index: index, ProductName: amdStringAt(row, "asic.market_name", "market_name", "gpu_name"), PersistenceMode: "N/A", @@ -111,6 +130,7 @@ func parseAMDStatic(data string) (*Info, error) { "limit.max_power_limit", "limit.max_power", ), + PowerLimit: amdMetricAt(row, "W", "limit.ppt0.socket_power_limit", "limit.socket_power_limit"), MemUsed: "N/A", MemTotal: "N/A", GPUUtil: "N/A", @@ -150,11 +170,31 @@ func applyAMDMetrics(info *Info, data string) error { continue } setAMDMetric(&device.GPUUtil, row, "%", "usage.gfx_activity", "usage.gfx", "gfx_activity", "gfx_usage") - setAMDMetric(&device.Temperature, row, "°C", "temperature.hotspot", "temperature.edge", "hotspot_temperature", "gpu_temperature", "gpu_temp") + setAMDMetric(&device.Temperature, row, "°C", "temperature.edge", "gpu_temperature", "gpu_temp") setAMDMetric(&device.PowerDraw, row, "W", "power.socket_power", "socket_power", "power_usage") setAMDMetric(&device.MemUsed, row, "MB", "mem_usage.used_vram", "vram.used", "used_vram", "vram_used") setAMDMetric(&device.MemTotal, row, "MB", "mem_usage.total_vram", "vram.total", "total_vram", "vram_total") - setAMDMetric(&device.FanSpeed, row, "%", "fan.speed", "fan_speed") + + setAMDMetric(&device.FanSpeed, row, "%", "fan.usage") + if amdMetricAt(row, "%", "fan.usage") == "" { + speed, speedErr := strconv.ParseFloat(amdStringAt(row, "fan.speed"), 64) + maximum, maxErr := strconv.ParseFloat(amdStringAt(row, "fan.max"), 64) + if speedErr == nil && maxErr == nil && !math.IsNaN(speed) && !math.IsInf(speed, 0) && maximum > 0 && !math.IsInf(maximum, 0) && speed >= 0 && speed <= maximum { + device.FanSpeed = fmt.Sprintf("%.2f %%", speed/maximum*100) + } + } + setAMDMetric(&device.FanRPM, row, "RPM", "fan.rpm") + setAMDMetric(&device.HotspotTemperature, row, "°C", "temperature.hotspot", "hotspot_temperature") + setAMDMetric(&device.MemoryTemperature, row, "°C", "temperature.mem") + setAMDMetric(&device.MemoryActivity, row, "%", "usage.umc_activity") + setAMDMetric(&device.MediaUtil, row, "%", "usage.mm_activity") + setAMDMetric(&device.Frequency, row, "MHz", "clock.gfx_0.clk") + setAMDMetric(&device.MemoryFrequency, row, "MHz", "clock.mem_0.clk") + correctable := amdStringAt(row, "ecc.total_correctable_count", "ecc.correctable_count") + uncorrectable := amdStringAt(row, "ecc.total_uncorrectable_count", "ecc.uncorrectable_count") + if correctable != "" || uncorrectable != "" { + device.ECCErrors = []ECCError{{Scope: "Total", Correctable: correctable, Uncorrectable: uncorrectable}} + } if value := amdStringAt(row, "perf_level", "performance_level"); value != "" { device.PerformanceState = value } @@ -177,6 +217,7 @@ func applyAMDProcesses(info *Info, data string) error { if !ok { continue } + device.ProcessStatus = "ok" processList, _ := amdValueAt(row, "process_list") items := amdObjectList(processList) if len(items) == 0 { diff --git a/agent/utils/ai_tools/gpu/client.go b/agent/utils/ai_tools/gpu/client.go index dd1095740..641bc7f66 100644 --- a/agent/utils/ai_tools/gpu/client.go +++ b/agent/utils/ai_tools/gpu/client.go @@ -64,6 +64,7 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { if result.info == nil { continue } + merged.Warnings = append(merged.Warnings, result.info.Warnings...) if result.info.Type != "" { types = append(types, result.info.Type) } @@ -86,6 +87,9 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { if len(merged.Devices) == 0 && len(errs) > 0 { return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...)) } + for _, err := range errs { + merged.Warnings = append(merged.Warnings, err.Error()) + } return merged, nil } diff --git a/agent/utils/ai_tools/gpu/nvidia.go b/agent/utils/ai_tools/gpu/nvidia.go index a7193f577..62ba1ae2f 100644 --- a/agent/utils/ai_tools/gpu/nvidia.go +++ b/agent/utils/ai_tools/gpu/nvidia.go @@ -1,16 +1,12 @@ package gpu import ( - "bytes" "context" "encoding/xml" - "errors" "fmt" - "io" "strings" "time" - "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/cmd" ) @@ -24,41 +20,10 @@ func (n nvidiaSMI) LoadInfo(ctx context.Context) (*Info, error) { if err != nil { return nil, fmt.Errorf("calling %s failed: %w", nvidiaSMICommand, err) } - data := []byte(itemData) - version := "v11" - - buf := bytes.NewBuffer(data) - decoder := xml.NewDecoder(buf) - for { - token, err := decoder.Token() - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return nil, fmt.Errorf("reading token failed: %w", err) - } - d, ok := token.(xml.Directive) - if !ok { - continue - } - directive := string(d) - if !strings.HasPrefix(directive, "DOCTYPE") { - continue - } - parts := strings.Split(directive, " ") - s := strings.Trim(parts[len(parts)-1], "\" ") - if strings.HasPrefix(s, "nvsmi_device_") && strings.HasSuffix(s, ".dtd") { - version = strings.TrimSuffix(strings.TrimPrefix(s, "nvsmi_device_"), ".dtd") - } else { - global.LOG.Debugf("Cannot find schema version in %q", directive) - } - break - } - - return parseNvidiaSMI(data, version) + return parseNvidiaSMI([]byte(itemData)) } -func parseNvidiaSMI(buf []byte, version string) (*Info, error) { +func parseNvidiaSMI(buf []byte) (*Info, error) { var ( s nvidiaSMIResponse info Info @@ -72,42 +37,84 @@ func parseNvidiaSMI(buf []byte, version string) (*Info, error) { info.DriverVersion = s.DriverVersion for i := range s.Gpu { gpuItem := Device{ - Type: "nvidia", - Index: uint(i), - ProductName: s.Gpu[i].ProductName, - PersistenceMode: s.Gpu[i].PersistenceMode, - BusID: s.Gpu[i].ID, - DisplayActive: s.Gpu[i].DisplayActive, - ECC: s.Gpu[i].EccErrors.Volatile.DramUncorrectable, - FanSpeed: s.Gpu[i].FanSpeed, - Temperature: s.Gpu[i].Temperature.GpuTemp, - PerformanceState: s.Gpu[i].PerformanceState, - MemUsed: s.Gpu[i].FbMemoryUsage.Used, - MemTotal: s.Gpu[i].FbMemoryUsage.Total, - GPUUtil: s.Gpu[i].Utilization.GpuUtil, - ComputeMode: s.Gpu[i].ComputeMode, - MigMode: s.Gpu[i].MigMode.CurrentMig, - } - if version == "v12" || version == "v13" { - gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.PowerDraw - if gpuItem.PowerDraw == "" { - gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.InstantPowerDraw - } - gpuItem.MaxPowerLimit = s.Gpu[i].GpuPowerReadings.CurrentPowerLimit - } else { - gpuItem.PowerDraw = s.Gpu[i].PowerReadings.PowerDraw - gpuItem.MaxPowerLimit = s.Gpu[i].PowerReadings.MaxPowerLimit + Type: "nvidia", + MemoryActivity: s.Gpu[i].Utilization.MemoryUtil, + EncoderUtil: s.Gpu[i].Utilization.EncoderUtil, + DecoderUtil: s.Gpu[i].Utilization.DecoderUtil, + JPEGUtil: s.Gpu[i].Utilization.JpegUtil, + OFAUtil: s.Gpu[i].Utilization.OfaUtil, + MediaFrequency: s.Gpu[i].Clocks.VideoClock, + UUID: s.Gpu[i].UUID, + DriverVersion: s.DriverVersion, + Architecture: s.Gpu[i].ProductArchitecture, + Frequency: s.Gpu[i].Clocks.GraphicsClock, + MemoryFrequency: s.Gpu[i].Clocks.MemClock, + MemoryTemperature: s.Gpu[i].Temperature.MemoryTemp, + MemoryFree: s.Gpu[i].FbMemoryUsage.Free, + MemoryReserved: s.Gpu[i].FbMemoryUsage.Reserved, + PCIeGeneration: firstSMIValue(s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.DeviceCurrentLinkGen, s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.CurrentLinkGen), + PCIeMaxGeneration: firstSMIValue(s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.MaxDeviceLinkGen, s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.MaxLinkGen), + PCIeWidth: s.Gpu[i].Pci.PciGpuLinkInfo.LinkWidths.CurrentLinkWidth, + PCIeMaxWidth: s.Gpu[i].Pci.PciGpuLinkInfo.LinkWidths.MaxLinkWidth, + ProcessStatus: "unavailable", + Index: uint(i), + ProductName: s.Gpu[i].ProductName, + PersistenceMode: s.Gpu[i].PersistenceMode, + BusID: s.Gpu[i].ID, + DisplayActive: s.Gpu[i].DisplayActive, + ECC: s.Gpu[i].EccMode.CurrentEcc, + FanSpeed: s.Gpu[i].FanSpeed, + Temperature: s.Gpu[i].Temperature.GpuTemp, + PerformanceState: s.Gpu[i].PerformanceState, + MemUsed: s.Gpu[i].FbMemoryUsage.Used, + MemTotal: s.Gpu[i].FbMemoryUsage.Total, + GPUUtil: s.Gpu[i].Utilization.GpuUtil, + ComputeMode: s.Gpu[i].ComputeMode, + MigMode: s.Gpu[i].MigMode.CurrentMig, } + gpuItem.ECCPending = s.Gpu[i].EccMode.PendingEcc + gpuItem.ECCErrors = []ECCError{ + {Scope: "Volatile Total", Correctable: s.Gpu[i].EccErrors.Volatile.SingleBit.Total, Uncorrectable: s.Gpu[i].EccErrors.Volatile.DoubleBit.Total}, + {Scope: "Aggregate Total", Correctable: s.Gpu[i].EccErrors.Aggregate.SingleBit.Total, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.DoubleBit.Total}, - for _, process := range s.Gpu[i].Processes.ProcessInfo { - gpuItem.Processes = append(gpuItem.Processes, Process{ - PID: process.Pid, - Type: process.Type, - ProcessName: process.ProcessName, - UsedMemory: process.UsedMemory, - }) + {Scope: "Volatile DRAM", Correctable: s.Gpu[i].EccErrors.Volatile.DramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Volatile.DramUncorrectable}, + {Scope: "Volatile SRAM", Correctable: s.Gpu[i].EccErrors.Volatile.SramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Volatile.SramUncorrectable}, + {Scope: "Aggregate DRAM", Correctable: s.Gpu[i].EccErrors.Aggregate.DramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.DramUncorrectable}, + {Scope: "Aggregate SRAM", Correctable: s.Gpu[i].EccErrors.Aggregate.SramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.SramUncorrectable}, + } + gpuItem.PowerDraw = firstSMIValue(s.Gpu[i].GpuPowerReadings.PowerDraw, s.Gpu[i].GpuPowerReadings.InstantPowerDraw, s.Gpu[i].PowerReadings.PowerDraw) + gpuItem.PowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.CurrentPowerLimit, s.Gpu[i].PowerReadings.EnforcedPowerLimit, s.Gpu[i].PowerReadings.PowerLimit) + gpuItem.MaxPowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.MaxPowerLimit, s.Gpu[i].PowerReadings.MaxPowerLimit) + gpuItem.DefaultPowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.DefaultPowerLimit, s.Gpu[i].PowerReadings.DefaultPowerLimit) + for _, event := range append(s.Gpu[i].ClocksEventReasons.Reasons, s.Gpu[i].ClocksThrottleReasons.Reasons...) { + if strings.EqualFold(strings.TrimSpace(event.Value), "Active") { + gpuItem.ClockEvents = append(gpuItem.ClockEvents, strings.TrimPrefix(strings.TrimPrefix(event.XMLName.Local, "clocks_event_reason_"), "clocks_throttle_reason_")) + } + } + if s.Gpu[i].Processes != nil && strings.TrimSpace(s.Gpu[i].Processes.Text) == "" { + gpuItem.ProcessStatus = "ok" + } + if s.Gpu[i].Processes != nil { + for _, process := range s.Gpu[i].Processes.ProcessInfo { + gpuItem.Processes = append(gpuItem.Processes, Process{ + PID: process.Pid, + Type: process.Type, + ProcessName: process.ProcessName, + UsedMemory: process.UsedMemory, + }) + } } info.Devices = append(info.Devices, gpuItem) } return &info, nil } + +func firstSMIValue(values ...string) string { + for _, value := range values { + value = strings.TrimSpace(value) + if value != "" && !strings.EqualFold(value, "N/A") && !strings.EqualFold(value, "Not Supported") { + return value + } + } + return "" +} diff --git a/agent/utils/ai_tools/gpu/nvidia_types.go b/agent/utils/ai_tools/gpu/nvidia_types.go index 7c0ac568e..a4050c47e 100644 --- a/agent/utils/ai_tools/gpu/nvidia_types.go +++ b/agent/utils/ai_tools/gpu/nvidia_types.go @@ -1,9 +1,12 @@ package gpu +import "encoding/xml" + type nvidiaSMIResponse struct { - AttachedGpus string `xml:"attached_gpus"` - CudaVersion string `xml:"cuda_version"` - DriverVersion string `xml:"driver_version"` + XMLName xml.Name `xml:"nvidia_smi_log"` + AttachedGpus string `xml:"attached_gpus"` + CudaVersion string `xml:"cuda_version"` + DriverVersion string `xml:"driver_version"` Gpu []struct { ID string `xml:"id,attr"` AccountedProcesses struct{} `xml:"accounted_processes"` @@ -37,16 +40,17 @@ type nvidiaSMIResponse struct { VideoClock string `xml:"video_clock"` } `xml:"clocks"` ClocksEventReasons struct { - ClocksEventReasonApplicationsClocksSetting string `xml:"clocks_event_reason_applications_clocks_setting"` - ClocksEventReasonDisplayClocksSetting string `xml:"clocks_event_reason_display_clocks_setting"` - ClocksEventReasonGpuIdle string `xml:"clocks_event_reason_gpu_idle"` - ClocksEventReasonHwPowerBrakeSlowdown string `xml:"clocks_event_reason_hw_power_brake_slowdown"` - ClocksEventReasonHwSlowdown string `xml:"clocks_event_reason_hw_slowdown"` - ClocksEventReasonHwThermalSlowdown string `xml:"clocks_event_reason_hw_thermal_slowdown"` - ClocksEventReasonSwPowerCap string `xml:"clocks_event_reason_sw_power_cap"` - ClocksEventReasonSwThermalSlowdown string `xml:"clocks_event_reason_sw_thermal_slowdown"` - ClocksEventReasonSyncBoost string `xml:"clocks_event_reason_sync_boost"` + Reasons []struct { + XMLName xml.Name + Value string `xml:",chardata"` + } `xml:",any"` } `xml:"clocks_event_reasons"` + ClocksThrottleReasons struct { + Reasons []struct { + XMLName xml.Name + Value string `xml:",chardata"` + } `xml:",any"` + } `xml:"clocks_throttle_reasons"` ComputeMode string `xml:"compute_mode"` DefaultApplicationsClocks struct { GraphicsClock string `xml:"graphics_clock"` @@ -63,12 +67,24 @@ type nvidiaSMIResponse struct { } `xml:"driver_model"` EccErrors struct { Aggregate struct { + SingleBit struct { + Total string `xml:"total"` + } `xml:"single_bit"` + DoubleBit struct { + Total string `xml:"total"` + } `xml:"double_bit"` DramCorrectable string `xml:"dram_correctable"` DramUncorrectable string `xml:"dram_uncorrectable"` SramCorrectable string `xml:"sram_correctable"` SramUncorrectable string `xml:"sram_uncorrectable"` } `xml:"aggregate"` Volatile struct { + SingleBit struct { + Total string `xml:"total"` + } `xml:"single_bit"` + DoubleBit struct { + Total string `xml:"total"` + } `xml:"double_bit"` DramCorrectable string `xml:"dram_correctable"` DramUncorrectable string `xml:"dram_uncorrectable"` SramCorrectable string `xml:"sram_correctable"` @@ -226,7 +242,8 @@ type nvidiaSMIResponse struct { MinPowerLimit string `xml:"min_power_limit"` MaxPowerLimit string `xml:"max_power_limit"` } `xml:"power_readings"` - Processes struct { + Processes *struct { + Text string `xml:",chardata"` ProcessInfo []struct { Pid string `xml:"pid"` Type string `xml:"type"` diff --git a/agent/utils/ai_tools/gpu/types.go b/agent/utils/ai_tools/gpu/types.go index 495e17cb1..5e96a97f7 100644 --- a/agent/utils/ai_tools/gpu/types.go +++ b/agent/utils/ai_tools/gpu/types.go @@ -1,6 +1,8 @@ package gpu type Info struct { + Warnings []string `json:"warnings"` + CudaVersion string `json:"cudaVersion"` DriverVersion string `json:"driverVersion"` Type string `json:"type"` @@ -8,7 +10,42 @@ type Info struct { Devices []Device `json:"gpu"` } +type ECCError struct { + Scope string `json:"scope"` + Correctable string `json:"correctable"` + Uncorrectable string `json:"uncorrectable"` +} + type Device struct { + ECCPending string `json:"eccPending"` + ECCErrors []ECCError `json:"eccErrors"` + MemoryActivity string `json:"memoryActivity"` + EncoderUtil string `json:"encoderUtil"` + DecoderUtil string `json:"decoderUtil"` + JPEGUtil string `json:"jpegUtil"` + OFAUtil string `json:"ofaUtil"` + MediaUtil string `json:"mediaUtil"` + HotspotTemperature string `json:"hotspotTemperature"` + FanRPM string `json:"fanRPM"` + MediaFrequency string `json:"mediaFrequency"` + + UUID string `json:"uuid"` + DriverVersion string `json:"driverVersion"` + Architecture string `json:"architecture"` + Frequency string `json:"frequency"` + MemoryFrequency string `json:"memoryFrequency"` + MemoryTemperature string `json:"memoryTemperature"` + MemoryFree string `json:"memoryFree"` + MemoryReserved string `json:"memoryReserved"` + PowerLimit string `json:"powerLimit"` + DefaultPowerLimit string `json:"defaultPowerLimit"` + PCIeGeneration string `json:"pcieGeneration"` + PCIeMaxGeneration string `json:"pcieMaxGeneration"` + PCIeWidth string `json:"pcieWidth"` + PCIeMaxWidth string `json:"pcieMaxWidth"` + ProcessStatus string `json:"processStatus"` + ClockEvents []string `json:"clockEvents"` + Type string `json:"type"` Index uint `json:"index"` ProductName string `json:"productName"` diff --git a/agent/utils/ai_tools/npu/npu.go b/agent/utils/ai_tools/npu/npu.go index eb529e2c5..c5ee211a1 100644 --- a/agent/utils/ai_tools/npu/npu.go +++ b/agent/utils/ai_tools/npu/npu.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "sync" "time" "github.com/1Panel-dev/1Panel/agent/utils/cmd" @@ -34,7 +35,31 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { if err != nil { return nil, fmt.Errorf("calling %s failed: %w", ascendSMICommand, err) } - return parseAscendSMI(itemData), nil + + info := parseAscendSMI(itemData) + var wg sync.WaitGroup + warnings := make([]string, len(info.Devices)) + for i := range info.Devices { + wg.Add(1) + go func(index int) { + defer wg.Done() + device := &info.Devices[index] + mgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second)) + data, err := mgr.RunWithStdout(ascendSMICommand, "info", "-t", "usages", "-i", strconv.FormatUint(uint64(device.NPUIndex), 10), "-c", strconv.FormatUint(uint64(device.ChipIndex), 10)) + if err != nil { + warnings[index] = fmt.Sprintf("npu-smi usages %d/%d: %v", device.NPUIndex, device.ChipIndex, err) + return + } + applyAscendUsages(device, data) + }(i) + } + wg.Wait() + for _, warning := range warnings { + if warning != "" { + info.Warnings = append(info.Warnings, warning) + } + } + return info, nil } func parseAscendSMI(data string) *Info { @@ -158,6 +183,12 @@ func parseAscendSMI(data string) *Info { pending = nil } + for i := range info.Devices { + info.Devices[i].ProcessStatus = "unavailable" + if processSection { + info.Devices[i].ProcessStatus = "ok" + } + } return info } @@ -208,6 +239,9 @@ func ascendMemoryPools(value, header string) (string, string, string, string) { } hbm := usage[len(usage)-1] + if !strings.Contains(normalizedHeader, "MEMORYUSAGE") { + memoryUsed, memoryTotal = "", "" + } return memoryUsed, memoryTotal, hbm[0] + " MB", hbm[1] + " MB" } @@ -233,3 +267,29 @@ func ascendValueWithUnit(value, unit string) string { } return value + " " + unit } + +func applyAscendUsages(device *Device, data string) { + for _, line := range strings.Split(data, "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + key = strings.ToLower(strings.Join(strings.Fields(key), "")) + value = strings.TrimSpace(value) + switch key { + case "aicoreusagerate(%)": + usage, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimSuffix(value, "%")), 64) + if err == nil && usage >= 0 && usage <= 100 { + device.AICore = ascendValueWithUnit(value, "%") + } + case "aicpuusagerate(%)": + device.AICPUUtil = ascendValueWithUnit(value, "%") + case "ctrlcpuusagerate(%)": + device.CtrlCPUUtil = ascendValueWithUnit(value, "%") + case "ddrbandwidthusagerate(%)", "memorybandwidthusagerate(%)": + device.DDRBandwidth = ascendValueWithUnit(value, "%") + case "hbmbandwidthusagerate(%)": + device.HBMBandwidth = ascendValueWithUnit(value, "%") + } + } +} diff --git a/agent/utils/ai_tools/npu/types.go b/agent/utils/ai_tools/npu/types.go index 92de01645..6b1ebff6b 100644 --- a/agent/utils/ai_tools/npu/types.go +++ b/agent/utils/ai_tools/npu/types.go @@ -1,12 +1,20 @@ package npu type Info struct { + Warnings []string `json:"warnings"` Type string `json:"type"` DriverVersion string `json:"driverVersion"` Devices []Device `json:"npu"` } type Device struct { + AICPUUtil string `json:"aiCPUUtil"` + CtrlCPUUtil string `json:"ctrlCPUUtil"` + DDRBandwidth string `json:"ddrBandwidth"` + HBMBandwidth string `json:"hbmBandwidth"` + + ProcessStatus string `json:"processStatus"` + Type string `json:"type"` Index uint `json:"index"` NPUIndex uint `json:"npuIndex"` diff --git a/agent/utils/ai_tools/xpu/types.go b/agent/utils/ai_tools/xpu/types.go index 94e22c9e6..94f50369e 100644 --- a/agent/utils/ai_tools/xpu/types.go +++ b/agent/utils/ai_tools/xpu/types.go @@ -33,16 +33,29 @@ type discoveryInfo struct { } type DeviceLevelMetric struct { - MetricsType string `json:"metrics_type"` - Value float64 `json:"value"` + MetricsType string `json:"metrics_type"` + Value *float64 `json:"value"` +} + +type TileMetrics struct { + TileID int `json:"tile_id"` + DataList []DeviceLevelMetric `json:"data_list"` +} + +type TileStats struct { + TileID int `json:"tileID"` + Stats Stats `json:"stats"` } type DeviceStats struct { + TileLevel []TileMetrics `json:"tile_level"` DeviceID int `json:"device_id"` DeviceLevel []DeviceLevelMetric `json:"device_level"` } type Info struct { + Warnings []string `json:"warnings"` + Type string `json:"type"` DriverVersion string `json:"driverVersion"` @@ -50,12 +63,17 @@ type Info struct { } type Device struct { + Tiles []TileStats `json:"tiles"` + ProcessStatus string `json:"processStatus"` + Basic Basic `json:"basic"` Stats Stats `json:"stats"` Processes []Process `json:"processes"` } type Basic struct { + UUID string `json:"uuid"` + DeviceID int `json:"deviceID"` DeviceName string `json:"deviceName"` VendorName string `json:"vendorName"` @@ -66,6 +84,14 @@ type Basic struct { } type Stats struct { + MediaUtil string `json:"mediaUtil"` + ComputeUtil string `json:"computeUtil"` + CopyUtil string `json:"copyUtil"` + MediaFrequency string `json:"mediaFrequency"` + MemoryTemperature string `json:"memoryTemperature"` + + MemoryBandwidthUtil string `json:"memoryBandwidthUtil"` + Power string `json:"power"` GPUUtil string `json:"gpuUtil"` Frequency string `json:"frequency"` diff --git a/agent/utils/ai_tools/xpu/xpu.go b/agent/utils/ai_tools/xpu/xpu.go index 73a612c3a..b45c10d6e 100644 --- a/agent/utils/ai_tools/xpu/xpu.go +++ b/agent/utils/ai_tools/xpu/xpu.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "math" "sort" "strconv" "sync" @@ -51,12 +52,15 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) { processData, err := cmdMgr.RunWithStdout(xpuSMICommand, "ps", "-j") if err != nil { - global.LOG.Warnf("calling xpu-smi ps failed, process information will be omitted: %v", err) + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi ps: %v", err)) } else { var psList DeviceUtilByProcList if err := json.Unmarshal([]byte(processData), &psList); err != nil { - global.LOG.Warnf("processData json unmarshal failed, process information will be omitted: %v", err) + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi ps output: %v", err)) } else { + for i := range res.Devices { + res.Devices[i].ProcessStatus = "ok" + } for _, ps := range psList.DeviceUtilByProcList { process := Process{ PID: ps.ProcessID, @@ -87,7 +91,9 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg * defer wg.Done() xpu := Device{ + ProcessStatus: "unavailable", Basic: Basic{ + UUID: device.UUID, DeviceID: device.DeviceID, DeviceName: device.DeviceName, VendorName: device.VendorName, @@ -115,13 +121,19 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg * wgCmd.Wait() if xpuErr != nil { - global.LOG.Errorf("calling xpu-smi discovery failed for device %d, %v", device.DeviceID, xpuErr) + mu.Lock() + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi device %d: %v", device.DeviceID, xpuErr)) + res.Devices = append(res.Devices, xpu) + mu.Unlock() return } var info discoveryDevice if err := json.Unmarshal([]byte(xpuData), &info); err != nil { - global.LOG.Errorf("xpuData json unmarshal failed for device %d, err: %v", device.DeviceID, err) + mu.Lock() + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi device %d output: %v", device.DeviceID, err)) + res.Devices = append(res.Devices, xpu) + mu.Unlock() return } @@ -142,13 +154,22 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg * } if statsErr != nil { - global.LOG.Warnf("calling xpu-smi stats failed for device %d, metrics will be omitted: %v", device.DeviceID, statsErr) + mu.Lock() + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi stats %d: %v", device.DeviceID, statsErr)) + mu.Unlock() } else { var stats DeviceStats if err := json.Unmarshal([]byte(statsData), &stats); err != nil { - global.LOG.Warnf("statsData json unmarshal failed for device %d, metrics will be omitted: %v", device.DeviceID, err) + mu.Lock() + res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi stats %d output: %v", device.DeviceID, err)) + mu.Unlock() } else { loadStats(&xpu.Stats, stats.DeviceLevel) + for _, tile := range stats.TileLevel { + item := TileStats{TileID: tile.TileID} + loadStats(&item.Stats, tile.DataList) + xpu.Tiles = append(xpu.Tiles, item) + } } } @@ -162,19 +183,38 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg * func loadStats(stats *Stats, metrics []DeviceLevelMetric) { for _, stat := range metrics { + if stat.Value == nil || math.IsNaN(*stat.Value) || math.IsInf(*stat.Value, 0) { + continue + } + value := *stat.Value + if value < 0 && stat.MetricsType != "XPUM_STATS_GPU_CORE_TEMPERATURE" && stat.MetricsType != "XPUM_STATS_MEMORY_TEMPERATURE" { + continue + } switch stat.MetricsType { + case "XPUM_STATS_MEMORY_TEMPERATURE": + stats.MemoryTemperature = fmt.Sprintf("%.1f°C", value) + case "XPUM_STATS_ENGINE_GROUP_COMPUTE_ALL_UTILIZATION": + stats.ComputeUtil = fmt.Sprintf("%.1f%%", value) + case "XPUM_STATS_ENGINE_GROUP_MEDIA_ALL_UTILIZATION": + stats.MediaUtil = fmt.Sprintf("%.1f%%", value) + case "XPUM_STATS_ENGINE_GROUP_COPY_ALL_UTILIZATION": + stats.CopyUtil = fmt.Sprintf("%.1f%%", value) + case "XPUM_STATS_MEDIA_ENGINE_FREQUENCY": + stats.MediaFrequency = fmt.Sprintf("%.1fMHz", value) case "XPUM_STATS_POWER": - stats.Power = fmt.Sprintf("%.1fW", stat.Value) + stats.Power = fmt.Sprintf("%.1fW", value) case "XPUM_STATS_GPU_UTILIZATION": - stats.GPUUtil = fmt.Sprintf("%.1f%%", stat.Value) + stats.GPUUtil = fmt.Sprintf("%.1f%%", value) case "XPUM_STATS_GPU_FREQUENCY": - stats.Frequency = fmt.Sprintf("%.1fMHz", stat.Value) + stats.Frequency = fmt.Sprintf("%.1fMHz", value) case "XPUM_STATS_GPU_CORE_TEMPERATURE": - stats.Temperature = fmt.Sprintf("%.1f°C", stat.Value) + stats.Temperature = fmt.Sprintf("%.1f°C", value) case "XPUM_STATS_MEMORY_USED": - stats.MemoryUsed = fmt.Sprintf("%.1f MiB", stat.Value) - case "XPUM_STATS_MEMORY_UTILIZATION", "XPUM_STATS_MEMORY_BANDWIDTH", "XPUM_STATS_MEMORY_BANDWIDTH_UTILIZATION": - stats.MemoryUtil = fmt.Sprintf("%.1f%%", stat.Value) + stats.MemoryUsed = fmt.Sprintf("%.1f MiB", value) + case "XPUM_STATS_MEMORY_UTILIZATION": + stats.MemoryUtil = fmt.Sprintf("%.1f%%", value) + case "XPUM_STATS_MEMORY_BANDWIDTH", "XPUM_STATS_MEMORY_BANDWIDTH_UTILIZATION": + stats.MemoryBandwidthUtil = fmt.Sprintf("%.1f%%", value) } } } diff --git a/frontend/src/api/interface/ai.ts b/frontend/src/api/interface/ai.ts index 74d7449e6..63dd4b5ee 100644 --- a/frontend/src/api/interface/ai.ts +++ b/frontend/src/api/interface/ai.ts @@ -22,6 +22,8 @@ export namespace AI { } export interface Info { + collectedAt?: string; + warnings?: string[]; cudaVersion: string; driverVersion: string; type: string; @@ -31,6 +33,34 @@ export namespace AI { xpu: XpuInfo['xpu']; } export interface GPU { + eccPending?: string; + eccErrors?: { scope: string; correctable: string; uncorrectable: string }[]; + memoryActivity?: string; + encoderUtil?: string; + decoderUtil?: string; + jpegUtil?: string; + ofaUtil?: string; + mediaUtil?: string; + hotspotTemperature?: string; + fanRPM?: string; + mediaFrequency?: string; + + uuid?: string; + driverVersion?: string; + architecture?: string; + frequency?: string; + memoryFrequency?: string; + memoryTemperature?: string; + memoryFree?: string; + memoryReserved?: string; + powerLimit?: string; + defaultPowerLimit?: string; + pcieGeneration?: string; + pcieMaxGeneration?: string; + pcieWidth?: string; + pcieMaxWidth?: string; + clockEvents?: string[]; + processStatus?: string; type: 'nvidia' | 'amd'; index: number; productName: string; @@ -52,6 +82,12 @@ export namespace AI { processes: Process[]; } export interface NPU { + aiCPUUtil?: string; + ctrlCPUUtil?: string; + ddrBandwidth?: string; + hbmBandwidth?: string; + + processStatus?: string; type: 'ascend'; index: number; npuIndex: number; @@ -73,6 +109,9 @@ export namespace AI { processes: NPUProcess[]; } export interface MonitorGPUSearch { + aggregation?: 'avg' | 'max'; + deviceID?: string; + legacy?: boolean; productName: string; startTime: Date; endTime: Date; @@ -83,8 +122,10 @@ export namespace AI { chartHide: Array; } export interface ChartHide { + deviceID?: string; + legacy?: boolean; productName: string; - type?: 'gpu' | 'xpu'; + type?: 'gpu' | 'npu' | 'xpu'; process: boolean; gpu: boolean; memory: boolean; @@ -94,17 +135,45 @@ export namespace AI { speed: boolean; } export interface MonitorGPUData { + memoryActivity: Array; + encoderUtil: Array; + decoderUtil: Array; + jpegUtil: Array; + ofaUtil: Array; + mediaUtil: Array; + computeUtil: Array; + copyUtil: Array; + hotspotTemperature: Array; + fanRPM: Array; + aiCPUUtil: Array; + ctrlCPUUtil: Array; + ddrUsed: Array; + ddrTotal: Array; + hbmUsed: Array; + hbmTotal: Array; + ddrBandwidth: Array; + hbmBandwidth: Array; + memoryBandwidth: Array; + mediaFrequency: Array; + hugepagesUsed: Array; + hugepagesTotal: Array; + + bucketSeconds: number; + sampleCount: number; + memoryTemperatureValue: Array; + frequencyValue: Array; + memoryFrequencyValue: Array; date: Array; - gpuValue: Array; - temperatureValue: Array; - powerTotal: Array; - powerUsed: Array; - powerPercent: Array; - memoryTotal: Array; - memoryUsed: Array; - memoryPercent: Array; - speedValue: Array; - processCount: Array; + gpuValue: Array; + temperatureValue: Array; + powerTotal: Array; + powerUsed: Array; + powerPercent: Array; + memoryTotal: Array; + memoryUsed: Array; + memoryPercent: Array; + speedValue: Array; + processCount: Array; gpuProcesses: Array>; } export interface GPUProcess { @@ -132,12 +201,15 @@ export namespace AI { } interface Xpu { + tiles?: { tileID: number; stats: Stats }[]; + processStatus?: string; basic: Basic; stats: Stats; processes: XpuProcess[]; } interface Basic { + uuid?: string; deviceID: number; deviceName: string; vendorName: string; @@ -148,6 +220,13 @@ export namespace AI { } interface Stats { + mediaUtil?: string; + computeUtil?: string; + copyUtil?: string; + mediaFrequency?: string; + memoryTemperature?: string; + + memoryBandwidthUtil?: string; power: string; gpuUtil: string; frequency: string; diff --git a/frontend/src/components/v-charts/components/Line.vue b/frontend/src/components/v-charts/components/Line.vue index ab11b6044..3a75b9e8d 100644 --- a/frontend/src/components/v-charts/components/Line.vue +++ b/frontend/src/components/v-charts/components/Line.vue @@ -134,10 +134,11 @@ function initChart() { series.push({ name: item?.name, type: 'line', - itemStyle: seriesStyle[index + 2], - areaStyle: seriesStyle[index], + itemStyle: item.itemStyle ?? seriesStyle[index + 2], + areaStyle: item.areaStyle ?? seriesStyle[index], data: item?.data, - showSymbol: false, + showSymbol: item.showSymbol ?? false, + symbolSize: item.symbolSize, yAxisIndex: item.yAxisIndex ? 1 : null, }); }); @@ -307,7 +308,9 @@ function initChart() { series: series, dataZoom: [ { - ...(props.option.xData?.[0] ? { startValue: props.option.xData[0] } : {}), + ...((props.option.xAxis?.min ?? props.option.xData?.[0]) != null + ? { startValue: props.option.xAxis?.min ?? props.option.xData[0] } + : {}), left: grid.left, right: grid.right, show: props.dataZoom, diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index c1e7950af..5254005bb 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1508,6 +1508,59 @@ const message = { extraHeaders: 'Extra Headers', }, gpu: { + memoryActivity: 'Memory read/write activity', + hotspotTemperature: 'Hotspot temperature', + fanRPM: 'Fan speed (RPM)', + eccCorrectable: 'Correctable errors', + eccUncorrectable: 'Uncorrectable errors', + eccPending: 'Pending ECC mode', + engineUtil: 'Engine utilization', + cpuUtil: 'CPU · utilization', + aiCore: 'AICore · utilization', + aiCPUUtil: 'AICPU · utilization', + ctrlCPUUtil: 'CtrlCPU · utilization', + mediaFrequency: 'Media · frequency', + encoderUtil: 'Encoder · utilization', + decoderUtil: 'Decoder · utilization', + jpegUtil: 'JPEG · utilization', + ofaUtil: 'OFA · utilization', + mediaUtil: 'Media · utilization', + computeUtil: 'Compute · utilization', + copyUtil: 'Copy · utilization', + ddrUsage: 'DDR · capacity', + ddrUsed: 'DDR · used', + ddrTotal: 'DDR · total', + ddrBandwidth: 'DDR · bandwidth utilization', + hbmUsage: 'HBM · capacity', + hbmUsed: 'HBM · used', + hbmTotal: 'HBM · total', + hbmBandwidth: 'HBM · bandwidth utilization', + hugepagesUsage: 'Huge pages · count', + hugepagesUsed: 'Huge pages · used', + hugepagesTotal: 'Huge pages · total', + + historyAverage: 'Average', + historyPeak: 'Peak', + historyAggregated: + '{count} samples aggregated into {seconds}-second intervals; processes show counts only.', + historyRaw: '{count} original samples.', + historyMetricEmpty: 'No data for this metric in the selected time range', + historyLoadFailed: 'Failed to load history. Please retry.', + collectedAt: 'Collected at', + processUnavailable: 'Process information is unavailable', + architecture: 'Architecture', + memoryFrequency: 'Memory frequency', + memoryTemperature: 'Memory temperature', + memoryReserved: 'Reserved memory', + powerDefaultLimit: 'Default power limit', + powerMaxLimit: 'Maximum configurable power limit', + clockEvents: 'Clock limiting reasons', + memoryBandwidth: 'Memory bandwidth utilization', + legacyDevice: 'Legacy record', + legacyHistory: + 'These records have no stable device ID. Their identity and missing values cannot be verified.', + pcieGeneration: 'PCIe generation (current / maximum)', + pcieWidth: 'PCIe width (current / maximum)', gpu: 'GPU Monitoring', gpuHelper: 'No supported GPU/XPU/NPU management command was detected. Please check and try again!', process: 'Process Information', @@ -1523,7 +1576,7 @@ const message = { temperature: 'Temperature', performanceState: 'Performance State', powerUsage: 'Power Consumption', - memoryUsage: 'Memory Utilization', + memoryUsage: 'Memory capacity usage', fanSpeed: 'Fan Speed', power: 'Power', powerCurrent: 'Current Power', diff --git a/frontend/src/lang/modules/es-es.ts b/frontend/src/lang/modules/es-es.ts index 909deeb36..df78f4de5 100644 --- a/frontend/src/lang/modules/es-es.ts +++ b/frontend/src/lang/modules/es-es.ts @@ -1517,6 +1517,59 @@ const message = { extraHeaders: 'Encabezados adicionales', }, gpu: { + memoryActivity: 'Actividad de lectura/escritura de memoria', + hotspotTemperature: 'Temperatura del punto caliente', + fanRPM: 'Velocidad del ventilador (RPM)', + eccCorrectable: 'Errores corregibles', + eccUncorrectable: 'Errores no corregibles', + eccPending: 'Modo ECC pendiente', + engineUtil: 'Utilización de motores', + cpuUtil: 'CPU · utilización', + aiCore: 'AICore · utilización', + aiCPUUtil: 'AICPU · utilización', + ctrlCPUUtil: 'CtrlCPU · utilización', + mediaFrequency: 'Medios · frecuencia', + encoderUtil: 'Codificador · utilización', + decoderUtil: 'Decodificador · utilización', + jpegUtil: 'JPEG · utilización', + ofaUtil: 'OFA · utilización', + mediaUtil: 'Medios · utilización', + computeUtil: 'Cómputo · utilización', + copyUtil: 'Copia · utilización', + ddrUsage: 'DDR · capacidad', + ddrUsed: 'DDR · usado', + ddrTotal: 'DDR · total', + ddrBandwidth: 'DDR · utilización del ancho de banda', + hbmUsage: 'HBM · capacidad', + hbmUsed: 'HBM · usado', + hbmTotal: 'HBM · total', + hbmBandwidth: 'HBM · utilización del ancho de banda', + hugepagesUsage: 'Páginas grandes · cantidad', + hugepagesUsed: 'Páginas grandes · usado', + hugepagesTotal: 'Páginas grandes · total', + + historyAverage: 'Promedio', + historyPeak: 'Máximo', + historyAggregated: + '{count} muestras agrupadas en intervalos de {seconds} segundos; los procesos solo muestran cantidades.', + historyRaw: '{count} muestras originales.', + historyMetricEmpty: 'No hay datos de esta métrica en el período seleccionado', + historyLoadFailed: 'No se pudo cargar el historial. Inténtelo de nuevo.', + collectedAt: 'Fecha de recopilación', + processUnavailable: 'Información de procesos no disponible', + architecture: 'Arquitectura', + memoryFrequency: 'Frecuencia de memoria', + memoryTemperature: 'Temperatura de memoria', + memoryReserved: 'Memoria reservada', + powerDefaultLimit: 'Límite de potencia predeterminado', + powerMaxLimit: 'Potencia máxima configurable', + clockEvents: 'Motivos de limitación de frecuencia', + memoryBandwidth: 'Uso del ancho de banda de memoria', + legacyDevice: 'Registro antiguo', + legacyHistory: + 'Estos registros no tienen un ID de dispositivo estable. No se pueden verificar su identidad ni los valores ausentes.', + pcieGeneration: 'Generación PCIe (actual / máxima)', + pcieWidth: 'Ancho PCIe (actual / máximo)', gpu: 'Monitoreo de GPU', gpuHelper: 'No se detectó ningún comando de administración de GPU/XPU/NPU compatible. ¡Compruebe e inténtelo de nuevo!', @@ -1532,7 +1585,7 @@ const message = { temperature: 'Temperatura', performanceState: 'Estado de Rendimiento', powerUsage: 'Consumo de Energía', - memoryUsage: 'Utilización de Memoria', + memoryUsage: 'Uso de capacidad de memoria', fanSpeed: 'Velocidad del Ventilador', power: 'Energía', powerCurrent: 'Energía Actual', diff --git a/frontend/src/lang/modules/fa.ts b/frontend/src/lang/modules/fa.ts index 1ae1a727d..978ff0186 100644 --- a/frontend/src/lang/modules/fa.ts +++ b/frontend/src/lang/modules/fa.ts @@ -1493,6 +1493,58 @@ const message = { extraHeaders: 'هدرهای اضافی', }, gpu: { + memoryActivity: 'فعالیت خواندن/نوشتن حافظه', + hotspotTemperature: 'دمای نقطه داغ', + fanRPM: 'سرعت فن (RPM)', + eccCorrectable: 'خطاهای قابل اصلاح', + eccUncorrectable: 'خطاهای غیرقابل اصلاح', + eccPending: 'حالت ECC در انتظار اعمال', + engineUtil: 'میزان استفاده از موتور', + cpuUtil: 'CPU · میزان استفاده', + aiCore: 'AICore · میزان استفاده', + aiCPUUtil: 'AICPU · میزان استفاده', + ctrlCPUUtil: 'CtrlCPU · میزان استفاده', + mediaFrequency: 'رسانه · فرکانس', + encoderUtil: 'رمزگذار · میزان استفاده', + decoderUtil: 'رمزگشا · میزان استفاده', + jpegUtil: 'JPEG · میزان استفاده', + ofaUtil: 'OFA · میزان استفاده', + mediaUtil: 'رسانه · میزان استفاده', + computeUtil: 'محاسبه · میزان استفاده', + copyUtil: 'کپی · میزان استفاده', + ddrUsage: 'DDR · ظرفیت', + ddrUsed: 'DDR · استفاده‌شده', + ddrTotal: 'DDR · کل', + ddrBandwidth: 'DDR · میزان استفاده از پهنای باند', + hbmUsage: 'HBM · ظرفیت', + hbmUsed: 'HBM · استفاده‌شده', + hbmTotal: 'HBM · کل', + hbmBandwidth: 'HBM · میزان استفاده از پهنای باند', + hugepagesUsage: 'صفحات بزرگ · تعداد', + hugepagesUsed: 'صفحات بزرگ · استفاده‌شده', + hugepagesTotal: 'صفحات بزرگ · کل', + + historyAverage: 'میانگین', + historyPeak: 'بیشینه', + historyAggregated: + '{count} نمونه در بازه‌های {seconds} ثانیه‌ای تجمیع شده‌اند؛ فقط تعداد فرایندها نمایش داده می‌شود.', + historyRaw: '{count} نمونه اصلی.', + historyMetricEmpty: 'در بازه زمانی انتخاب‌شده داده‌ای برای این شاخص وجود ندارد', + historyLoadFailed: 'بارگیری سابقه ناموفق بود. دوباره تلاش کنید.', + collectedAt: 'زمان گردآوری', + processUnavailable: 'اطلاعات فرایند در دسترس نیست', + architecture: 'معماری', + memoryFrequency: 'فرکانس حافظه', + memoryTemperature: 'دمای حافظه', + memoryReserved: 'حافظه رزروشده', + powerDefaultLimit: 'محدودیت توان پیش‌فرض', + powerMaxLimit: 'حداکثر توان قابل تنظیم', + clockEvents: 'دلایل محدودیت فرکانس', + memoryBandwidth: 'میزان استفاده از پهنای باند حافظه', + legacyDevice: 'رکورد قدیمی', + legacyHistory: 'این رکوردها شناسه ثابت دستگاه ندارند. هویت دستگاه و مقادیر مفقود قابل تأیید نیستند.', + pcieGeneration: 'نسل PCIe (فعلی / حداکثر)', + pcieWidth: 'عرض PCIe (فعلی / حداکثر)', gpu: 'نظارت بر GPU', gpuHelper: 'هیچ دستور مدیریت GPU/XPU/NPU پشتیبانی‌شده‌ای شناسایی نشد. لطفاً بررسی و دوباره تلاش کنید!', process: 'اطلاعات فرآیند', @@ -1508,7 +1560,7 @@ const message = { temperature: 'دما', performanceState: 'وضعیت عملکرد', powerUsage: 'مصرف برق', - memoryUsage: 'استفاده از حافظه', + memoryUsage: 'میزان اشغال ظرفیت حافظه', fanSpeed: 'سرعت فن', power: 'برق', powerCurrent: 'برق فعلی', diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts index 0b72aac5b..b219d06d7 100644 --- a/frontend/src/lang/modules/ja.ts +++ b/frontend/src/lang/modules/ja.ts @@ -1496,6 +1496,57 @@ const message = { extraHeaders: '追加ヘッダー', }, gpu: { + memoryActivity: 'メモリ読み書き稼働率', + hotspotTemperature: 'ホットスポット温度', + fanRPM: 'ファン回転数（RPM）', + eccCorrectable: '訂正可能なエラー数', + eccUncorrectable: '訂正不能なエラー数', + eccPending: '適用待ちの ECC モード', + engineUtil: 'エンジン使用率', + cpuUtil: 'CPU · 使用率', + aiCore: 'AICore · 使用率', + aiCPUUtil: 'AICPU · 使用率', + ctrlCPUUtil: 'CtrlCPU · 使用率', + mediaFrequency: 'メディア · 周波数', + encoderUtil: 'エンコーダー · 使用率', + decoderUtil: 'デコーダー · 使用率', + jpegUtil: 'JPEG · 使用率', + ofaUtil: 'OFA · 使用率', + mediaUtil: 'メディア · 使用率', + computeUtil: '計算 · 使用率', + copyUtil: 'コピー · 使用率', + ddrUsage: 'DDR · 容量', + ddrUsed: 'DDR · 使用量', + ddrTotal: 'DDR · 総容量', + ddrBandwidth: 'DDR · 帯域使用率', + hbmUsage: 'HBM · 容量', + hbmUsed: 'HBM · 使用量', + hbmTotal: 'HBM · 総容量', + hbmBandwidth: 'HBM · 帯域使用率', + hugepagesUsage: 'ヒュージページ · 数', + hugepagesUsed: 'ヒュージページ · 使用量', + hugepagesTotal: 'ヒュージページ · 総容量', + + historyAverage: '平均値', + historyPeak: '最大値', + historyAggregated: '{count} 件のサンプルを {seconds} 秒間隔で集計。プロセスは件数のみ表示します。', + historyRaw: '元のサンプル {count} 件。', + historyMetricEmpty: '選択した期間にこの指標のデータはありません', + historyLoadFailed: '履歴の読み込みに失敗しました。再試行してください。', + collectedAt: '収集日時', + processUnavailable: 'プロセス情報を取得できません', + architecture: 'アーキテクチャ', + memoryFrequency: 'メモリ周波数', + memoryTemperature: 'メモリ温度', + memoryReserved: '予約済みメモリ', + powerDefaultLimit: '既定の電力制限', + powerMaxLimit: '設定可能な最大電力', + clockEvents: 'クロック制限の理由', + memoryBandwidth: 'メモリ帯域幅使用率', + legacyDevice: '旧記録', + legacyHistory: 'これらの記録には固定のデバイス ID がなく、デバイスの識別や欠損値を確認できません。', + pcieGeneration: 'PCIe 世代（現在 / 最大）', + pcieWidth: 'PCIe 幅（現在 / 最大）', gpu: 'GPU 監視', gpuHelper: '対応する GPU/XPU/NPU 管理コマンドが検出されませんでした。確認して再試行してください！', process: 'プロセス情報', @@ -1510,7 +1561,7 @@ const message = { temperature: '温度', performanceState: 'パフォーマンス状態', powerUsage: '消費電力', - memoryUsage: 'メモリ使用率', + memoryUsage: 'メモリ容量使用率', fanSpeed: 'ファン速度', power: '電力', powerCurrent: '現在の電力', diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts index d31b00b7c..6f52b5d40 100644 --- a/frontend/src/lang/modules/ko.ts +++ b/frontend/src/lang/modules/ko.ts @@ -1482,6 +1482,57 @@ const message = { extraHeaders: '추가 헤더', }, gpu: { + memoryActivity: '메모리 읽기/쓰기 활성률', + hotspotTemperature: '핫스폿 온도', + fanRPM: '팬 회전 속도 (RPM)', + eccCorrectable: '수정 가능한 오류 수', + eccUncorrectable: '수정 불가능한 오류 수', + eccPending: '적용 대기 중인 ECC 모드', + engineUtil: '엔진 사용률', + cpuUtil: 'CPU · 사용률', + aiCore: 'AICore · 사용률', + aiCPUUtil: 'AICPU · 사용률', + ctrlCPUUtil: 'CtrlCPU · 사용률', + mediaFrequency: '미디어 · 주파수', + encoderUtil: '인코더 · 사용률', + decoderUtil: '디코더 · 사용률', + jpegUtil: 'JPEG · 사용률', + ofaUtil: 'OFA · 사용률', + mediaUtil: '미디어 · 사용률', + computeUtil: '컴퓨팅 · 사용률', + copyUtil: '복사 · 사용률', + ddrUsage: 'DDR · 용량', + ddrUsed: 'DDR · 사용량', + ddrTotal: 'DDR · 총량', + ddrBandwidth: 'DDR · 대역폭 사용률', + hbmUsage: 'HBM · 용량', + hbmUsed: 'HBM · 사용량', + hbmTotal: 'HBM · 총량', + hbmBandwidth: 'HBM · 대역폭 사용률', + hugepagesUsage: '대용량 페이지 · 수', + hugepagesUsed: '대용량 페이지 · 사용량', + hugepagesTotal: '대용량 페이지 · 총량', + + historyAverage: '평균', + historyPeak: '최댓값', + historyAggregated: '샘플 {count}개를 {seconds}초 간격으로 집계합니다. 프로세스는 개수만 표시합니다.', + historyRaw: '원본 샘플 {count}개.', + historyMetricEmpty: '선택한 기간에 이 지표의 데이터가 없습니다', + historyLoadFailed: '기록을 불러오지 못했습니다. 다시 시도하세요.', + collectedAt: '수집 시간', + processUnavailable: '프로세스 정보를 사용할 수 없습니다', + architecture: '아키텍처', + memoryFrequency: '메모리 주파수', + memoryTemperature: '메모리 온도', + memoryReserved: '예약된 메모리', + powerDefaultLimit: '기본 전력 제한', + powerMaxLimit: '설정 가능한 최대 전력', + clockEvents: '클럭 제한 이유', + memoryBandwidth: '메모리 대역폭 사용률', + legacyDevice: '이전 기록', + legacyHistory: '이 기록에는 고정 장치 ID가 없어 장치 식별 및 누락 값을 확인할 수 없습니다.', + pcieGeneration: 'PCIe 세대 (현재 / 최대)', + pcieWidth: 'PCIe 너비 (현재 / 최대)', gpu: 'GPU 모니터링', gpuHelper: '지원되는 GPU/XPU/NPU 관리 명령을 감지하지 못했습니다. 확인하고 다시 시도하세요!', process: '프로세스 정보', @@ -1496,7 +1547,7 @@ const message = { temperature: '온도', performanceState: '성능 상태', powerUsage: '전력 소비', - memoryUsage: '메모리 사용률', + memoryUsage: '메모리 용량 사용률', fanSpeed: '팬 속도', power: '전력', powerCurrent: '현재 전력', diff --git a/frontend/src/lang/modules/lo.ts b/frontend/src/lang/modules/lo.ts index 7bc6018a9..731ffee83 100644 --- a/frontend/src/lang/modules/lo.ts +++ b/frontend/src/lang/modules/lo.ts @@ -1475,6 +1475,57 @@ const message = { extraHeaders: 'Headers ເພີ່ມເຕີມ', }, gpu: { + memoryActivity: 'ກິດຈະກຳອ່ານ/ຂຽນໜ່ວຍຄວາມຈຳ', + hotspotTemperature: 'ອຸນຫະພູມຈຸດຮ້ອນ', + fanRPM: 'ຄວາມໄວພັດລົມ (RPM)', + eccCorrectable: 'ຂໍ້ຜິດພາດທີ່ແກ້ໄຂໄດ້', + eccUncorrectable: 'ຂໍ້ຜິດພາດທີ່ແກ້ໄຂບໍ່ໄດ້', + eccPending: 'ໂໝດ ECC ທີ່ລໍຖ້າ', + engineUtil: 'ອັດຕາການໃຊ້ເຄື່ອງຈັກ', + cpuUtil: 'CPU · ອັດຕາການໃຊ້', + aiCore: 'AICore · ອັດຕາການໃຊ້', + aiCPUUtil: 'AICPU · ອັດຕາການໃຊ້', + ctrlCPUUtil: 'CtrlCPU · ອັດຕາການໃຊ້', + mediaFrequency: 'ສື່ · ຄວາມຖີ່', + encoderUtil: 'ຕົວເຂົ້າລະຫັດ · ອັດຕາການໃຊ້', + decoderUtil: 'ຕົວຖອດລະຫັດ · ອັດຕາການໃຊ້', + jpegUtil: 'JPEG · ອັດຕາການໃຊ້', + ofaUtil: 'OFA · ອັດຕາການໃຊ້', + mediaUtil: 'ສື່ · ອັດຕາການໃຊ້', + computeUtil: 'ຄຳນວນ · ອັດຕາການໃຊ້', + copyUtil: 'ສຳເນົາ · ອັດຕາການໃຊ້', + ddrUsage: 'DDR · ຄວາມຈຸ', + ddrUsed: 'DDR · ໃຊ້ແລ້ວ', + ddrTotal: 'DDR · ທັງໝົດ', + ddrBandwidth: 'DDR · ອັດຕາການໃຊ້ແບນວິດ', + hbmUsage: 'HBM · ຄວາມຈຸ', + hbmUsed: 'HBM · ໃຊ້ແລ້ວ', + hbmTotal: 'HBM · ທັງໝົດ', + hbmBandwidth: 'HBM · ອັດຕາການໃຊ້ແບນວິດ', + hugepagesUsage: 'ໜ້າຂະໜາດໃຫຍ່ · ຈຳນວນ', + hugepagesUsed: 'ໜ້າຂະໜາດໃຫຍ່ · ໃຊ້ແລ້ວ', + hugepagesTotal: 'ໜ້າຂະໜາດໃຫຍ່ · ທັງໝົດ', + + historyAverage: 'ຄ່າສະເລ່ຍ', + historyPeak: 'ຄ່າສູງສຸດ', + historyAggregated: 'ລວມ {count} ຕົວຢ່າງເປັນຊ່ວງ {seconds} ວິນາທີ; ສະແດງພຽງຈຳນວນໂປຣເຊສ.', + historyRaw: '{count} ຕົວຢ່າງຕົ້ນສະບັບ.', + historyMetricEmpty: 'ບໍ່ມີຂໍ້ມູນຕົວຊີ້ວັດນີ້ໃນຊ່ວງເວລາທີ່ເລືອກ', + historyLoadFailed: 'ໂຫຼດປະຫວັດບໍ່ສຳເລັດ. ກະລຸນາລອງໃໝ່.', + collectedAt: 'ເວລາເກັບຂໍ້ມູນ', + processUnavailable: 'ບໍ່ມີຂໍ້ມູນໂປຣເຊສ', + architecture: 'ສະຖາປັດຕະຍະກຳ', + memoryFrequency: 'ຄວາມຖີ່ໜ່ວຍຄວາມຈຳ', + memoryTemperature: 'ອຸນຫະພູມໜ່ວຍຄວາມຈຳ', + memoryReserved: 'ໜ່ວຍຄວາມຈຳທີ່ສະຫງວນ', + powerDefaultLimit: 'ຂີດຈຳກັດພະລັງງານເລີ່ມຕົ້ນ', + powerMaxLimit: 'ພະລັງງານສູງສຸດທີ່ຕັ້ງໄດ້', + clockEvents: 'ເຫດຜົນທີ່ຈຳກັດຄວາມຖີ່', + memoryBandwidth: 'ການໃຊ້ແບນວິດໜ່ວຍຄວາມຈຳ', + legacyDevice: 'ບັນທຶກເກົ່າ', + legacyHistory: 'ບັນທຶກເຫຼົ່ານີ້ບໍ່ມີ ID ອຸປະກອນຄົງທີ່. ບໍ່ສາມາດຢືນຢັນອຸປະກອນ ແລະ ຄ່າທີ່ຂາດຫາຍໄດ້.', + pcieGeneration: 'ລຸ້ນ PCIe (ປັດຈຸບັນ / ສູງສຸດ)', + pcieWidth: 'ຄວາມກວ້າງ PCIe (ປັດຈຸບັນ / ສູງສຸດ)', gpu: 'ຕິດຕາມ GPU', gpuHelper: 'ກວດບໍ່ພົບຄຳສັ່ງຈັດການ GPU/XPU/NPU ທີ່ຮອງຮັບ. ກະລຸນາກວດສອບ ແລະ ລອງໃໝ່!', process: 'ຂໍ້ມູນໂປຣເຊສ', @@ -1490,7 +1541,7 @@ const message = { temperature: 'ອຸນຫະພູມ', performanceState: 'ສະຖານະປະສິດທິພາບ', powerUsage: 'ການໃຊ້ພະລັງງານ', - memoryUsage: 'ການໃຊ້ໜ່ວຍຄວາມຈຳ', + memoryUsage: 'ການໃຊ້ຄວາມຈຸໜ່ວຍຄວາມຈຳ', fanSpeed: 'ຄວາມໄວພັດລົມ', power: 'ພະລັງງານ', powerCurrent: 'ພະລັງງານປັດຈຸບັນ', diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts index b17a54895..f91da8600 100644 --- a/frontend/src/lang/modules/ms.ts +++ b/frontend/src/lang/modules/ms.ts @@ -1522,6 +1522,59 @@ const message = { extraHeaders: 'Pengepala Tambahan', }, gpu: { + memoryActivity: 'Aktiviti baca/tulis memori', + hotspotTemperature: 'Suhu titik panas', + fanRPM: 'Kelajuan kipas (RPM)', + eccCorrectable: 'Ralat boleh dibetulkan', + eccUncorrectable: 'Ralat tidak boleh dibetulkan', + eccPending: 'Mod ECC menunggu', + engineUtil: 'Penggunaan enjin', + cpuUtil: 'CPU · penggunaan', + aiCore: 'AICore · penggunaan', + aiCPUUtil: 'AICPU · penggunaan', + ctrlCPUUtil: 'CtrlCPU · penggunaan', + mediaFrequency: 'Media · frekuensi', + encoderUtil: 'Pengekod · penggunaan', + decoderUtil: 'Penyahkod · penggunaan', + jpegUtil: 'JPEG · penggunaan', + ofaUtil: 'OFA · penggunaan', + mediaUtil: 'Media · penggunaan', + computeUtil: 'Pengiraan · penggunaan', + copyUtil: 'Salinan · penggunaan', + ddrUsage: 'DDR · kapasiti', + ddrUsed: 'DDR · digunakan', + ddrTotal: 'DDR · jumlah', + ddrBandwidth: 'DDR · penggunaan lebar jalur', + hbmUsage: 'HBM · kapasiti', + hbmUsed: 'HBM · digunakan', + hbmTotal: 'HBM · jumlah', + hbmBandwidth: 'HBM · penggunaan lebar jalur', + hugepagesUsage: 'Halaman besar · bilangan', + hugepagesUsed: 'Halaman besar · digunakan', + hugepagesTotal: 'Halaman besar · jumlah', + + historyAverage: 'Purata', + historyPeak: 'Puncak', + historyAggregated: + '{count} sampel diagregatkan dalam sela {seconds} saat; proses hanya memaparkan bilangan.', + historyRaw: '{count} sampel asal.', + historyMetricEmpty: 'Tiada data untuk metrik ini dalam julat masa yang dipilih', + historyLoadFailed: 'Gagal memuatkan sejarah. Sila cuba lagi.', + collectedAt: 'Masa pengumpulan', + processUnavailable: 'Maklumat proses tidak tersedia', + architecture: 'Seni bina', + memoryFrequency: 'Frekuensi memori', + memoryTemperature: 'Suhu memori', + memoryReserved: 'Memori terpelihara', + powerDefaultLimit: 'Had kuasa lalai', + powerMaxLimit: 'Kuasa maksimum boleh ditetapkan', + clockEvents: 'Sebab had frekuensi', + memoryBandwidth: 'Penggunaan lebar jalur memori', + legacyDevice: 'Rekod lama', + legacyHistory: + 'Rekod ini tiada ID peranti yang tetap. Identiti peranti dan nilai yang hilang tidak dapat disahkan.', + pcieGeneration: 'Generasi PCIe (semasa / maksimum)', + pcieWidth: 'Lebar PCIe (semasa / maksimum)', gpu: 'Pemantauan GPU', gpuHelper: 'Tiada arahan pengurusan GPU/XPU/NPU yang disokong dikesan. Sila periksa dan cuba lagi!', process: 'Maklumat Proses', @@ -1536,7 +1589,7 @@ const message = { temperature: 'Suhu', performanceState: 'Status Prestasi', powerUsage: 'Penggunaan Kuasa', - memoryUsage: 'Penggunaan Memori', + memoryUsage: 'Penggunaan kapasiti memori', fanSpeed: 'Kelajuan Kipas', power: 'Kuasa', powerCurrent: 'Kuasa Semasa', diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts index 033cb4f81..d40f29f0c 100644 --- a/frontend/src/lang/modules/pt-br.ts +++ b/frontend/src/lang/modules/pt-br.ts @@ -1518,6 +1518,59 @@ const message = { extraHeaders: 'Cabeçalhos extras', }, gpu: { + memoryActivity: 'Atividade de leitura/escrita da memória', + hotspotTemperature: 'Temperatura do ponto quente', + fanRPM: 'Rotação da ventoinha (RPM)', + eccCorrectable: 'Erros corrigíveis', + eccUncorrectable: 'Erros incorrigíveis', + eccPending: 'Modo ECC pendente', + engineUtil: 'Utilização dos motores', + cpuUtil: 'CPU · utilização', + aiCore: 'AICore · utilização', + aiCPUUtil: 'AICPU · utilização', + ctrlCPUUtil: 'CtrlCPU · utilização', + mediaFrequency: 'Mídia · frequência', + encoderUtil: 'Codificador · utilização', + decoderUtil: 'Decodificador · utilização', + jpegUtil: 'JPEG · utilização', + ofaUtil: 'OFA · utilização', + mediaUtil: 'Mídia · utilização', + computeUtil: 'Computação · utilização', + copyUtil: 'Cópia · utilização', + ddrUsage: 'DDR · capacidade', + ddrUsed: 'DDR · usado', + ddrTotal: 'DDR · total', + ddrBandwidth: 'DDR · utilização da largura de banda', + hbmUsage: 'HBM · capacidade', + hbmUsed: 'HBM · usado', + hbmTotal: 'HBM · total', + hbmBandwidth: 'HBM · utilização da largura de banda', + hugepagesUsage: 'Páginas grandes · quantidade', + hugepagesUsed: 'Páginas grandes · usado', + hugepagesTotal: 'Páginas grandes · total', + + historyAverage: 'Média', + historyPeak: 'Pico', + historyAggregated: + '{count} amostras agregadas em intervalos de {seconds} segundos; processos exibem apenas contagens.', + historyRaw: '{count} amostras originais.', + historyMetricEmpty: 'Sem dados desta métrica no período selecionado', + historyLoadFailed: 'Falha ao carregar o histórico. Tente novamente.', + collectedAt: 'Data da coleta', + processUnavailable: 'Informações de processos indisponíveis', + architecture: 'Arquitetura', + memoryFrequency: 'Frequência da memória', + memoryTemperature: 'Temperatura da memória', + memoryReserved: 'Memória reservada', + powerDefaultLimit: 'Limite de potência padrão', + powerMaxLimit: 'Potência máxima configurável', + clockEvents: 'Motivos de limitação de frequência', + memoryBandwidth: 'Uso da largura de banda da memória', + legacyDevice: 'Registro antigo', + legacyHistory: + 'Estes registros não têm um ID de dispositivo estável. Não é possível verificar a identidade nem os valores ausentes.', + pcieGeneration: 'Geração PCIe (atual / máxima)', + pcieWidth: 'Largura PCIe (atual / máxima)', gpu: 'Monitoramento de GPU', gpuHelper: 'Nenhum comando de gerenciamento de GPU/XPU/NPU compatível foi detectado. Verifique e tente novamente!', @@ -1533,7 +1586,7 @@ const message = { temperature: 'Temperatura', performanceState: 'Estado de Desempenho', powerUsage: 'Consumo de Energia', - memoryUsage: 'Utilização de Memória', + memoryUsage: 'Uso da capacidade de memória', fanSpeed: 'Velocidade do Ventilador', power: 'Energia', powerCurrent: 'Energia Atual', diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts index 993215f84..1d9a6e109 100644 --- a/frontend/src/lang/modules/ru.ts +++ b/frontend/src/lang/modules/ru.ts @@ -1511,6 +1511,59 @@ const message = { extraHeaders: 'Дополнительные заголовки', }, gpu: { + memoryActivity: 'Активность чтения/записи памяти', + hotspotTemperature: 'Температура горячей точки', + fanRPM: 'Скорость вентилятора (об/мин)', + eccCorrectable: 'Исправимые ошибки', + eccUncorrectable: 'Неисправимые ошибки', + eccPending: 'Ожидающий режим ECC', + engineUtil: 'Загрузка движков', + cpuUtil: 'CPU · загрузка', + aiCore: 'AICore · загрузка', + aiCPUUtil: 'AICPU · загрузка', + ctrlCPUUtil: 'CtrlCPU · загрузка', + mediaFrequency: 'Медиа · частота', + encoderUtil: 'Кодировщик · загрузка', + decoderUtil: 'Декодировщик · загрузка', + jpegUtil: 'JPEG · загрузка', + ofaUtil: 'OFA · загрузка', + mediaUtil: 'Медиа · загрузка', + computeUtil: 'Вычисления · загрузка', + copyUtil: 'Копирование · загрузка', + ddrUsage: 'DDR · объём', + ddrUsed: 'DDR · использовано', + ddrTotal: 'DDR · всего', + ddrBandwidth: 'DDR · использование пропускной способности', + hbmUsage: 'HBM · объём', + hbmUsed: 'HBM · использовано', + hbmTotal: 'HBM · всего', + hbmBandwidth: 'HBM · использование пропускной способности', + hugepagesUsage: 'Большие страницы · количество', + hugepagesUsed: 'Большие страницы · использовано', + hugepagesTotal: 'Большие страницы · всего', + + historyAverage: 'Среднее', + historyPeak: 'Максимум', + historyAggregated: + '{count} измерений сгруппированы по {seconds} с; для процессов показано только количество.', + historyRaw: 'Исходных измерений: {count}.', + historyMetricEmpty: 'Нет данных этой метрики за выбранный период', + historyLoadFailed: 'Не удалось загрузить историю. Повторите попытку.', + collectedAt: 'Время сбора', + processUnavailable: 'Информация о процессах недоступна', + architecture: 'Архитектура', + memoryFrequency: 'Частота памяти', + memoryTemperature: 'Температура памяти', + memoryReserved: 'Зарезервированная память', + powerDefaultLimit: 'Лимит мощности по умолчанию', + powerMaxLimit: 'Максимальная настраиваемая мощность', + clockEvents: 'Причины ограничения частоты', + memoryBandwidth: 'Использование пропускной способности памяти', + legacyDevice: 'Старая запись', + legacyHistory: + 'У этих записей нет постоянного ID устройства. Нельзя проверить принадлежность устройству и отсутствующие значения.', + pcieGeneration: 'Поколение PCIe (текущее / максимальное)', + pcieWidth: 'Ширина PCIe (текущая / максимальная)', gpu: 'Мониторинг GPU', gpuHelper: 'Поддерживаемая команда управления GPU/XPU/NPU не обнаружена. Проверьте и повторите попытку!', process: 'Информация о Процессе', @@ -1525,7 +1578,7 @@ const message = { temperature: 'Температура', performanceState: 'Состояние Производительности', powerUsage: 'Потребление Мощности', - memoryUsage: 'Использование Памяти', + memoryUsage: 'Использование объёма памяти', fanSpeed: 'Скорость Вентилятора', power: 'Мощность', powerCurrent: 'Текущая Мощность', diff --git a/frontend/src/lang/modules/tr.ts b/frontend/src/lang/modules/tr.ts index a5ede94c0..1ccff27ca 100644 --- a/frontend/src/lang/modules/tr.ts +++ b/frontend/src/lang/modules/tr.ts @@ -1518,6 +1518,58 @@ const message = { extraHeaders: 'Ek Başlıklar', }, gpu: { + memoryActivity: 'Bellek okuma/yazma etkinliği', + hotspotTemperature: 'Sıcak nokta sıcaklığı', + fanRPM: 'Fan hızı (RPM)', + eccCorrectable: 'Düzeltilebilir hatalar', + eccUncorrectable: 'Düzeltilemeyen hatalar', + eccPending: 'Bekleyen ECC modu', + engineUtil: 'Motor kullanımı', + cpuUtil: 'CPU · kullanım', + aiCore: 'AICore · kullanım', + aiCPUUtil: 'AICPU · kullanım', + ctrlCPUUtil: 'CtrlCPU · kullanım', + mediaFrequency: 'Medya · frekans', + encoderUtil: 'Kodlayıcı · kullanım', + decoderUtil: 'Kod çözücü · kullanım', + jpegUtil: 'JPEG · kullanım', + ofaUtil: 'OFA · kullanım', + mediaUtil: 'Medya · kullanım', + computeUtil: 'Hesaplama · kullanım', + copyUtil: 'Kopyalama · kullanım', + ddrUsage: 'DDR · kapasite', + ddrUsed: 'DDR · kullanılan', + ddrTotal: 'DDR · toplam', + ddrBandwidth: 'DDR · bant genişliği kullanımı', + hbmUsage: 'HBM · kapasite', + hbmUsed: 'HBM · kullanılan', + hbmTotal: 'HBM · toplam', + hbmBandwidth: 'HBM · bant genişliği kullanımı', + hugepagesUsage: 'Büyük sayfalar · sayı', + hugepagesUsed: 'Büyük sayfalar · kullanılan', + hugepagesTotal: 'Büyük sayfalar · toplam', + + historyAverage: 'Ortalama', + historyPeak: 'En yüksek', + historyAggregated: + '{count} örnek {seconds} saniyelik aralıklarla toplandı; işlemler için yalnızca sayılar gösterilir.', + historyRaw: '{count} ham örnek.', + historyMetricEmpty: 'Seçilen zaman aralığında bu metriğe ait veri yok', + historyLoadFailed: 'Geçmiş yüklenemedi. Lütfen tekrar deneyin.', + collectedAt: 'Toplanma zamanı', + processUnavailable: 'İşlem bilgileri kullanılamıyor', + architecture: 'Mimari', + memoryFrequency: 'Bellek frekansı', + memoryTemperature: 'Bellek sıcaklığı', + memoryReserved: 'Ayrılmış bellek', + powerDefaultLimit: 'Varsayılan güç sınırı', + powerMaxLimit: 'Yapılandırılabilir en yüksek güç', + clockEvents: 'Saat frekansı sınırlama nedenleri', + memoryBandwidth: 'Bellek bant genişliği kullanımı', + legacyDevice: 'Eski kayıt', + legacyHistory: 'Bu kayıtların sabit cihaz kimliği yoktur. Cihaz kimliği ve eksik değerler doğrulanamaz.', + pcieGeneration: 'PCIe nesli (geçerli / en yüksek)', + pcieWidth: 'PCIe genişliği (geçerli / en yüksek)', gpu: 'GPU İzleme', gpuHelper: 'Desteklenen bir GPU/XPU/NPU yönetim komutu algılanmadı. Lütfen kontrol edip tekrar deneyin!', process: 'İşlem Bilgisi', @@ -1532,7 +1584,7 @@ const message = { temperature: 'Sıcaklık', performanceState: 'Performans Durumu', powerUsage: 'Güç Tüketimi', - memoryUsage: 'Bellek Kullanımı', + memoryUsage: 'Bellek kapasitesi kullanımı', fanSpeed: 'Fan Hızı', power: 'Güç', powerCurrent: 'Mevcut Güç', diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts index 24376da15..bf868465f 100644 --- a/frontend/src/lang/modules/zh-Hant.ts +++ b/frontend/src/lang/modules/zh-Hant.ts @@ -1429,6 +1429,57 @@ const message = { extraHeaders: '額外請求標頭', }, gpu: { + memoryActivity: '顯示記憶體讀寫活躍率', + hotspotTemperature: '熱點溫度', + fanRPM: '風扇轉速（RPM）', + eccCorrectable: '可修正錯誤數', + eccUncorrectable: '不可修正錯誤數', + eccPending: '待生效 ECC 模式', + engineUtil: '引擎使用率', + cpuUtil: 'CPU · 使用率', + aiCore: 'AICore · 使用率', + aiCPUUtil: 'AICPU · 使用率', + ctrlCPUUtil: 'CtrlCPU · 使用率', + mediaFrequency: '媒體引擎 · 頻率', + encoderUtil: '編碼器 · 使用率', + decoderUtil: '解碼器 · 使用率', + jpegUtil: 'JPEG · 使用率', + ofaUtil: 'OFA · 使用率', + mediaUtil: '媒體引擎 · 使用率', + computeUtil: '計算引擎 · 使用率', + copyUtil: '複製引擎 · 使用率', + ddrUsage: 'DDR · 容量', + ddrUsed: 'DDR · 已用', + ddrTotal: 'DDR · 總量', + ddrBandwidth: 'DDR · 頻寬使用率', + hbmUsage: 'HBM · 容量', + hbmUsed: 'HBM · 已用', + hbmTotal: 'HBM · 總量', + hbmBandwidth: 'HBM · 頻寬使用率', + hugepagesUsage: '大頁 · 數量', + hugepagesUsed: '大頁 · 已用', + hugepagesTotal: '大頁 · 總量', + + historyAverage: '平均值', + historyPeak: '峰值', + historyAggregated: '共 {count} 筆取樣，按 {seconds} 秒彙整；程序僅顯示統計數量。', + historyRaw: '共 {count} 筆原始取樣。', + historyMetricEmpty: '所選時間範圍內沒有此指標的資料', + historyLoadFailed: '歷史記錄載入失敗，請重試', + collectedAt: '採集時間', + processUnavailable: '程序資訊不可用', + architecture: '架構', + memoryFrequency: '顯存頻率', + memoryTemperature: '顯存溫度', + memoryReserved: '保留顯存', + powerDefaultLimit: '預設功率限制', + powerMaxLimit: '最大可設定功率', + clockEvents: '頻率限制原因', + memoryBandwidth: '顯存頻寬使用率', + legacyDevice: '舊記錄', + legacyHistory: '這些記錄沒有穩定裝置識別碼，無法確認其裝置身分和缺失值。', + pcieGeneration: 'PCIe 世代（目前 / 最大）', + pcieWidth: 'PCIe 寬度（目前 / 最大）', gpu: 'GPU 監控', gpuHelper: '目前系統未偵測到支援的 GPU/XPU/NPU 管理指令，請檢查後重試', process: '行程資訊', @@ -1443,7 +1494,7 @@ const message = { temperature: '溫度', performanceState: '效能狀態', powerUsage: '功耗', - memoryUsage: '顯存使用率', + memoryUsage: '顯示記憶體容量占用率', fanSpeed: '風扇轉速', power: '功率', powerCurrent: '目前功率', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index cdbfabd35..59bba8f5e 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1442,6 +1442,57 @@ const message = { extraHeaders: '额外请求头', }, gpu: { + memoryActivity: '显存读写活跃率', + hotspotTemperature: '热点温度', + fanRPM: '风扇转速（RPM）', + eccCorrectable: '可纠正错误数', + eccUncorrectable: '不可纠正错误数', + eccPending: '待生效 ECC 模式', + engineUtil: '引擎使用率', + cpuUtil: 'CPU · 使用率', + aiCore: 'AICore · 使用率', + aiCPUUtil: 'AICPU · 使用率', + ctrlCPUUtil: 'CtrlCPU · 使用率', + mediaFrequency: '媒体引擎 · 频率', + encoderUtil: '编码器 · 使用率', + decoderUtil: '解码器 · 使用率', + jpegUtil: 'JPEG · 使用率', + ofaUtil: 'OFA · 使用率', + mediaUtil: '媒体引擎 · 使用率', + computeUtil: '计算引擎 · 使用率', + copyUtil: '复制引擎 · 使用率', + ddrUsage: 'DDR · 容量', + ddrUsed: 'DDR · 已用', + ddrTotal: 'DDR · 总量', + ddrBandwidth: 'DDR · 带宽利用率', + hbmUsage: 'HBM · 容量', + hbmUsed: 'HBM · 已用', + hbmTotal: 'HBM · 总量', + hbmBandwidth: 'HBM · 带宽利用率', + hugepagesUsage: '大页 · 数量', + hugepagesUsed: '大页 · 已用', + hugepagesTotal: '大页 · 总量', + + historyAverage: '平均值', + historyPeak: '峰值', + historyAggregated: '共 {count} 条采样，按 {seconds} 秒聚合；进程仅展示统计数量。', + historyRaw: '共 {count} 条原始采样。', + historyMetricEmpty: '所选时间范围内没有该指标的数据', + historyLoadFailed: '历史记录加载失败，请重试', + collectedAt: '采集时间', + processUnavailable: '进程信息不可用', + architecture: '架构', + memoryFrequency: '显存频率', + memoryTemperature: '显存温度', + memoryReserved: '保留显存', + powerDefaultLimit: '默认功率限制', + powerMaxLimit: '最大可设置功率', + clockEvents: '频率限制原因', + memoryBandwidth: '显存带宽利用率', + legacyDevice: '旧记录', + legacyHistory: '这些记录没有稳定设备标识，无法确认其设备身份和缺失值。', + pcieGeneration: 'PCIe 代际（当前 / 最大）', + pcieWidth: 'PCIe 宽度（当前 / 最大）', gpu: 'GPU 监控', gpuHelper: '当前系统未检测到支持的 GPU/XPU/NPU 管理指令，请检查后重试！', process: '进程信息', @@ -1457,7 +1508,7 @@ const message = { temperature: '温度', performanceState: '性能状态', powerUsage: '功耗', - memoryUsage: '显存使用率', + memoryUsage: '显存容量占用率', fanSpeed: '风扇转速', power: '功率', powerCurrent: '当前功率', diff --git a/frontend/src/views/ai/gpu/current/components/DeviceDetails.vue b/frontend/src/views/ai/gpu/current/components/DeviceDetails.vue new file mode 100644 index 000000000..be24eec06 --- /dev/null +++ b/frontend/src/views/ai/gpu/current/components/DeviceDetails.vue @@ -0,0 +1,90 @@ + + + + + diff --git a/frontend/src/views/ai/gpu/current/index.vue b/frontend/src/views/ai/gpu/current/index.vue index 9abfd3f0d..2b786c5ab 100644 --- a/frontend/src/views/ai/gpu/current/index.vue +++ b/frontend/src/views/ai/gpu/current/index.vue @@ -2,12 +2,7 @@
- +
-
+
{{ item.type === 'ascend' ? 'AICore(%)' : $t('aiTools.gpu.gpuUtil') }} - {{ deviceUtil(item) }} + {{ deviceUtil(item) || 'N/A' }} {{ formatTemperature(item.temperature) }}
-
-
-
-
- {{ $t('aiTools.gpu.runtimeInfo') }} -
-
-
- - {{ $t('aiTools.gpu.fanSpeed') }} - - {{ item.fanSpeed }} -
-
- - {{ $t('aiTools.gpu.performanceState') }} - - - - - {{ item.performanceState }} -
-
- - {{ $t('aiTools.gpu.powerUsage') }} - - {{ formatPower(item) }} -
-
-
-
-
- {{ $t('aiTools.gpu.deviceInfo') }} -
-
-
- - {{ $t('aiTools.gpu.persistenceMode') }} - - - - - - {{ - $t('aiTools.gpu.' + item.persistenceMode.toLowerCase()) - }} - -
-
- - {{ $t('aiTools.gpu.busID') }} - - {{ item.busID }} -
-
- - {{ $t('aiTools.gpu.displayActive') }} - - - {{ - lowerCase(item.displayActive) === 'disabled' - ? $t('aiTools.gpu.displayActiveF') - : $t('aiTools.gpu.displayActiveT') - }} - -
-
- - Uncorr. ECC - - - - - {{ loadEcc(item.ecc) }} -
-
- - {{ $t('aiTools.gpu.computeMode') }} - - - - - - {{ loadComputeMode(item.computeMode) }} -
-
- - MIG M. - - - - - - {{ - item.migMode === 'N/A' - ? $t('aiTools.gpu.migModeNA') - : $t('aiTools.gpu.' + lowerCase(item.migMode)) - }} - -
-
-
+
+
+ {{ $t('aiTools.gpu.powerUsage') }} + {{ formatPower(item) || 'N/A' }}
-
-
-
- {{ $t('aiTools.gpu.runtimeInfo') }} -
-
-
- - {{ $t('aiTools.gpu.powerUsage') }} - - {{ formatPower(item) }} -
-
- Hugepages-Usage(page) - - {{ formatUsage(item.hugepagesUsed, item.hugepagesTotal) }} - -
-
- HBM-Usage - - {{ formatMemory(item.hbmUsed, item.hbmTotal) }} - -
-
-
-
-
- {{ $t('aiTools.gpu.deviceInfo') }} -
-
-
- - {{ $t('aiTools.gpu.busID') }} - - {{ item.busID }} -
-
-
+
+ + {{ + item.type === 'ascend' + ? $t('aiTools.gpu.aiCPUUtil') + : $t('aiTools.gpu.frequency') + }} + + + {{ + (item.type === 'ascend' ? item.aiCPUUtil : item.frequency) || 'N/A' + }} + +
+
+ + {{ + item.type === 'ascend' + ? $t('commons.table.status') + : $t('aiTools.gpu.fanSpeed') + }} + + {{ item.health || 'N/A' }} + + {{ + [item.fanSpeed, item.fanRPM].filter(isAvailable).join(' / ') || + 'N/A' + }} +
@@ -283,16 +168,30 @@ XPU {{ item.basic.deviceID }} · {{ item.basic.deviceName }}
- - {{ $t('aiTools.gpu.processCount') }}: {{ item.processes?.length || 0 }} - +
+ + {{ $t('aiTools.gpu.processCount') }}: + {{ + item.processStatus === 'unavailable' + ? 'N/A' + : item.processes?.length || 0 + }} + + + {{ $t('commons.button.view') }} + +
@@ -330,52 +229,18 @@ {{ formatTemperature(item.stats.temperature) }}
-
-
-
-
- {{ $t('aiTools.gpu.runtimeInfo') }} -
-
-
- - {{ $t('aiTools.gpu.freeMemory') }} - - {{ item.basic.freeMemory }} -
-
- - {{ $t('aiTools.gpu.powerUsage') }} - - {{ item.stats.power }} -
-
- - {{ $t('aiTools.gpu.frequency') }} - - {{ item.stats.frequency }} -
-
- - {{ $t('aiTools.gpu.memoryUsage') }} - - {{ item.stats.memoryUtil }} -
-
-
-
-
- {{ $t('aiTools.gpu.deviceInfo') }} -
-
-
- - {{ $t('aiTools.gpu.busID') }} - - {{ item.basic.pciBdfAddress }} -
-
-
+
+
+ {{ $t('aiTools.gpu.powerUsage') }} + {{ item.stats.power || 'N/A' }} +
+
+ {{ $t('aiTools.gpu.frequency') }} + {{ item.stats.frequency || 'N/A' }} +
+
+ {{ $t('aiTools.gpu.memoryTemperature') }} + {{ formatTemperature(item.stats.memoryTemperature || '') }}
@@ -385,26 +250,61 @@
+ + + + +