feat: enhance native GPU NPU and XPU monitoring (#13939)

This commit is contained in:
ssongliu
2026-09-28 18:32:12 +08:00
committed by GitHub
parent d2bb3813d9
commit d27d6db1ea
38 changed files with 2492 additions and 1265 deletions
+44 -11
View File
@@ -44,6 +44,8 @@ type MonitorGPUOptions struct {
Options []string `json:"options"`
}
type GPUChartHide struct {
DeviceID string `json:"deviceID"`
Legacy bool `json:"legacy"`
ProductName string `json:"productName"`
Type string `json:"type"`
Process bool `json:"process"`
@@ -55,23 +57,54 @@ type GPUChartHide struct {
Speed bool `json:"speed"`
}
type MonitorGPUSearch struct {
Aggregation string `json:"aggregation" validate:"omitempty,oneof=avg max"`
DeviceID string `json:"deviceID"`
Legacy bool `json:"legacy"`
ProductName string `json:"productName"`
StartTime time.Time `json:"startTime"`
EndTime time.Time `json:"endTime"`
}
type MonitorGPUData struct {
Date []time.Time `json:"date"`
GPUValue []float64 `json:"gpuValue"`
TemperatureValue []float64 `json:"temperatureValue"`
PowerTotal []float64 `json:"powerTotal"`
PowerUsed []float64 `json:"powerUsed"`
PowerPercent []float64 `json:"powerPercent"`
MemoryTotal []float64 `json:"memoryTotal"`
MemoryUsed []float64 `json:"memoryUsed"`
MemoryPercent []float64 `json:"memoryPercent"`
SpeedValue []int `json:"speedValue"`
MemoryActivity []*float64 `json:"memoryActivity"`
EncoderUtil []*float64 `json:"encoderUtil"`
DecoderUtil []*float64 `json:"decoderUtil"`
JPEGUtil []*float64 `json:"jpegUtil"`
OFAUtil []*float64 `json:"ofaUtil"`
MediaUtil []*float64 `json:"mediaUtil"`
ComputeUtil []*float64 `json:"computeUtil"`
CopyUtil []*float64 `json:"copyUtil"`
HotspotTemperature []*float64 `json:"hotspotTemperature"`
FanRPM []*float64 `json:"fanRPM"`
AICPUUtil []*float64 `json:"aiCPUUtil"`
CtrlCPUUtil []*float64 `json:"ctrlCPUUtil"`
DDRUsed []*float64 `json:"ddrUsed"`
DDRTotal []*float64 `json:"ddrTotal"`
HBMUsed []*float64 `json:"hbmUsed"`
HBMTotal []*float64 `json:"hbmTotal"`
DDRBandwidth []*float64 `json:"ddrBandwidth"`
HBMBandwidth []*float64 `json:"hbmBandwidth"`
MemoryBandwidth []*float64 `json:"memoryBandwidth"`
MediaFrequency []*float64 `json:"mediaFrequency"`
HugepagesUsed []*float64 `json:"hugepagesUsed"`
HugepagesTotal []*float64 `json:"hugepagesTotal"`
ProcessCount []int `json:"processCount"`
BucketSeconds int64 `json:"bucketSeconds"`
SampleCount int64 `json:"sampleCount"`
MemoryTemperatureValue []*float64 `json:"memoryTemperatureValue"`
FrequencyValue []*float64 `json:"frequencyValue"`
MemoryFrequencyValue []*float64 `json:"memoryFrequencyValue"`
Date []time.Time `json:"date"`
GPUValue []*float64 `json:"gpuValue"`
TemperatureValue []*float64 `json:"temperatureValue"`
PowerTotal []*float64 `json:"powerTotal"`
PowerUsed []*float64 `json:"powerUsed"`
PowerPercent []*float64 `json:"powerPercent"`
MemoryTotal []*float64 `json:"memoryTotal"`
MemoryUsed []*float64 `json:"memoryUsed"`
MemoryPercent []*float64 `json:"memoryPercent"`
SpeedValue []*float64 `json:"speedValue"`
ProcessCount []*float64 `json:"processCount"`
GPUProcesses [][]GPUProcess `json:"gpuProcesses"`
}
+40 -9
View File
@@ -33,14 +33,45 @@ type MonitorNetwork struct {
}
type MonitorGPU struct {
MemoryUtil *float64 `json:"memoryUtil"`
MemoryActivity *float64 `json:"memoryActivity"`
EncoderUtil *float64 `json:"encoderUtil"`
DecoderUtil *float64 `json:"decoderUtil"`
JPEGUtil *float64 `json:"jpegUtil"`
OFAUtil *float64 `json:"ofaUtil"`
MediaUtil *float64 `json:"mediaUtil"`
ComputeUtil *float64 `json:"computeUtil"`
CopyUtil *float64 `json:"copyUtil"`
HotspotTemperature *float64 `json:"hotspotTemperature"`
FanRPM *float64 `json:"fanRPM"`
AICPUUtil *float64 `json:"aiCPUUtil"`
CtrlCPUUtil *float64 `json:"ctrlCPUUtil"`
DDRUsed *float64 `json:"ddrUsed"`
DDRTotal *float64 `json:"ddrTotal"`
HBMUsed *float64 `json:"hbmUsed"`
HBMTotal *float64 `json:"hbmTotal"`
DDRBandwidth *float64 `json:"ddrBandwidth"`
HBMBandwidth *float64 `json:"hbmBandwidth"`
MemoryBandwidth *float64 `json:"memoryBandwidth"`
MediaFrequency *float64 `json:"mediaFrequency"`
HugepagesUsed *float64 `json:"hugepagesUsed"`
HugepagesTotal *float64 `json:"hugepagesTotal"`
MemoryTemperature *float64 `json:"memoryTemperature"`
DeviceID string `json:"deviceID"`
DeviceType string `json:"deviceType"`
ProcessStatus string `json:"processStatus"`
Frequency *float64 `json:"frequency"`
MemoryFrequency *float64 `json:"memoryFrequency"`
IntervalSeconds int `json:"intervalSeconds"`
BaseModel
ProductName string `json:"productName"`
GPUUtil float64 `json:"gpuUtil"`
Temperature float64 `json:"temperature"`
PowerDraw float64 `json:"powerDraw"`
MaxPowerLimit float64 `json:"maxPowerLimit"`
MemUsed float64 `json:"memUsed"`
MemTotal float64 `json:"memTotal"`
FanSpeed int `json:"fanSpeed"`
Processes string `json:"processes"`
ProductName string `json:"productName"`
GPUUtil *float64 `json:"gpuUtil"`
Temperature *float64 `json:"temperature"`
PowerDraw *float64 `json:"powerDraw"`
MaxPowerLimit *float64 `json:"maxPowerLimit"`
MemUsed *float64 `json:"memUsed"`
MemTotal *float64 `json:"memTotal"`
FanSpeed *float64 `json:"fanSpeed"`
Processes string `json:"processes"`
}
+79
View File
@@ -1,6 +1,8 @@
package repo
import (
"fmt"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/app/model"
@@ -10,9 +12,20 @@ import (
type MonitorRepo struct{}
type GPUHistoryPoint struct {
model.MonitorGPU
Bucket int64
PowerPercent *float64
MemoryPercent *float64
ProcessCount *float64
}
type IMonitorRepo interface {
GetBase(opts ...DBOption) ([]model.MonitorBase, error)
GetGPU(opts ...DBOption) ([]model.MonitorGPU, error)
CountGPU(opts ...DBOption) (int64, error)
GetGPUHistory(start time.Time, bucketSeconds int64, aggregation string, opts ...DBOption) ([]GPUHistoryPoint, error)
GetGPUDevices() ([]model.MonitorGPU, error)
GetIO(opts ...DBOption) ([]model.MonitorIO, error)
GetNetwork(opts ...DBOption) ([]model.MonitorNetwork, error)
@@ -26,6 +39,7 @@ type IMonitorRepo interface {
DelMonitorNet(timeForDelete time.Time) error
WithByProductName(name string) DBOption
WithByGPUDevice(deviceID, name string, legacy bool) DBOption
}
func NewIMonitorRepo() IMonitorRepo {
@@ -102,3 +116,68 @@ func (s *MonitorRepo) WithByProductName(name string) DBOption {
return g.Where("product_name = ?", name)
}
}
func (u *MonitorRepo) GetGPUDevices() ([]model.MonitorGPU, error) {
var data []model.MonitorGPU
err := global.GPUMonitorDB.Model(&model.MonitorGPU{}).Select("device_id, product_name, device_type").Group("device_id, product_name, device_type").Order("product_name, device_id").Find(&data).Error
return data, err
}
func (u *MonitorRepo) WithByGPUDevice(deviceID, name string, legacy bool) DBOption {
return func(db *gorm.DB) *gorm.DB {
if deviceID != "" {
return db.Where("device_id = ?", deviceID)
}
db = db.Where("product_name = ?", name)
if legacy {
db = db.Where("device_id IS NULL OR device_id = ''")
}
return db
}
}
func (u *MonitorRepo) CountGPU(opts ...DBOption) (int64, error) {
db := global.GPUMonitorDB.Model(&model.MonitorGPU{})
for _, opt := range opts {
db = opt(db)
}
var count int64
err := db.Count(&count).Error
return count, err
}
func (u *MonitorRepo) GetGPUHistory(start time.Time, bucketSeconds int64, aggregation string, opts ...DBOption) ([]GPUHistoryPoint, error) {
db := global.GPUMonitorDB.Model(&model.MonitorGPU{})
for _, opt := range opts {
db = opt(db)
}
expressions := []string{
"CASE WHEN max_power_limit > 0 THEN 100.0 * power_draw / max_power_limit END",
"CASE WHEN mem_total > 0 AND mem_used IS NOT NULL THEN 100.0 * mem_used / mem_total ELSE memory_util END",
"CASE WHEN (process_status = 'ok' OR process_status IS NULL OR process_status = '') AND json_valid(processes) THEN CASE WHEN json_type(processes) = 'array' THEN json_array_length(processes) END END",
}
aliases := []string{"power_percent", "memory_percent", "process_count"}
columns := []string{"*"}
if bucketSeconds > 0 {
operation := "AVG"
if aggregation == "max" {
operation = "MAX"
}
columns = []string{fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d AS bucket", start.Unix(), bucketSeconds)}
for _, column := range []string{"memory_activity", "encoder_util", "decoder_util", "jpeg_util", "ofa_util", "media_util", "compute_util", "copy_util", "hotspot_temperature", "fan_rpm", "ai_cpu_util", "ctrl_cpu_util", "ddr_used", "ddr_total", "hbm_used", "hbm_total", "ddr_bandwidth", "hbm_bandwidth", "memory_bandwidth", "media_frequency", "hugepages_used", "hugepages_total", "gpu_util", "temperature", "memory_temperature", "power_draw", "max_power_limit", "mem_used", "mem_total", "frequency", "memory_frequency", "fan_speed"} {
columns = append(columns, operation+"("+column+") AS "+column)
}
for i := range expressions {
expressions[i] = operation + "(" + expressions[i] + ")"
}
db = db.Group("bucket").Order("bucket ASC")
} else {
db = db.Order("created_at ASC, id ASC")
}
for i, expression := range expressions {
columns = append(columns, expression+" AS "+aliases[i])
}
var data []GPUHistoryPoint
err := db.Select(strings.Join(columns, ", ")).Scan(&data).Error
return data, err
}
+8 -1
View File
@@ -588,6 +588,9 @@ func loadAcceleratorInfo() ([]dto.GPUInfo, []dto.NPUInfo, []dto.XPUInfo) {
xpuData []dto.XPUInfo
)
for _, device := range snapshot.Devices {
if device.ParentID != "" {
continue
}
switch device.Kind {
case accelerator.KindGPU:
if device.GPU == nil {
@@ -597,7 +600,11 @@ func loadAcceleratorInfo() ([]dto.GPUInfo, []dto.NPUInfo, []dto.XPUInfo) {
if err := copier.Copy(&dataItem, device.GPU); err != nil {
continue
}
dataItem.PowerUsage = dataItem.PowerDraw + " / " + dataItem.MaxPowerLimit
dataItem.MaxPowerLimit = device.GPU.PowerLimit
dataItem.PowerUsage = dataItem.PowerDraw
if dataItem.MaxPowerLimit != "" {
dataItem.PowerUsage += " / " + dataItem.MaxPowerLimit
}
dataItem.MemoryUsage = dataItem.MemUsed + " / " + dataItem.MemTotal
gpuData = append(gpuData, dataItem)
case accelerator.KindNPU:
+216 -104
View File
@@ -128,115 +128,135 @@ func (m *MonitorService) LoadMonitorData(req dto.MonitorSearch) ([]dto.MonitorDa
func (m *MonitorService) LoadGPUOptions() dto.MonitorGPUOptions {
var data dto.MonitorGPUOptions
exist, client := accelerator.New()
if !exist {
return data
}
snapshot, err := client.Collect(context.Background())
if err != nil {
global.LOG.Errorf("Load accelerator info failed, err: %v", err)
return data
}
if warning := snapshot.Warning(); warning != nil {
global.LOG.Warnf("Load accelerator info partially failed, err: %v", warning)
}
return loadGPUOptions(snapshot)
}
func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions {
var data dto.MonitorGPUOptions
hasGPUOrNPU := false
hasXPU := false
for _, item := range snapshot.Devices {
if item.Kind == accelerator.KindXPU {
hasXPU = true
seen := make(map[string]bool)
if exist, client := accelerator.New(); exist {
snapshot, err := client.Collect(context.Background())
if err != nil {
global.LOG.Warnf("Load accelerator options failed: %v", err)
} else {
hasGPUOrNPU = true
data = loadGPUOptions(snapshot)
for _, item := range data.ChartHide {
seen[item.DeviceID] = true
}
}
}
switch {
case hasGPUOrNPU && hasXPU:
data.GPUType = "mixed"
case hasXPU:
data.GPUType = "xpu"
case hasGPUOrNPU:
data.GPUType = "gpu"
devices, err := monitorRepo.GetGPUDevices()
if err != nil {
global.LOG.Warnf("Load accelerator history options failed: %v", err)
return data
}
sort.Slice(snapshot.Devices, func(i, j int) bool {
if snapshot.Devices[i].Kind != snapshot.Devices[j].Kind {
return snapshot.Devices[i].Kind < snapshot.Devices[j].Kind
for _, device := range devices {
key := device.DeviceID
if key == "" {
key = "legacy:" + device.ProductName
}
if snapshot.Devices[i].Vendor != snapshot.Devices[j].Vendor {
return snapshot.Devices[i].Vendor < snapshot.Devices[j].Vendor
if seen[key] {
continue
}
if snapshot.Devices[i].NPUIndex != snapshot.Devices[j].NPUIndex {
return snapshot.Devices[i].NPUIndex < snapshot.Devices[j].NPUIndex
}
if snapshot.Devices[i].ChipIndex != snapshot.Devices[j].ChipIndex {
return snapshot.Devices[i].ChipIndex < snapshot.Devices[j].ChipIndex
}
return snapshot.Devices[i].Index < snapshot.Devices[j].Index
})
for _, item := range snapshot.Devices {
optionType := "gpu"
if item.Kind == accelerator.KindXPU {
optionType = "xpu"
}
chartHide := dto.GPUChartHide{
ProductName: item.Label,
Type: optionType,
GPU: !item.Capabilities.Utilization,
Memory: !item.Capabilities.Memory,
Power: !item.Capabilities.Power,
PowerLimit: !item.Capabilities.PowerLimit,
Temperature: !item.Capabilities.Temperature,
Speed: !item.Capabilities.FanSpeed,
}
data.ChartHide = append(data.ChartHide, chartHide)
data.Options = append(data.Options, chartHide.ProductName)
seen[key] = true
data.ChartHide = append(data.ChartHide, dto.GPUChartHide{DeviceID: device.DeviceID, ProductName: device.ProductName, Type: device.DeviceType, Legacy: device.DeviceID == ""})
data.Options = append(data.Options, device.ProductName)
}
return data
}
func (m *MonitorService) LoadGPUMonitorData(req dto.MonitorGPUSearch) (dto.MonitorGPUData, error) {
loc, _ := time.LoadLocation(common.LoadTimeZoneByCmd())
req.StartTime = req.StartTime.In(loc)
req.EndTime = req.EndTime.In(loc)
var data dto.MonitorGPUData
gpuList, err := monitorRepo.GetGPU(repo.WithByCreatedAt(req.StartTime, req.EndTime), monitorRepo.WithByProductName(req.ProductName))
if req.StartTime.IsZero() || req.EndTime.IsZero() || !req.EndTime.After(req.StartTime) {
return data, fmt.Errorf("invalid GPU history time range")
}
if req.DeviceID == "" && req.ProductName == "" {
return data, fmt.Errorf("GPU history requires a device")
}
if req.Aggregation != "" && req.Aggregation != "avg" && req.Aggregation != "max" {
return data, fmt.Errorf("invalid GPU history aggregation")
}
loc, err := time.LoadLocation(common.LoadTimeZoneByCmd())
if err != nil {
return data, err
}
for _, gpu := range gpuList {
data.Date = append(data.Date, gpu.CreatedAt)
data.GPUValue = append(data.GPUValue, gpu.GPUUtil)
data.TemperatureValue = append(data.TemperatureValue, gpu.Temperature)
data.PowerUsed = append(data.PowerUsed, gpu.PowerDraw)
data.PowerTotal = append(data.PowerTotal, gpu.MaxPowerLimit)
if gpu.MaxPowerLimit != 0 {
data.PowerPercent = append(data.PowerPercent, gpu.PowerDraw/gpu.MaxPowerLimit*100)
} else {
data.PowerPercent = append(data.PowerPercent, float64(0))
req.StartTime, req.EndTime = req.StartTime.In(loc), req.EndTime.In(loc)
opts := []repo.DBOption{repo.WithByCreatedAt(req.StartTime, req.EndTime), monitorRepo.WithByGPUDevice(req.DeviceID, req.ProductName, req.Legacy)}
data.SampleCount, err = monitorRepo.CountGPU(opts...)
if err != nil || data.SampleCount == 0 {
return data, err
}
if data.SampleCount > 1200 {
seconds := req.EndTime.Unix() - req.StartTime.Unix() + 1
data.BucketSeconds = (seconds + 599) / 600
}
points, err := monitorRepo.GetGPUHistory(req.StartTime, data.BucketSeconds, req.Aggregation, opts...)
if err != nil {
return data, err
}
samples := make([]repo.GPUHistoryPoint, 0, len(points))
if data.BucketSeconds > 0 {
next := 0
for bucket := int64(0); bucket <= (req.EndTime.Unix()-req.StartTime.Unix())/data.BucketSeconds; bucket++ {
point := repo.GPUHistoryPoint{}
if next < len(points) && points[next].Bucket == bucket {
point = points[next]
next++
}
point.CreatedAt = time.Unix(req.StartTime.Unix()+bucket*data.BucketSeconds, 0).In(loc)
if bucket == 0 {
point.CreatedAt = req.StartTime
}
samples = append(samples, point)
}
data.MemoryTotal = append(data.MemoryTotal, gpu.MemTotal)
data.MemoryUsed = append(data.MemoryUsed, gpu.MemUsed)
if gpu.MemTotal != 0 {
data.MemoryPercent = append(data.MemoryPercent, gpu.MemUsed/gpu.MemTotal*100)
} else {
data.MemoryPercent = append(data.MemoryPercent, float64(0))
} else {
for i, point := range points {
if i > 0 && points[i-1].IntervalSeconds > 0 {
interval := time.Duration(points[i-1].IntervalSeconds) * time.Second
if point.CreatedAt.Sub(points[i-1].CreatedAt) > 2*interval {
samples = append(samples, repo.GPUHistoryPoint{MonitorGPU: model.MonitorGPU{BaseModel: model.BaseModel{CreatedAt: points[i-1].CreatedAt.Add(interval)}}})
}
}
samples = append(samples, point)
}
var process []dto.GPUProcess
if err := json.Unmarshal([]byte(gpu.Processes), &process); err == nil {
data.ProcessCount = append(data.ProcessCount, len(process))
data.GPUProcesses = append(data.GPUProcesses, process)
} else {
data.ProcessCount = append(data.ProcessCount, 0)
data.GPUProcesses = append(data.GPUProcesses, []dto.GPUProcess{})
}
for _, point := range samples {
data.Date = append(data.Date, point.CreatedAt)
data.MemoryActivity = append(data.MemoryActivity, point.MemoryActivity)
data.EncoderUtil = append(data.EncoderUtil, point.EncoderUtil)
data.DecoderUtil = append(data.DecoderUtil, point.DecoderUtil)
data.JPEGUtil = append(data.JPEGUtil, point.JPEGUtil)
data.OFAUtil = append(data.OFAUtil, point.OFAUtil)
data.MediaUtil = append(data.MediaUtil, point.MediaUtil)
data.ComputeUtil = append(data.ComputeUtil, point.ComputeUtil)
data.CopyUtil = append(data.CopyUtil, point.CopyUtil)
data.HotspotTemperature = append(data.HotspotTemperature, point.HotspotTemperature)
data.FanRPM = append(data.FanRPM, point.FanRPM)
data.AICPUUtil = append(data.AICPUUtil, point.AICPUUtil)
data.CtrlCPUUtil = append(data.CtrlCPUUtil, point.CtrlCPUUtil)
data.DDRUsed = append(data.DDRUsed, point.DDRUsed)
data.DDRTotal = append(data.DDRTotal, point.DDRTotal)
data.HBMUsed = append(data.HBMUsed, point.HBMUsed)
data.HBMTotal = append(data.HBMTotal, point.HBMTotal)
data.DDRBandwidth = append(data.DDRBandwidth, point.DDRBandwidth)
data.HBMBandwidth = append(data.HBMBandwidth, point.HBMBandwidth)
data.MemoryBandwidth = append(data.MemoryBandwidth, point.MemoryBandwidth)
data.MediaFrequency = append(data.MediaFrequency, point.MediaFrequency)
data.HugepagesUsed = append(data.HugepagesUsed, point.HugepagesUsed)
data.HugepagesTotal = append(data.HugepagesTotal, point.HugepagesTotal)
data.GPUValue = append(data.GPUValue, point.GPUUtil)
data.TemperatureValue = append(data.TemperatureValue, point.Temperature)
data.MemoryTemperatureValue = append(data.MemoryTemperatureValue, point.MemoryTemperature)
data.PowerUsed = append(data.PowerUsed, point.PowerDraw)
data.PowerTotal = append(data.PowerTotal, point.MaxPowerLimit)
data.PowerPercent = append(data.PowerPercent, point.PowerPercent)
data.MemoryPercent = append(data.MemoryPercent, point.MemoryPercent)
data.MemoryTotal = append(data.MemoryTotal, point.MemTotal)
data.MemoryUsed = append(data.MemoryUsed, point.MemUsed)
data.SpeedValue = append(data.SpeedValue, point.FanSpeed)
data.FrequencyValue = append(data.FrequencyValue, point.Frequency)
data.MemoryFrequencyValue = append(data.MemoryFrequencyValue, point.MemoryFrequency)
data.ProcessCount = append(data.ProcessCount, point.ProcessCount)
var processes []dto.GPUProcess
if data.BucketSeconds == 0 && point.ProcessCount != nil {
_ = json.Unmarshal([]byte(point.Processes), &processes)
}
data.SpeedValue = append(data.SpeedValue, gpu.FanSpeed)
data.GPUProcesses = append(data.GPUProcesses, processes)
}
return data, nil
}
@@ -307,7 +327,6 @@ func (m *MonitorService) CleanData() error {
}
func (m *MonitorService) Run() {
saveAcceleratorDataToDB()
var itemModel model.MonitorBase
totalPercent, _ := cpu.Percent(3*time.Second, false)
if len(totalPercent) == 1 {
@@ -343,6 +362,7 @@ func (m *MonitorService) Run() {
m.loadDiskIO()
m.loadNetIO()
m.saveGPUData()
MonitorStoreDays, err := settingRepo.Get(settingRepo.WithByKey("MonitorStoreDays"))
if err != nil {
@@ -602,7 +622,62 @@ func StartMonitor(removeBefore bool, interval string) error {
return nil
}
func saveAcceleratorDataToDB() {
func loadGPUOptions(snapshot *accelerator.Snapshot) dto.MonitorGPUOptions {
var data dto.MonitorGPUOptions
hasGPUOrNPU := false
hasXPU := false
for _, item := range snapshot.Devices {
if item.Kind == accelerator.KindXPU {
hasXPU = true
} else {
hasGPUOrNPU = true
}
}
switch {
case hasGPUOrNPU && hasXPU:
data.GPUType = "mixed"
case hasXPU:
data.GPUType = "xpu"
case hasGPUOrNPU:
data.GPUType = "gpu"
}
sort.Slice(snapshot.Devices, func(i, j int) bool {
if snapshot.Devices[i].Kind != snapshot.Devices[j].Kind {
return snapshot.Devices[i].Kind < snapshot.Devices[j].Kind
}
if snapshot.Devices[i].Vendor != snapshot.Devices[j].Vendor {
return snapshot.Devices[i].Vendor < snapshot.Devices[j].Vendor
}
if snapshot.Devices[i].NPUIndex != snapshot.Devices[j].NPUIndex {
return snapshot.Devices[i].NPUIndex < snapshot.Devices[j].NPUIndex
}
if snapshot.Devices[i].ChipIndex != snapshot.Devices[j].ChipIndex {
return snapshot.Devices[i].ChipIndex < snapshot.Devices[j].ChipIndex
}
return snapshot.Devices[i].Index < snapshot.Devices[j].Index
})
for _, item := range snapshot.Devices {
chartHide := dto.GPUChartHide{
DeviceID: item.ID,
ProductName: item.Label,
Type: string(item.Kind),
}
data.ChartHide = append(data.ChartHide, chartHide)
data.Options = append(data.Options, chartHide.ProductName)
}
return data
}
func (m *MonitorService) saveGPUData() {
status, err := settingRepo.GetValueByKey("MonitorStatus")
if err != nil {
global.LOG.Errorf("load monitor status failed: %v", err)
return
}
if status != constant.StatusEnable {
return
}
exist, client := accelerator.New()
if !exist {
return
@@ -615,27 +690,64 @@ func saveAcceleratorDataToDB() {
if warning := snapshot.Warning(); warning != nil {
global.LOG.Warnf("load accelerator monitor data partially failed, err: %v", warning)
}
intervalSeconds := 0
if setting, err := settingRepo.Get(settingRepo.WithByKey("MonitorInterval")); err == nil {
intervalSeconds, _ = strconv.Atoi(setting.Value)
}
list := make([]model.MonitorGPU, 0, len(snapshot.Devices))
for _, device := range snapshot.Devices {
list = append(list, newMonitorGPU(device))
item := newMonitorGPU(device)
item.CreatedAt = snapshot.Info.CollectedAt
item.IntervalSeconds = intervalSeconds
list = append(list, item)
}
if err := repo.NewIMonitorRepo().BatchCreateMonitorGPU(list); err != nil {
if err := monitorRepo.BatchCreateMonitorGPU(list); err != nil {
global.LOG.Errorf("batch create accelerator monitor data failed, err: %v", err)
}
}
func newMonitorGPU(device accelerator.Device) model.MonitorGPU {
item := model.MonitorGPU{
ProductName: device.Label,
GPUUtil: device.Metrics.Utilization.ValueOrZero(),
Temperature: device.Metrics.Temperature.ValueOrZero(),
PowerDraw: device.Metrics.Power.ValueOrZero(),
MaxPowerLimit: device.Metrics.PowerLimit.ValueOrZero(),
MemUsed: device.Metrics.MemoryUsed.ValueOrZero(),
MemTotal: device.Metrics.MemoryTotal.ValueOrZero(),
FanSpeed: int(device.Metrics.FanSpeed.ValueOrZero()),
MemoryUtil: device.Metrics.MemoryUtil.Value,
MemoryActivity: device.Metrics.MemoryActivity.Value,
EncoderUtil: device.Metrics.EncoderUtil.Value,
DecoderUtil: device.Metrics.DecoderUtil.Value,
JPEGUtil: device.Metrics.JPEGUtil.Value,
OFAUtil: device.Metrics.OFAUtil.Value,
MediaUtil: device.Metrics.MediaUtil.Value,
ComputeUtil: device.Metrics.ComputeUtil.Value,
CopyUtil: device.Metrics.CopyUtil.Value,
HotspotTemperature: device.Metrics.HotspotTemperature.Value,
FanRPM: device.Metrics.FanRPM.Value,
AICPUUtil: device.Metrics.AICPUUtil.Value,
CtrlCPUUtil: device.Metrics.CtrlCPUUtil.Value,
DDRUsed: device.Metrics.DDRUsed.Value,
DDRTotal: device.Metrics.DDRTotal.Value,
HBMUsed: device.Metrics.HBMUsed.Value,
HBMTotal: device.Metrics.HBMTotal.Value,
DDRBandwidth: device.Metrics.DDRBandwidth.Value,
HBMBandwidth: device.Metrics.HBMBandwidth.Value,
MemoryBandwidth: device.Metrics.MemoryBandwidth.Value,
MediaFrequency: device.Metrics.MediaFrequency.Value,
HugepagesUsed: device.Metrics.HugepagesUsed.Value,
HugepagesTotal: device.Metrics.HugepagesTotal.Value,
ProductName: device.Label,
DeviceID: device.ID,
DeviceType: string(device.Kind),
ProcessStatus: device.ProcessStatus,
Frequency: device.Metrics.Frequency.Value,
MemoryFrequency: device.Metrics.MemoryFrequency.Value,
MemoryTemperature: device.Metrics.MemoryTemperature.Value,
GPUUtil: device.Metrics.Utilization.Value,
Temperature: device.Metrics.Temperature.Value,
PowerDraw: device.Metrics.Power.Value,
MaxPowerLimit: device.Metrics.PowerLimit.Value,
MemUsed: device.Metrics.MemoryUsed.Value,
MemTotal: device.Metrics.MemoryTotal.Value,
FanSpeed: device.Metrics.FanSpeed.Value,
}
if len(device.Processes) == 0 {
if device.ProcessStatus != "ok" {
return item
}
processes := make([]dto.GPUProcess, 0, len(device.Processes))
+1
View File
@@ -111,6 +111,7 @@ func agentDBMigrations() []*gormigrate.Migration {
migrations.SimplifyFirewallRulePolicy,
migrations.AddDockerPortGuardReadOnly,
migrations.MigrateFirewallPortWhitelistSources,
migrations.AddAcceleratorMetrics,
}
}
+7
View File
@@ -1908,3 +1908,10 @@ var AddDockerPortGuardReadOnly = &gormigrate.Migration{
return tx.Migrator().CreateIndex(&model.DockerPortGuardPolicy{}, "idx_docker_port_guard_endpoint")
},
}
var AddAcceleratorMetrics = &gormigrate.Migration{
ID: "20260928-accelerator-vendor-metrics",
Migrate: func(tx *gorm.DB) error {
return global.GPUMonitorDB.AutoMigrate(&model.MonitorGPU{})
},
}
@@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu"
@@ -52,6 +53,11 @@ func (c Client) LoadInfo() (*Info, error) {
}
func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
results := make([]providerResult, len(c.providers))
var wg sync.WaitGroup
for index, item := range c.providers {
@@ -66,8 +72,11 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
}()
}
wg.Wait()
if err := ctx.Err(); err != nil {
return nil, err
}
snapshot := &Snapshot{DriverVersions: make(map[string]string)}
snapshot := &Snapshot{}
var (
errs []error
active []*ProviderSnapshot
@@ -78,6 +87,11 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
errs = append(errs, result.err)
continue
}
if result.snapshot != nil {
for _, warning := range result.snapshot.Warnings {
errs = append(errs, errors.New(warning))
}
}
if result.snapshot == nil || len(result.snapshot.Devices) == 0 {
continue
}
@@ -90,11 +104,8 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
if item.CudaVersion != "" {
snapshot.Info.CudaVersion = item.CudaVersion
}
if item.DriverVersion != "" {
snapshot.DriverVersions[item.Type] = item.DriverVersion
if item.Type == "xpu" {
xpuVersion = item.DriverVersion
}
if item.Type == "xpu" {
xpuVersion = item.DriverVersion
}
}
snapshot.Warnings = append(snapshot.Warnings, errs...)
@@ -102,6 +113,10 @@ func (c Client) Collect(ctx context.Context) (*Snapshot, error) {
return nil, fmt.Errorf("calling accelerator monitoring tools failed: %w", errors.Join(errs...))
}
snapshot.Info.CollectedAt = time.Now()
for _, err := range errs {
snapshot.Info.Warnings = append(snapshot.Info.Warnings, err.Error())
}
snapshot.Info.XPUDriverVersion = xpuVersion
snapshot.Info.Type, snapshot.Info.DriverVersion = mergeProviderMetadata(active)
return snapshot, nil
+3 -2
View File
@@ -1,6 +1,7 @@
package accelerator
import (
"math"
"strconv"
"strings"
@@ -18,11 +19,11 @@ func metric(display, unit string) Metric {
return result
}
parsed, err := strconv.ParseFloat(matched[1], 64)
if err != nil {
if err != nil || math.IsNaN(parsed) || math.IsInf(parsed, 0) || (parsed < 0 && unit != "°C") {
return result
}
normalized, ok := convertMetricUnit(parsed, matched[2], unit)
if !ok {
if !ok || math.IsNaN(normalized) || math.IsInf(normalized, 0) || (unit == "%" && normalized > 100) {
return result
}
result.Value = &normalized
+79 -63
View File
@@ -12,26 +12,38 @@ import (
func normalizeGPU(item *gpu.Device) Device {
metrics := Metrics{
Utilization: metric(item.GPUUtil, "%"),
Temperature: metric(item.Temperature, "°C"),
Power: metric(item.PowerDraw, "W"),
PowerLimit: metric(item.MaxPowerLimit, "W"),
MemoryUsed: memoryMetric(item.MemUsed),
MemoryTotal: memoryMetric(item.MemTotal),
FanSpeed: metric(item.FanSpeed, "%"),
MemoryActivity: metric(item.MemoryActivity, "%"),
EncoderUtil: metric(item.EncoderUtil, "%"),
DecoderUtil: metric(item.DecoderUtil, "%"),
JPEGUtil: metric(item.JPEGUtil, "%"),
OFAUtil: metric(item.OFAUtil, "%"),
MediaUtil: metric(item.MediaUtil, "%"),
HotspotTemperature: metric(item.HotspotTemperature, "°C"),
FanRPM: metric(item.FanRPM, "RPM"),
MediaFrequency: metric(item.MediaFrequency, "MHz"),
Utilization: metric(item.GPUUtil, "%"),
Temperature: metric(item.Temperature, "°C"),
MemoryTemperature: metric(item.MemoryTemperature, "°C"),
Power: metric(item.PowerDraw, "W"),
PowerLimit: metric(item.PowerLimit, "W"),
Frequency: metric(item.Frequency, "MHz"),
MemoryFrequency: metric(item.MemoryFrequency, "MHz"),
MemoryUsed: memoryMetric(item.MemUsed),
MemoryTotal: memoryMetric(item.MemTotal),
FanSpeed: metric(item.FanSpeed, "%"),
}
device := Device{
ID: stableID(item.Type, item.BusID, strconv.FormatUint(uint64(item.Index), 10)),
Kind: KindGPU,
Vendor: item.Type,
Index: int(item.Index),
Name: item.ProductName,
Label: fmt.Sprintf("%d - %s", item.Index, item.ProductName),
BusID: item.BusID,
Metrics: metrics,
GPU: item,
ProcessStatus: item.ProcessStatus,
ID: stableID(item.Type, item.UUID, stableID("pci", item.BusID, strconv.FormatUint(uint64(item.Index), 10))),
Kind: KindGPU,
Vendor: item.Type,
Index: int(item.Index),
Name: item.ProductName,
Label: fmt.Sprintf("%d - %s", item.Index, item.ProductName),
BusID: item.BusID,
Metrics: metrics,
GPU: item,
}
device.Capabilities = capabilities(metrics)
for _, process := range item.Processes {
device.Processes = append(device.Processes, Process{
PID: process.PID,
@@ -45,26 +57,36 @@ func normalizeGPU(item *gpu.Device) Device {
func normalizeNPU(item *npu.Device) Device {
metrics := Metrics{
Utilization: metric(item.AICore, "%"),
Temperature: metric(item.Temperature, "°C"),
Power: metric(item.PowerDraw, "W"),
MemoryUsed: memoryMetric(item.MemUsed),
MemoryTotal: memoryMetric(item.MemTotal),
AICPUUtil: metric(item.AICPUUtil, "%"),
CtrlCPUUtil: metric(item.CtrlCPUUtil, "%"),
DDRBandwidth: metric(item.DDRBandwidth, "%"),
HBMBandwidth: metric(item.HBMBandwidth, "%"),
DDRUsed: metric(item.MemoryUsed, "MiB"),
DDRTotal: metric(item.MemoryTotal, "MiB"),
HBMUsed: metric(item.HBMUsed, "MiB"),
HBMTotal: metric(item.HBMTotal, "MiB"),
HugepagesUsed: metric(item.HugepagesUsed, "pages"),
HugepagesTotal: metric(item.HugepagesTotal, "pages"),
Utilization: metric(item.AICore, "%"),
Temperature: metric(item.Temperature, "°C"),
Power: metric(item.PowerDraw, "W"),
MemoryUsed: memoryMetric(item.MemUsed),
MemoryTotal: memoryMetric(item.MemTotal),
}
device := Device{
ID: fmt.Sprintf("ascend:%d:%d", item.NPUIndex, item.ChipIndex),
Kind: KindNPU,
Vendor: "ascend",
Index: int(item.Index),
NPUIndex: int(item.NPUIndex),
ChipIndex: int(item.ChipIndex),
Name: item.ProductName,
Label: fmt.Sprintf("NPU %d / Chip %d - %s", item.NPUIndex, item.ChipIndex, item.ProductName),
BusID: item.BusID,
Metrics: metrics,
NPU: item,
ProcessStatus: item.ProcessStatus,
ID: fmt.Sprintf("%s:%d", stableID("ascend", item.BusID, strconv.FormatUint(uint64(item.NPUIndex), 10)), item.ChipIndex),
Kind: KindNPU,
Vendor: "ascend",
Index: int(item.Index),
NPUIndex: int(item.NPUIndex),
ChipIndex: int(item.ChipIndex),
Name: item.ProductName,
Label: fmt.Sprintf("NPU %d / Chip %d - %s", item.NPUIndex, item.ChipIndex, item.ProductName),
BusID: item.BusID,
Metrics: metrics,
NPU: item,
}
device.Capabilities = capabilities(metrics)
for _, process := range item.Processes {
device.Processes = append(device.Processes, Process{
PID: process.PID,
@@ -78,26 +100,32 @@ func normalizeNPU(item *npu.Device) Device {
func normalizeXPU(item *xpu.Device) Device {
metrics := Metrics{
Utilization: metric(item.Stats.GPUUtil, "%"),
Temperature: metric(item.Stats.Temperature, "°C"),
Power: metric(item.Stats.Power, "W"),
MemoryUsed: memoryMetric(item.Stats.MemoryUsed),
MemoryTotal: memoryMetric(item.Basic.Memory),
MemoryUtil: metric(item.Stats.MemoryUtil, "%"),
Frequency: metric(item.Stats.Frequency, "MHz"),
MediaUtil: metric(item.Stats.MediaUtil, "%"),
ComputeUtil: metric(item.Stats.ComputeUtil, "%"),
CopyUtil: metric(item.Stats.CopyUtil, "%"),
MediaFrequency: metric(item.Stats.MediaFrequency, "MHz"),
MemoryTemperature: metric(item.Stats.MemoryTemperature, "°C"),
MemoryBandwidth: metric(item.Stats.MemoryBandwidthUtil, "%"),
Utilization: metric(item.Stats.GPUUtil, "%"),
Temperature: metric(item.Stats.Temperature, "°C"),
Power: metric(item.Stats.Power, "W"),
MemoryUsed: memoryMetric(item.Stats.MemoryUsed),
MemoryTotal: memoryMetric(item.Basic.Memory),
MemoryUtil: metric(item.Stats.MemoryUtil, "%"),
Frequency: metric(item.Stats.Frequency, "MHz"),
}
device := Device{
ID: stableID("xpu", item.Basic.PciBdfAddress, strconv.Itoa(item.Basic.DeviceID)),
Kind: KindXPU,
Vendor: item.Basic.VendorName,
Index: item.Basic.DeviceID,
Name: item.Basic.DeviceName,
Label: fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName),
BusID: item.Basic.PciBdfAddress,
Metrics: metrics,
XPU: item,
ProcessStatus: item.ProcessStatus,
ID: stableID("xpu", item.Basic.UUID, stableID("pci", item.Basic.PciBdfAddress, strconv.Itoa(item.Basic.DeviceID))),
Kind: KindXPU,
Vendor: item.Basic.VendorName,
Index: item.Basic.DeviceID,
Name: item.Basic.DeviceName,
Label: fmt.Sprintf("%d - %s", item.Basic.DeviceID, item.Basic.DeviceName),
BusID: item.Basic.PciBdfAddress,
Metrics: metrics,
XPU: item,
}
device.Capabilities = capabilities(metrics)
for _, process := range item.Processes {
device.Processes = append(device.Processes, Process{
PID: strconv.Itoa(process.PID),
@@ -110,18 +138,6 @@ func normalizeXPU(item *xpu.Device) Device {
return device
}
func capabilities(metrics Metrics) Capabilities {
return Capabilities{
Utilization: metrics.Utilization.Available(),
Temperature: metrics.Temperature.Available(),
Power: metrics.Power.Available(),
PowerLimit: metrics.PowerLimit.Available(),
Memory: metrics.MemoryUsed.Available() || metrics.MemoryTotal.Available(),
FanSpeed: metrics.FanSpeed.Available(),
Frequency: metrics.Frequency.Available(),
}
}
func stableID(vendor, busID, fallback string) string {
if busID != "" && !strings.EqualFold(busID, "N/A") {
return vendor + ":" + busID
+17 -1
View File
@@ -2,6 +2,7 @@ package accelerator
import (
"context"
"fmt"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu"
@@ -26,6 +27,7 @@ func (p gpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) {
}
result := &ProviderSnapshot{
Type: info.Type,
Warnings: info.Warnings,
DriverVersion: info.DriverVersion,
CudaVersion: info.CudaVersion,
GPUs: info.Devices,
@@ -49,6 +51,7 @@ func (p npuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) {
}
result := &ProviderSnapshot{
Type: info.Type,
Warnings: info.Warnings,
DriverVersion: info.DriverVersion,
NPUs: info.Devices,
}
@@ -71,11 +74,24 @@ func (p xpuProvider) Collect(ctx context.Context) (*ProviderSnapshot, error) {
}
result := &ProviderSnapshot{
Type: info.Type,
Warnings: info.Warnings,
DriverVersion: info.DriverVersion,
XPUs: info.Devices,
}
for index := range result.XPUs {
result.Devices = append(result.Devices, normalizeXPU(&result.XPUs[index]))
parent := normalizeXPU(&result.XPUs[index])
result.Devices = append(result.Devices, parent)
for _, tile := range result.XPUs[index].Tiles {
raw := xpu.Device{Basic: result.XPUs[index].Basic, Stats: tile.Stats, ProcessStatus: "unavailable"}
raw.Basic.Memory = ""
raw.Basic.FreeMemory = ""
device := normalizeXPU(&raw)
device.ParentID = parent.ID
device.ID = fmt.Sprintf("%s:tile:%d", parent.ID, tile.TileID)
device.Label = fmt.Sprintf("%s / Tile %d", parent.Label, tile.TileID)
result.Devices = append(result.Devices, device)
}
}
return result, nil
}
+54 -34
View File
@@ -2,6 +2,7 @@ package accelerator
import (
"errors"
"time"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/gpu"
"github.com/1Panel-dev/1Panel/agent/utils/ai_tools/npu"
@@ -17,6 +18,9 @@ const (
)
type Info struct {
CollectedAt time.Time `json:"collectedAt"`
Warnings []string `json:"warnings"`
Type string `json:"type"`
CudaVersion string `json:"cudaVersion"`
DriverVersion string `json:"driverVersion"`
@@ -44,25 +48,40 @@ func (m Metric) ValueOrZero() float64 {
}
type Metrics struct {
Utilization Metric
Temperature Metric
Power Metric
PowerLimit Metric
MemoryUsed Metric
MemoryTotal Metric
MemoryUtil Metric
FanSpeed Metric
Frequency Metric
}
MemoryActivity Metric
EncoderUtil Metric
DecoderUtil Metric
JPEGUtil Metric
OFAUtil Metric
MediaUtil Metric
ComputeUtil Metric
CopyUtil Metric
HotspotTemperature Metric
FanRPM Metric
AICPUUtil Metric
CtrlCPUUtil Metric
DDRUsed Metric
DDRTotal Metric
HBMUsed Metric
HBMTotal Metric
DDRBandwidth Metric
HBMBandwidth Metric
MemoryBandwidth Metric
MediaFrequency Metric
HugepagesUsed Metric
HugepagesTotal Metric
type Capabilities struct {
Utilization bool
Temperature bool
Power bool
PowerLimit bool
Memory bool
FanSpeed bool
Frequency bool
MemoryTemperature Metric
Utilization Metric
Temperature Metric
Power Metric
PowerLimit Metric
MemoryUsed Metric
MemoryTotal Metric
MemoryUtil Metric
FanSpeed Metric
Frequency Metric
MemoryFrequency Metric
}
type Process struct {
@@ -74,18 +93,19 @@ type Process struct {
}
type Device struct {
ID string
Kind Kind
Vendor string
Index int
NPUIndex int
ChipIndex int
Name string
Label string
BusID string
Metrics Metrics
Capabilities Capabilities
Processes []Process
ParentID string
ProcessStatus string
ID string
Kind Kind
Vendor string
Index int
NPUIndex int
ChipIndex int
Name string
Label string
BusID string
Metrics Metrics
Processes []Process
GPU *gpu.Device `json:"-"`
NPU *npu.Device `json:"-"`
@@ -93,10 +113,9 @@ type Device struct {
}
type Snapshot struct {
Info Info
Devices []Device
DriverVersions map[string]string
Warnings []error
Info Info
Devices []Device
Warnings []error
}
func (s Snapshot) Warning() error {
@@ -104,6 +123,7 @@ func (s Snapshot) Warning() error {
}
type ProviderSnapshot struct {
Warnings []string
Type string
DriverVersion string
CudaVersion string
+58 -17
View File
@@ -3,12 +3,13 @@ package gpu
import (
"context"
"fmt"
"math"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
)
@@ -32,16 +33,22 @@ func findAMDSMI() (string, bool) {
func (a amdSMI) LoadInfo(ctx context.Context) (*Info, error) {
var (
staticData string
metricData string
processData string
staticErr error
metricErr error
processErr error
wg sync.WaitGroup
staticData string
metricData string
extendedData string
extendedErr error
processData string
staticErr error
metricErr error
processErr error
wg sync.WaitGroup
)
wg.Add(3)
wg.Add(4)
go func() {
defer wg.Done()
extendedData, extendedErr = runAMDSMI(ctx, a.command, "metric", "--clock", "--ecc", "--json")
}()
go func() {
defer wg.Done()
staticData, staticErr = runAMDSMI(ctx, a.command, "static", "--asic", "--bus", "--driver", "--limit", "--json")
@@ -64,16 +71,25 @@ func (a amdSMI) LoadInfo(ctx context.Context) (*Info, error) {
return nil, fmt.Errorf("parsing %s static output failed: %w", a.command, err)
}
if metricErr == nil {
metricErr = applyAMDMetrics(info, metricData)
}
if metricErr != nil {
global.LOG.Warnf("calling %s metric failed, metrics will be omitted: %v", a.command, metricErr)
} else if err := applyAMDMetrics(info, metricData); err != nil {
global.LOG.Warnf("parsing %s metric output failed, metrics will be omitted: %v", a.command, err)
info.Warnings = append(info.Warnings, fmt.Sprintf("%s metrics: %v", a.command, metricErr))
}
if extendedErr == nil {
extendedErr = applyAMDMetrics(info, extendedData)
}
if extendedErr != nil {
info.Warnings = append(info.Warnings, fmt.Sprintf("%s extended metrics: %v", a.command, extendedErr))
}
if processErr == nil {
processErr = applyAMDProcesses(info, processData)
}
if processErr != nil {
global.LOG.Warnf("calling %s process failed, process information will be omitted: %v", a.command, processErr)
} else if err := applyAMDProcesses(info, processData); err != nil {
global.LOG.Warnf("parsing %s process output failed, process information will be omitted: %v", a.command, err)
info.Warnings = append(info.Warnings, fmt.Sprintf("%s processes: %v", a.command, processErr))
}
return info, nil
}
@@ -96,6 +112,9 @@ func parseAMDStatic(data string) (*Info, error) {
}
device := Device{
Type: "amd",
UUID: amdStringAt(row, "uuid", "asic.uuid"),
DriverVersion: amdStringAt(row, "driver.version", "driver_version", "amdgpu_version"),
ProcessStatus: "unavailable",
Index: index,
ProductName: amdStringAt(row, "asic.market_name", "market_name", "gpu_name"),
PersistenceMode: "N/A",
@@ -111,6 +130,7 @@ func parseAMDStatic(data string) (*Info, error) {
"limit.max_power_limit",
"limit.max_power",
),
PowerLimit: amdMetricAt(row, "W", "limit.ppt0.socket_power_limit", "limit.socket_power_limit"),
MemUsed: "N/A",
MemTotal: "N/A",
GPUUtil: "N/A",
@@ -150,11 +170,31 @@ func applyAMDMetrics(info *Info, data string) error {
continue
}
setAMDMetric(&device.GPUUtil, row, "%", "usage.gfx_activity", "usage.gfx", "gfx_activity", "gfx_usage")
setAMDMetric(&device.Temperature, row, "°C", "temperature.hotspot", "temperature.edge", "hotspot_temperature", "gpu_temperature", "gpu_temp")
setAMDMetric(&device.Temperature, row, "°C", "temperature.edge", "gpu_temperature", "gpu_temp")
setAMDMetric(&device.PowerDraw, row, "W", "power.socket_power", "socket_power", "power_usage")
setAMDMetric(&device.MemUsed, row, "MB", "mem_usage.used_vram", "vram.used", "used_vram", "vram_used")
setAMDMetric(&device.MemTotal, row, "MB", "mem_usage.total_vram", "vram.total", "total_vram", "vram_total")
setAMDMetric(&device.FanSpeed, row, "%", "fan.speed", "fan_speed")
setAMDMetric(&device.FanSpeed, row, "%", "fan.usage")
if amdMetricAt(row, "%", "fan.usage") == "" {
speed, speedErr := strconv.ParseFloat(amdStringAt(row, "fan.speed"), 64)
maximum, maxErr := strconv.ParseFloat(amdStringAt(row, "fan.max"), 64)
if speedErr == nil && maxErr == nil && !math.IsNaN(speed) && !math.IsInf(speed, 0) && maximum > 0 && !math.IsInf(maximum, 0) && speed >= 0 && speed <= maximum {
device.FanSpeed = fmt.Sprintf("%.2f %%", speed/maximum*100)
}
}
setAMDMetric(&device.FanRPM, row, "RPM", "fan.rpm")
setAMDMetric(&device.HotspotTemperature, row, "°C", "temperature.hotspot", "hotspot_temperature")
setAMDMetric(&device.MemoryTemperature, row, "°C", "temperature.mem")
setAMDMetric(&device.MemoryActivity, row, "%", "usage.umc_activity")
setAMDMetric(&device.MediaUtil, row, "%", "usage.mm_activity")
setAMDMetric(&device.Frequency, row, "MHz", "clock.gfx_0.clk")
setAMDMetric(&device.MemoryFrequency, row, "MHz", "clock.mem_0.clk")
correctable := amdStringAt(row, "ecc.total_correctable_count", "ecc.correctable_count")
uncorrectable := amdStringAt(row, "ecc.total_uncorrectable_count", "ecc.uncorrectable_count")
if correctable != "" || uncorrectable != "" {
device.ECCErrors = []ECCError{{Scope: "Total", Correctable: correctable, Uncorrectable: uncorrectable}}
}
if value := amdStringAt(row, "perf_level", "performance_level"); value != "" {
device.PerformanceState = value
}
@@ -177,6 +217,7 @@ func applyAMDProcesses(info *Info, data string) error {
if !ok {
continue
}
device.ProcessStatus = "ok"
processList, _ := amdValueAt(row, "process_list")
items := amdObjectList(processList)
if len(items) == 0 {
+4
View File
@@ -64,6 +64,7 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) {
if result.info == nil {
continue
}
merged.Warnings = append(merged.Warnings, result.info.Warnings...)
if result.info.Type != "" {
types = append(types, result.info.Type)
}
@@ -86,6 +87,9 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) {
if len(merged.Devices) == 0 && len(errs) > 0 {
return nil, fmt.Errorf("calling GPU monitoring tools failed: %w", errors.Join(errs...))
}
for _, err := range errs {
merged.Warnings = append(merged.Warnings, err.Error())
}
return merged, nil
}
+76 -69
View File
@@ -1,16 +1,12 @@
package gpu
import (
"bytes"
"context"
"encoding/xml"
"errors"
"fmt"
"io"
"strings"
"time"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
)
@@ -24,41 +20,10 @@ func (n nvidiaSMI) LoadInfo(ctx context.Context) (*Info, error) {
if err != nil {
return nil, fmt.Errorf("calling %s failed: %w", nvidiaSMICommand, err)
}
data := []byte(itemData)
version := "v11"
buf := bytes.NewBuffer(data)
decoder := xml.NewDecoder(buf)
for {
token, err := decoder.Token()
if err != nil {
if errors.Is(err, io.EOF) {
break
}
return nil, fmt.Errorf("reading token failed: %w", err)
}
d, ok := token.(xml.Directive)
if !ok {
continue
}
directive := string(d)
if !strings.HasPrefix(directive, "DOCTYPE") {
continue
}
parts := strings.Split(directive, " ")
s := strings.Trim(parts[len(parts)-1], "\" ")
if strings.HasPrefix(s, "nvsmi_device_") && strings.HasSuffix(s, ".dtd") {
version = strings.TrimSuffix(strings.TrimPrefix(s, "nvsmi_device_"), ".dtd")
} else {
global.LOG.Debugf("Cannot find schema version in %q", directive)
}
break
}
return parseNvidiaSMI(data, version)
return parseNvidiaSMI([]byte(itemData))
}
func parseNvidiaSMI(buf []byte, version string) (*Info, error) {
func parseNvidiaSMI(buf []byte) (*Info, error) {
var (
s nvidiaSMIResponse
info Info
@@ -72,42 +37,84 @@ func parseNvidiaSMI(buf []byte, version string) (*Info, error) {
info.DriverVersion = s.DriverVersion
for i := range s.Gpu {
gpuItem := Device{
Type: "nvidia",
Index: uint(i),
ProductName: s.Gpu[i].ProductName,
PersistenceMode: s.Gpu[i].PersistenceMode,
BusID: s.Gpu[i].ID,
DisplayActive: s.Gpu[i].DisplayActive,
ECC: s.Gpu[i].EccErrors.Volatile.DramUncorrectable,
FanSpeed: s.Gpu[i].FanSpeed,
Temperature: s.Gpu[i].Temperature.GpuTemp,
PerformanceState: s.Gpu[i].PerformanceState,
MemUsed: s.Gpu[i].FbMemoryUsage.Used,
MemTotal: s.Gpu[i].FbMemoryUsage.Total,
GPUUtil: s.Gpu[i].Utilization.GpuUtil,
ComputeMode: s.Gpu[i].ComputeMode,
MigMode: s.Gpu[i].MigMode.CurrentMig,
}
if version == "v12" || version == "v13" {
gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.PowerDraw
if gpuItem.PowerDraw == "" {
gpuItem.PowerDraw = s.Gpu[i].GpuPowerReadings.InstantPowerDraw
}
gpuItem.MaxPowerLimit = s.Gpu[i].GpuPowerReadings.CurrentPowerLimit
} else {
gpuItem.PowerDraw = s.Gpu[i].PowerReadings.PowerDraw
gpuItem.MaxPowerLimit = s.Gpu[i].PowerReadings.MaxPowerLimit
Type: "nvidia",
MemoryActivity: s.Gpu[i].Utilization.MemoryUtil,
EncoderUtil: s.Gpu[i].Utilization.EncoderUtil,
DecoderUtil: s.Gpu[i].Utilization.DecoderUtil,
JPEGUtil: s.Gpu[i].Utilization.JpegUtil,
OFAUtil: s.Gpu[i].Utilization.OfaUtil,
MediaFrequency: s.Gpu[i].Clocks.VideoClock,
UUID: s.Gpu[i].UUID,
DriverVersion: s.DriverVersion,
Architecture: s.Gpu[i].ProductArchitecture,
Frequency: s.Gpu[i].Clocks.GraphicsClock,
MemoryFrequency: s.Gpu[i].Clocks.MemClock,
MemoryTemperature: s.Gpu[i].Temperature.MemoryTemp,
MemoryFree: s.Gpu[i].FbMemoryUsage.Free,
MemoryReserved: s.Gpu[i].FbMemoryUsage.Reserved,
PCIeGeneration: firstSMIValue(s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.DeviceCurrentLinkGen, s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.CurrentLinkGen),
PCIeMaxGeneration: firstSMIValue(s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.MaxDeviceLinkGen, s.Gpu[i].Pci.PciGpuLinkInfo.PcieGen.MaxLinkGen),
PCIeWidth: s.Gpu[i].Pci.PciGpuLinkInfo.LinkWidths.CurrentLinkWidth,
PCIeMaxWidth: s.Gpu[i].Pci.PciGpuLinkInfo.LinkWidths.MaxLinkWidth,
ProcessStatus: "unavailable",
Index: uint(i),
ProductName: s.Gpu[i].ProductName,
PersistenceMode: s.Gpu[i].PersistenceMode,
BusID: s.Gpu[i].ID,
DisplayActive: s.Gpu[i].DisplayActive,
ECC: s.Gpu[i].EccMode.CurrentEcc,
FanSpeed: s.Gpu[i].FanSpeed,
Temperature: s.Gpu[i].Temperature.GpuTemp,
PerformanceState: s.Gpu[i].PerformanceState,
MemUsed: s.Gpu[i].FbMemoryUsage.Used,
MemTotal: s.Gpu[i].FbMemoryUsage.Total,
GPUUtil: s.Gpu[i].Utilization.GpuUtil,
ComputeMode: s.Gpu[i].ComputeMode,
MigMode: s.Gpu[i].MigMode.CurrentMig,
}
gpuItem.ECCPending = s.Gpu[i].EccMode.PendingEcc
gpuItem.ECCErrors = []ECCError{
{Scope: "Volatile Total", Correctable: s.Gpu[i].EccErrors.Volatile.SingleBit.Total, Uncorrectable: s.Gpu[i].EccErrors.Volatile.DoubleBit.Total},
{Scope: "Aggregate Total", Correctable: s.Gpu[i].EccErrors.Aggregate.SingleBit.Total, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.DoubleBit.Total},
for _, process := range s.Gpu[i].Processes.ProcessInfo {
gpuItem.Processes = append(gpuItem.Processes, Process{
PID: process.Pid,
Type: process.Type,
ProcessName: process.ProcessName,
UsedMemory: process.UsedMemory,
})
{Scope: "Volatile DRAM", Correctable: s.Gpu[i].EccErrors.Volatile.DramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Volatile.DramUncorrectable},
{Scope: "Volatile SRAM", Correctable: s.Gpu[i].EccErrors.Volatile.SramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Volatile.SramUncorrectable},
{Scope: "Aggregate DRAM", Correctable: s.Gpu[i].EccErrors.Aggregate.DramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.DramUncorrectable},
{Scope: "Aggregate SRAM", Correctable: s.Gpu[i].EccErrors.Aggregate.SramCorrectable, Uncorrectable: s.Gpu[i].EccErrors.Aggregate.SramUncorrectable},
}
gpuItem.PowerDraw = firstSMIValue(s.Gpu[i].GpuPowerReadings.PowerDraw, s.Gpu[i].GpuPowerReadings.InstantPowerDraw, s.Gpu[i].PowerReadings.PowerDraw)
gpuItem.PowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.CurrentPowerLimit, s.Gpu[i].PowerReadings.EnforcedPowerLimit, s.Gpu[i].PowerReadings.PowerLimit)
gpuItem.MaxPowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.MaxPowerLimit, s.Gpu[i].PowerReadings.MaxPowerLimit)
gpuItem.DefaultPowerLimit = firstSMIValue(s.Gpu[i].GpuPowerReadings.DefaultPowerLimit, s.Gpu[i].PowerReadings.DefaultPowerLimit)
for _, event := range append(s.Gpu[i].ClocksEventReasons.Reasons, s.Gpu[i].ClocksThrottleReasons.Reasons...) {
if strings.EqualFold(strings.TrimSpace(event.Value), "Active") {
gpuItem.ClockEvents = append(gpuItem.ClockEvents, strings.TrimPrefix(strings.TrimPrefix(event.XMLName.Local, "clocks_event_reason_"), "clocks_throttle_reason_"))
}
}
if s.Gpu[i].Processes != nil && strings.TrimSpace(s.Gpu[i].Processes.Text) == "" {
gpuItem.ProcessStatus = "ok"
}
if s.Gpu[i].Processes != nil {
for _, process := range s.Gpu[i].Processes.ProcessInfo {
gpuItem.Processes = append(gpuItem.Processes, Process{
PID: process.Pid,
Type: process.Type,
ProcessName: process.ProcessName,
UsedMemory: process.UsedMemory,
})
}
}
info.Devices = append(info.Devices, gpuItem)
}
return &info, nil
}
func firstSMIValue(values ...string) string {
for _, value := range values {
value = strings.TrimSpace(value)
if value != "" && !strings.EqualFold(value, "N/A") && !strings.EqualFold(value, "Not Supported") {
return value
}
}
return ""
}
+30 -13
View File
@@ -1,9 +1,12 @@
package gpu
import "encoding/xml"
type nvidiaSMIResponse struct {
AttachedGpus string `xml:"attached_gpus"`
CudaVersion string `xml:"cuda_version"`
DriverVersion string `xml:"driver_version"`
XMLName xml.Name `xml:"nvidia_smi_log"`
AttachedGpus string `xml:"attached_gpus"`
CudaVersion string `xml:"cuda_version"`
DriverVersion string `xml:"driver_version"`
Gpu []struct {
ID string `xml:"id,attr"`
AccountedProcesses struct{} `xml:"accounted_processes"`
@@ -37,16 +40,17 @@ type nvidiaSMIResponse struct {
VideoClock string `xml:"video_clock"`
} `xml:"clocks"`
ClocksEventReasons struct {
ClocksEventReasonApplicationsClocksSetting string `xml:"clocks_event_reason_applications_clocks_setting"`
ClocksEventReasonDisplayClocksSetting string `xml:"clocks_event_reason_display_clocks_setting"`
ClocksEventReasonGpuIdle string `xml:"clocks_event_reason_gpu_idle"`
ClocksEventReasonHwPowerBrakeSlowdown string `xml:"clocks_event_reason_hw_power_brake_slowdown"`
ClocksEventReasonHwSlowdown string `xml:"clocks_event_reason_hw_slowdown"`
ClocksEventReasonHwThermalSlowdown string `xml:"clocks_event_reason_hw_thermal_slowdown"`
ClocksEventReasonSwPowerCap string `xml:"clocks_event_reason_sw_power_cap"`
ClocksEventReasonSwThermalSlowdown string `xml:"clocks_event_reason_sw_thermal_slowdown"`
ClocksEventReasonSyncBoost string `xml:"clocks_event_reason_sync_boost"`
Reasons []struct {
XMLName xml.Name
Value string `xml:",chardata"`
} `xml:",any"`
} `xml:"clocks_event_reasons"`
ClocksThrottleReasons struct {
Reasons []struct {
XMLName xml.Name
Value string `xml:",chardata"`
} `xml:",any"`
} `xml:"clocks_throttle_reasons"`
ComputeMode string `xml:"compute_mode"`
DefaultApplicationsClocks struct {
GraphicsClock string `xml:"graphics_clock"`
@@ -63,12 +67,24 @@ type nvidiaSMIResponse struct {
} `xml:"driver_model"`
EccErrors struct {
Aggregate struct {
SingleBit struct {
Total string `xml:"total"`
} `xml:"single_bit"`
DoubleBit struct {
Total string `xml:"total"`
} `xml:"double_bit"`
DramCorrectable string `xml:"dram_correctable"`
DramUncorrectable string `xml:"dram_uncorrectable"`
SramCorrectable string `xml:"sram_correctable"`
SramUncorrectable string `xml:"sram_uncorrectable"`
} `xml:"aggregate"`
Volatile struct {
SingleBit struct {
Total string `xml:"total"`
} `xml:"single_bit"`
DoubleBit struct {
Total string `xml:"total"`
} `xml:"double_bit"`
DramCorrectable string `xml:"dram_correctable"`
DramUncorrectable string `xml:"dram_uncorrectable"`
SramCorrectable string `xml:"sram_correctable"`
@@ -226,7 +242,8 @@ type nvidiaSMIResponse struct {
MinPowerLimit string `xml:"min_power_limit"`
MaxPowerLimit string `xml:"max_power_limit"`
} `xml:"power_readings"`
Processes struct {
Processes *struct {
Text string `xml:",chardata"`
ProcessInfo []struct {
Pid string `xml:"pid"`
Type string `xml:"type"`
+37
View File
@@ -1,6 +1,8 @@
package gpu
type Info struct {
Warnings []string `json:"warnings"`
CudaVersion string `json:"cudaVersion"`
DriverVersion string `json:"driverVersion"`
Type string `json:"type"`
@@ -8,7 +10,42 @@ type Info struct {
Devices []Device `json:"gpu"`
}
type ECCError struct {
Scope string `json:"scope"`
Correctable string `json:"correctable"`
Uncorrectable string `json:"uncorrectable"`
}
type Device struct {
ECCPending string `json:"eccPending"`
ECCErrors []ECCError `json:"eccErrors"`
MemoryActivity string `json:"memoryActivity"`
EncoderUtil string `json:"encoderUtil"`
DecoderUtil string `json:"decoderUtil"`
JPEGUtil string `json:"jpegUtil"`
OFAUtil string `json:"ofaUtil"`
MediaUtil string `json:"mediaUtil"`
HotspotTemperature string `json:"hotspotTemperature"`
FanRPM string `json:"fanRPM"`
MediaFrequency string `json:"mediaFrequency"`
UUID string `json:"uuid"`
DriverVersion string `json:"driverVersion"`
Architecture string `json:"architecture"`
Frequency string `json:"frequency"`
MemoryFrequency string `json:"memoryFrequency"`
MemoryTemperature string `json:"memoryTemperature"`
MemoryFree string `json:"memoryFree"`
MemoryReserved string `json:"memoryReserved"`
PowerLimit string `json:"powerLimit"`
DefaultPowerLimit string `json:"defaultPowerLimit"`
PCIeGeneration string `json:"pcieGeneration"`
PCIeMaxGeneration string `json:"pcieMaxGeneration"`
PCIeWidth string `json:"pcieWidth"`
PCIeMaxWidth string `json:"pcieMaxWidth"`
ProcessStatus string `json:"processStatus"`
ClockEvents []string `json:"clockEvents"`
Type string `json:"type"`
Index uint `json:"index"`
ProductName string `json:"productName"`
+61 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"strconv"
"strings"
"sync"
"time"
"github.com/1Panel-dev/1Panel/agent/utils/cmd"
@@ -34,7 +35,31 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) {
if err != nil {
return nil, fmt.Errorf("calling %s failed: %w", ascendSMICommand, err)
}
return parseAscendSMI(itemData), nil
info := parseAscendSMI(itemData)
var wg sync.WaitGroup
warnings := make([]string, len(info.Devices))
for i := range info.Devices {
wg.Add(1)
go func(index int) {
defer wg.Done()
device := &info.Devices[index]
mgr := cmd.NewCommandMgr(cmd.WithContext(ctx), cmd.WithTimeout(5*time.Second))
data, err := mgr.RunWithStdout(ascendSMICommand, "info", "-t", "usages", "-i", strconv.FormatUint(uint64(device.NPUIndex), 10), "-c", strconv.FormatUint(uint64(device.ChipIndex), 10))
if err != nil {
warnings[index] = fmt.Sprintf("npu-smi usages %d/%d: %v", device.NPUIndex, device.ChipIndex, err)
return
}
applyAscendUsages(device, data)
}(i)
}
wg.Wait()
for _, warning := range warnings {
if warning != "" {
info.Warnings = append(info.Warnings, warning)
}
}
return info, nil
}
func parseAscendSMI(data string) *Info {
@@ -158,6 +183,12 @@ func parseAscendSMI(data string) *Info {
pending = nil
}
for i := range info.Devices {
info.Devices[i].ProcessStatus = "unavailable"
if processSection {
info.Devices[i].ProcessStatus = "ok"
}
}
return info
}
@@ -208,6 +239,9 @@ func ascendMemoryPools(value, header string) (string, string, string, string) {
}
hbm := usage[len(usage)-1]
if !strings.Contains(normalizedHeader, "MEMORYUSAGE") {
memoryUsed, memoryTotal = "", ""
}
return memoryUsed, memoryTotal, hbm[0] + " MB", hbm[1] + " MB"
}
@@ -233,3 +267,29 @@ func ascendValueWithUnit(value, unit string) string {
}
return value + " " + unit
}
func applyAscendUsages(device *Device, data string) {
for _, line := range strings.Split(data, "\n") {
key, value, ok := strings.Cut(line, ":")
if !ok {
continue
}
key = strings.ToLower(strings.Join(strings.Fields(key), ""))
value = strings.TrimSpace(value)
switch key {
case "aicoreusagerate(%)":
usage, err := strconv.ParseFloat(strings.TrimSpace(strings.TrimSuffix(value, "%")), 64)
if err == nil && usage >= 0 && usage <= 100 {
device.AICore = ascendValueWithUnit(value, "%")
}
case "aicpuusagerate(%)":
device.AICPUUtil = ascendValueWithUnit(value, "%")
case "ctrlcpuusagerate(%)":
device.CtrlCPUUtil = ascendValueWithUnit(value, "%")
case "ddrbandwidthusagerate(%)", "memorybandwidthusagerate(%)":
device.DDRBandwidth = ascendValueWithUnit(value, "%")
case "hbmbandwidthusagerate(%)":
device.HBMBandwidth = ascendValueWithUnit(value, "%")
}
}
}
+8
View File
@@ -1,12 +1,20 @@
package npu
type Info struct {
Warnings []string `json:"warnings"`
Type string `json:"type"`
DriverVersion string `json:"driverVersion"`
Devices []Device `json:"npu"`
}
type Device struct {
AICPUUtil string `json:"aiCPUUtil"`
CtrlCPUUtil string `json:"ctrlCPUUtil"`
DDRBandwidth string `json:"ddrBandwidth"`
HBMBandwidth string `json:"hbmBandwidth"`
ProcessStatus string `json:"processStatus"`
Type string `json:"type"`
Index uint `json:"index"`
NPUIndex uint `json:"npuIndex"`
+28 -2
View File
@@ -33,16 +33,29 @@ type discoveryInfo struct {
}
type DeviceLevelMetric struct {
MetricsType string `json:"metrics_type"`
Value float64 `json:"value"`
MetricsType string `json:"metrics_type"`
Value *float64 `json:"value"`
}
type TileMetrics struct {
TileID int `json:"tile_id"`
DataList []DeviceLevelMetric `json:"data_list"`
}
type TileStats struct {
TileID int `json:"tileID"`
Stats Stats `json:"stats"`
}
type DeviceStats struct {
TileLevel []TileMetrics `json:"tile_level"`
DeviceID int `json:"device_id"`
DeviceLevel []DeviceLevelMetric `json:"device_level"`
}
type Info struct {
Warnings []string `json:"warnings"`
Type string `json:"type"`
DriverVersion string `json:"driverVersion"`
@@ -50,12 +63,17 @@ type Info struct {
}
type Device struct {
Tiles []TileStats `json:"tiles"`
ProcessStatus string `json:"processStatus"`
Basic Basic `json:"basic"`
Stats Stats `json:"stats"`
Processes []Process `json:"processes"`
}
type Basic struct {
UUID string `json:"uuid"`
DeviceID int `json:"deviceID"`
DeviceName string `json:"deviceName"`
VendorName string `json:"vendorName"`
@@ -66,6 +84,14 @@ type Basic struct {
}
type Stats struct {
MediaUtil string `json:"mediaUtil"`
ComputeUtil string `json:"computeUtil"`
CopyUtil string `json:"copyUtil"`
MediaFrequency string `json:"mediaFrequency"`
MemoryTemperature string `json:"memoryTemperature"`
MemoryBandwidthUtil string `json:"memoryBandwidthUtil"`
Power string `json:"power"`
GPUUtil string `json:"gpuUtil"`
Frequency string `json:"frequency"`
+53 -13
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"sort"
"strconv"
"sync"
@@ -51,12 +52,15 @@ func (c Client) LoadInfoContext(ctx context.Context) (*Info, error) {
processData, err := cmdMgr.RunWithStdout(xpuSMICommand, "ps", "-j")
if err != nil {
global.LOG.Warnf("calling xpu-smi ps failed, process information will be omitted: %v", err)
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi ps: %v", err))
} else {
var psList DeviceUtilByProcList
if err := json.Unmarshal([]byte(processData), &psList); err != nil {
global.LOG.Warnf("processData json unmarshal failed, process information will be omitted: %v", err)
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi ps output: %v", err))
} else {
for i := range res.Devices {
res.Devices[i].ProcessStatus = "ok"
}
for _, ps := range psList.DeviceUtilByProcList {
process := Process{
PID: ps.ProcessID,
@@ -87,7 +91,9 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg *
defer wg.Done()
xpu := Device{
ProcessStatus: "unavailable",
Basic: Basic{
UUID: device.UUID,
DeviceID: device.DeviceID,
DeviceName: device.DeviceName,
VendorName: device.VendorName,
@@ -115,13 +121,19 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg *
wgCmd.Wait()
if xpuErr != nil {
global.LOG.Errorf("calling xpu-smi discovery failed for device %d, %v", device.DeviceID, xpuErr)
mu.Lock()
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi device %d: %v", device.DeviceID, xpuErr))
res.Devices = append(res.Devices, xpu)
mu.Unlock()
return
}
var info discoveryDevice
if err := json.Unmarshal([]byte(xpuData), &info); err != nil {
global.LOG.Errorf("xpuData json unmarshal failed for device %d, err: %v", device.DeviceID, err)
mu.Lock()
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi device %d output: %v", device.DeviceID, err))
res.Devices = append(res.Devices, xpu)
mu.Unlock()
return
}
@@ -142,13 +154,22 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg *
}
if statsErr != nil {
global.LOG.Warnf("calling xpu-smi stats failed for device %d, metrics will be omitted: %v", device.DeviceID, statsErr)
mu.Lock()
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi stats %d: %v", device.DeviceID, statsErr))
mu.Unlock()
} else {
var stats DeviceStats
if err := json.Unmarshal([]byte(statsData), &stats); err != nil {
global.LOG.Warnf("statsData json unmarshal failed for device %d, metrics will be omitted: %v", device.DeviceID, err)
mu.Lock()
res.Warnings = append(res.Warnings, fmt.Sprintf("xpu-smi stats %d output: %v", device.DeviceID, err))
mu.Unlock()
} else {
loadStats(&xpu.Stats, stats.DeviceLevel)
for _, tile := range stats.TileLevel {
item := TileStats{TileID: tile.TileID}
loadStats(&item.Stats, tile.DataList)
xpu.Tiles = append(xpu.Tiles, item)
}
}
}
@@ -162,19 +183,38 @@ func (c Client) loadDeviceInfo(ctx context.Context, device discoveryDevice, wg *
func loadStats(stats *Stats, metrics []DeviceLevelMetric) {
for _, stat := range metrics {
if stat.Value == nil || math.IsNaN(*stat.Value) || math.IsInf(*stat.Value, 0) {
continue
}
value := *stat.Value
if value < 0 && stat.MetricsType != "XPUM_STATS_GPU_CORE_TEMPERATURE" && stat.MetricsType != "XPUM_STATS_MEMORY_TEMPERATURE" {
continue
}
switch stat.MetricsType {
case "XPUM_STATS_MEMORY_TEMPERATURE":
stats.MemoryTemperature = fmt.Sprintf("%.1f°C", value)
case "XPUM_STATS_ENGINE_GROUP_COMPUTE_ALL_UTILIZATION":
stats.ComputeUtil = fmt.Sprintf("%.1f%%", value)
case "XPUM_STATS_ENGINE_GROUP_MEDIA_ALL_UTILIZATION":
stats.MediaUtil = fmt.Sprintf("%.1f%%", value)
case "XPUM_STATS_ENGINE_GROUP_COPY_ALL_UTILIZATION":
stats.CopyUtil = fmt.Sprintf("%.1f%%", value)
case "XPUM_STATS_MEDIA_ENGINE_FREQUENCY":
stats.MediaFrequency = fmt.Sprintf("%.1fMHz", value)
case "XPUM_STATS_POWER":
stats.Power = fmt.Sprintf("%.1fW", stat.Value)
stats.Power = fmt.Sprintf("%.1fW", value)
case "XPUM_STATS_GPU_UTILIZATION":
stats.GPUUtil = fmt.Sprintf("%.1f%%", stat.Value)
stats.GPUUtil = fmt.Sprintf("%.1f%%", value)
case "XPUM_STATS_GPU_FREQUENCY":
stats.Frequency = fmt.Sprintf("%.1fMHz", stat.Value)
stats.Frequency = fmt.Sprintf("%.1fMHz", value)
case "XPUM_STATS_GPU_CORE_TEMPERATURE":
stats.Temperature = fmt.Sprintf("%.1f°C", stat.Value)
stats.Temperature = fmt.Sprintf("%.1f°C", value)
case "XPUM_STATS_MEMORY_USED":
stats.MemoryUsed = fmt.Sprintf("%.1f MiB", stat.Value)
case "XPUM_STATS_MEMORY_UTILIZATION", "XPUM_STATS_MEMORY_BANDWIDTH", "XPUM_STATS_MEMORY_BANDWIDTH_UTILIZATION":
stats.MemoryUtil = fmt.Sprintf("%.1f%%", stat.Value)
stats.MemoryUsed = fmt.Sprintf("%.1f MiB", value)
case "XPUM_STATS_MEMORY_UTILIZATION":
stats.MemoryUtil = fmt.Sprintf("%.1f%%", value)
case "XPUM_STATS_MEMORY_BANDWIDTH", "XPUM_STATS_MEMORY_BANDWIDTH_UTILIZATION":
stats.MemoryBandwidthUtil = fmt.Sprintf("%.1f%%", value)
}
}
}
+90 -11
View File
@@ -22,6 +22,8 @@ export namespace AI {
}
export interface Info {
collectedAt?: string;
warnings?: string[];
cudaVersion: string;
driverVersion: string;
type: string;
@@ -31,6 +33,34 @@ export namespace AI {
xpu: XpuInfo['xpu'];
}
export interface GPU {
eccPending?: string;
eccErrors?: { scope: string; correctable: string; uncorrectable: string }[];
memoryActivity?: string;
encoderUtil?: string;
decoderUtil?: string;
jpegUtil?: string;
ofaUtil?: string;
mediaUtil?: string;
hotspotTemperature?: string;
fanRPM?: string;
mediaFrequency?: string;
uuid?: string;
driverVersion?: string;
architecture?: string;
frequency?: string;
memoryFrequency?: string;
memoryTemperature?: string;
memoryFree?: string;
memoryReserved?: string;
powerLimit?: string;
defaultPowerLimit?: string;
pcieGeneration?: string;
pcieMaxGeneration?: string;
pcieWidth?: string;
pcieMaxWidth?: string;
clockEvents?: string[];
processStatus?: string;
type: 'nvidia' | 'amd';
index: number;
productName: string;
@@ -52,6 +82,12 @@ export namespace AI {
processes: Process[];
}
export interface NPU {
aiCPUUtil?: string;
ctrlCPUUtil?: string;
ddrBandwidth?: string;
hbmBandwidth?: string;
processStatus?: string;
type: 'ascend';
index: number;
npuIndex: number;
@@ -73,6 +109,9 @@ export namespace AI {
processes: NPUProcess[];
}
export interface MonitorGPUSearch {
aggregation?: 'avg' | 'max';
deviceID?: string;
legacy?: boolean;
productName: string;
startTime: Date;
endTime: Date;
@@ -83,8 +122,10 @@ export namespace AI {
chartHide: Array<ChartHide>;
}
export interface ChartHide {
deviceID?: string;
legacy?: boolean;
productName: string;
type?: 'gpu' | 'xpu';
type?: 'gpu' | 'npu' | 'xpu';
process: boolean;
gpu: boolean;
memory: boolean;
@@ -94,17 +135,45 @@ export namespace AI {
speed: boolean;
}
export interface MonitorGPUData {
memoryActivity: Array<number | null>;
encoderUtil: Array<number | null>;
decoderUtil: Array<number | null>;
jpegUtil: Array<number | null>;
ofaUtil: Array<number | null>;
mediaUtil: Array<number | null>;
computeUtil: Array<number | null>;
copyUtil: Array<number | null>;
hotspotTemperature: Array<number | null>;
fanRPM: Array<number | null>;
aiCPUUtil: Array<number | null>;
ctrlCPUUtil: Array<number | null>;
ddrUsed: Array<number | null>;
ddrTotal: Array<number | null>;
hbmUsed: Array<number | null>;
hbmTotal: Array<number | null>;
ddrBandwidth: Array<number | null>;
hbmBandwidth: Array<number | null>;
memoryBandwidth: Array<number | null>;
mediaFrequency: Array<number | null>;
hugepagesUsed: Array<number | null>;
hugepagesTotal: Array<number | null>;
bucketSeconds: number;
sampleCount: number;
memoryTemperatureValue: Array<number | null>;
frequencyValue: Array<number | null>;
memoryFrequencyValue: Array<number | null>;
date: Array<Date>;
gpuValue: Array<number>;
temperatureValue: Array<number>;
powerTotal: Array<number>;
powerUsed: Array<number>;
powerPercent: Array<number>;
memoryTotal: Array<number>;
memoryUsed: Array<number>;
memoryPercent: Array<number>;
speedValue: Array<number>;
processCount: Array<number>;
gpuValue: Array<number | null>;
temperatureValue: Array<number | null>;
powerTotal: Array<number | null>;
powerUsed: Array<number | null>;
powerPercent: Array<number | null>;
memoryTotal: Array<number | null>;
memoryUsed: Array<number | null>;
memoryPercent: Array<number | null>;
speedValue: Array<number | null>;
processCount: Array<number | null>;
gpuProcesses: Array<Array<GPUProcess>>;
}
export interface GPUProcess {
@@ -132,12 +201,15 @@ export namespace AI {
}
interface Xpu {
tiles?: { tileID: number; stats: Stats }[];
processStatus?: string;
basic: Basic;
stats: Stats;
processes: XpuProcess[];
}
interface Basic {
uuid?: string;
deviceID: number;
deviceName: string;
vendorName: string;
@@ -148,6 +220,13 @@ export namespace AI {
}
interface Stats {
mediaUtil?: string;
computeUtil?: string;
copyUtil?: string;
mediaFrequency?: string;
memoryTemperature?: string;
memoryBandwidthUtil?: string;
power: string;
gpuUtil: string;
frequency: string;
@@ -134,10 +134,11 @@ function initChart() {
series.push({
name: item?.name,
type: 'line',
itemStyle: seriesStyle[index + 2],
areaStyle: seriesStyle[index],
itemStyle: item.itemStyle ?? seriesStyle[index + 2],
areaStyle: item.areaStyle ?? seriesStyle[index],
data: item?.data,
showSymbol: false,
showSymbol: item.showSymbol ?? false,
symbolSize: item.symbolSize,
yAxisIndex: item.yAxisIndex ? 1 : null,
});
});
@@ -307,7 +308,9 @@ function initChart() {
series: series,
dataZoom: [
{
...(props.option.xData?.[0] ? { startValue: props.option.xData[0] } : {}),
...((props.option.xAxis?.min ?? props.option.xData?.[0]) != null
? { startValue: props.option.xAxis?.min ?? props.option.xData[0] }
: {}),
left: grid.left,
right: grid.right,
show: props.dataZoom,
+54 -1
View File
@@ -1508,6 +1508,59 @@ const message = {
extraHeaders: 'Extra Headers',
},
gpu: {
memoryActivity: 'Memory read/write activity',
hotspotTemperature: 'Hotspot temperature',
fanRPM: 'Fan speed (RPM)',
eccCorrectable: 'Correctable errors',
eccUncorrectable: 'Uncorrectable errors',
eccPending: 'Pending ECC mode',
engineUtil: 'Engine utilization',
cpuUtil: 'CPU · utilization',
aiCore: 'AICore · utilization',
aiCPUUtil: 'AICPU · utilization',
ctrlCPUUtil: 'CtrlCPU · utilization',
mediaFrequency: 'Media · frequency',
encoderUtil: 'Encoder · utilization',
decoderUtil: 'Decoder · utilization',
jpegUtil: 'JPEG · utilization',
ofaUtil: 'OFA · utilization',
mediaUtil: 'Media · utilization',
computeUtil: 'Compute · utilization',
copyUtil: 'Copy · utilization',
ddrUsage: 'DDR · capacity',
ddrUsed: 'DDR · used',
ddrTotal: 'DDR · total',
ddrBandwidth: 'DDR · bandwidth utilization',
hbmUsage: 'HBM · capacity',
hbmUsed: 'HBM · used',
hbmTotal: 'HBM · total',
hbmBandwidth: 'HBM · bandwidth utilization',
hugepagesUsage: 'Huge pages · count',
hugepagesUsed: 'Huge pages · used',
hugepagesTotal: 'Huge pages · total',
historyAverage: 'Average',
historyPeak: 'Peak',
historyAggregated:
'{count} samples aggregated into {seconds}-second intervals; processes show counts only.',
historyRaw: '{count} original samples.',
historyMetricEmpty: 'No data for this metric in the selected time range',
historyLoadFailed: 'Failed to load history. Please retry.',
collectedAt: 'Collected at',
processUnavailable: 'Process information is unavailable',
architecture: 'Architecture',
memoryFrequency: 'Memory frequency',
memoryTemperature: 'Memory temperature',
memoryReserved: 'Reserved memory',
powerDefaultLimit: 'Default power limit',
powerMaxLimit: 'Maximum configurable power limit',
clockEvents: 'Clock limiting reasons',
memoryBandwidth: 'Memory bandwidth utilization',
legacyDevice: 'Legacy record',
legacyHistory:
'These records have no stable device ID. Their identity and missing values cannot be verified.',
pcieGeneration: 'PCIe generation (current / maximum)',
pcieWidth: 'PCIe width (current / maximum)',
gpu: 'GPU Monitoring',
gpuHelper: 'No supported GPU/XPU/NPU management command was detected. Please check and try again!',
process: 'Process Information',
@@ -1523,7 +1576,7 @@ const message = {
temperature: 'Temperature',
performanceState: 'Performance State',
powerUsage: 'Power Consumption',
memoryUsage: 'Memory Utilization',
memoryUsage: 'Memory capacity usage',
fanSpeed: 'Fan Speed',
power: 'Power',
powerCurrent: 'Current Power',
+54 -1
View File
@@ -1517,6 +1517,59 @@ const message = {
extraHeaders: 'Encabezados adicionales',
},
gpu: {
memoryActivity: 'Actividad de lectura/escritura de memoria',
hotspotTemperature: 'Temperatura del punto caliente',
fanRPM: 'Velocidad del ventilador (RPM)',
eccCorrectable: 'Errores corregibles',
eccUncorrectable: 'Errores no corregibles',
eccPending: 'Modo ECC pendiente',
engineUtil: 'Utilización de motores',
cpuUtil: 'CPU · utilización',
aiCore: 'AICore · utilización',
aiCPUUtil: 'AICPU · utilización',
ctrlCPUUtil: 'CtrlCPU · utilización',
mediaFrequency: 'Medios · frecuencia',
encoderUtil: 'Codificador · utilización',
decoderUtil: 'Decodificador · utilización',
jpegUtil: 'JPEG · utilización',
ofaUtil: 'OFA · utilización',
mediaUtil: 'Medios · utilización',
computeUtil: 'Cómputo · utilización',
copyUtil: 'Copia · utilización',
ddrUsage: 'DDR · capacidad',
ddrUsed: 'DDR · usado',
ddrTotal: 'DDR · total',
ddrBandwidth: 'DDR · utilización del ancho de banda',
hbmUsage: 'HBM · capacidad',
hbmUsed: 'HBM · usado',
hbmTotal: 'HBM · total',
hbmBandwidth: 'HBM · utilización del ancho de banda',
hugepagesUsage: 'Páginas grandes · cantidad',
hugepagesUsed: 'Páginas grandes · usado',
hugepagesTotal: 'Páginas grandes · total',
historyAverage: 'Promedio',
historyPeak: 'Máximo',
historyAggregated:
'{count} muestras agrupadas en intervalos de {seconds} segundos; los procesos solo muestran cantidades.',
historyRaw: '{count} muestras originales.',
historyMetricEmpty: 'No hay datos de esta métrica en el período seleccionado',
historyLoadFailed: 'No se pudo cargar el historial. Inténtelo de nuevo.',
collectedAt: 'Fecha de recopilación',
processUnavailable: 'Información de procesos no disponible',
architecture: 'Arquitectura',
memoryFrequency: 'Frecuencia de memoria',
memoryTemperature: 'Temperatura de memoria',
memoryReserved: 'Memoria reservada',
powerDefaultLimit: 'Límite de potencia predeterminado',
powerMaxLimit: 'Potencia máxima configurable',
clockEvents: 'Motivos de limitación de frecuencia',
memoryBandwidth: 'Uso del ancho de banda de memoria',
legacyDevice: 'Registro antiguo',
legacyHistory:
'Estos registros no tienen un ID de dispositivo estable. No se pueden verificar su identidad ni los valores ausentes.',
pcieGeneration: 'Generación PCIe (actual / máxima)',
pcieWidth: 'Ancho PCIe (actual / máximo)',
gpu: 'Monitoreo de GPU',
gpuHelper:
'No se detectó ningún comando de administración de GPU/XPU/NPU compatible. ¡Compruebe e inténtelo de nuevo!',
@@ -1532,7 +1585,7 @@ const message = {
temperature: 'Temperatura',
performanceState: 'Estado de Rendimiento',
powerUsage: 'Consumo de Energía',
memoryUsage: 'Utilización de Memoria',
memoryUsage: 'Uso de capacidad de memoria',
fanSpeed: 'Velocidad del Ventilador',
power: 'Energía',
powerCurrent: 'Energía Actual',
+53 -1
View File
@@ -1493,6 +1493,58 @@ const message = {
extraHeaders: 'هدرهای اضافی',
},
gpu: {
memoryActivity: 'فعالیت خواندن/نوشتن حافظه',
hotspotTemperature: 'دمای نقطه داغ',
fanRPM: 'سرعت فن (RPM)',
eccCorrectable: 'خطاهای قابل اصلاح',
eccUncorrectable: 'خطاهای غیرقابل اصلاح',
eccPending: 'حالت ECC در انتظار اعمال',
engineUtil: 'میزان استفاده از موتور',
cpuUtil: 'CPU · میزان استفاده',
aiCore: 'AICore · میزان استفاده',
aiCPUUtil: 'AICPU · میزان استفاده',
ctrlCPUUtil: 'CtrlCPU · میزان استفاده',
mediaFrequency: 'رسانه · فرکانس',
encoderUtil: 'رمزگذار · میزان استفاده',
decoderUtil: 'رمزگشا · میزان استفاده',
jpegUtil: 'JPEG · میزان استفاده',
ofaUtil: 'OFA · میزان استفاده',
mediaUtil: 'رسانه · میزان استفاده',
computeUtil: 'محاسبه · میزان استفاده',
copyUtil: 'کپی · میزان استفاده',
ddrUsage: 'DDR · ظرفیت',
ddrUsed: 'DDR · استفاده‌شده',
ddrTotal: 'DDR · کل',
ddrBandwidth: 'DDR · میزان استفاده از پهنای باند',
hbmUsage: 'HBM · ظرفیت',
hbmUsed: 'HBM · استفاده‌شده',
hbmTotal: 'HBM · کل',
hbmBandwidth: 'HBM · میزان استفاده از پهنای باند',
hugepagesUsage: 'صفحات بزرگ · تعداد',
hugepagesUsed: 'صفحات بزرگ · استفاده‌شده',
hugepagesTotal: 'صفحات بزرگ · کل',
historyAverage: 'میانگین',
historyPeak: 'بیشینه',
historyAggregated:
'{count} نمونه در بازه‌های {seconds} ثانیه‌ای تجمیع شده‌اند؛ فقط تعداد فرایندها نمایش داده می‌شود.',
historyRaw: '{count} نمونه اصلی.',
historyMetricEmpty: 'در بازه زمانی انتخاب‌شده داده‌ای برای این شاخص وجود ندارد',
historyLoadFailed: 'بارگیری سابقه ناموفق بود. دوباره تلاش کنید.',
collectedAt: 'زمان گردآوری',
processUnavailable: 'اطلاعات فرایند در دسترس نیست',
architecture: 'معماری',
memoryFrequency: 'فرکانس حافظه',
memoryTemperature: 'دمای حافظه',
memoryReserved: 'حافظه رزروشده',
powerDefaultLimit: 'محدودیت توان پیش‌فرض',
powerMaxLimit: 'حداکثر توان قابل تنظیم',
clockEvents: 'دلایل محدودیت فرکانس',
memoryBandwidth: 'میزان استفاده از پهنای باند حافظه',
legacyDevice: 'رکورد قدیمی',
legacyHistory: 'این رکوردها شناسه ثابت دستگاه ندارند. هویت دستگاه و مقادیر مفقود قابل تأیید نیستند.',
pcieGeneration: 'نسل PCIe (فعلی / حداکثر)',
pcieWidth: 'عرض PCIe (فعلی / حداکثر)',
gpu: 'نظارت بر GPU',
gpuHelper: 'هیچ دستور مدیریت GPU/XPU/NPU پشتیبانی‌شده‌ای شناسایی نشد. لطفاً بررسی و دوباره تلاش کنید!',
process: 'اطلاعات فرآیند',
@@ -1508,7 +1560,7 @@ const message = {
temperature: 'دما',
performanceState: 'وضعیت عملکرد',
powerUsage: 'مصرف برق',
memoryUsage: 'استفاده از حافظه',
memoryUsage: 'میزان اشغال ظرفیت حافظه',
fanSpeed: 'سرعت فن',
power: 'برق',
powerCurrent: 'برق فعلی',
+52 -1
View File
@@ -1496,6 +1496,57 @@ const message = {
extraHeaders: '追加ヘッダー',
},
gpu: {
memoryActivity: 'メモリ読み書き稼働率',
hotspotTemperature: 'ホットスポット温度',
fanRPM: 'ファン回転数(RPM)',
eccCorrectable: '訂正可能なエラー数',
eccUncorrectable: '訂正不能なエラー数',
eccPending: '適用待ちの ECC モード',
engineUtil: 'エンジン使用率',
cpuUtil: 'CPU · 使用率',
aiCore: 'AICore · 使用率',
aiCPUUtil: 'AICPU · 使用率',
ctrlCPUUtil: 'CtrlCPU · 使用率',
mediaFrequency: 'メディア · 周波数',
encoderUtil: 'エンコーダー · 使用率',
decoderUtil: 'デコーダー · 使用率',
jpegUtil: 'JPEG · 使用率',
ofaUtil: 'OFA · 使用率',
mediaUtil: 'メディア · 使用率',
computeUtil: '計算 · 使用率',
copyUtil: 'コピー · 使用率',
ddrUsage: 'DDR · 容量',
ddrUsed: 'DDR · 使用量',
ddrTotal: 'DDR · 総容量',
ddrBandwidth: 'DDR · 帯域使用率',
hbmUsage: 'HBM · 容量',
hbmUsed: 'HBM · 使用量',
hbmTotal: 'HBM · 総容量',
hbmBandwidth: 'HBM · 帯域使用率',
hugepagesUsage: 'ヒュージページ · 数',
hugepagesUsed: 'ヒュージページ · 使用量',
hugepagesTotal: 'ヒュージページ · 総容量',
historyAverage: '平均値',
historyPeak: '最大値',
historyAggregated: '{count} 件のサンプルを {seconds} 秒間隔で集計。プロセスは件数のみ表示します。',
historyRaw: '元のサンプル {count} 件。',
historyMetricEmpty: '選択した期間にこの指標のデータはありません',
historyLoadFailed: '履歴の読み込みに失敗しました。再試行してください。',
collectedAt: '収集日時',
processUnavailable: 'プロセス情報を取得できません',
architecture: 'アーキテクチャ',
memoryFrequency: 'メモリ周波数',
memoryTemperature: 'メモリ温度',
memoryReserved: '予約済みメモリ',
powerDefaultLimit: '既定の電力制限',
powerMaxLimit: '設定可能な最大電力',
clockEvents: 'クロック制限の理由',
memoryBandwidth: 'メモリ帯域幅使用率',
legacyDevice: '旧記録',
legacyHistory: 'これらの記録には固定のデバイス ID がなく、デバイスの識別や欠損値を確認できません。',
pcieGeneration: 'PCIe 世代(現在 / 最大)',
pcieWidth: 'PCIe 幅(現在 / 最大)',
gpu: 'GPU 監視',
gpuHelper: '対応する GPU/XPU/NPU 管理コマンドが検出されませんでした。確認して再試行してください!',
process: 'プロセス情報',
@@ -1510,7 +1561,7 @@ const message = {
temperature: '温度',
performanceState: 'パフォーマンス状態',
powerUsage: '消費電力',
memoryUsage: 'メモリ使用率',
memoryUsage: 'メモリ容量使用率',
fanSpeed: 'ファン速度',
power: '電力',
powerCurrent: '現在の電力',
+52 -1
View File
@@ -1482,6 +1482,57 @@ const message = {
extraHeaders: '추가 헤더',
},
gpu: {
memoryActivity: '메모리 읽기/쓰기 활성률',
hotspotTemperature: '핫스폿 온도',
fanRPM: '팬 회전 속도 (RPM)',
eccCorrectable: '수정 가능한 오류 수',
eccUncorrectable: '수정 불가능한 오류 수',
eccPending: '적용 대기 중인 ECC 모드',
engineUtil: '엔진 사용률',
cpuUtil: 'CPU · 사용률',
aiCore: 'AICore · 사용률',
aiCPUUtil: 'AICPU · 사용률',
ctrlCPUUtil: 'CtrlCPU · 사용률',
mediaFrequency: '미디어 · 주파수',
encoderUtil: '인코더 · 사용률',
decoderUtil: '디코더 · 사용률',
jpegUtil: 'JPEG · 사용률',
ofaUtil: 'OFA · 사용률',
mediaUtil: '미디어 · 사용률',
computeUtil: '컴퓨팅 · 사용률',
copyUtil: '복사 · 사용률',
ddrUsage: 'DDR · 용량',
ddrUsed: 'DDR · 사용량',
ddrTotal: 'DDR · 총량',
ddrBandwidth: 'DDR · 대역폭 사용률',
hbmUsage: 'HBM · 용량',
hbmUsed: 'HBM · 사용량',
hbmTotal: 'HBM · 총량',
hbmBandwidth: 'HBM · 대역폭 사용률',
hugepagesUsage: '대용량 페이지 · 수',
hugepagesUsed: '대용량 페이지 · 사용량',
hugepagesTotal: '대용량 페이지 · 총량',
historyAverage: '평균',
historyPeak: '최댓값',
historyAggregated: '샘플 {count}개를 {seconds}초 간격으로 집계합니다. 프로세스는 개수만 표시합니다.',
historyRaw: '원본 샘플 {count}개.',
historyMetricEmpty: '선택한 기간에 이 지표의 데이터가 없습니다',
historyLoadFailed: '기록을 불러오지 못했습니다. 다시 시도하세요.',
collectedAt: '수집 시간',
processUnavailable: '프로세스 정보를 사용할 수 없습니다',
architecture: '아키텍처',
memoryFrequency: '메모리 주파수',
memoryTemperature: '메모리 온도',
memoryReserved: '예약된 메모리',
powerDefaultLimit: '기본 전력 제한',
powerMaxLimit: '설정 가능한 최대 전력',
clockEvents: '클럭 제한 이유',
memoryBandwidth: '메모리 대역폭 사용률',
legacyDevice: '이전 기록',
legacyHistory: '이 기록에는 고정 장치 ID가 없어 장치 식별 및 누락 값을 확인할 수 없습니다.',
pcieGeneration: 'PCIe 세대 (현재 / 최대)',
pcieWidth: 'PCIe 너비 (현재 / 최대)',
gpu: 'GPU 모니터링',
gpuHelper: '지원되는 GPU/XPU/NPU 관리 명령을 감지하지 못했습니다. 확인하고 다시 시도하세요!',
process: '프로세스 정보',
@@ -1496,7 +1547,7 @@ const message = {
temperature: '온도',
performanceState: '성능 상태',
powerUsage: '전력 소비',
memoryUsage: '메모리 사용률',
memoryUsage: '메모리 용량 사용률',
fanSpeed: '팬 속도',
power: '전력',
powerCurrent: '현재 전력',
+52 -1
View File
@@ -1475,6 +1475,57 @@ const message = {
extraHeaders: 'Headers ເພີ່ມເຕີມ',
},
gpu: {
memoryActivity: 'ກິດຈະກຳອ່ານ/ຂຽນໜ່ວຍຄວາມຈຳ',
hotspotTemperature: 'ອຸນຫະພູມຈຸດຮ້ອນ',
fanRPM: 'ຄວາມໄວພັດລົມ (RPM)',
eccCorrectable: 'ຂໍ້ຜິດພາດທີ່ແກ້ໄຂໄດ້',
eccUncorrectable: 'ຂໍ້ຜິດພາດທີ່ແກ້ໄຂບໍ່ໄດ້',
eccPending: 'ໂໝດ ECC ທີ່ລໍຖ້າ',
engineUtil: 'ອັດຕາການໃຊ້ເຄື່ອງຈັກ',
cpuUtil: 'CPU · ອັດຕາການໃຊ້',
aiCore: 'AICore · ອັດຕາການໃຊ້',
aiCPUUtil: 'AICPU · ອັດຕາການໃຊ້',
ctrlCPUUtil: 'CtrlCPU · ອັດຕາການໃຊ້',
mediaFrequency: 'ສື່ · ຄວາມຖີ່',
encoderUtil: 'ຕົວເຂົ້າລະຫັດ · ອັດຕາການໃຊ້',
decoderUtil: 'ຕົວຖອດລະຫັດ · ອັດຕາການໃຊ້',
jpegUtil: 'JPEG · ອັດຕາການໃຊ້',
ofaUtil: 'OFA · ອັດຕາການໃຊ້',
mediaUtil: 'ສື່ · ອັດຕາການໃຊ້',
computeUtil: 'ຄຳນວນ · ອັດຕາການໃຊ້',
copyUtil: 'ສຳເນົາ · ອັດຕາການໃຊ້',
ddrUsage: 'DDR · ຄວາມຈຸ',
ddrUsed: 'DDR · ໃຊ້ແລ້ວ',
ddrTotal: 'DDR · ທັງໝົດ',
ddrBandwidth: 'DDR · ອັດຕາການໃຊ້ແບນວິດ',
hbmUsage: 'HBM · ຄວາມຈຸ',
hbmUsed: 'HBM · ໃຊ້ແລ້ວ',
hbmTotal: 'HBM · ທັງໝົດ',
hbmBandwidth: 'HBM · ອັດຕາການໃຊ້ແບນວິດ',
hugepagesUsage: 'ໜ້າຂະໜາດໃຫຍ່ · ຈຳນວນ',
hugepagesUsed: 'ໜ້າຂະໜາດໃຫຍ່ · ໃຊ້ແລ້ວ',
hugepagesTotal: 'ໜ້າຂະໜາດໃຫຍ່ · ທັງໝົດ',
historyAverage: 'ຄ່າສະເລ່ຍ',
historyPeak: 'ຄ່າສູງສຸດ',
historyAggregated: 'ລວມ {count} ຕົວຢ່າງເປັນຊ່ວງ {seconds} ວິນາທີ; ສະແດງພຽງຈຳນວນໂປຣເຊສ.',
historyRaw: '{count} ຕົວຢ່າງຕົ້ນສະບັບ.',
historyMetricEmpty: 'ບໍ່ມີຂໍ້ມູນຕົວຊີ້ວັດນີ້ໃນຊ່ວງເວລາທີ່ເລືອກ',
historyLoadFailed: 'ໂຫຼດປະຫວັດບໍ່ສຳເລັດ. ກະລຸນາລອງໃໝ່.',
collectedAt: 'ເວລາເກັບຂໍ້ມູນ',
processUnavailable: 'ບໍ່ມີຂໍ້ມູນໂປຣເຊສ',
architecture: 'ສະຖາປັດຕະຍະກຳ',
memoryFrequency: 'ຄວາມຖີ່ໜ່ວຍຄວາມຈຳ',
memoryTemperature: 'ອຸນຫະພູມໜ່ວຍຄວາມຈຳ',
memoryReserved: 'ໜ່ວຍຄວາມຈຳທີ່ສະຫງວນ',
powerDefaultLimit: 'ຂີດຈຳກັດພະລັງງານເລີ່ມຕົ້ນ',
powerMaxLimit: 'ພະລັງງານສູງສຸດທີ່ຕັ້ງໄດ້',
clockEvents: 'ເຫດຜົນທີ່ຈຳກັດຄວາມຖີ່',
memoryBandwidth: 'ການໃຊ້ແບນວິດໜ່ວຍຄວາມຈຳ',
legacyDevice: 'ບັນທຶກເກົ່າ',
legacyHistory: 'ບັນທຶກເຫຼົ່ານີ້ບໍ່ມີ ID ອຸປະກອນຄົງທີ່. ບໍ່ສາມາດຢືນຢັນອຸປະກອນ ແລະ ຄ່າທີ່ຂາດຫາຍໄດ້.',
pcieGeneration: 'ລຸ້ນ PCIe (ປັດຈຸບັນ / ສູງສຸດ)',
pcieWidth: 'ຄວາມກວ້າງ PCIe (ປັດຈຸບັນ / ສູງສຸດ)',
gpu: 'ຕິດຕາມ GPU',
gpuHelper: 'ກວດບໍ່ພົບຄຳສັ່ງຈັດການ GPU/XPU/NPU ທີ່ຮອງຮັບ. ກະລຸນາກວດສອບ ແລະ ລອງໃໝ່!',
process: 'ຂໍ້ມູນໂປຣເຊສ',
@@ -1490,7 +1541,7 @@ const message = {
temperature: 'ອຸນຫະພູມ',
performanceState: 'ສະຖານະປະສິດທິພາບ',
powerUsage: 'ການໃຊ້ພະລັງງານ',
memoryUsage: 'ການໃຊ້ໜ່ວຍຄວາມຈຳ',
memoryUsage: 'ການໃຊ້ຄວາມຈຸໜ່ວຍຄວາມຈຳ',
fanSpeed: 'ຄວາມໄວພັດລົມ',
power: 'ພະລັງງານ',
powerCurrent: 'ພະລັງງານປັດຈຸບັນ',
+54 -1
View File
@@ -1522,6 +1522,59 @@ const message = {
extraHeaders: 'Pengepala Tambahan',
},
gpu: {
memoryActivity: 'Aktiviti baca/tulis memori',
hotspotTemperature: 'Suhu titik panas',
fanRPM: 'Kelajuan kipas (RPM)',
eccCorrectable: 'Ralat boleh dibetulkan',
eccUncorrectable: 'Ralat tidak boleh dibetulkan',
eccPending: 'Mod ECC menunggu',
engineUtil: 'Penggunaan enjin',
cpuUtil: 'CPU · penggunaan',
aiCore: 'AICore · penggunaan',
aiCPUUtil: 'AICPU · penggunaan',
ctrlCPUUtil: 'CtrlCPU · penggunaan',
mediaFrequency: 'Media · frekuensi',
encoderUtil: 'Pengekod · penggunaan',
decoderUtil: 'Penyahkod · penggunaan',
jpegUtil: 'JPEG · penggunaan',
ofaUtil: 'OFA · penggunaan',
mediaUtil: 'Media · penggunaan',
computeUtil: 'Pengiraan · penggunaan',
copyUtil: 'Salinan · penggunaan',
ddrUsage: 'DDR · kapasiti',
ddrUsed: 'DDR · digunakan',
ddrTotal: 'DDR · jumlah',
ddrBandwidth: 'DDR · penggunaan lebar jalur',
hbmUsage: 'HBM · kapasiti',
hbmUsed: 'HBM · digunakan',
hbmTotal: 'HBM · jumlah',
hbmBandwidth: 'HBM · penggunaan lebar jalur',
hugepagesUsage: 'Halaman besar · bilangan',
hugepagesUsed: 'Halaman besar · digunakan',
hugepagesTotal: 'Halaman besar · jumlah',
historyAverage: 'Purata',
historyPeak: 'Puncak',
historyAggregated:
'{count} sampel diagregatkan dalam sela {seconds} saat; proses hanya memaparkan bilangan.',
historyRaw: '{count} sampel asal.',
historyMetricEmpty: 'Tiada data untuk metrik ini dalam julat masa yang dipilih',
historyLoadFailed: 'Gagal memuatkan sejarah. Sila cuba lagi.',
collectedAt: 'Masa pengumpulan',
processUnavailable: 'Maklumat proses tidak tersedia',
architecture: 'Seni bina',
memoryFrequency: 'Frekuensi memori',
memoryTemperature: 'Suhu memori',
memoryReserved: 'Memori terpelihara',
powerDefaultLimit: 'Had kuasa lalai',
powerMaxLimit: 'Kuasa maksimum boleh ditetapkan',
clockEvents: 'Sebab had frekuensi',
memoryBandwidth: 'Penggunaan lebar jalur memori',
legacyDevice: 'Rekod lama',
legacyHistory:
'Rekod ini tiada ID peranti yang tetap. Identiti peranti dan nilai yang hilang tidak dapat disahkan.',
pcieGeneration: 'Generasi PCIe (semasa / maksimum)',
pcieWidth: 'Lebar PCIe (semasa / maksimum)',
gpu: 'Pemantauan GPU',
gpuHelper: 'Tiada arahan pengurusan GPU/XPU/NPU yang disokong dikesan. Sila periksa dan cuba lagi!',
process: 'Maklumat Proses',
@@ -1536,7 +1589,7 @@ const message = {
temperature: 'Suhu',
performanceState: 'Status Prestasi',
powerUsage: 'Penggunaan Kuasa',
memoryUsage: 'Penggunaan Memori',
memoryUsage: 'Penggunaan kapasiti memori',
fanSpeed: 'Kelajuan Kipas',
power: 'Kuasa',
powerCurrent: 'Kuasa Semasa',
+54 -1
View File
@@ -1518,6 +1518,59 @@ const message = {
extraHeaders: 'Cabeçalhos extras',
},
gpu: {
memoryActivity: 'Atividade de leitura/escrita da memória',
hotspotTemperature: 'Temperatura do ponto quente',
fanRPM: 'Rotação da ventoinha (RPM)',
eccCorrectable: 'Erros corrigíveis',
eccUncorrectable: 'Erros incorrigíveis',
eccPending: 'Modo ECC pendente',
engineUtil: 'Utilização dos motores',
cpuUtil: 'CPU · utilização',
aiCore: 'AICore · utilização',
aiCPUUtil: 'AICPU · utilização',
ctrlCPUUtil: 'CtrlCPU · utilização',
mediaFrequency: 'Mídia · frequência',
encoderUtil: 'Codificador · utilização',
decoderUtil: 'Decodificador · utilização',
jpegUtil: 'JPEG · utilização',
ofaUtil: 'OFA · utilização',
mediaUtil: 'Mídia · utilização',
computeUtil: 'Computação · utilização',
copyUtil: 'Cópia · utilização',
ddrUsage: 'DDR · capacidade',
ddrUsed: 'DDR · usado',
ddrTotal: 'DDR · total',
ddrBandwidth: 'DDR · utilização da largura de banda',
hbmUsage: 'HBM · capacidade',
hbmUsed: 'HBM · usado',
hbmTotal: 'HBM · total',
hbmBandwidth: 'HBM · utilização da largura de banda',
hugepagesUsage: 'Páginas grandes · quantidade',
hugepagesUsed: 'Páginas grandes · usado',
hugepagesTotal: 'Páginas grandes · total',
historyAverage: 'Média',
historyPeak: 'Pico',
historyAggregated:
'{count} amostras agregadas em intervalos de {seconds} segundos; processos exibem apenas contagens.',
historyRaw: '{count} amostras originais.',
historyMetricEmpty: 'Sem dados desta métrica no período selecionado',
historyLoadFailed: 'Falha ao carregar o histórico. Tente novamente.',
collectedAt: 'Data da coleta',
processUnavailable: 'Informações de processos indisponíveis',
architecture: 'Arquitetura',
memoryFrequency: 'Frequência da memória',
memoryTemperature: 'Temperatura da memória',
memoryReserved: 'Memória reservada',
powerDefaultLimit: 'Limite de potência padrão',
powerMaxLimit: 'Potência máxima configurável',
clockEvents: 'Motivos de limitação de frequência',
memoryBandwidth: 'Uso da largura de banda da memória',
legacyDevice: 'Registro antigo',
legacyHistory:
'Estes registros não têm um ID de dispositivo estável. Não é possível verificar a identidade nem os valores ausentes.',
pcieGeneration: 'Geração PCIe (atual / máxima)',
pcieWidth: 'Largura PCIe (atual / máxima)',
gpu: 'Monitoramento de GPU',
gpuHelper:
'Nenhum comando de gerenciamento de GPU/XPU/NPU compatível foi detectado. Verifique e tente novamente!',
@@ -1533,7 +1586,7 @@ const message = {
temperature: 'Temperatura',
performanceState: 'Estado de Desempenho',
powerUsage: 'Consumo de Energia',
memoryUsage: 'Utilização de Memória',
memoryUsage: 'Uso da capacidade de memória',
fanSpeed: 'Velocidade do Ventilador',
power: 'Energia',
powerCurrent: 'Energia Atual',
+54 -1
View File
@@ -1511,6 +1511,59 @@ const message = {
extraHeaders: 'Дополнительные заголовки',
},
gpu: {
memoryActivity: 'Активность чтения/записи памяти',
hotspotTemperature: 'Температура горячей точки',
fanRPM: 'Скорость вентилятора (об/мин)',
eccCorrectable: 'Исправимые ошибки',
eccUncorrectable: 'Неисправимые ошибки',
eccPending: 'Ожидающий режим ECC',
engineUtil: 'Загрузка движков',
cpuUtil: 'CPU · загрузка',
aiCore: 'AICore · загрузка',
aiCPUUtil: 'AICPU · загрузка',
ctrlCPUUtil: 'CtrlCPU · загрузка',
mediaFrequency: 'Медиа · частота',
encoderUtil: 'Кодировщик · загрузка',
decoderUtil: 'Декодировщик · загрузка',
jpegUtil: 'JPEG · загрузка',
ofaUtil: 'OFA · загрузка',
mediaUtil: 'Медиа · загрузка',
computeUtil: 'Вычисления · загрузка',
copyUtil: 'Копирование · загрузка',
ddrUsage: 'DDR · объём',
ddrUsed: 'DDR · использовано',
ddrTotal: 'DDR · всего',
ddrBandwidth: 'DDR · использование пропускной способности',
hbmUsage: 'HBM · объём',
hbmUsed: 'HBM · использовано',
hbmTotal: 'HBM · всего',
hbmBandwidth: 'HBM · использование пропускной способности',
hugepagesUsage: 'Большие страницы · количество',
hugepagesUsed: 'Большие страницы · использовано',
hugepagesTotal: 'Большие страницы · всего',
historyAverage: 'Среднее',
historyPeak: 'Максимум',
historyAggregated:
'{count} измерений сгруппированы по {seconds} с; для процессов показано только количество.',
historyRaw: 'Исходных измерений: {count}.',
historyMetricEmpty: 'Нет данных этой метрики за выбранный период',
historyLoadFailed: 'Не удалось загрузить историю. Повторите попытку.',
collectedAt: 'Время сбора',
processUnavailable: 'Информация о процессах недоступна',
architecture: 'Архитектура',
memoryFrequency: 'Частота памяти',
memoryTemperature: 'Температура памяти',
memoryReserved: 'Зарезервированная память',
powerDefaultLimit: 'Лимит мощности по умолчанию',
powerMaxLimit: 'Максимальная настраиваемая мощность',
clockEvents: 'Причины ограничения частоты',
memoryBandwidth: 'Использование пропускной способности памяти',
legacyDevice: 'Старая запись',
legacyHistory:
'У этих записей нет постоянного ID устройства. Нельзя проверить принадлежность устройству и отсутствующие значения.',
pcieGeneration: 'Поколение PCIe (текущее / максимальное)',
pcieWidth: 'Ширина PCIe (текущая / максимальная)',
gpu: 'Мониторинг GPU',
gpuHelper: 'Поддерживаемая команда управления GPU/XPU/NPU не обнаружена. Проверьте и повторите попытку!',
process: 'Информация о Процессе',
@@ -1525,7 +1578,7 @@ const message = {
temperature: 'Температура',
performanceState: 'Состояние Производительности',
powerUsage: 'Потребление Мощности',
memoryUsage: 'Использование Памяти',
memoryUsage: 'Использование объёма памяти',
fanSpeed: 'Скорость Вентилятора',
power: 'Мощность',
powerCurrent: 'Текущая Мощность',
+53 -1
View File
@@ -1518,6 +1518,58 @@ const message = {
extraHeaders: 'Ek Başlıklar',
},
gpu: {
memoryActivity: 'Bellek okuma/yazma etkinliği',
hotspotTemperature: 'Sıcak nokta sıcaklığı',
fanRPM: 'Fan hızı (RPM)',
eccCorrectable: 'Düzeltilebilir hatalar',
eccUncorrectable: 'Düzeltilemeyen hatalar',
eccPending: 'Bekleyen ECC modu',
engineUtil: 'Motor kullanımı',
cpuUtil: 'CPU · kullanım',
aiCore: 'AICore · kullanım',
aiCPUUtil: 'AICPU · kullanım',
ctrlCPUUtil: 'CtrlCPU · kullanım',
mediaFrequency: 'Medya · frekans',
encoderUtil: 'Kodlayıcı · kullanım',
decoderUtil: 'Kod çözücü · kullanım',
jpegUtil: 'JPEG · kullanım',
ofaUtil: 'OFA · kullanım',
mediaUtil: 'Medya · kullanım',
computeUtil: 'Hesaplama · kullanım',
copyUtil: 'Kopyalama · kullanım',
ddrUsage: 'DDR · kapasite',
ddrUsed: 'DDR · kullanılan',
ddrTotal: 'DDR · toplam',
ddrBandwidth: 'DDR · bant genişliği kullanımı',
hbmUsage: 'HBM · kapasite',
hbmUsed: 'HBM · kullanılan',
hbmTotal: 'HBM · toplam',
hbmBandwidth: 'HBM · bant genişliği kullanımı',
hugepagesUsage: 'Büyük sayfalar · sayı',
hugepagesUsed: 'Büyük sayfalar · kullanılan',
hugepagesTotal: 'Büyük sayfalar · toplam',
historyAverage: 'Ortalama',
historyPeak: 'En yüksek',
historyAggregated:
'{count} örnek {seconds} saniyelik aralıklarla toplandı; işlemler için yalnızca sayılar gösterilir.',
historyRaw: '{count} ham örnek.',
historyMetricEmpty: 'Seçilen zaman aralığında bu metriğe ait veri yok',
historyLoadFailed: 'Geçmiş yüklenemedi. Lütfen tekrar deneyin.',
collectedAt: 'Toplanma zamanı',
processUnavailable: 'İşlem bilgileri kullanılamıyor',
architecture: 'Mimari',
memoryFrequency: 'Bellek frekansı',
memoryTemperature: 'Bellek sıcaklığı',
memoryReserved: 'Ayrılmış bellek',
powerDefaultLimit: 'Varsayılan güç sınırı',
powerMaxLimit: 'Yapılandırılabilir en yüksek güç',
clockEvents: 'Saat frekansı sınırlama nedenleri',
memoryBandwidth: 'Bellek bant genişliği kullanımı',
legacyDevice: 'Eski kayıt',
legacyHistory: 'Bu kayıtların sabit cihaz kimliği yoktur. Cihaz kimliği ve eksik değerler doğrulanamaz.',
pcieGeneration: 'PCIe nesli (geçerli / en yüksek)',
pcieWidth: 'PCIe genişliği (geçerli / en yüksek)',
gpu: 'GPU İzleme',
gpuHelper: 'Desteklenen bir GPU/XPU/NPU yönetim komutu algılanmadı. Lütfen kontrol edip tekrar deneyin!',
process: 'İşlem Bilgisi',
@@ -1532,7 +1584,7 @@ const message = {
temperature: 'Sıcaklık',
performanceState: 'Performans Durumu',
powerUsage: 'Güç Tüketimi',
memoryUsage: 'Bellek Kullanımı',
memoryUsage: 'Bellek kapasitesi kullanımı',
fanSpeed: 'Fan Hızı',
power: 'Güç',
powerCurrent: 'Mevcut Güç',
+52 -1
View File
@@ -1429,6 +1429,57 @@ const message = {
extraHeaders: '額外請求標頭',
},
gpu: {
memoryActivity: '顯示記憶體讀寫活躍率',
hotspotTemperature: '熱點溫度',
fanRPM: '風扇轉速(RPM)',
eccCorrectable: '可修正錯誤數',
eccUncorrectable: '不可修正錯誤數',
eccPending: '待生效 ECC 模式',
engineUtil: '引擎使用率',
cpuUtil: 'CPU · 使用率',
aiCore: 'AICore · 使用率',
aiCPUUtil: 'AICPU · 使用率',
ctrlCPUUtil: 'CtrlCPU · 使用率',
mediaFrequency: '媒體引擎 · 頻率',
encoderUtil: '編碼器 · 使用率',
decoderUtil: '解碼器 · 使用率',
jpegUtil: 'JPEG · 使用率',
ofaUtil: 'OFA · 使用率',
mediaUtil: '媒體引擎 · 使用率',
computeUtil: '計算引擎 · 使用率',
copyUtil: '複製引擎 · 使用率',
ddrUsage: 'DDR · 容量',
ddrUsed: 'DDR · 已用',
ddrTotal: 'DDR · 總量',
ddrBandwidth: 'DDR · 頻寬使用率',
hbmUsage: 'HBM · 容量',
hbmUsed: 'HBM · 已用',
hbmTotal: 'HBM · 總量',
hbmBandwidth: 'HBM · 頻寬使用率',
hugepagesUsage: '大頁 · 數量',
hugepagesUsed: '大頁 · 已用',
hugepagesTotal: '大頁 · 總量',
historyAverage: '平均值',
historyPeak: '峰值',
historyAggregated: '共 {count} 筆取樣,按 {seconds} 秒彙整;程序僅顯示統計數量。',
historyRaw: '共 {count} 筆原始取樣。',
historyMetricEmpty: '所選時間範圍內沒有此指標的資料',
historyLoadFailed: '歷史記錄載入失敗,請重試',
collectedAt: '採集時間',
processUnavailable: '程序資訊不可用',
architecture: '架構',
memoryFrequency: '顯存頻率',
memoryTemperature: '顯存溫度',
memoryReserved: '保留顯存',
powerDefaultLimit: '預設功率限制',
powerMaxLimit: '最大可設定功率',
clockEvents: '頻率限制原因',
memoryBandwidth: '顯存頻寬使用率',
legacyDevice: '舊記錄',
legacyHistory: '這些記錄沒有穩定裝置識別碼,無法確認其裝置身分和缺失值。',
pcieGeneration: 'PCIe 世代(目前 / 最大)',
pcieWidth: 'PCIe 寬度(目前 / 最大)',
gpu: 'GPU 監控',
gpuHelper: '目前系統未偵測到支援的 GPU/XPU/NPU 管理指令,請檢查後重試',
process: '行程資訊',
@@ -1443,7 +1494,7 @@ const message = {
temperature: '溫度',
performanceState: '效能狀態',
powerUsage: '功耗',
memoryUsage: '顯存使用率',
memoryUsage: '顯示記憶體容量占用率',
fanSpeed: '風扇轉速',
power: '功率',
powerCurrent: '目前功率',
+52 -1
View File
@@ -1442,6 +1442,57 @@ const message = {
extraHeaders: '额外请求头',
},
gpu: {
memoryActivity: '显存读写活跃率',
hotspotTemperature: '热点温度',
fanRPM: '风扇转速(RPM)',
eccCorrectable: '可纠正错误数',
eccUncorrectable: '不可纠正错误数',
eccPending: '待生效 ECC 模式',
engineUtil: '引擎使用率',
cpuUtil: 'CPU · 使用率',
aiCore: 'AICore · 使用率',
aiCPUUtil: 'AICPU · 使用率',
ctrlCPUUtil: 'CtrlCPU · 使用率',
mediaFrequency: '媒体引擎 · 频率',
encoderUtil: '编码器 · 使用率',
decoderUtil: '解码器 · 使用率',
jpegUtil: 'JPEG · 使用率',
ofaUtil: 'OFA · 使用率',
mediaUtil: '媒体引擎 · 使用率',
computeUtil: '计算引擎 · 使用率',
copyUtil: '复制引擎 · 使用率',
ddrUsage: 'DDR · 容量',
ddrUsed: 'DDR · 已用',
ddrTotal: 'DDR · 总量',
ddrBandwidth: 'DDR · 带宽利用率',
hbmUsage: 'HBM · 容量',
hbmUsed: 'HBM · 已用',
hbmTotal: 'HBM · 总量',
hbmBandwidth: 'HBM · 带宽利用率',
hugepagesUsage: '大页 · 数量',
hugepagesUsed: '大页 · 已用',
hugepagesTotal: '大页 · 总量',
historyAverage: '平均值',
historyPeak: '峰值',
historyAggregated: '共 {count} 条采样,按 {seconds} 秒聚合;进程仅展示统计数量。',
historyRaw: '共 {count} 条原始采样。',
historyMetricEmpty: '所选时间范围内没有该指标的数据',
historyLoadFailed: '历史记录加载失败,请重试',
collectedAt: '采集时间',
processUnavailable: '进程信息不可用',
architecture: '架构',
memoryFrequency: '显存频率',
memoryTemperature: '显存温度',
memoryReserved: '保留显存',
powerDefaultLimit: '默认功率限制',
powerMaxLimit: '最大可设置功率',
clockEvents: '频率限制原因',
memoryBandwidth: '显存带宽利用率',
legacyDevice: '旧记录',
legacyHistory: '这些记录没有稳定设备标识,无法确认其设备身份和缺失值。',
pcieGeneration: 'PCIe 代际(当前 / 最大)',
pcieWidth: 'PCIe 宽度(当前 / 最大)',
gpu: 'GPU 监控',
gpuHelper: '当前系统未检测到支持的 GPU/XPU/NPU 管理指令,请检查后重试!',
process: '进程信息',
@@ -1457,7 +1508,7 @@ const message = {
temperature: '温度',
performanceState: '性能状态',
powerUsage: '功耗',
memoryUsage: '显存使用率',
memoryUsage: '显存容量占用率',
fanSpeed: '风扇转速',
power: '功率',
powerCurrent: '当前功率',
@@ -0,0 +1,90 @@
<template>
<div v-if="sections.length" class="device-details">
<section v-for="section in sections" :key="section.id" class="detail-section">
<h3>{{ section.title }}</h3>
<div class="detail-grid">
<div v-for="item in section.items" :key="item.label" class="detail-item">
<div class="detail-label">
<span>{{ item.label }}</span>
<el-tooltip v-if="item.help" placement="top">
<template #content>
<span class="detail-help">{{ item.help }}</span>
</template>
<el-icon tabindex="0" :aria-label="item.help"><InfoFilled /></el-icon>
</el-tooltip>
</div>
<el-tooltip v-if="item.ellipsis" :content="item.value" placement="top">
<strong class="detail-ellipsis" tabindex="0">{{ item.value }}</strong>
</el-tooltip>
<strong v-else>{{ item.value }}</strong>
</div>
</div>
</section>
</div>
</template>
<script lang="ts" setup>
import { InfoFilled } from '@element-plus/icons-vue';
export interface DetailSection {
id: string;
title: string;
items: { label: string; value?: string; help?: string; ellipsis?: boolean }[];
}
defineProps<{ sections: DetailSection[] }>();
</script>
<style lang="scss" scoped>
.detail-help {
white-space: pre-line;
}
.device-details {
display: grid;
gap: 24px;
}
.detail-section {
& + & {
padding-top: 24px;
border-top: 1px solid var(--el-border-color-lighter);
}
h3 {
margin: 0 0 16px;
color: var(--el-text-color-primary);
font-size: 14px;
font-weight: 600;
}
}
.detail-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(min(180px, 100%), 1fr));
gap: 16px 20px;
}
.detail-item {
min-width: 0;
.detail-label {
display: flex;
align-items: baseline;
gap: 6px;
margin-bottom: 5px;
color: var(--el-text-color-secondary);
font-size: 12px;
line-height: 18px;
overflow-wrap: anywhere;
}
strong {
display: block;
color: var(--el-text-color-primary);
font-size: 13px;
font-weight: 500;
font-variant-numeric: tabular-nums;
line-height: 20px;
overflow-wrap: anywhere;
}
.detail-ellipsis {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
}
</style>
File diff suppressed because it is too large Load Diff
+299 -484
View File
@@ -1,500 +1,322 @@
<template>
<div v-loading="loading">
<RouterMenu />
<div class="content-container__search" v-if="options.length !== 0">
<el-card>
<div>
<el-date-picker
@change="search()"
v-model="timeRangeGlobal"
type="datetimerange"
range-separator="-"
:start-placeholder="$t('commons.search.timeStart')"
:end-placeholder="$t('commons.search.timeEnd')"
:shortcuts="shortcuts"
style="max-width: 360px; width: 100%"
:size="isMobile ? 'small' : 'default'"
></el-date-picker>
<el-select class="p-w-300 ml-2" v-model="searchInfo.productName" @change="search()">
<el-option v-for="item in options" :key="item" :label="item" :value="item" />
</el-select>
<TableRefresh class="float-right" @search="search()" />
<el-button icon="Setting" class="float-right mr-2" @click="quickJump()" />
</div>
</el-card>
</div>
<el-row :gutter="7" v-if="options.length !== 0">
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div :class="isMobile ? 'flx-wrap' : 'flex justify-between'">
<span class="title">{{ $t('aiTools.gpu.memoryUsage') }}</span>
</div>
</template>
<div class="chart" v-if="!currentHide?.memory">
<v-charts
height="400px"
id="loadMemoryChart"
type="line"
:option="chartsOption['loadMemoryChart']"
v-if="chartsOption['loadMemoryChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
</el-card>
</el-col>
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div :class="isMobile ? 'flx-wrap' : 'flex justify-between'">
<span class="title">{{ $t('aiTools.gpu.gpuUtil') }}</span>
</div>
</template>
<div class="chart" v-if="!currentHide?.gpu">
<v-charts
height="400px"
id="loadGPUChart"
type="line"
:option="chartsOption['loadGPUChart']"
v-if="chartsOption['loadGPUChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
</el-card>
</el-col>
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div :class="isMobile ? 'flx-wrap' : 'flex justify-between'">
<span class="title">{{ $t('aiTools.gpu.process') }}</span>
</div>
</template>
<div class="chart" v-if="!currentHide?.process">
<v-charts
height="400px"
id="loadProcessChart"
type="line"
:option="chartsOption['loadProcessChart']"
v-if="chartsOption['loadProcessChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
</el-card>
</el-col>
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div :class="isMobile ? 'flx-wrap' : 'flex justify-between'">
<span class="title">{{ $t('aiTools.gpu.powerUsage') }}</span>
</div>
</template>
<div class="chart" v-if="!currentHide?.power">
<v-charts
height="400px"
id="loadPowerChart"
type="line"
:option="chartsOption['loadPowerChart']"
v-if="chartsOption['loadPowerChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
</el-card>
</el-col>
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div>
{{ $t('aiTools.gpu.temperature') }}
<el-tooltip placement="top" :content="$t('aiTools.gpu.temperatureHelper')">
<el-icon size="15"><InfoFilled /></el-icon>
</el-tooltip>
</div>
</template>
<div class="chart" v-if="!currentHide?.temperature">
<v-charts
height="400px"
id="loadTemperatureChart"
type="line"
:option="chartsOption['loadTemperatureChart']"
v-if="chartsOption['loadTemperatureChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
</el-card>
</el-col>
<el-col :xs="24" :sm="24" :md="12" :lg="12" :xl="12">
<el-card class="card-interval" style="overflow: inherit">
<template #header>
<div :class="isMobile ? 'flx-wrap' : 'flex justify-between'">
<span class="title">{{ $t('aiTools.gpu.fanSpeed') }}</span>
</div>
</template>
<div class="chart" v-if="!currentHide?.speed">
<v-charts
height="400px"
id="loadSpeedChart"
type="line"
:option="chartsOption['loadSpeedChart']"
v-if="chartsOption['loadSpeedChart']"
:dataZoom="true"
/>
</div>
<el-empty class="chart" v-else :description="$t('aiTools.gpu.notSupport')" />
<el-card class="history-toolbar">
<div class="flex flex-wrap gap-3 items-center">
<el-date-picker
v-model="timeRangeGlobal"
type="datetimerange"
style="max-width: 100%; width: 360px; flex-grow: 0"
range-separator="-"
:start-placeholder="$t('commons.search.timeStart')"
:end-placeholder="$t('commons.search.timeEnd')"
:shortcuts="shortcuts"
:clearable="false"
:size="isMobile ? 'small' : 'default'"
@change="changeTimeRange"
/>
<el-select style="max-width: 100%" class="p-w-300" v-model="selectedDevice" @change="search">
<el-option v-for="item in options" :key="item.value" :label="item.label" :value="item.value" />
</el-select>
<el-radio-group v-if="history && history.bucketSeconds > 0" v-model="aggregation" @change="search">
<el-radio-button value="avg">{{ $t('aiTools.gpu.historyAverage') }}</el-radio-button>
<el-radio-button value="max">{{ $t('aiTools.gpu.historyPeak') }}</el-radio-button>
</el-radio-group>
<TableRefresh @search="refresh" />
</div>
<div v-if="history?.sampleCount" class="history-summary">
{{
history.bucketSeconds > 0
? $t('aiTools.gpu.historyAggregated', {
seconds: history.bucketSeconds,
count: history.sampleCount,
})
: $t('aiTools.gpu.historyRaw', { count: history.sampleCount })
}}
</div>
</el-card>
<el-alert
v-if="currentDevice?.legacy"
:title="$t('aiTools.gpu.legacyHistory')"
type="info"
:closable="false"
show-icon
/>
<el-alert v-if="failed" :title="$t('aiTools.gpu.historyLoadFailed')" type="error" :closable="false" show-icon />
<el-empty v-else-if="!loading && !history?.sampleCount" :description="$t('commons.msg.noneData')" />
<el-empty
v-else-if="history?.sampleCount && !charts.length"
:description="$t('aiTools.gpu.historyMetricEmpty')"
/>
<el-row v-else-if="history?.sampleCount" :gutter="12">
<el-col v-for="chart in charts" :key="chart.id" :xs="24" :md="12">
<el-card class="card-interval">
<template #header>{{ chart.title }}</template>
<v-charts :id="chart.id" height="320px" type="line" :option="chart.option" :dataZoom="true" />
</el-card>
</el-col>
</el-row>
<LayoutContent :title="$t('aiTools.gpu.gpu')" :divider="true" v-else>
<template #main>
<div class="app-warn">
<div class="flx-center">
<span>{{ $t('aiTools.gpu.gpuHelper') }}</span>
</div>
<div>
<img src="@/assets/images/no_app.svg" />
</div>
</div>
</template>
</LayoutContent>
</div>
</template>
<script setup lang="ts">
import { ref, reactive, onMounted } from 'vue';
import { ref, computed, onMounted } from 'vue';
import { loadGPUMonitor, getGPUOptions } from '@/api/modules/ai';
import { dateFormatWithoutYear } from '@/utils/date';
import RouterMenu from '@/views/ai/gpu/index.vue';
import { shortcuts } from '@/utils/shortcuts';
import i18n from '@/lang';
import { routerToName } from '@/utils/router';
import { AI } from '@/api/interface/ai';
import { useGlobalStore } from '@/composables/useGlobalStore';
const { isMobile } = useGlobalStore();
const loading = ref(false);
const options = ref<string[]>([]);
const gpuType = ref('gpu');
const failed = ref(false);
const options = ref<(AI.ChartHide & { value: string; label: string })[]>([]);
const selectedDevice = ref('');
const aggregation = ref<'avg' | 'max'>('avg');
const history = ref<AI.MonitorGPUData>();
const currentDevice = computed(() => options.value.find((item) => item.value === selectedDevice.value));
const timeRangeGlobal = ref<[Date, Date]>([new Date(new Date().setHours(0, 0, 0, 0)), new Date()]);
const chartsOption = ref({
loadPowerChart: null,
loadGPUChart: null,
loadMemoryChart: null,
loadTemperatureChart: null,
loadSpeedChart: null,
let followNow = true;
let requestID = 0;
type MetricKey = Exclude<keyof AI.MonitorGPUData, 'date' | 'gpuProcesses' | 'sampleCount' | 'bucketSeconds'>;
type Series = { key: MetricKey; label: string; unit: string; percent?: boolean };
const charts = computed(() => {
const data = history.value;
if (!data) return [];
const definitions: { id: string; title: string; series: Series[] }[] = [
{
id: 'gpu-util',
title: currentDevice.value?.type === 'npu' ? 'aiCore' : 'gpuUtil',
series: [{ key: 'gpuValue', label: currentDevice.value?.type === 'npu' ? 'aiCore' : 'gpuUtil', unit: '%' }],
},
{
id: 'gpu-memory',
title: 'memory',
series: [
{ key: 'memoryUsed', label: 'memoryUsed', unit: 'MiB' },
{ key: 'memoryTotal', label: 'memoryTotal', unit: 'MiB' },
{ key: 'memoryPercent', label: 'percent', unit: '%', percent: true },
],
},
{
id: 'gpu-power',
title: 'powerUsage',
series: [
{ key: 'powerUsed', label: 'powerCurrent', unit: 'W' },
{ key: 'powerTotal', label: 'powerLimit', unit: 'W' },
{ key: 'powerPercent', label: 'percent', unit: '%', percent: true },
],
},
{
id: 'gpu-temperature',
title: 'temperature',
series: [
{ key: 'temperatureValue', label: 'temperature', unit: '°C' },
{ key: 'memoryTemperatureValue', label: 'memoryTemperature', unit: '°C' },
{ key: 'hotspotTemperature', label: 'hotspotTemperature', unit: '°C' },
],
},
{
id: 'gpu-frequency',
title: 'frequency',
series: [
{ key: 'frequencyValue', label: 'frequency', unit: 'MHz' },
{ key: 'memoryFrequencyValue', label: 'memoryFrequency', unit: 'MHz' },
{ key: 'mediaFrequency', label: 'mediaFrequency', unit: 'MHz' },
],
},
{ id: 'gpu-fan', title: 'fanSpeed', series: [{ key: 'speedValue', label: 'fanSpeed', unit: '%' }] },
{
id: 'gpu-process',
title: 'processCount',
series: [{ key: 'processCount', label: 'processCount', unit: '' }],
},
{
id: 'memory-activity',
title: 'memoryActivity',
series: [{ key: 'memoryActivity', label: 'memoryActivity', unit: '%' }],
},
{
id: 'engine-util',
title: 'engineUtil',
series: [
{ key: 'encoderUtil', label: 'encoderUtil', unit: '%' },
{ key: 'decoderUtil', label: 'decoderUtil', unit: '%' },
{ key: 'jpegUtil', label: 'jpegUtil', unit: '%' },
{ key: 'ofaUtil', label: 'ofaUtil', unit: '%' },
{ key: 'mediaUtil', label: 'mediaUtil', unit: '%' },
{ key: 'computeUtil', label: 'computeUtil', unit: '%' },
{ key: 'copyUtil', label: 'copyUtil', unit: '%' },
],
},
{
id: 'memory-bandwidth',
title: 'memoryBandwidth',
series: [
{ key: 'memoryBandwidth', label: 'memoryBandwidth', unit: '%' },
{ key: 'ddrBandwidth', label: 'ddrBandwidth', unit: '%' },
{ key: 'hbmBandwidth', label: 'hbmBandwidth', unit: '%' },
],
},
{
id: 'npu-cpu',
title: 'cpuUtil',
series: [
{ key: 'aiCPUUtil', label: 'aiCPUUtil', unit: '%' },
{ key: 'ctrlCPUUtil', label: 'ctrlCPUUtil', unit: '%' },
],
},
{
id: 'npu-ddr',
title: 'ddrUsage',
series: [
{ key: 'ddrUsed', label: 'ddrUsed', unit: 'MiB' },
{ key: 'ddrTotal', label: 'ddrTotal', unit: 'MiB' },
],
},
{
id: 'npu-hbm',
title: 'hbmUsage',
series: [
{ key: 'hbmUsed', label: 'hbmUsed', unit: 'MiB' },
{ key: 'hbmTotal', label: 'hbmTotal', unit: 'MiB' },
],
},
{
id: 'npu-hugepages',
title: 'hugepagesUsage',
series: [
{ key: 'hugepagesUsed', label: 'hugepagesUsed', unit: 'pages' },
{ key: 'hugepagesTotal', label: 'hugepagesTotal', unit: 'pages' },
],
},
{ id: 'fan-rpm', title: 'fanRPM', series: [{ key: 'fanRPM', label: 'fanRPM', unit: 'RPM' }] },
];
const dates = (data.date || []).map((date) => new Date(date).getTime());
return definitions.flatMap((chart) => {
const series = chart.series.filter((item) => data[item.key]?.some((value) => value != null));
if (!series.length) return [];
const hasPercent = series.some((item) => item.percent) && series.some((item) => !item.percent);
const primaryUnit = (series.find((item) => !item.percent) || series[0]).unit;
return [
{
id: chart.id,
title: i18n.global.t('aiTools.gpu.' + chart.title),
option: {
xData: dates,
xAxis: {
type: 'time',
splitNumber: isMobile.value ? 3 : 6,
axisLabel: { hideOverlap: true },
min: timeRangeGlobal.value[0].getTime(),
max: timeRangeGlobal.value[1].getTime(),
},
yData: series.map((item, index) => ({
itemStyle: {
color: ['#409eff', '#67c23a', '#e6a23c', '#f56c6c', '#8b5cf6', '#0891b2', '#db2777'][
index % 7
],
},
areaStyle: { opacity: 0 },
name: i18n.global.t('aiTools.gpu.' + item.label),
data: dates.map((date, index) => [date, data[item.key]?.[index] ?? null]),
yAxisIndex: item.percent && hasPercent ? 1 : 0,
showSymbol: true,
symbolSize: 3,
})),
yAxis: [
{
type: 'value',
name: primaryUnit,
...(primaryUnit === '°C' ? {} : { min: 0 }),
...(primaryUnit === '%' ? { max: 100 } : {}),
},
...(hasPercent ? [{ type: 'value', name: '%', position: 'right', min: 0 }] : []),
],
grid: { left: isMobile.value ? 55 : 65, right: hasPercent ? 55 : 25, bottom: 80, top: 55 },
legend: { type: 'scroll', top: 0, bottom: 'auto' },
tooltip: {
trigger: 'axis',
formatter: (items: any[]) => {
const tooltip = document.createElement('div');
const title = document.createElement('div');
title.textContent = items.length ? new Date(items[0].value[0]).toLocaleString() : '';
tooltip.appendChild(title);
for (const item of items) {
const line = document.createElement('div');
const value = item.value[1];
line.textContent = `${item.seriesName}: ${value == null ? 'N/A' : Number(value.toFixed(2))} ${series[item.seriesIndex]?.unit || ''}`;
tooltip.appendChild(line);
}
if (chart.id === 'gpu-process' && data.bucketSeconds === 0 && items.length) {
const processes = data.gpuProcesses?.[items[0].dataIndex];
if (processes?.length) {
const typeTitle = i18n.global.t(
currentDevice.value?.type === 'xpu' ? 'aiTools.gpu.shr' : 'aiTools.gpu.type',
);
appendProcessTable(tooltip, processes, typeTitle);
}
}
return tooltip;
},
},
},
},
];
});
});
const chartHide = ref<AI.ChartHide[]>([]);
const currentHide = ref<AI.ChartHide>();
const searchInfo = reactive<AI.MonitorGPUSearch>({
productName: '',
startTime: new Date(new Date().setHours(0, 0, 0, 0)),
endTime: new Date(),
});
const loadOptions = async () => {
loading.value = true;
await getGPUOptions()
.then((res) => {
gpuType.value = res.data.gpuType || 'gpu';
options.value = res.data.options || [];
chartHide.value = res.data.chartHide || [];
searchInfo.productName = options.value.length > 0 ? options.value[0] : '';
search();
})
.catch(() => {
loading.value = false;
options.value = [];
});
};
const loadCurrentHide = () => {
currentHide.value = undefined;
for (const item of chartHide.value) {
if (item.productName === searchInfo.productName) {
currentHide.value = item;
break;
}
}
};
const search = async () => {
if (timeRangeGlobal.value && timeRangeGlobal.value.length === 2) {
searchInfo.startTime = timeRangeGlobal.value[0];
searchInfo.endTime = timeRangeGlobal.value[1];
}
const id = ++requestID;
history.value = undefined;
failed.value = false;
const device = currentDevice.value;
if (!device || !timeRangeGlobal.value) return;
loading.value = true;
loadCurrentHide();
await loadGPUMonitor(searchInfo)
.then((res) => {
loading.value = false;
let baseDate = res.data.date || [];
if (baseDate.length === 0) {
baseDate = loadEmptyDate(timeRangeGlobal.value);
}
let date = baseDate.map(function (item: any) {
return dateFormatWithoutYear(item);
});
if (!currentHide.value || !currentHide.value.gpu) {
initCPUCharts(date, res.data.gpuValue || []);
}
if (!currentHide.value || !currentHide.value.memory) {
initMemoryCharts(date, res.data);
}
if (!currentHide.value || !currentHide.value.power) {
initPowerCharts(date, res.data);
}
if (!currentHide.value || !currentHide.value.speed) {
initSpeedCharts(date, res.data.speedValue || []);
}
if (!currentHide.value || !currentHide.value.temperature) {
initTemperatureCharts(date, res.data.temperatureValue || []);
}
if (!currentHide.value || !currentHide.value.process) {
initProcessCharts(date, res.data);
}
})
.catch(() => {
loading.value = false;
try {
const response = await loadGPUMonitor({
deviceID: device.deviceID,
productName: device.productName,
legacy: device.legacy,
startTime: timeRangeGlobal.value[0],
endTime: timeRangeGlobal.value[1],
aggregation: aggregation.value,
});
if (id === requestID) history.value = response.data;
} catch {
if (id === requestID) failed.value = true;
} finally {
if (id === requestID) loading.value = false;
}
};
function initCPUCharts(baseDate: any, items: any) {
let percents = items.map(function (item: any) {
return Number(item.toFixed(2));
});
let data = percents.length === 0 ? loadEmptyData() : percents;
chartsOption.value['loadGPUChart'] = {
xData: baseDate,
yData: [
{
name: i18n.global.t('aiTools.gpu.gpuUtil'),
data: data,
},
],
formatStr: '%',
};
}
function initMemoryCharts(baseDate: any, data: any) {
chartsOption.value['loadMemoryChart'] = {
xData: baseDate,
yData: [
{
name: i18n.global.t('aiTools.gpu.memoryUsed'),
data: data.memoryUsed,
},
{
name: i18n.global.t('aiTools.gpu.memoryTotal'),
data: data.memoryTotal,
},
{
name: i18n.global.t('aiTools.gpu.percent'),
data: data.memoryPercent,
yAxisIndex: 1,
},
],
yAxis: [
{ type: 'value', name: i18n.global.t('aiTools.gpu.memory') },
{
type: 'value',
name: i18n.global.t('aiTools.gpu.percent') + ' ( % )',
position: 'right',
alignTicks: true,
},
],
grid: isMobile.value ? { left: '15%', right: '15%', bottom: '20%' } : null,
tooltip: {
trigger: 'axis',
formatter: function (list: any) {
return withMemoryProcess(list);
},
},
formatStr: '%',
};
}
function initPowerCharts(baseDate: any, data: any) {
const yData: any[] = [
{
name: i18n.global.t('aiTools.gpu.powerCurrent'),
data: data.powerUsed,
},
];
const yAxis: any[] = [{ type: 'value', name: i18n.global.t('aiTools.gpu.power') }];
if (!currentHide.value?.powerLimit) {
yData.push(
{
name: i18n.global.t('aiTools.gpu.powerLimit'),
data: data.powerTotal,
},
{
name: i18n.global.t('aiTools.gpu.percent'),
data: data.powerPercent,
yAxisIndex: 1,
},
);
yAxis.push({
type: 'value',
name: i18n.global.t('aiTools.gpu.percent') + ' ( % )',
position: 'right',
alignTicks: true,
});
}
chartsOption.value['loadPowerChart'] = {
xData: baseDate,
yData,
yAxis,
grid: isMobile.value ? { left: '15%', right: '15%', bottom: '20%' } : null,
tooltip: {
trigger: 'axis',
formatter: function (list: any) {
const tooltip = createTooltip(list[0].name);
for (const item of list) {
if (
item.seriesName === i18n.global.t('aiTools.gpu.powerCurrent') ||
item.seriesName === i18n.global.t('aiTools.gpu.powerLimit')
) {
appendSeries(tooltip, item, item.data, 'W');
} else {
appendSeries(tooltip, item, Number(item.data.toFixed(2)), '%');
}
}
return tooltip;
},
},
formatStr: currentHide.value?.powerLimit ? 'W' : '%',
};
}
const quickJump = () => {
routerToName('HostMonitorSetting');
const changeTimeRange = () => {
followNow = false;
void search();
};
function initTemperatureCharts(baseDate: any, items: any) {
let temperatures = items.map(function (item: any) {
return Number(item);
});
temperatures = temperatures.length === 0 ? loadEmptyData() : temperatures;
chartsOption.value['loadTemperatureChart'] = {
xData: baseDate,
yData: [
{
name: i18n.global.t('aiTools.gpu.temperature'),
data: temperatures,
},
],
formatStr: '°C',
};
}
function initProcessCharts(baseDate: any, data: any) {
let process = data.processCount.map(function (item: any) {
return Number(item);
});
process = process.length === 0 ? loadEmptyData() : process;
chartsOption.value['loadProcessChart'] = {
xData: baseDate,
yData: [
{
name: i18n.global.t('aiTools.gpu.processCount'),
data: process,
},
],
tooltip: {
trigger: 'axis',
formatter: function (list: any) {
const param = list[0];
const index = param.dataIndex;
let process = data.gpuProcesses?.length > index ? data.gpuProcesses[index] : [];
return withProcess(list, process);
},
},
formatStr: '-',
};
}
function initSpeedCharts(baseDate: any, items: any) {
let speeds = items.map(function (item: any) {
return Number(item);
});
speeds = speeds.length === 0 ? loadEmptyData() : speeds;
chartsOption.value['loadSpeedChart'] = {
xData: baseDate,
yData: [
{
name: i18n.global.t('aiTools.gpu.fanSpeed'),
data: speeds,
},
],
formatStr: '%',
};
}
function loadEmptyDate(timeRange: any) {
if (timeRange.length != 2) {
return;
const loadOptions = async () => {
loading.value = true;
failed.value = false;
try {
const response = await getGPUOptions();
options.value = (response.data.chartHide || []).map((item) => ({
...item,
value: item.deviceID || `legacy:${item.productName}`,
label: `${item.productName} · ${item.deviceID || i18n.global.t('aiTools.gpu.legacyDevice')}`,
}));
if (!currentDevice.value) selectedDevice.value = options.value[0]?.value || '';
await search();
} catch {
failed.value = true;
} finally {
loading.value = false;
}
let date1 = new Date(timeRange[0]);
let date2 = new Date(timeRange[1]);
return [date1, date2];
}
function loadEmptyData() {
return [0, 0];
}
function withMemoryProcess(list: any) {
const tooltip = createTooltip(list[0].name);
for (const item of list) {
if (
item.seriesName === i18n.global.t('aiTools.gpu.memoryUsed') ||
item.seriesName === i18n.global.t('aiTools.gpu.memoryTotal')
) {
appendSeries(tooltip, item, item.data, 'MiB');
} else {
appendSeries(tooltip, item, Number(item.data.toFixed(2)), '%');
}
}
return tooltip;
}
function withProcess(list: any, process: any) {
const tooltip = createTooltip(list[0].name);
for (const item of list) {
appendSeries(tooltip, item, item.data, '');
}
const type = currentHide.value?.type || gpuType.value;
const title = type === 'xpu' ? i18n.global.t('aiTools.gpu.shr') : i18n.global.t('aiTools.gpu.type');
appendProcessTable(tooltip, process || [], title);
return tooltip;
}
function createTooltip(name: unknown): HTMLDivElement {
const tooltip = document.createElement('div');
const date = document.createElement('div');
date.style.display = 'inline-block';
date.style.width = '100%';
date.style.paddingBottom = '10px';
date.textContent = `${i18n.global.t('commons.search.date')}: ${String(name ?? '').replaceAll('\n', ' ')}`;
tooltip.appendChild(date);
return tooltip;
}
function appendSeries(tooltip: HTMLElement, item: any, data: unknown, unit: string) {
const line = document.createElement('div');
line.style.width = '100%';
const marker = document.createElement('span');
marker.textContent = '●';
if (typeof item.color === 'string') {
marker.style.color = item.color;
}
line.appendChild(marker);
line.appendChild(document.createTextNode(` ${String(item.seriesName ?? '')}: ${String(data ?? '')} ${unit}`));
tooltip.appendChild(line);
}
};
const refresh = () => {
if (followNow) timeRangeGlobal.value = [timeRangeGlobal.value[0], new Date()];
void loadOptions();
};
function appendProcessTable(tooltip: HTMLElement, process: AI.GPUProcess[], typeTitle: string) {
const separator = document.createElement('div');
@@ -541,24 +363,17 @@ const loadProcessType = (val: string) => {
return val;
};
onMounted(() => {
loadOptions();
});
onMounted(loadOptions);
</script>
<style scoped lang="scss">
.content-container__search {
margin-top: 7px;
.el-card {
--el-card-padding: 12px;
}
.history-toolbar {
margin: 7px 0 12px;
--el-card-padding: 12px;
}
.title {
font-size: 16px;
font-weight: 500;
}
.chart {
width: 100%;
height: 400px;
.history-summary {
margin-top: 12px;
color: var(--el-text-color-secondary);
font-size: 13px;
}
</style>