From afcb94f6eff08ce2060b2ccd99035bb2612de35c Mon Sep 17 00:00:00 2001 From: ssongliu Date: Mon, 12 Oct 2026 15:54:42 +0800 Subject: [PATCH] feat(vllm): add request metrics and optimize monitor history (#14032) --- agent/app/api/v2/monitor_vllm.go | 2 +- agent/app/dto/monitor.go | 1 + agent/app/model/monitor.go | 20 ++- agent/app/repo/monitor_vllm.go | 72 +++++++-- agent/app/service/monitor.go | 2 +- agent/app/service/monitor_vllm.go | 34 ++-- agent/init/migration/migrate.go | 1 + agent/init/migration/migrations/init.go | 7 + agent/utils/ai_tools/vllm/history.go | 156 +++++++++++++++++++ agent/utils/ai_tools/vllm/metrics.go | 78 +++++----- agent/utils/ai_tools/vllm/request_metrics.go | 65 ++++++++ frontend/src/lang/modules/en.ts | 18 +++ frontend/src/lang/modules/es-es.ts | 18 +++ frontend/src/lang/modules/fa.ts | 17 ++ frontend/src/lang/modules/ja.ts | 17 ++ frontend/src/lang/modules/ko.ts | 17 ++ frontend/src/lang/modules/lo.ts | 17 ++ frontend/src/lang/modules/ms.ts | 18 +++ frontend/src/lang/modules/pt-br.ts | 18 +++ frontend/src/lang/modules/ru.ts | 18 +++ frontend/src/lang/modules/tr.ts | 17 ++ frontend/src/lang/modules/zh-Hant.ts | 17 ++ frontend/src/lang/modules/zh.ts | 17 ++ 23 files changed, 577 insertions(+), 70 deletions(-) create mode 100644 agent/utils/ai_tools/vllm/history.go create mode 100644 agent/utils/ai_tools/vllm/request_metrics.go diff --git a/agent/app/api/v2/monitor_vllm.go b/agent/app/api/v2/monitor_vllm.go index 199554af5..9a36feb30 100644 --- a/agent/app/api/v2/monitor_vllm.go +++ b/agent/app/api/v2/monitor_vllm.go @@ -11,7 +11,7 @@ func (b *BaseApi) LoadVLLMMonitor(c *gin.Context) { if err := helper.CheckBindAndValidate(&req, c); err != nil { return } - data, err := monitorService.LoadVLLMMonitorData(req) + data, err := monitorService.LoadVLLMMonitorData(c.Request.Context(), req) if err != nil { helper.InternalServer(c, err) return diff --git a/agent/app/dto/monitor.go b/agent/app/dto/monitor.go index 0aa83f4cb..6392e5564 100644 --- a/agent/app/dto/monitor.go +++ b/agent/app/dto/monitor.go @@ -127,6 +127,7 @@ type GPUProcess struct { } type MonitorVLLMSearch struct { + Group string `json:"group" validate:"omitempty,oneof=overview latency requests"` AppInstallID uint `json:"appInstallID" validate:"required"` StartTime time.Time `json:"startTime" validate:"required"` EndTime time.Time `json:"endTime" validate:"required"` diff --git a/agent/app/model/monitor.go b/agent/app/model/monitor.go index 43ad63ec8..ebe2f3cb7 100644 --- a/agent/app/model/monitor.go +++ b/agent/app/model/monitor.go @@ -79,12 +79,20 @@ type MonitorGPU struct { } type MonitorVLLM struct { - ID uint `json:"-" gorm:"primarykey;autoIncrement"` - CreatedAt time.Time `json:"createdAt"` - AppInstallID uint `json:"appInstallID"` - Status string `json:"status"` - RawMetrics string `json:"-"` - HistogramDeltas string `json:"-"` + ID uint `json:"-" gorm:"primarykey;autoIncrement"` + CreatedAt time.Time `json:"createdAt"` + AppInstallID uint `json:"appInstallID"` + Status string `json:"status"` + RawMetrics string `json:"-"` + HistogramDeltas string `json:"-"` + IntervalSeconds float64 `json:"intervalSeconds" gorm:"-"` + RequestDeltas string `json:"-"` + QueueTime *float64 `json:"queueTime"` + AverageGenerationTokens *float64 `json:"averageGenerationTokens"` + PrefixCacheHitRate *float64 `json:"prefixCacheHitRate"` + FinishReasons map[string]float64 `json:"finishReasons" gorm:"-"` + PromptDistribution map[string]float64 `json:"promptDistribution" gorm:"-"` + GenerationDistribution map[string]float64 `json:"generationDistribution" gorm:"-"` Running *float64 `json:"running"` Waiting *float64 `json:"waiting"` diff --git a/agent/app/repo/monitor_vllm.go b/agent/app/repo/monitor_vllm.go index a03a79c8a..a8c2087ec 100644 --- a/agent/app/repo/monitor_vllm.go +++ b/agent/app/repo/monitor_vllm.go @@ -1,6 +1,7 @@ package repo import ( + "context" "errors" "fmt" "strings" @@ -15,8 +16,7 @@ type VLLMMonitorRepo struct{} type VLLMHistoryPoint struct { model.MonitorVLLM - Bucket int64 - HistogramSamples string + Bucket int64 } func (r *VLLMMonitorRepo) Create(point *model.MonitorVLLM) error { @@ -41,16 +41,30 @@ func (r *VLLMMonitorRepo) DeleteBefore(before time.Time) error { return global.VLLMMonitorDB.Where("created_at < ?", before).Delete(&model.MonitorVLLM{}).Error } -func (r *VLLMMonitorRepo) Count(id uint, start, end time.Time) (int64, error) { +func (r *VLLMMonitorRepo) Count(ctx context.Context, id uint, start, end time.Time) (int64, error) { var count int64 - db := global.VLLMMonitorDB.Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end) + db := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end) err := db.Count(&count).Error return count, err } -func (r *VLLMMonitorRepo) History(id uint, start, end time.Time, seconds int64, aggregation string) ([]VLLMHistoryPoint, error) { - db := global.VLLMMonitorDB.Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end) - metrics := []string{"running", "waiting", "cache_usage", "prompt_throughput", "generation_throughput", "request_throughput", "time_to_first_token", "time_per_output_token", "request_latency", "prefill_time", "decode_time", "time_to_first_token_p50", "time_to_first_token_p90", "time_to_first_token_p95", "time_to_first_token_p99", "time_per_output_token_p50", "time_per_output_token_p90", "time_per_output_token_p95", "time_per_output_token_p99", "request_latency_p50", "request_latency_p90", "request_latency_p95", "request_latency_p99"} +func (r *VLLMMonitorRepo) History(ctx context.Context, id uint, start, end time.Time, seconds int64, aggregation, group string) ([]VLLMHistoryPoint, error) { + db := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end) + var metrics []string + if group == "" || group == "overview" { + metrics = append(metrics, "running", "waiting", "cache_usage", "prompt_throughput", "generation_throughput", "request_throughput") + if seconds <= 0 || aggregation == "max" { + metrics = append(metrics, "prefix_cache_hit_rate") + } + } + if seconds <= 0 || aggregation == "max" { + if group == "" || group == "latency" { + metrics = append(metrics, "queue_time", "time_to_first_token", "time_per_output_token", "request_latency", "prefill_time", "decode_time", "time_to_first_token_p50", "time_to_first_token_p90", "time_to_first_token_p95", "time_to_first_token_p99", "time_per_output_token_p50", "time_per_output_token_p90", "time_per_output_token_p95", "time_per_output_token_p99", "request_latency_p50", "request_latency_p90", "request_latency_p95", "request_latency_p99") + } + if group == "" || group == "requests" { + metrics = append(metrics, "average_generation_tokens") + } + } var columns []string if seconds > 0 { operation := "AVG" @@ -59,20 +73,52 @@ func (r *VLLMMonitorRepo) History(id uint, start, end time.Time, seconds int64, } columns = []string{fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d AS bucket", start.Unix(), seconds)} for _, column := range metrics { - if aggregation != "max" && (strings.HasPrefix(column, "time_to_first_token_p") || strings.HasPrefix(column, "time_per_output_token_p") || strings.HasPrefix(column, "request_latency_p")) { - continue - } columns = append(columns, operation+"("+column+") AS "+column) } - if aggregation != "max" { - columns = append(columns, "json_group_array(json(NULLIF(histogram_deltas, ''))) AS histogram_samples") - } db = db.Group("bucket").Order("bucket ASC") } else { columns = append([]string{"id", "created_at", "app_install_id", "status"}, metrics...) + if group == "" || group == "requests" { + columns = append(columns, "histogram_deltas", "request_deltas") + } db = db.Order("created_at ASC, id ASC") } var points []VLLMHistoryPoint err := db.Select(strings.Join(columns, ", ")).Scan(&points).Error return points, err } + +func (r *VLLMMonitorRepo) WalkDeltas(ctx context.Context, id uint, start, end time.Time, seconds int64, aggregation, group string, visit func(int64, string, string) error) error { + histograms, requests := "''", "''" + if group == "" || group == "requests" || (group == "latency" && aggregation != "max") { + histograms = "COALESCE(histogram_deltas, '')" + } + if group == "" || group == "requests" || (group == "overview" && aggregation != "max") { + requests = "COALESCE(request_deltas, '')" + } + if histograms == "''" && requests == "''" { + return ctx.Err() + } + bucket := fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d", start.Unix(), seconds) + rows, err := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}). + Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end). + Select(bucket + ", " + histograms + ", " + requests).Rows() + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + if err := ctx.Err(); err != nil { + return err + } + var index int64 + var histograms, requests string + if err := rows.Scan(&index, &histograms, &requests); err != nil { + return err + } + if err := visit(index, histograms, requests); err != nil { + return err + } + } + return rows.Err() +} diff --git a/agent/app/service/monitor.go b/agent/app/service/monitor.go index bba77821f..852efb06a 100644 --- a/agent/app/service/monitor.go +++ b/agent/app/service/monitor.go @@ -54,7 +54,7 @@ type IMonitorService interface { LoadIOOptions() []string LoadNetworkOptions() []string - LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) + LoadVLLMMonitorData(ctx context.Context, req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) LoadVLLMCurrent(ctx context.Context, req dto.MonitorVLLMCurrent) (model.MonitorVLLM, error) CleanVLLMMonitor(req dto.MonitorVLLMClean) error LoadGPUOptions() dto.MonitorGPUOptions diff --git a/agent/app/service/monitor_vllm.go b/agent/app/service/monitor_vllm.go index de1be89e7..d780edf52 100644 --- a/agent/app/service/monitor_vllm.go +++ b/agent/app/service/monitor_vllm.go @@ -31,12 +31,12 @@ var vllmMetricsClient = &http.Client{ CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse }, } -func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) { +func (m *MonitorService) LoadVLLMMonitorData(ctx context.Context, req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) { data := dto.MonitorVLLMData{Points: []model.MonitorVLLM{}} if !req.EndTime.After(req.StartTime) { return data, fmt.Errorf("invalid vLLM history request") } - install, err := appInstallRepo.GetFirst(repo.WithByID(req.AppInstallID)) + install, err := appInstallRepo.GetFirstByCtx(ctx, repo.WithByID(req.AppInstallID)) if err != nil { return data, err } @@ -49,23 +49,31 @@ func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.Mon return data, err } req.StartTime, req.EndTime = req.StartTime.In(loc), req.EndTime.In(loc) - data.SampleCount, err = vllmMonitorRepo.Count(req.AppInstallID, req.StartTime, req.EndTime) + data.SampleCount, err = vllmMonitorRepo.Count(ctx, req.AppInstallID, req.StartTime, req.EndTime) if err != nil || data.SampleCount == 0 { return data, err } if data.SampleCount > 1200 { data.BucketSeconds = (req.EndTime.Unix() - req.StartTime.Unix() + 600) / 600 } - points, err := vllmMonitorRepo.History(req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation) + points, err := vllmMonitorRepo.History(ctx, req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation, req.Group) if err != nil { return data, err } if data.BucketSeconds > 0 { - if req.Aggregation != "max" { - for i := range points { - if err := vllm.AggregateHistograms(points[i].HistogramSamples, &points[i].MonitorVLLM); err != nil { - return data, err - } + aggregates := map[int64]*vllm.HistoryAccumulator{} + err = vllmMonitorRepo.WalkDeltas(ctx, req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation, req.Group, func(bucket int64, histograms, requests string) error { + if aggregates[bucket] == nil { + aggregates[bucket] = &vllm.HistoryAccumulator{} + } + return aggregates[bucket].Add(histograms, requests, req.Aggregation, req.Group) + }) + if err != nil { + return data, err + } + for i := range points { + if aggregate := aggregates[points[i].Bucket]; aggregate != nil { + aggregate.Apply(&points[i].MonitorVLLM, req.Aggregation) } } next := 0 @@ -93,6 +101,14 @@ func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.Mon } interval := time.Duration(intervalSeconds) * time.Second for i, point := range points { + if err := ctx.Err(); err != nil { + return data, err + } + var aggregate vllm.HistoryAccumulator + if err := aggregate.Add(point.HistogramDeltas, point.RequestDeltas, "max", req.Group); err != nil { + return data, err + } + aggregate.Apply(&point.MonitorVLLM, "max") if i > 0 && point.CreatedAt.Sub(points[i-1].CreatedAt) > 2*interval { data.Points = append(data.Points, model.MonitorVLLM{CreatedAt: points[i-1].CreatedAt.Add(interval), AppInstallID: req.AppInstallID}) } diff --git a/agent/init/migration/migrate.go b/agent/init/migration/migrate.go index 017c29b60..a750287fa 100644 --- a/agent/init/migration/migrate.go +++ b/agent/init/migration/migrate.go @@ -113,6 +113,7 @@ func agentDBMigrations() []*gormigrate.Migration { migrations.AddAcceleratorMetrics, migrations.AddVLLMMonitor, migrations.AddMonitorSettings, + migrations.AddVLLMRequestMetrics, } } diff --git a/agent/init/migration/migrations/init.go b/agent/init/migration/migrations/init.go index 5fe2da537..78f846423 100644 --- a/agent/init/migration/migrations/init.go +++ b/agent/init/migration/migrations/init.go @@ -1874,3 +1874,10 @@ var AddMonitorSettings = &gormigrate.Migration{ return nil }, } + +var AddVLLMRequestMetrics = &gormigrate.Migration{ + ID: "20261010-vllm-request-metrics", + Migrate: func(tx *gorm.DB) error { + return global.VLLMMonitorDB.AutoMigrate(&model.MonitorVLLM{}) + }, +} diff --git a/agent/utils/ai_tools/vllm/history.go b/agent/utils/ai_tools/vllm/history.go new file mode 100644 index 000000000..5ee067734 --- /dev/null +++ b/agent/utils/ai_tools/vllm/history.go @@ -0,0 +1,156 @@ +package vllm + +import ( + "encoding/json" + "math" + + "github.com/1Panel-dev/1Panel/agent/app/model" +) + +type HistoryAccumulator struct { + histograms map[string]histogram + prefixHits float64 + prefixQueries float64 + finishReasons map[string]float64 + finishSeconds map[string]float64 +} + +func (a *HistoryAccumulator) Add(histograms, requests, aggregation, group string) error { + if histograms != "" { + var sample map[string]histogram + if aggregation == "max" || group == "requests" { + var distributions struct { + Prompt *histogram `json:"request_prompt_tokens"` + Generation *histogram `json:"request_generation_tokens"` + } + if err := json.Unmarshal([]byte(histograms), &distributions); err != nil { + return err + } + sample = map[string]histogram{} + if distributions.Prompt != nil { + sample["request_prompt_tokens"] = *distributions.Prompt + } + if distributions.Generation != nil { + sample["request_generation_tokens"] = *distributions.Generation + } + } else if group == "latency" { + var latency struct { + FirstToken *histogram `json:"time_to_first_token_seconds"` + InterToken *histogram `json:"inter_token_latency_seconds"` + Request *histogram `json:"e2e_request_latency_seconds"` + Prefill *histogram `json:"request_prefill_time_seconds"` + Decode *histogram `json:"request_decode_time_seconds"` + Queue *histogram `json:"request_queue_time_seconds"` + } + if err := json.Unmarshal([]byte(histograms), &latency); err != nil { + return err + } + sample = map[string]histogram{} + for name, item := range map[string]*histogram{ + "time_to_first_token_seconds": latency.FirstToken, + "inter_token_latency_seconds": latency.InterToken, + "e2e_request_latency_seconds": latency.Request, + "request_prefill_time_seconds": latency.Prefill, + "request_decode_time_seconds": latency.Decode, + "request_queue_time_seconds": latency.Queue, + } { + if item != nil { + sample[name] = *item + } + } + } else if err := json.Unmarshal([]byte(histograms), &sample); err != nil { + return err + } + if a.histograms == nil { + a.histograms = map[string]histogram{} + } + for name, item := range sample { + switch name { + case "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds": + item.Buckets = nil + } + total, exists := a.histograms[name] + if !exists { + a.histograms[name] = item + continue + } + total.Seconds += item.Seconds + total.Sum += item.Sum + total.Count += item.Count + if len(total.Buckets) != len(item.Buckets) { + total.Buckets = nil + } + for bound, count := range total.Buckets { + addition, ok := item.Buckets[bound] + if !ok { + total.Buckets = nil + break + } + total.Buckets[bound] = count + addition + } + a.histograms[name] = total + } + } + if requests == "" { + return nil + } + if group == "overview" { + var prefix struct { + Seconds float64 `json:"seconds"` + Hits *float64 `json:"prefixHits"` + Queries *float64 `json:"prefixQueries"` + } + if err := json.Unmarshal([]byte(requests), &prefix); err != nil { + return err + } + if prefix.Seconds > 0 && prefix.Hits != nil && prefix.Queries != nil { + a.prefixHits += *prefix.Hits + a.prefixQueries += *prefix.Queries + } + return nil + } + var sample requestMetrics + if err := json.Unmarshal([]byte(requests), &sample); err != nil { + return err + } + if sample.Seconds <= 0 { + return nil + } + if group == "" && aggregation != "max" && sample.PrefixHits != nil && sample.PrefixQueries != nil { + a.prefixHits += *sample.PrefixHits + a.prefixQueries += *sample.PrefixQueries + } + for reason, count := range sample.FinishReasons { + if a.finishReasons == nil { + a.finishReasons = map[string]float64{} + } + if aggregation == "max" { + a.finishReasons[reason] = math.Max(a.finishReasons[reason], count/sample.Seconds) + } else { + if a.finishSeconds == nil { + a.finishSeconds = map[string]float64{} + } + a.finishReasons[reason] += count + a.finishSeconds[reason] += sample.Seconds + } + } + return nil +} + +func (a *HistoryAccumulator) Apply(result *model.MonitorVLLM, aggregation string) { + applyHistograms(result, a.histograms, aggregation != "max") + for reason, count := range a.finishReasons { + if result.FinishReasons == nil { + result.FinishReasons = map[string]float64{} + } + if aggregation == "max" { + result.FinishReasons[reason] = count + } else { + result.FinishReasons[reason] = count / a.finishSeconds[reason] + } + } + if aggregation != "max" && a.prefixQueries > 0 { + rate := a.prefixHits / a.prefixQueries * 100 + result.PrefixCacheHitRate = &rate + } +} diff --git a/agent/utils/ai_tools/vllm/metrics.go b/agent/utils/ai_tools/vllm/metrics.go index 352f7bb0b..f4624c470 100644 --- a/agent/utils/ai_tools/vllm/metrics.go +++ b/agent/utils/ai_tools/vllm/metrics.go @@ -17,6 +17,7 @@ import ( type Metrics map[string]map[string]float64 type histogram struct { + Seconds float64 `json:"seconds,omitempty"` Sum float64 `json:"sum"` Count float64 `json:"count"` Buckets map[string]float64 `json:"buckets,omitempty"` @@ -37,14 +38,16 @@ func Parse(reader io.Reader) (Metrics, error) { } name := strings.TrimPrefix(match[1], "vllm:") switch name { - case "num_requests_running", "num_requests_waiting", "kv_cache_usage_perc", "gpu_cache_usage_perc", "prompt_tokens_total", "generation_tokens_total", "request_success_total": + case "request_prefill_time_seconds_bucket", "request_decode_time_seconds_bucket", "request_queue_time_seconds_bucket": + continue + case "num_requests_running", "num_requests_waiting", "kv_cache_usage_perc", "gpu_cache_usage_perc", "prompt_tokens_total", "generation_tokens_total", "request_success_total", "prefix_cache_hits_total", "prefix_cache_queries_total": default: base := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_sum"), "_count"), "_bucket") if base == name { continue } switch base { - case "time_to_first_token_seconds", "time_per_output_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds": + case "time_to_first_token_seconds", "time_per_output_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds", "request_prompt_tokens", "request_generation_tokens": default: continue } @@ -117,7 +120,7 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e } } histograms := map[string]histogram{} - for _, name := range []string{"time_to_first_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds"} { + for _, name := range []string{"time_to_first_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds", "request_prompt_tokens", "request_generation_tokens"} { source := name if name == "inter_token_latency_seconds" && len(current[name+"_count"]) == 0 { source = "time_per_output_token_seconds" @@ -127,9 +130,21 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e if sum == nil || count == nil { continue } - histograms[name] = histogram{Sum: *sum, Count: *count, Buckets: bucketDeltas(current[source+"_bucket"], previous[source+"_bucket"], *count)} + item := histogram{Sum: *sum, Count: *count} + switch name { + case "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds": + default: + item.Buckets = bucketDeltas(current[source+"_bucket"], previous[source+"_bucket"], *count) + } + if name == "request_prompt_tokens" || name == "request_generation_tokens" { + item.Seconds = elapsed + } + histograms[name] = item + } + applyHistograms(&result, histograms, true) + if err := calculateRequestMetrics(current, previous, elapsed, &result); err != nil { + return model.MonitorVLLM{}, err } - applyHistograms(&result, histograms) encoded, err := json.Marshal(histograms) if err != nil { return model.MonitorVLLM{}, err @@ -138,43 +153,24 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e return result, nil } -func AggregateHistograms(encoded string, result *model.MonitorVLLM) error { - var samples []map[string]histogram - if err := json.Unmarshal([]byte(encoded), &samples); err != nil { - return err - } - merged := map[string]histogram{} - for _, sample := range samples { - for name, item := range sample { - if item.Count == 0 { - continue - } - total, exists := merged[name] - if !exists { - merged[name] = item - continue - } - total.Sum += item.Sum - total.Count += item.Count - if len(total.Buckets) != len(item.Buckets) { - total.Buckets = nil - } - for bound, count := range total.Buckets { - addition, ok := item.Buckets[bound] - if !ok { - total.Buckets = nil - break - } - total.Buckets[bound] = count + addition - } - merged[name] = total +func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram, includeMeans bool) { + for name, target := range map[string]*map[string]float64{ + "request_prompt_tokens": &result.PromptDistribution, + "request_generation_tokens": &result.GenerationDistribution, + } { + item := histograms[name] + if item.Seconds <= 0 || len(item.Buckets) == 0 { + continue + } + result.IntervalSeconds = math.Max(result.IntervalSeconds, item.Seconds) + *target = make(map[string]float64, len(item.Buckets)) + for bound, count := range item.Buckets { + (*target)[bound] = count / item.Seconds } } - applyHistograms(result, merged) - return nil -} - -func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram) { + if !includeMeans { + return + } for _, item := range []struct { name string mean **float64 @@ -185,6 +181,8 @@ func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram) {"e2e_request_latency_seconds", &result.RequestLatency, []**float64{&result.RequestLatencyP50, &result.RequestLatencyP90, &result.RequestLatencyP95, &result.RequestLatencyP99}}, {"request_prefill_time_seconds", &result.PrefillTime, nil}, {"request_decode_time_seconds", &result.DecodeTime, nil}, + {"request_queue_time_seconds", &result.QueueTime, nil}, + {"request_generation_tokens", &result.AverageGenerationTokens, nil}, } { histogram, ok := histograms[item.name] if !ok || histogram.Count <= 0 { diff --git a/agent/utils/ai_tools/vllm/request_metrics.go b/agent/utils/ai_tools/vllm/request_metrics.go new file mode 100644 index 000000000..d26bb1eca --- /dev/null +++ b/agent/utils/ai_tools/vllm/request_metrics.go @@ -0,0 +1,65 @@ +package vllm + +import ( + "encoding/json" + "strconv" + + "github.com/1Panel-dev/1Panel/agent/app/model" + "github.com/1Panel-dev/1Panel/agent/utils/re" +) + +type requestMetrics struct { + Seconds float64 `json:"seconds"` + PrefixHits *float64 `json:"prefixHits,omitempty"` + PrefixQueries *float64 `json:"prefixQueries,omitempty"` + FinishReasons map[string]float64 `json:"finishReasons,omitempty"` +} + +func calculateRequestMetrics(current, previous Metrics, elapsed float64, result *model.MonitorVLLM) error { + sample := requestMetrics{Seconds: elapsed, FinishReasons: map[string]float64{}} + hits := counterDelta(current["prefix_cache_hits_total"], previous["prefix_cache_hits_total"]) + queries := counterDelta(current["prefix_cache_queries_total"], previous["prefix_cache_queries_total"]) + if hits != nil && queries != nil && *hits <= *queries { + sample.PrefixHits, sample.PrefixQueries = hits, queries + if *queries > 0 { + rate := *hits / *queries * 100 + result.PrefixCacheHitRate = &rate + } + } + before := groupFinishReasons(previous["request_success_total"]) + for reason, values := range groupFinishReasons(current["request_success_total"]) { + if delta := counterDelta(values, before[reason]); delta != nil { + sample.FinishReasons[reason] = *delta + if result.FinishReasons == nil { + result.FinishReasons = map[string]float64{} + } + result.FinishReasons[reason] = *delta / elapsed + } + } + encoded, err := json.Marshal(sample) + if err != nil { + return err + } + result.RequestDeltas = string(encoded) + return nil +} + +func groupFinishReasons(samples map[string]float64) map[string]map[string]float64 { + groups := map[string]map[string]float64{} + for labels, value := range samples { + for _, label := range re.GetRegex(re.VLLMMetricLabelPattern).FindAllStringSubmatch(labels, -1) { + if label[1] != "finished_reason" { + continue + } + reason, err := strconv.Unquote(label[2]) + if err != nil || reason == "" { + continue + } + if groups[reason] == nil { + groups[reason] = map[string]float64{} + } + groups[reason][labels] = value + } + } + return groups +} diff --git a/frontend/src/lang/modules/en.ts b/frontend/src/lang/modules/en.ts index 550bf9894..1c3418b0a 100644 --- a/frontend/src/lang/modules/en.ts +++ b/frontend/src/lang/modules/en.ts @@ -1517,6 +1517,24 @@ const message = { extraHeaders: 'Extra Headers', }, vllmMonitor: { + tokensPerSecond: 'tokens/s', + requestsPerSecond: 'Requests/s', + finishReasonLabels: { + stop: 'Normal completion', + length: 'Length limit reached', + abort: 'Aborted', + error: 'Error', + repetition: 'Repetition', + }, + requestAnalysis: 'Request analysis', + promptDistribution: 'Prompt token length distribution', + generationDistribution: 'Generation token length distribution', + finishReasons: 'Request finish reasons', + queueTime: 'Mean queue time', + averageGenerationTokens: 'Mean output tokens per request', + prefixCacheHitRate: 'Prefix cache hit rate', + distributionHelper: + 'Length distributions show request rates and are unaffected by the average/peak selection.', cleanHelper: 'Clear monitoring history only for this vLLM instance? Other instances, host and GPU monitoring are unaffected.', overview: 'Overview', diff --git a/frontend/src/lang/modules/es-es.ts b/frontend/src/lang/modules/es-es.ts index f3dd9c2f0..cc3df07d5 100644 --- a/frontend/src/lang/modules/es-es.ts +++ b/frontend/src/lang/modules/es-es.ts @@ -1526,6 +1526,24 @@ const message = { extraHeaders: 'Encabezados adicionales', }, vllmMonitor: { + tokensPerSecond: 'tokens/s', + requestsPerSecond: 'Solicitudes/s', + finishReasonLabels: { + stop: 'Finalización normal', + length: 'Límite de longitud alcanzado', + abort: 'Cancelada', + error: 'Error', + repetition: 'Repetición', + }, + requestAnalysis: 'Análisis de solicitudes', + promptDistribution: 'Distribución de longitud de tokens de entrada', + generationDistribution: 'Distribución de longitud de tokens de salida', + finishReasons: 'Motivos de finalización de solicitudes', + queueTime: 'Tiempo medio en cola', + averageGenerationTokens: 'Tokens de salida medios por solicitud', + prefixCacheHitRate: 'Tasa de aciertos de caché de prefijos', + distributionHelper: + 'Las distribuciones de longitud muestran tasas de solicitudes y no cambian al seleccionar media o pico.', cleanHelper: '¿Borrar solo el historial de monitoreo de esta instancia de vLLM? No afecta a otras instancias ni al monitoreo del host o de GPU.', overview: 'Resumen', diff --git a/frontend/src/lang/modules/fa.ts b/frontend/src/lang/modules/fa.ts index 08fe4ed59..93764f4a0 100644 --- a/frontend/src/lang/modules/fa.ts +++ b/frontend/src/lang/modules/fa.ts @@ -1502,6 +1502,23 @@ const message = { extraHeaders: 'هدرهای اضافی', }, vllmMonitor: { + tokensPerSecond: 'tokens/ثانیه', + requestsPerSecond: 'درخواست/ثانیه', + finishReasonLabels: { + stop: 'پایان عادی', + length: 'رسیدن به حد طول', + abort: 'لغوشده', + error: 'خطا', + repetition: 'تکرار خروجی', + }, + requestAnalysis: 'تحلیل درخواست‌ها', + promptDistribution: 'توزیع طول توکن‌های ورودی', + generationDistribution: 'توزیع طول توکن‌های خروجی', + finishReasons: 'دلایل پایان درخواست‌ها', + queueTime: 'میانگین زمان انتظار در صف', + averageGenerationTokens: 'میانگین توکن‌های خروجی هر درخواست', + prefixCacheHitRate: 'نرخ اصابت حافظه نهان پیشوند', + distributionHelper: 'توزیع طول، نرخ درخواست‌ها را نشان می‌دهد و از انتخاب میانگین یا اوج تأثیر نمی‌پذیرد.', cleanHelper: 'فقط سابقه نظارت این نمونه vLLM پاک شود؟ سایر نمونه\u200cها و نظارت میزبان و GPU تغییری نمی\u200cکنند.', overview: 'نمای کلی', diff --git a/frontend/src/lang/modules/ja.ts b/frontend/src/lang/modules/ja.ts index c62b15d48..d09f283a7 100644 --- a/frontend/src/lang/modules/ja.ts +++ b/frontend/src/lang/modules/ja.ts @@ -1505,6 +1505,23 @@ const message = { extraHeaders: '追加ヘッダー', }, vllmMonitor: { + tokensPerSecond: 'tokens/秒', + requestsPerSecond: 'リクエスト/秒', + finishReasonLabels: { + stop: '正常終了', + length: '長さ上限に到達', + abort: '中止', + error: 'エラー', + repetition: '繰り返し出力', + }, + requestAnalysis: 'リクエスト分析', + promptDistribution: '入力トークン長の分布', + generationDistribution: '出力トークン長の分布', + finishReasons: 'リクエスト終了理由', + queueTime: '平均キュー待機時間', + averageGenerationTokens: 'リクエストあたりの平均出力トークン数', + prefixCacheHitRate: 'プレフィックスキャッシュヒット率', + distributionHelper: '長さの分布はリクエストレートで集計され、平均／ピークの切り替えには影響されません。', cleanHelper: 'この vLLM インスタンスの監視履歴のみを削除しますか？他のインスタンス、ホスト、GPU の監視には影響しません。', overview: '概要', diff --git a/frontend/src/lang/modules/ko.ts b/frontend/src/lang/modules/ko.ts index 8176531f5..5b810db33 100644 --- a/frontend/src/lang/modules/ko.ts +++ b/frontend/src/lang/modules/ko.ts @@ -1490,6 +1490,23 @@ const message = { extraHeaders: '추가 헤더', }, vllmMonitor: { + tokensPerSecond: 'tokens/초', + requestsPerSecond: '요청/초', + finishReasonLabels: { + stop: '정상 종료', + length: '길이 제한 도달', + abort: '중단', + error: '오류', + repetition: '반복 출력', + }, + requestAnalysis: '요청 분석', + promptDistribution: '입력 토큰 길이 분포', + generationDistribution: '출력 토큰 길이 분포', + finishReasons: '요청 종료 사유', + queueTime: '평균 대기 시간', + averageGenerationTokens: '요청당 평균 출력 토큰 수', + prefixCacheHitRate: '접두사 캐시 적중률', + distributionHelper: '길이 분포는 요청 속도로 집계되며 평균/최댓값 선택의 영향을 받지 않습니다.', cleanHelper: '현재 vLLM 인스턴스의 모니터링 기록만 삭제하시겠습니까? 다른 인스턴스, 호스트 및 GPU 모니터링에는 영향을 주지 않습니다.', overview: '개요', diff --git a/frontend/src/lang/modules/lo.ts b/frontend/src/lang/modules/lo.ts index 6b4223ad6..d8338fa04 100644 --- a/frontend/src/lang/modules/lo.ts +++ b/frontend/src/lang/modules/lo.ts @@ -1484,6 +1484,23 @@ const message = { extraHeaders: 'Headers ເພີ່ມເຕີມ', }, vllmMonitor: { + tokensPerSecond: 'tokens/ວິນາທີ', + requestsPerSecond: 'ຄຳຮ້ອງຂໍ/ວິນາທີ', + finishReasonLabels: { + stop: 'ສຳເລັດປົກກະຕິ', + length: 'ເຖິງຂີດຈຳກັດຄວາມຍາວ', + abort: 'ຍົກເລີກ', + error: 'ຂໍ້ຜິດພາດ', + repetition: 'ຜົນອອກຊ້ຳ', + }, + requestAnalysis: 'ການວິເຄາະຄຳຮ້ອງຂໍ', + promptDistribution: 'ການແຈກຢາຍຄວາມຍາວໂທເຄັນຂາເຂົ້າ', + generationDistribution: 'ການແຈກຢາຍຄວາມຍາວໂທເຄັນຂາອອກ', + finishReasons: 'ເຫດຜົນທີ່ຄຳຮ້ອງຂໍສິ້ນສຸດ', + queueTime: 'ເວລາລໍຖ້າໃນຄິວສະເລ່ຍ', + averageGenerationTokens: 'ຈຳນວນໂທເຄັນຂາອອກສະເລ່ຍຕໍ່ຄຳຮ້ອງຂໍ', + prefixCacheHitRate: 'ອັດຕາການພົບໃນແຄດຄຳນຳໜ້າ', + distributionHelper: 'ການແຈກຢາຍຄວາມຍາວສະແດງອັດຕາຄຳຮ້ອງຂໍ ແລະ ບໍ່ປ່ຽນຕາມການເລືອກຄ່າສະເລ່ຍ ຫຼື ຄ່າສູງສຸດ.', cleanHelper: 'ລຶບສະເພາະປະຫວັດການຕິດຕາມຂອງ vLLM ນີ້ບໍ? ບໍ່ກະທົບຕໍ່ອິນສະແຕນອື່ນ, ໂຮສ ຫຼື GPU.', overview: 'ພາບລວມ', latencyAnalysis: 'ການວິເຄາະຄວາມລ່າຊ້າ', diff --git a/frontend/src/lang/modules/ms.ts b/frontend/src/lang/modules/ms.ts index 446d8d5f3..99184d90a 100644 --- a/frontend/src/lang/modules/ms.ts +++ b/frontend/src/lang/modules/ms.ts @@ -1531,6 +1531,24 @@ const message = { extraHeaders: 'Pengepala Tambahan', }, vllmMonitor: { + tokensPerSecond: 'tokens/s', + requestsPerSecond: 'Permintaan/s', + finishReasonLabels: { + stop: 'Selesai secara normal', + length: 'Had panjang dicapai', + abort: 'Dibatalkan', + error: 'Ralat', + repetition: 'Pengulangan', + }, + requestAnalysis: 'Analisis permintaan', + promptDistribution: 'Taburan panjang token input', + generationDistribution: 'Taburan panjang token output', + finishReasons: 'Sebab permintaan tamat', + queueTime: 'Purata masa dalam baris gilir', + averageGenerationTokens: 'Purata token output setiap permintaan', + prefixCacheHitRate: 'Kadar padanan cache awalan', + distributionHelper: + 'Taburan panjang menunjukkan kadar permintaan dan tidak dipengaruhi oleh pilihan purata/puncak.', cleanHelper: 'Padamkan sejarah pemantauan untuk instans vLLM ini sahaja? Instans lain serta pemantauan hos dan GPU tidak terjejas.', overview: 'Gambaran keseluruhan', diff --git a/frontend/src/lang/modules/pt-br.ts b/frontend/src/lang/modules/pt-br.ts index 81378baee..5845b0049 100644 --- a/frontend/src/lang/modules/pt-br.ts +++ b/frontend/src/lang/modules/pt-br.ts @@ -1527,6 +1527,24 @@ const message = { extraHeaders: 'Cabeçalhos extras', }, vllmMonitor: { + tokensPerSecond: 'tokens/s', + requestsPerSecond: 'Solicitações/s', + finishReasonLabels: { + stop: 'Conclusão normal', + length: 'Limite de comprimento atingido', + abort: 'Abortada', + error: 'Erro', + repetition: 'Repetição', + }, + requestAnalysis: 'Análise de solicitações', + promptDistribution: 'Distribuição do comprimento dos tokens de entrada', + generationDistribution: 'Distribuição do comprimento dos tokens de saída', + finishReasons: 'Motivos de conclusão das solicitações', + queueTime: 'Tempo médio na fila', + averageGenerationTokens: 'Média de tokens de saída por solicitação', + prefixCacheHitRate: 'Taxa de acertos do cache de prefixos', + distributionHelper: + 'As distribuições de comprimento mostram taxas de solicitações e não mudam com a seleção de média ou pico.', cleanHelper: 'Limpar apenas o histórico de monitoramento desta instância do vLLM? Outras instâncias e o monitoramento do host e da GPU não serão afetados.', overview: 'Visão geral', diff --git a/frontend/src/lang/modules/ru.ts b/frontend/src/lang/modules/ru.ts index e991afb08..571df7578 100644 --- a/frontend/src/lang/modules/ru.ts +++ b/frontend/src/lang/modules/ru.ts @@ -1520,6 +1520,24 @@ const message = { extraHeaders: 'Дополнительные заголовки', }, vllmMonitor: { + tokensPerSecond: 'tokens/с', + requestsPerSecond: 'Запросов/с', + finishReasonLabels: { + stop: 'Нормальное завершение', + length: 'Достигнут лимит длины', + abort: 'Прервано', + error: 'Ошибка', + repetition: 'Повторение', + }, + requestAnalysis: 'Анализ запросов', + promptDistribution: 'Распределение длины входных токенов', + generationDistribution: 'Распределение длины выходных токенов', + finishReasons: 'Причины завершения запросов', + queueTime: 'Среднее время в очереди', + averageGenerationTokens: 'Среднее число выходных токенов на запрос', + prefixCacheHitRate: 'Доля попаданий в кэш префиксов', + distributionHelper: + 'Распределения длины показывают частоту запросов и не зависят от выбора среднего или пикового значения.', cleanHelper: 'Удалить историю мониторинга только этого экземпляра vLLM? Другие экземпляры, мониторинг хоста и GPU не будут затронуты.', overview: 'Обзор', diff --git a/frontend/src/lang/modules/tr.ts b/frontend/src/lang/modules/tr.ts index dcc6ae4c3..5e57478bd 100644 --- a/frontend/src/lang/modules/tr.ts +++ b/frontend/src/lang/modules/tr.ts @@ -1527,6 +1527,23 @@ const message = { extraHeaders: 'Ek Başlıklar', }, vllmMonitor: { + tokensPerSecond: 'tokens/sn', + requestsPerSecond: 'İstek/sn', + finishReasonLabels: { + stop: 'Normal tamamlanma', + length: 'Uzunluk sınırına ulaşıldı', + abort: 'İptal edildi', + error: 'Hata', + repetition: 'Tekrarlama', + }, + requestAnalysis: 'İstek analizi', + promptDistribution: 'Girdi belirteci uzunluk dağılımı', + generationDistribution: 'Çıktı belirteci uzunluk dağılımı', + finishReasons: 'İstek bitiş nedenleri', + queueTime: 'Ortalama kuyruk süresi', + averageGenerationTokens: 'İstek başına ortalama çıktı belirteci', + prefixCacheHitRate: 'Önek önbelleği isabet oranı', + distributionHelper: 'Uzunluk dağılımları istek hızlarını gösterir ve ortalama/tepe seçiminden etkilenmez.', cleanHelper: 'Yalnızca bu vLLM örneğinin izleme geçmişi temizlensin mi? Diğer örnekler, ana makine ve GPU izlemesi etkilenmez.', overview: 'Genel bakış', diff --git a/frontend/src/lang/modules/zh-Hant.ts b/frontend/src/lang/modules/zh-Hant.ts index 0bbe88ae8..8400ceee4 100644 --- a/frontend/src/lang/modules/zh-Hant.ts +++ b/frontend/src/lang/modules/zh-Hant.ts @@ -1437,6 +1437,23 @@ const message = { extraHeaders: '額外請求標頭', }, vllmMonitor: { + tokensPerSecond: 'tokens/秒', + requestsPerSecond: '請求/秒', + finishReasonLabels: { + stop: '正常結束', + length: '達到長度限制', + abort: '中止', + error: '錯誤', + repetition: '重複輸出', + }, + requestAnalysis: '請求分析', + promptDistribution: '輸入 Token 長度分佈', + generationDistribution: '輸出 Token 長度分佈', + finishReasons: '請求結束原因', + queueTime: '平均排隊耗時', + averageGenerationTokens: '每請求平均輸出 Token 數', + prefixCacheHitRate: '前綴快取命中率', + distributionHelper: '長度分佈按請求速率彙總，不受平均／峰值切換影響。', cleanHelper: '僅清空目前 vLLM 實例的監控歷史，不影響其他實例、主機或 GPU 監控，是否繼續？', overview: '概覽', latencyAnalysis: '延遲分析', diff --git a/frontend/src/lang/modules/zh.ts b/frontend/src/lang/modules/zh.ts index fc44ba8ce..6cdcdc9cc 100644 --- a/frontend/src/lang/modules/zh.ts +++ b/frontend/src/lang/modules/zh.ts @@ -1450,6 +1450,23 @@ const message = { extraHeaders: '额外请求头', }, vllmMonitor: { + tokensPerSecond: 'tokens/秒', + requestsPerSecond: '请求/秒', + finishReasonLabels: { + stop: '正常结束', + length: '达到长度限制', + abort: '中止', + error: '错误', + repetition: '重复输出', + }, + requestAnalysis: '请求分析', + promptDistribution: '输入 Token 长度分布', + generationDistribution: '输出 Token 长度分布', + finishReasons: '请求结束原因', + queueTime: '平均排队耗时', + averageGenerationTokens: '每请求平均输出 Token 数', + prefixCacheHitRate: '前缀缓存命中率', + distributionHelper: '长度分布按请求速率汇总，不受平均／峰值切换影响。', cleanHelper: '仅清空当前 vLLM 实例的监控历史，不影响其他实例、主机或 GPU 监控，是否继续？', overview: '概览', latencyAnalysis: '延迟分析',