starred/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2026-10-12 08:00:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(vllm): add request metrics and optimize monitor history (#14032)

Browse Source
This commit is contained in:
ssongliu
2026-10-12 15:54:42 +08:00
committed by GitHub
parent 5a2e430dff
commit afcb94f6ef
23 changed files with 577 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/app/api/v2/monitor_vllm.go
+1 -1
View File
@@ -11,7 +11,7 @@ func (b *BaseApi) LoadVLLMMonitor(c *gin.Context) {
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}
data, err := monitorService.LoadVLLMMonitorData(req)
data, err := monitorService.LoadVLLMMonitorData(c.Request.Context(), req)
if err != nil {
helper.InternalServer(c, err)
return
agent/app/dto/monitor.go
+1
View File
@@ -127,6 +127,7 @@ type GPUProcess struct {
}
type MonitorVLLMSearch struct {
Group string `json:"group" validate:"omitempty,oneof=overview latency requests"`
AppInstallID uint `json:"appInstallID" validate:"required"`
StartTime time.Time `json:"startTime" validate:"required"`
EndTime time.Time `json:"endTime" validate:"required"`
agent/app/model/monitor.go
+14 -6
View File
@@ -79,12 +79,20 @@ type MonitorGPU struct {
}
type MonitorVLLM struct {
ID uint `json:"-" gorm:"primarykey;autoIncrement"`
CreatedAt time.Time `json:"createdAt"`
AppInstallID uint `json:"appInstallID"`
Status string `json:"status"`
RawMetrics string `json:"-"`
HistogramDeltas string `json:"-"`
ID uint `json:"-" gorm:"primarykey;autoIncrement"`
CreatedAt time.Time `json:"createdAt"`
AppInstallID uint `json:"appInstallID"`
Status string `json:"status"`
RawMetrics string `json:"-"`
HistogramDeltas string `json:"-"`
IntervalSeconds float64 `json:"intervalSeconds" gorm:"-"`
RequestDeltas string `json:"-"`
QueueTime *float64 `json:"queueTime"`
AverageGenerationTokens *float64 `json:"averageGenerationTokens"`
PrefixCacheHitRate *float64 `json:"prefixCacheHitRate"`
FinishReasons map[string]float64 `json:"finishReasons" gorm:"-"`
PromptDistribution map[string]float64 `json:"promptDistribution" gorm:"-"`
GenerationDistribution map[string]float64 `json:"generationDistribution" gorm:"-"`
Running *float64 `json:"running"`
Waiting *float64 `json:"waiting"`
agent/app/repo/monitor_vllm.go
+59 -13
View File
@@ -1,6 +1,7 @@
package repo
import (
"context"
"errors"
"fmt"
"strings"
@@ -15,8 +16,7 @@ type VLLMMonitorRepo struct{}
type VLLMHistoryPoint struct {
model.MonitorVLLM
Bucket int64
HistogramSamples string
Bucket int64
}
func (r *VLLMMonitorRepo) Create(point *model.MonitorVLLM) error {
@@ -41,16 +41,30 @@ func (r *VLLMMonitorRepo) DeleteBefore(before time.Time) error {
return global.VLLMMonitorDB.Where("created_at < ?", before).Delete(&model.MonitorVLLM{}).Error
}
func (r *VLLMMonitorRepo) Count(id uint, start, end time.Time) (int64, error) {
func (r *VLLMMonitorRepo) Count(ctx context.Context, id uint, start, end time.Time) (int64, error) {
var count int64
db := global.VLLMMonitorDB.Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end)
db := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end)
err := db.Count(&count).Error
return count, err
}
func (r *VLLMMonitorRepo) History(id uint, start, end time.Time, seconds int64, aggregation string) ([]VLLMHistoryPoint, error) {
db := global.VLLMMonitorDB.Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end)
metrics := []string{"running", "waiting", "cache_usage", "prompt_throughput", "generation_throughput", "request_throughput", "time_to_first_token", "time_per_output_token", "request_latency", "prefill_time", "decode_time", "time_to_first_token_p50", "time_to_first_token_p90", "time_to_first_token_p95", "time_to_first_token_p99", "time_per_output_token_p50", "time_per_output_token_p90", "time_per_output_token_p95", "time_per_output_token_p99", "request_latency_p50", "request_latency_p90", "request_latency_p95", "request_latency_p99"}
func (r *VLLMMonitorRepo) History(ctx context.Context, id uint, start, end time.Time, seconds int64, aggregation, group string) ([]VLLMHistoryPoint, error) {
db := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}).Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end)
var metrics []string
if group == "" || group == "overview" {
metrics = append(metrics, "running", "waiting", "cache_usage", "prompt_throughput", "generation_throughput", "request_throughput")
if seconds <= 0 || aggregation == "max" {
metrics = append(metrics, "prefix_cache_hit_rate")
}
}
if seconds <= 0 || aggregation == "max" {
if group == "" || group == "latency" {
metrics = append(metrics, "queue_time", "time_to_first_token", "time_per_output_token", "request_latency", "prefill_time", "decode_time", "time_to_first_token_p50", "time_to_first_token_p90", "time_to_first_token_p95", "time_to_first_token_p99", "time_per_output_token_p50", "time_per_output_token_p90", "time_per_output_token_p95", "time_per_output_token_p99", "request_latency_p50", "request_latency_p90", "request_latency_p95", "request_latency_p99")
}
if group == "" || group == "requests" {
metrics = append(metrics, "average_generation_tokens")
}
}
var columns []string
if seconds > 0 {
operation := "AVG"
@@ -59,20 +73,52 @@ func (r *VLLMMonitorRepo) History(id uint, start, end time.Time, seconds int64,
}
columns = []string{fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d AS bucket", start.Unix(), seconds)}
for _, column := range metrics {
if aggregation != "max" && (strings.HasPrefix(column, "time_to_first_token_p") || strings.HasPrefix(column, "time_per_output_token_p") || strings.HasPrefix(column, "request_latency_p")) {
continue
}
columns = append(columns, operation+"("+column+") AS "+column)
}
if aggregation != "max" {
columns = append(columns, "json_group_array(json(NULLIF(histogram_deltas, ''))) AS histogram_samples")
}
db = db.Group("bucket").Order("bucket ASC")
} else {
columns = append([]string{"id", "created_at", "app_install_id", "status"}, metrics...)
if group == "" || group == "requests" {
columns = append(columns, "histogram_deltas", "request_deltas")
}
db = db.Order("created_at ASC, id ASC")
}
var points []VLLMHistoryPoint
err := db.Select(strings.Join(columns, ", ")).Scan(&points).Error
return points, err
}
func (r *VLLMMonitorRepo) WalkDeltas(ctx context.Context, id uint, start, end time.Time, seconds int64, aggregation, group string, visit func(int64, string, string) error) error {
histograms, requests := "''", "''"
if group == "" || group == "requests" || (group == "latency" && aggregation != "max") {
histograms = "COALESCE(histogram_deltas, '')"
}
if group == "" || group == "requests" || (group == "overview" && aggregation != "max") {
requests = "COALESCE(request_deltas, '')"
}
if histograms == "''" && requests == "''" {
return ctx.Err()
}
bucket := fmt.Sprintf("(CAST(strftime('%%s', created_at) AS INTEGER) - %d) / %d", start.Unix(), seconds)
rows, err := global.VLLMMonitorDB.WithContext(ctx).Model(&model.MonitorVLLM{}).
Where("app_install_id = ? AND created_at >= ? AND created_at <= ?", id, start, end).
Select(bucket + ", " + histograms + ", " + requests).Rows()
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
if err := ctx.Err(); err != nil {
return err
}
var index int64
var histograms, requests string
if err := rows.Scan(&index, &histograms, &requests); err != nil {
return err
}
if err := visit(index, histograms, requests); err != nil {
return err
}
}
return rows.Err()
}
agent/app/service/monitor.go
+1 -1
View File
@@ -54,7 +54,7 @@ type IMonitorService interface {
LoadIOOptions() []string
LoadNetworkOptions() []string
LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error)
LoadVLLMMonitorData(ctx context.Context, req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error)
LoadVLLMCurrent(ctx context.Context, req dto.MonitorVLLMCurrent) (model.MonitorVLLM, error)
CleanVLLMMonitor(req dto.MonitorVLLMClean) error
LoadGPUOptions() dto.MonitorGPUOptions
agent/app/service/monitor_vllm.go
+25 -9
View File
@@ -31,12 +31,12 @@ var vllmMetricsClient = &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse },
}
func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) {
func (m *MonitorService) LoadVLLMMonitorData(ctx context.Context, req dto.MonitorVLLMSearch) (dto.MonitorVLLMData, error) {
data := dto.MonitorVLLMData{Points: []model.MonitorVLLM{}}
if !req.EndTime.After(req.StartTime) {
return data, fmt.Errorf("invalid vLLM history request")
}
install, err := appInstallRepo.GetFirst(repo.WithByID(req.AppInstallID))
install, err := appInstallRepo.GetFirstByCtx(ctx, repo.WithByID(req.AppInstallID))
if err != nil {
return data, err
}
@@ -49,23 +49,31 @@ func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.Mon
return data, err
}
req.StartTime, req.EndTime = req.StartTime.In(loc), req.EndTime.In(loc)
data.SampleCount, err = vllmMonitorRepo.Count(req.AppInstallID, req.StartTime, req.EndTime)
data.SampleCount, err = vllmMonitorRepo.Count(ctx, req.AppInstallID, req.StartTime, req.EndTime)
if err != nil || data.SampleCount == 0 {
return data, err
}
if data.SampleCount > 1200 {
data.BucketSeconds = (req.EndTime.Unix() - req.StartTime.Unix() + 600) / 600
}
points, err := vllmMonitorRepo.History(req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation)
points, err := vllmMonitorRepo.History(ctx, req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation, req.Group)
if err != nil {
return data, err
}
if data.BucketSeconds > 0 {
if req.Aggregation != "max" {
for i := range points {
if err := vllm.AggregateHistograms(points[i].HistogramSamples, &points[i].MonitorVLLM); err != nil {
return data, err
}
aggregates := map[int64]*vllm.HistoryAccumulator{}
err = vllmMonitorRepo.WalkDeltas(ctx, req.AppInstallID, req.StartTime, req.EndTime, data.BucketSeconds, req.Aggregation, req.Group, func(bucket int64, histograms, requests string) error {
if aggregates[bucket] == nil {
aggregates[bucket] = &vllm.HistoryAccumulator{}
}
return aggregates[bucket].Add(histograms, requests, req.Aggregation, req.Group)
})
if err != nil {
return data, err
}
for i := range points {
if aggregate := aggregates[points[i].Bucket]; aggregate != nil {
aggregate.Apply(&points[i].MonitorVLLM, req.Aggregation)
}
}
next := 0
@@ -93,6 +101,14 @@ func (m *MonitorService) LoadVLLMMonitorData(req dto.MonitorVLLMSearch) (dto.Mon
}
interval := time.Duration(intervalSeconds) * time.Second
for i, point := range points {
if err := ctx.Err(); err != nil {
return data, err
}
var aggregate vllm.HistoryAccumulator
if err := aggregate.Add(point.HistogramDeltas, point.RequestDeltas, "max", req.Group); err != nil {
return data, err
}
aggregate.Apply(&point.MonitorVLLM, "max")
if i > 0 && point.CreatedAt.Sub(points[i-1].CreatedAt) > 2*interval {
data.Points = append(data.Points, model.MonitorVLLM{CreatedAt: points[i-1].CreatedAt.Add(interval), AppInstallID: req.AppInstallID})
}
agent/init/migration/migrate.go
+1
View File
@@ -113,6 +113,7 @@ func agentDBMigrations() []*gormigrate.Migration {
migrations.AddAcceleratorMetrics,
migrations.AddVLLMMonitor,
migrations.AddMonitorSettings,
migrations.AddVLLMRequestMetrics,
}
}
agent/init/migration/migrations/init.go
+7
View File
@@ -1874,3 +1874,10 @@ var AddMonitorSettings = &gormigrate.Migration{
return nil
},
}
var AddVLLMRequestMetrics = &gormigrate.Migration{
ID: "20261010-vllm-request-metrics",
Migrate: func(tx *gorm.DB) error {
return global.VLLMMonitorDB.AutoMigrate(&model.MonitorVLLM{})
},
}
agent/utils/ai_tools/vllm/history.go
+156
View File
@@ -0,0 +1,156 @@
package vllm
import (
"encoding/json"
"math"
"github.com/1Panel-dev/1Panel/agent/app/model"
)
type HistoryAccumulator struct {
histograms map[string]histogram
prefixHits float64
prefixQueries float64
finishReasons map[string]float64
finishSeconds map[string]float64
}
func (a *HistoryAccumulator) Add(histograms, requests, aggregation, group string) error {
if histograms != "" {
var sample map[string]histogram
if aggregation == "max" || group == "requests" {
var distributions struct {
Prompt *histogram `json:"request_prompt_tokens"`
Generation *histogram `json:"request_generation_tokens"`
}
if err := json.Unmarshal([]byte(histograms), &distributions); err != nil {
return err
}
sample = map[string]histogram{}
if distributions.Prompt != nil {
sample["request_prompt_tokens"] = *distributions.Prompt
}
if distributions.Generation != nil {
sample["request_generation_tokens"] = *distributions.Generation
}
} else if group == "latency" {
var latency struct {
FirstToken *histogram `json:"time_to_first_token_seconds"`
InterToken *histogram `json:"inter_token_latency_seconds"`
Request *histogram `json:"e2e_request_latency_seconds"`
Prefill *histogram `json:"request_prefill_time_seconds"`
Decode *histogram `json:"request_decode_time_seconds"`
Queue *histogram `json:"request_queue_time_seconds"`
}
if err := json.Unmarshal([]byte(histograms), &latency); err != nil {
return err
}
sample = map[string]histogram{}
for name, item := range map[string]*histogram{
"time_to_first_token_seconds": latency.FirstToken,
"inter_token_latency_seconds": latency.InterToken,
"e2e_request_latency_seconds": latency.Request,
"request_prefill_time_seconds": latency.Prefill,
"request_decode_time_seconds": latency.Decode,
"request_queue_time_seconds": latency.Queue,
} {
if item != nil {
sample[name] = *item
}
}
} else if err := json.Unmarshal([]byte(histograms), &sample); err != nil {
return err
}
if a.histograms == nil {
a.histograms = map[string]histogram{}
}
for name, item := range sample {
switch name {
case "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds":
item.Buckets = nil
}
total, exists := a.histograms[name]
if !exists {
a.histograms[name] = item
continue
}
total.Seconds += item.Seconds
total.Sum += item.Sum
total.Count += item.Count
if len(total.Buckets) != len(item.Buckets) {
total.Buckets = nil
}
for bound, count := range total.Buckets {
addition, ok := item.Buckets[bound]
if !ok {
total.Buckets = nil
break
}
total.Buckets[bound] = count + addition
}
a.histograms[name] = total
}
}
if requests == "" {
return nil
}
if group == "overview" {
var prefix struct {
Seconds float64 `json:"seconds"`
Hits *float64 `json:"prefixHits"`
Queries *float64 `json:"prefixQueries"`
}
if err := json.Unmarshal([]byte(requests), &prefix); err != nil {
return err
}
if prefix.Seconds > 0 && prefix.Hits != nil && prefix.Queries != nil {
a.prefixHits += *prefix.Hits
a.prefixQueries += *prefix.Queries
}
return nil
}
var sample requestMetrics
if err := json.Unmarshal([]byte(requests), &sample); err != nil {
return err
}
if sample.Seconds <= 0 {
return nil
}
if group == "" && aggregation != "max" && sample.PrefixHits != nil && sample.PrefixQueries != nil {
a.prefixHits += *sample.PrefixHits
a.prefixQueries += *sample.PrefixQueries
}
for reason, count := range sample.FinishReasons {
if a.finishReasons == nil {
a.finishReasons = map[string]float64{}
}
if aggregation == "max" {
a.finishReasons[reason] = math.Max(a.finishReasons[reason], count/sample.Seconds)
} else {
if a.finishSeconds == nil {
a.finishSeconds = map[string]float64{}
}
a.finishReasons[reason] += count
a.finishSeconds[reason] += sample.Seconds
}
}
return nil
}
func (a *HistoryAccumulator) Apply(result *model.MonitorVLLM, aggregation string) {
applyHistograms(result, a.histograms, aggregation != "max")
for reason, count := range a.finishReasons {
if result.FinishReasons == nil {
result.FinishReasons = map[string]float64{}
}
if aggregation == "max" {
result.FinishReasons[reason] = count
} else {
result.FinishReasons[reason] = count / a.finishSeconds[reason]
}
}
if aggregation != "max" && a.prefixQueries > 0 {
rate := a.prefixHits / a.prefixQueries * 100
result.PrefixCacheHitRate = &rate
}
}
agent/utils/ai_tools/vllm/metrics.go
+38 -40
View File
@@ -17,6 +17,7 @@ import (
type Metrics map[string]map[string]float64
type histogram struct {
Seconds float64 `json:"seconds,omitempty"`
Sum float64 `json:"sum"`
Count float64 `json:"count"`
Buckets map[string]float64 `json:"buckets,omitempty"`
@@ -37,14 +38,16 @@ func Parse(reader io.Reader) (Metrics, error) {
}
name := strings.TrimPrefix(match[1], "vllm:")
switch name {
case "num_requests_running", "num_requests_waiting", "kv_cache_usage_perc", "gpu_cache_usage_perc", "prompt_tokens_total", "generation_tokens_total", "request_success_total":
case "request_prefill_time_seconds_bucket", "request_decode_time_seconds_bucket", "request_queue_time_seconds_bucket":
continue
case "num_requests_running", "num_requests_waiting", "kv_cache_usage_perc", "gpu_cache_usage_perc", "prompt_tokens_total", "generation_tokens_total", "request_success_total", "prefix_cache_hits_total", "prefix_cache_queries_total":
default:
base := strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(name, "_sum"), "_count"), "_bucket")
if base == name {
continue
}
switch base {
case "time_to_first_token_seconds", "time_per_output_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds":
case "time_to_first_token_seconds", "time_per_output_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds", "request_prompt_tokens", "request_generation_tokens":
default:
continue
}
@@ -117,7 +120,7 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e
}
}
histograms := map[string]histogram{}
for _, name := range []string{"time_to_first_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds"} {
for _, name := range []string{"time_to_first_token_seconds", "inter_token_latency_seconds", "e2e_request_latency_seconds", "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds", "request_prompt_tokens", "request_generation_tokens"} {
source := name
if name == "inter_token_latency_seconds" && len(current[name+"_count"]) == 0 {
source = "time_per_output_token_seconds"
@@ -127,9 +130,21 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e
if sum == nil || count == nil {
continue
}
histograms[name] = histogram{Sum: *sum, Count: *count, Buckets: bucketDeltas(current[source+"_bucket"], previous[source+"_bucket"], *count)}
item := histogram{Sum: *sum, Count: *count}
switch name {
case "request_prefill_time_seconds", "request_decode_time_seconds", "request_queue_time_seconds":
default:
item.Buckets = bucketDeltas(current[source+"_bucket"], previous[source+"_bucket"], *count)
}
if name == "request_prompt_tokens" || name == "request_generation_tokens" {
item.Seconds = elapsed
}
histograms[name] = item
}
applyHistograms(&result, histograms, true)
if err := calculateRequestMetrics(current, previous, elapsed, &result); err != nil {
return model.MonitorVLLM{}, err
}
applyHistograms(&result, histograms)
encoded, err := json.Marshal(histograms)
if err != nil {
return model.MonitorVLLM{}, err
@@ -138,43 +153,24 @@ func Calculate(current, previous Metrics, elapsed float64) (model.MonitorVLLM, e
return result, nil
}
func AggregateHistograms(encoded string, result *model.MonitorVLLM) error {
var samples []map[string]histogram
if err := json.Unmarshal([]byte(encoded), &samples); err != nil {
return err
}
merged := map[string]histogram{}
for _, sample := range samples {
for name, item := range sample {
if item.Count == 0 {
continue
}
total, exists := merged[name]
if !exists {
merged[name] = item
continue
}
total.Sum += item.Sum
total.Count += item.Count
if len(total.Buckets) != len(item.Buckets) {
total.Buckets = nil
}
for bound, count := range total.Buckets {
addition, ok := item.Buckets[bound]
if !ok {
total.Buckets = nil
break
}
total.Buckets[bound] = count + addition
}
merged[name] = total
func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram, includeMeans bool) {
for name, target := range map[string]*map[string]float64{
"request_prompt_tokens": &result.PromptDistribution,
"request_generation_tokens": &result.GenerationDistribution,
} {
item := histograms[name]
if item.Seconds <= 0 || len(item.Buckets) == 0 {
continue
}
result.IntervalSeconds = math.Max(result.IntervalSeconds, item.Seconds)
*target = make(map[string]float64, len(item.Buckets))
for bound, count := range item.Buckets {
(*target)[bound] = count / item.Seconds
}
}
applyHistograms(result, merged)
return nil
}
func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram) {
if !includeMeans {
return
}
for _, item := range []struct {
name string
mean **float64
@@ -185,6 +181,8 @@ func applyHistograms(result *model.MonitorVLLM, histograms map[string]histogram)
{"e2e_request_latency_seconds", &result.RequestLatency, []**float64{&result.RequestLatencyP50, &result.RequestLatencyP90, &result.RequestLatencyP95, &result.RequestLatencyP99}},
{"request_prefill_time_seconds", &result.PrefillTime, nil},
{"request_decode_time_seconds", &result.DecodeTime, nil},
{"request_queue_time_seconds", &result.QueueTime, nil},
{"request_generation_tokens", &result.AverageGenerationTokens, nil},
} {
histogram, ok := histograms[item.name]
if !ok || histogram.Count <= 0 {
agent/utils/ai_tools/vllm/request_metrics.go
+65
View File
@@ -0,0 +1,65 @@
package vllm
import (
"encoding/json"
"strconv"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/utils/re"
)
type requestMetrics struct {
Seconds float64 `json:"seconds"`
PrefixHits *float64 `json:"prefixHits,omitempty"`
PrefixQueries *float64 `json:"prefixQueries,omitempty"`
FinishReasons map[string]float64 `json:"finishReasons,omitempty"`
}
func calculateRequestMetrics(current, previous Metrics, elapsed float64, result *model.MonitorVLLM) error {
sample := requestMetrics{Seconds: elapsed, FinishReasons: map[string]float64{}}
hits := counterDelta(current["prefix_cache_hits_total"], previous["prefix_cache_hits_total"])
queries := counterDelta(current["prefix_cache_queries_total"], previous["prefix_cache_queries_total"])
if hits != nil && queries != nil && *hits <= *queries {
sample.PrefixHits, sample.PrefixQueries = hits, queries
if *queries > 0 {
rate := *hits / *queries * 100
result.PrefixCacheHitRate = &rate
}
}
before := groupFinishReasons(previous["request_success_total"])
for reason, values := range groupFinishReasons(current["request_success_total"]) {
if delta := counterDelta(values, before[reason]); delta != nil {
sample.FinishReasons[reason] = *delta
if result.FinishReasons == nil {
result.FinishReasons = map[string]float64{}
}
result.FinishReasons[reason] = *delta / elapsed
}
}
encoded, err := json.Marshal(sample)
if err != nil {
return err
}
result.RequestDeltas = string(encoded)
return nil
}
func groupFinishReasons(samples map[string]float64) map[string]map[string]float64 {
groups := map[string]map[string]float64{}
for labels, value := range samples {
for _, label := range re.GetRegex(re.VLLMMetricLabelPattern).FindAllStringSubmatch(labels, -1) {
if label[1] != "finished_reason" {
continue
}
reason, err := strconv.Unquote(label[2])
if err != nil || reason == "" {
continue
}
if groups[reason] == nil {
groups[reason] = map[string]float64{}
}
groups[reason][labels] = value
}
}
return groups
}
frontend/src/lang/modules/en.ts
+18
View File
@@ -1517,6 +1517,24 @@ const message = {
extraHeaders: 'Extra Headers',
},
vllmMonitor: {
tokensPerSecond: 'tokens/s',
requestsPerSecond: 'Requests/s',
finishReasonLabels: {
stop: 'Normal completion',
length: 'Length limit reached',
abort: 'Aborted',
error: 'Error',
repetition: 'Repetition',
},
requestAnalysis: 'Request analysis',
promptDistribution: 'Prompt token length distribution',
generationDistribution: 'Generation token length distribution',
finishReasons: 'Request finish reasons',
queueTime: 'Mean queue time',
averageGenerationTokens: 'Mean output tokens per request',
prefixCacheHitRate: 'Prefix cache hit rate',
distributionHelper:
'Length distributions show request rates and are unaffected by the average/peak selection.',
cleanHelper:
'Clear monitoring history only for this vLLM instance? Other instances, host and GPU monitoring are unaffected.',
overview: 'Overview',
frontend/src/lang/modules/es-es.ts
+18
View File
@@ -1526,6 +1526,24 @@ const message = {
extraHeaders: 'Encabezados adicionales',
},
vllmMonitor: {
tokensPerSecond: 'tokens/s',
requestsPerSecond: 'Solicitudes/s',
finishReasonLabels: {
stop: 'Finalización normal',
length: 'Límite de longitud alcanzado',
abort: 'Cancelada',
error: 'Error',
repetition: 'Repetición',
},
requestAnalysis: 'Análisis de solicitudes',
promptDistribution: 'Distribución de longitud de tokens de entrada',
generationDistribution: 'Distribución de longitud de tokens de salida',
finishReasons: 'Motivos de finalización de solicitudes',
queueTime: 'Tiempo medio en cola',
averageGenerationTokens: 'Tokens de salida medios por solicitud',
prefixCacheHitRate: 'Tasa de aciertos de caché de prefijos',
distributionHelper:
'Las distribuciones de longitud muestran tasas de solicitudes y no cambian al seleccionar media o pico.',
cleanHelper:
'¿Borrar solo el historial de monitoreo de esta instancia de vLLM? No afecta a otras instancias ni al monitoreo del host o de GPU.',
overview: 'Resumen',
frontend/src/lang/modules/fa.ts
+17
View File
@@ -1502,6 +1502,23 @@ const message = {
extraHeaders: 'هدرهای اضافی',
},
vllmMonitor: {
tokensPerSecond: 'tokens/ثانیه',
requestsPerSecond: 'درخواست/ثانیه',
finishReasonLabels: {
stop: 'پایان عادی',
length: 'رسیدن به حد طول',
abort: 'لغوشده',
error: 'خطا',
repetition: 'تکرار خروجی',
},
requestAnalysis: 'تحلیل درخواست‌ها',
promptDistribution: 'توزیع طول توکن‌های ورودی',
generationDistribution: 'توزیع طول توکن‌های خروجی',
finishReasons: 'دلایل پایان درخواست‌ها',
queueTime: 'میانگین زمان انتظار در صف',
averageGenerationTokens: 'میانگین توکن‌های خروجی هر درخواست',
prefixCacheHitRate: 'نرخ اصابت حافظه نهان پیشوند',
distributionHelper: 'توزیع طول، نرخ درخواست‌ها را نشان می‌دهد و از انتخاب میانگین یا اوج تأثیر نمی‌پذیرد.',
cleanHelper:
'فقط سابقه نظارت این نمونه vLLM پاک شود؟ سایر نمونه\u200cها و نظارت میزبان و GPU تغییری نمی\u200cکنند.',
overview: 'نمای کلی',
frontend/src/lang/modules/ja.ts
+17
View File
@@ -1505,6 +1505,23 @@ const message = {
extraHeaders: '追加ヘッダー',
},
vllmMonitor: {
tokensPerSecond: 'tokens/秒',
requestsPerSecond: 'リクエスト/秒',
finishReasonLabels: {
stop: '正常終了',
length: '長さ上限に到達',
abort: '中止',
error: 'エラー',
repetition: '繰り返し出力',
},
requestAnalysis: 'リクエスト分析',
promptDistribution: '入力トークン長の分布',
generationDistribution: '出力トークン長の分布',
finishReasons: 'リクエスト終了理由',
queueTime: '平均キュー待機時間',
averageGenerationTokens: 'リクエストあたりの平均出力トークン数',
prefixCacheHitRate: 'プレフィックスキャッシュヒット率',
distributionHelper: '長さの分布はリクエストレートで集計され、平均／ピークの切り替えには影響されません。',
cleanHelper:
'この vLLM インスタンスの監視履歴のみを削除しますか？他のインスタンス、ホスト、GPU の監視には影響しません。',
overview: '概要',
frontend/src/lang/modules/ko.ts
+17
View File
@@ -1490,6 +1490,23 @@ const message = {
extraHeaders: '추가 헤더',
},
vllmMonitor: {
tokensPerSecond: 'tokens/초',
requestsPerSecond: '요청/초',
finishReasonLabels: {
stop: '정상 종료',
length: '길이 제한 도달',
abort: '중단',
error: '오류',
repetition: '반복 출력',
},
requestAnalysis: '요청 분석',
promptDistribution: '입력 토큰 길이 분포',
generationDistribution: '출력 토큰 길이 분포',
finishReasons: '요청 종료 사유',
queueTime: '평균 대기 시간',
averageGenerationTokens: '요청당 평균 출력 토큰 수',
prefixCacheHitRate: '접두사 캐시 적중률',
distributionHelper: '길이 분포는 요청 속도로 집계되며 평균/최댓값 선택의 영향을 받지 않습니다.',
cleanHelper:
'현재 vLLM 인스턴스의 모니터링 기록만 삭제하시겠습니까? 다른 인스턴스, 호스트 및 GPU 모니터링에는 영향을 주지 않습니다.',
overview: '개요',
frontend/src/lang/modules/lo.ts
+17
View File
@@ -1484,6 +1484,23 @@ const message = {
extraHeaders: 'Headers ເພີ່ມເຕີມ',
},
vllmMonitor: {
tokensPerSecond: 'tokens/ວິນາທີ',
requestsPerSecond: 'ຄຳຮ້ອງຂໍ/ວິນາທີ',
finishReasonLabels: {
stop: 'ສຳເລັດປົກກະຕິ',
length: 'ເຖິງຂີດຈຳກັດຄວາມຍາວ',
abort: 'ຍົກເລີກ',
error: 'ຂໍ້ຜິດພາດ',
repetition: 'ຜົນອອກຊ້ຳ',
},
requestAnalysis: 'ການວິເຄາະຄຳຮ້ອງຂໍ',
promptDistribution: 'ການແຈກຢາຍຄວາມຍາວໂທເຄັນຂາເຂົ້າ',
generationDistribution: 'ການແຈກຢາຍຄວາມຍາວໂທເຄັນຂາອອກ',
finishReasons: 'ເຫດຜົນທີ່ຄຳຮ້ອງຂໍສິ້ນສຸດ',
queueTime: 'ເວລາລໍຖ້າໃນຄິວສະເລ່ຍ',
averageGenerationTokens: 'ຈຳນວນໂທເຄັນຂາອອກສະເລ່ຍຕໍ່ຄຳຮ້ອງຂໍ',
prefixCacheHitRate: 'ອັດຕາການພົບໃນແຄດຄຳນຳໜ້າ',
distributionHelper: 'ການແຈກຢາຍຄວາມຍາວສະແດງອັດຕາຄຳຮ້ອງຂໍ ແລະ ບໍ່ປ່ຽນຕາມການເລືອກຄ່າສະເລ່ຍ ຫຼື ຄ່າສູງສຸດ.',
cleanHelper: 'ລຶບສະເພາະປະຫວັດການຕິດຕາມຂອງ vLLM ນີ້ບໍ? ບໍ່ກະທົບຕໍ່ອິນສະແຕນອື່ນ, ໂຮສ ຫຼື GPU.',
overview: 'ພາບລວມ',
latencyAnalysis: 'ການວິເຄາະຄວາມລ່າຊ້າ',
frontend/src/lang/modules/ms.ts
+18
View File
@@ -1531,6 +1531,24 @@ const message = {
extraHeaders: 'Pengepala Tambahan',
},
vllmMonitor: {
tokensPerSecond: 'tokens/s',
requestsPerSecond: 'Permintaan/s',
finishReasonLabels: {
stop: 'Selesai secara normal',
length: 'Had panjang dicapai',
abort: 'Dibatalkan',
error: 'Ralat',
repetition: 'Pengulangan',
},
requestAnalysis: 'Analisis permintaan',
promptDistribution: 'Taburan panjang token input',
generationDistribution: 'Taburan panjang token output',
finishReasons: 'Sebab permintaan tamat',
queueTime: 'Purata masa dalam baris gilir',
averageGenerationTokens: 'Purata token output setiap permintaan',
prefixCacheHitRate: 'Kadar padanan cache awalan',
distributionHelper:
'Taburan panjang menunjukkan kadar permintaan dan tidak dipengaruhi oleh pilihan purata/puncak.',
cleanHelper:
'Padamkan sejarah pemantauan untuk instans vLLM ini sahaja? Instans lain serta pemantauan hos dan GPU tidak terjejas.',
overview: 'Gambaran keseluruhan',
frontend/src/lang/modules/pt-br.ts
+18
View File
@@ -1527,6 +1527,24 @@ const message = {
extraHeaders: 'Cabeçalhos extras',
},
vllmMonitor: {
tokensPerSecond: 'tokens/s',
requestsPerSecond: 'Solicitações/s',
finishReasonLabels: {
stop: 'Conclusão normal',
length: 'Limite de comprimento atingido',
abort: 'Abortada',
error: 'Erro',
repetition: 'Repetição',
},
requestAnalysis: 'Análise de solicitações',
promptDistribution: 'Distribuição do comprimento dos tokens de entrada',
generationDistribution: 'Distribuição do comprimento dos tokens de saída',
finishReasons: 'Motivos de conclusão das solicitações',
queueTime: 'Tempo médio na fila',
averageGenerationTokens: 'Média de tokens de saída por solicitação',
prefixCacheHitRate: 'Taxa de acertos do cache de prefixos',
distributionHelper:
'As distribuições de comprimento mostram taxas de solicitações e não mudam com a seleção de média ou pico.',
cleanHelper:
'Limpar apenas o histórico de monitoramento desta instância do vLLM? Outras instâncias e o monitoramento do host e da GPU não serão afetados.',
overview: 'Visão geral',
frontend/src/lang/modules/ru.ts
+18
View File
@@ -1520,6 +1520,24 @@ const message = {
extraHeaders: 'Дополнительные заголовки',
},
vllmMonitor: {
tokensPerSecond: 'tokens/с',
requestsPerSecond: 'Запросов/с',
finishReasonLabels: {
stop: 'Нормальное завершение',
length: 'Достигнут лимит длины',
abort: 'Прервано',
error: 'Ошибка',
repetition: 'Повторение',
},
requestAnalysis: 'Анализ запросов',
promptDistribution: 'Распределение длины входных токенов',
generationDistribution: 'Распределение длины выходных токенов',
finishReasons: 'Причины завершения запросов',
queueTime: 'Среднее время в очереди',
averageGenerationTokens: 'Среднее число выходных токенов на запрос',
prefixCacheHitRate: 'Доля попаданий в кэш префиксов',
distributionHelper:
'Распределения длины показывают частоту запросов и не зависят от выбора среднего или пикового значения.',
cleanHelper:
'Удалить историю мониторинга только этого экземпляра vLLM? Другие экземпляры, мониторинг хоста и GPU не будут затронуты.',
overview: 'Обзор',
frontend/src/lang/modules/tr.ts
+17
View File
@@ -1527,6 +1527,23 @@ const message = {
extraHeaders: 'Ek Başlıklar',
},
vllmMonitor: {
tokensPerSecond: 'tokens/sn',
requestsPerSecond: 'İstek/sn',
finishReasonLabels: {
stop: 'Normal tamamlanma',
length: 'Uzunluk sınırına ulaşıldı',
abort: 'İptal edildi',
error: 'Hata',
repetition: 'Tekrarlama',
},
requestAnalysis: 'İstek analizi',
promptDistribution: 'Girdi belirteci uzunluk dağılımı',
generationDistribution: 'Çıktı belirteci uzunluk dağılımı',
finishReasons: 'İstek bitiş nedenleri',
queueTime: 'Ortalama kuyruk süresi',
averageGenerationTokens: 'İstek başına ortalama çıktı belirteci',
prefixCacheHitRate: 'Önek önbelleği isabet oranı',
distributionHelper: 'Uzunluk dağılımları istek hızlarını gösterir ve ortalama/tepe seçiminden etkilenmez.',
cleanHelper:
'Yalnızca bu vLLM örneğinin izleme geçmişi temizlensin mi? Diğer örnekler, ana makine ve GPU izlemesi etkilenmez.',
overview: 'Genel bakış',
frontend/src/lang/modules/zh-Hant.ts
+17
View File
@@ -1437,6 +1437,23 @@ const message = {
extraHeaders: '額外請求標頭',
},
vllmMonitor: {
tokensPerSecond: 'tokens/秒',
requestsPerSecond: '請求/秒',
finishReasonLabels: {
stop: '正常結束',
length: '達到長度限制',
abort: '中止',
error: '錯誤',
repetition: '重複輸出',
},
requestAnalysis: '請求分析',
promptDistribution: '輸入 Token 長度分佈',
generationDistribution: '輸出 Token 長度分佈',
finishReasons: '請求結束原因',
queueTime: '平均排隊耗時',
averageGenerationTokens: '每請求平均輸出 Token 數',
prefixCacheHitRate: '前綴快取命中率',
distributionHelper: '長度分佈按請求速率彙總，不受平均／峰值切換影響。',
cleanHelper: '僅清空目前 vLLM 實例的監控歷史，不影響其他實例、主機或 GPU 監控，是否繼續？',
overview: '概覽',
latencyAnalysis: '延遲分析',
frontend/src/lang/modules/zh.ts
+17
View File
@@ -1450,6 +1450,23 @@ const message = {
extraHeaders: '额外请求头',
},
vllmMonitor: {
tokensPerSecond: 'tokens/秒',
requestsPerSecond: '请求/秒',
finishReasonLabels: {
stop: '正常结束',
length: '达到长度限制',
abort: '中止',
error: '错误',
repetition: '重复输出',
},
requestAnalysis: '请求分析',
promptDistribution: '输入 Token 长度分布',
generationDistribution: '输出 Token 长度分布',
finishReasons: '请求结束原因',
queueTime: '平均排队耗时',
averageGenerationTokens: '每请求平均输出 Token 数',
prefixCacheHitRate: '前缀缓存命中率',
distributionHelper: '长度分布按请求速率汇总，不受平均／峰值切换影响。',
cleanHelper: '仅清空当前 vLLM 实例的监控历史，不影响其他实例、主机或 GPU 监控，是否继续？',
overview: '概览',
latencyAnalysis: '延迟分析',