From 2f7bab67b3b082fb4d55aaba8514e50c09bd8736 Mon Sep 17 00:00:00 2001 From: ssongliu Date: Fri, 30 Oct 2026 17:03:21 +0800 Subject: [PATCH] feat(ai): add unified GPU configuration and collapsible deployment cards (#14253) --- agent/app/api/v2/gpu.go | 5 + agent/app/dto/ai.go | 22 + agent/app/dto/request/app.go | 4 + agent/app/dto/request/tensorrt_llm.go | 17 +- agent/app/dto/response/app.go | 17 +- agent/app/dto/response/tensorrt_llm.go | 14 +- agent/app/service/app.go | 20 + agent/app/service/app_install.go | 36 +- agent/app/service/model_gpu.go | 663 ++++++++++++++++++ agent/app/service/tensorrt_llm.go | 34 + agent/router/ro_ai.go | 1 + agent/utils/re/re.go | 14 + frontend/src/api/interface/ai.ts | 16 + frontend/src/api/interface/app.ts | 2 + frontend/src/api/modules/ai.ts | 4 + .../src/components/advanced-setting/index.vue | 4 +- frontend/src/lang/modules/en.ts | 25 +- frontend/src/lang/modules/es-es.ts | 25 +- frontend/src/lang/modules/fa.ts | 24 +- frontend/src/lang/modules/ja.ts | 24 +- frontend/src/lang/modules/ko.ts | 24 +- frontend/src/lang/modules/lo.ts | 24 +- frontend/src/lang/modules/ms.ts | 24 +- frontend/src/lang/modules/pt-br.ts | 25 +- frontend/src/lang/modules/ru.ts | 25 +- frontend/src/lang/modules/tr.ts | 24 +- frontend/src/lang/modules/zh-Hant.ts | 22 +- frontend/src/lang/modules/zh.ts | 22 +- .../components/account-address/index.vue | 3 +- .../components/deployment-manage/index.vue | 18 +- .../ai/model/components/form-section.scss | 41 ++ .../views/ai/model/components/form-section.ts | 7 + .../ai/model/components/gpu-config/index.vue | 246 +++++++ frontend/src/views/ai/model/index.vue | 10 - .../views/ai/model/ollama/operate/index.vue | 58 +- .../views/ai/model/tensorrt/operate/index.vue | 161 +++-- .../src/views/app-store/detail/form/index.vue | 220 +++--- .../app-store/installed/detail/index.vue | 341 +++++---- 38 files changed, 1868 insertions(+), 398 deletions(-) create mode 100644 agent/app/service/model_gpu.go create mode 100644 frontend/src/views/ai/model/components/form-section.scss create mode 100644 frontend/src/views/ai/model/components/form-section.ts create mode 100644 frontend/src/views/ai/model/components/gpu-config/index.vue diff --git a/agent/app/api/v2/gpu.go b/agent/app/api/v2/gpu.go index e9aedd740..b68658f66 100644 --- a/agent/app/api/v2/gpu.go +++ b/agent/app/api/v2/gpu.go @@ -3,11 +3,16 @@ package v2 import ( "github.com/1Panel-dev/1Panel/agent/app/api/v2/helper" "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/app/service" "github.com/1Panel-dev/1Panel/agent/global" "github.com/1Panel-dev/1Panel/agent/utils/ai_tools/accelerator" "github.com/gin-gonic/gin" ) +func (b *BaseApi) LoadModelGPUDevices(c *gin.Context) { + helper.SuccessWithData(c, service.LoadModelGPUDevices()) +} + // @Tags AI // @Summary Load gpu / xpu info // @Accept json diff --git a/agent/app/dto/ai.go b/agent/app/dto/ai.go index 0c9269cef..e841cb5bc 100644 --- a/agent/app/dto/ai.go +++ b/agent/app/dto/ai.go @@ -2,6 +2,28 @@ package dto import "time" +type ModelGPUDevice struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + UUID string `json:"uuid,omitempty"` + Available bool `json:"available"` +} + +type ModelGPUDevices struct { + Devices []ModelGPUDevice `json:"devices"` + Warnings []string `json:"warnings"` +} + +type ModelGPUConfig struct { + Mode string `json:"mode"` + Count int `json:"count,omitempty"` + Error string `json:"error,omitempty"` + Enabled bool `json:"enabled"` + DeviceIDs []string `json:"deviceIDs"` + Capabilities []string `json:"capabilities"` +} + type OllamaModelInfo struct { ID uint `json:"id"` Name string `json:"name"` diff --git a/agent/app/dto/request/app.go b/agent/app/dto/request/app.go index e69c00d8d..3928abcc7 100644 --- a/agent/app/dto/request/app.go +++ b/agent/app/dto/request/app.go @@ -22,6 +22,9 @@ type AppInstallCreate struct { Name string `json:"name" validate:"required"` Services map[string]string `json:"services"` TaskID string `json:"taskID"` + GPU *dto.ModelGPUConfig `json:"gpu"` + + GPUImageType string `json:"-"` SyncModelAccount *bool `json:"syncModelAccount"` ModelAccountBaseURLType string `json:"modelAccountBaseURLType" validate:"omitempty,oneof=container localhost systemIP custom"` @@ -128,6 +131,7 @@ type AppInstallDelete struct { } type AppInstalledUpdate struct { + GPU *dto.ModelGPUConfig `json:"gpu"` SyncModelAccount *bool `json:"syncModelAccount"` ModelAccountBaseURLType string `json:"modelAccountBaseURLType" validate:"omitempty,oneof=container localhost systemIP custom"` ModelAccountBaseURL string `json:"modelAccountBaseURL"` diff --git a/agent/app/dto/request/tensorrt_llm.go b/agent/app/dto/request/tensorrt_llm.go index 7a9604d9c..9abf00b10 100644 --- a/agent/app/dto/request/tensorrt_llm.go +++ b/agent/app/dto/request/tensorrt_llm.go @@ -8,14 +8,15 @@ type TensorRTLLMSearch struct { } type TensorRTLLMCreate struct { - Name string `json:"name" validate:"required"` - ContainerName string `json:"containerName" validate:"required"` - Version string `json:"version" validate:"required"` - ModelDir string `json:"modelDir" validate:"required"` - Image string `json:"image" validate:"required"` - Command string `json:"command" validate:"required"` - ModelType string `json:"modelType"` - ModelSpeedup bool `json:"modelSpeedup"` + Name string `json:"name" validate:"required"` + ContainerName string `json:"containerName" validate:"required"` + Version string `json:"version" validate:"required"` + ModelDir string `json:"modelDir" validate:"required"` + Image string `json:"image" validate:"required"` + Command string `json:"command" validate:"required"` + ModelType string `json:"modelType"` + ModelSpeedup bool `json:"modelSpeedup"` + GPU *dto.ModelGPUConfig `json:"gpu"` DockerConfig } diff --git a/agent/app/dto/response/app.go b/agent/app/dto/response/app.go index 7974e7681..2e50d12f6 100644 --- a/agent/app/dto/response/app.go +++ b/agent/app/dto/response/app.go @@ -186,13 +186,14 @@ type AppParam struct { } type AppConfig struct { - AppKey string `json:"appKey"` - Name string `json:"name"` - HttpPort int `json:"httpPort"` - SyncModelAccount bool `json:"syncModelAccount"` - ModelAccountBaseURLType string `json:"modelAccountBaseURLType"` - ModelAccountBaseURL string `json:"modelAccountBaseURL"` - Params []AppParam `json:"params"` - RawCompose string `json:"rawCompose"` + GPU *dto.ModelGPUConfig `json:"gpu,omitempty"` + AppKey string `json:"appKey"` + Name string `json:"name"` + HttpPort int `json:"httpPort"` + SyncModelAccount bool `json:"syncModelAccount"` + ModelAccountBaseURLType string `json:"modelAccountBaseURLType"` + ModelAccountBaseURL string `json:"modelAccountBaseURL"` + Params []AppParam `json:"params"` + RawCompose string `json:"rawCompose"` request.AppContainerConfig } diff --git a/agent/app/dto/response/tensorrt_llm.go b/agent/app/dto/response/tensorrt_llm.go index fdedaf050..babd20ff4 100644 --- a/agent/app/dto/response/tensorrt_llm.go +++ b/agent/app/dto/response/tensorrt_llm.go @@ -1,6 +1,7 @@ package response import ( + "github.com/1Panel-dev/1Panel/agent/app/dto" "github.com/1Panel-dev/1Panel/agent/app/dto/request" "github.com/1Panel-dev/1Panel/agent/app/model" ) @@ -12,12 +13,13 @@ type TensorRTLLMsRes struct { type TensorRTLLMDTO struct { model.TensorRTLLM - Version string `json:"version"` - Model string `json:"model"` - Dir string `json:"dir"` - ModelDir string `json:"modelDir"` - Image string `json:"image"` - Command string `json:"command"` + Version string `json:"version"` + Model string `json:"model"` + Dir string `json:"dir"` + ModelDir string `json:"modelDir"` + Image string `json:"image"` + Command string `json:"command"` + GPU *dto.ModelGPUConfig `json:"gpu"` ExposedPorts []request.ExposedPort `json:"exposedPorts"` Environments []request.Environment `json:"environments"` diff --git a/agent/app/service/app.go b/agent/app/service/app.go index 9dde9b482..c48f2a2f6 100644 --- a/agent/app/service/app.go +++ b/agent/app/service/app.go @@ -542,6 +542,26 @@ func (a AppService) installWithHooks(req request.AppInstallCreate, executeScript composeByte []byte paramByte []byte ) + if (app.Key == "ollama" || app.Key == "vllm") && req.GPU != nil { + gpuServiceName := appInstall.ServiceName + imageType := "nvidia" + if app.Key == "vllm" { + imageType = req.GPUImageType + if len(servicesMap) > 1 { + gpuServiceName = "vllm" + if _, ok := servicesMap[gpuServiceName]; !ok { + gpuServiceName = "vllm-dspark" + } + } + } + service, ok := servicesMap[gpuServiceName].(map[string]interface{}) + if !ok { + return nil, buserr.New("ErrFileParse") + } + if err = ApplyModelGPU(service, req.GPU, req.Params, imageType); err != nil { + return + } + } composeByte, err = yaml.Marshal(composeMap) if err != nil { diff --git a/agent/app/service/app_install.go b/agent/app/service/app_install.go index 5e9b89d07..9c3e5e1a2 100644 --- a/agent/app/service/app_install.go +++ b/agent/app/service/app_install.go @@ -493,9 +493,10 @@ func (a *AppInstallService) Update(req request.AppInstalledUpdate) error { } backupDockerCompose := installed.DockerCompose - if req.Advanced { + updateGPU := installed.App.Key == "ollama" && req.GPU != nil + if req.Advanced || updateGPU { composeMap := make(map[string]interface{}) - if req.EditCompose { + if req.Advanced && req.EditCompose { if err = yaml.Unmarshal([]byte(req.DockerCompose), &composeMap); err != nil { return err } @@ -504,16 +505,35 @@ func (a *AppInstallService) Update(req request.AppInstalledUpdate) error { return err } } - if !req.SkipComposeCommonConfig { + if req.Advanced && !req.SkipComposeCommonConfig { if err = addDockerComposeCommonParam(composeMap, installed.ServiceName, req.AppContainerConfig, req.Params); err != nil { return err } } + if updateGPU { + services, _ := composeMap["services"].(map[string]interface{}) + service, ok := services[installed.ServiceName].(map[string]interface{}) + if !ok { + return buserr.New("ErrFileParse") + } + gpuParams := map[string]interface{}{} + if err := json.Unmarshal([]byte(installed.Env), &gpuParams); err != nil { + return err + } + for key, value := range req.Params { + gpuParams[key] = value + } + if err := ApplyModelGPU(service, req.GPU, gpuParams, "nvidia"); err != nil { + return err + } + } composeByte, err := yaml.Marshal(composeMap) if err != nil { return err } installed.DockerCompose = string(composeByte) + } + if req.Advanced { if req.ContainerName == "" { req.Params[constant.ContainerName] = installed.ContainerName } else { @@ -1013,6 +1033,16 @@ func (a *AppInstallService) GetParams(id uint) (*response.AppConfig, error) { res.ModelAccountBaseURLType = install.ModelAccountBaseURLType res.ModelAccountBaseURL = install.ModelAccountBaseURL if install.App.Key == "ollama" { + var project map[string]interface{} + if err := yaml.Unmarshal([]byte(install.DockerCompose), &project); err != nil { + return nil, err + } + services, _ := project["services"].(map[string]interface{}) + service, ok := services[install.ServiceName].(map[string]interface{}) + if !ok { + return nil, buserr.New("ErrFileParse") + } + res.GPU = ReadModelGPU(service, envs, "nvidia") res.ModelAccountBaseURLType, res.ModelAccountBaseURL, err = loadOllamaModelAccountAddress(install) if err != nil { return nil, err diff --git a/agent/app/service/model_gpu.go b/agent/app/service/model_gpu.go new file mode 100644 index 000000000..dbbdba4f2 --- /dev/null +++ b/agent/app/service/model_gpu.go @@ -0,0 +1,663 @@ +package service + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/1Panel-dev/1Panel/agent/app/dto" + "github.com/1Panel-dev/1Panel/agent/utils/re" + "github.com/compose-spec/compose-go/v2/interpolation" + "github.com/compose-spec/compose-go/v2/template" +) + +func LoadModelGPUDevices() dto.ModelGPUDevices { + result := dto.ModelGPUDevices{Devices: []dto.ModelGPUDevice{}, Warnings: []string{}} + uuidPattern := re.GetRegex(re.NvidiaGPUUUIDPattern) + renderPattern := re.GetRegex(re.IntelGPURenderNamePattern) + ascendPattern := re.GetRegex(re.AscendGPUDeviceNamePattern) + for _, directory := range []string{"/proc/driver/nvidia/gpus", "/sys/class/drm", "/dev"} { + entries, err := os.ReadDir(directory) + if err != nil { + if !os.IsNotExist(err) { + result.Warnings = append(result.Warnings, err.Error()) + } + continue + } + for _, entry := range entries { + var device dto.ModelGPUDevice + var devicePath string + switch directory { + case "/proc/driver/nvidia/gpus": + informationPath := filepath.Join(directory, entry.Name(), "information") + content, err := os.ReadFile(informationPath) + if err != nil { + result.Warnings = append(result.Warnings, err.Error()) + continue + } + fields := map[string]string{} + for _, line := range strings.Split(string(content), "\n") { + key, value, found := strings.Cut(line, ":") + if found { + fields[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + } + uuid := fields["GPU UUID"] + minor, err := strconv.Atoi(fields["Device Minor"]) + if !uuidPattern.MatchString(uuid) || err != nil || minor < 0 { + result.Warnings = append(result.Warnings, fmt.Sprintf("invalid NVIDIA GPU UUID or device minor in %s", informationPath)) + continue + } + name := fields["Model"] + if name == "" { + name = "NVIDIA" + } + device = dto.ModelGPUDevice{ID: "nvidia:" + uuid, Name: name + " · " + entry.Name(), Type: "nvidia", UUID: uuid} + devicePath = filepath.Join("/dev", "nvidia"+strconv.Itoa(minor)) + case "/sys/class/drm": + if !renderPattern.MatchString(entry.Name()) { + continue + } + vendor, err := os.ReadFile(filepath.Join(directory, entry.Name(), "device/vendor")) + if err != nil { + result.Warnings = append(result.Warnings, err.Error()) + continue + } + if strings.TrimSpace(string(vendor)) != "0x8086" { + continue + } + devicePath = filepath.Join("/dev/dri", entry.Name()) + device = dto.ModelGPUDevice{ID: "intel:" + devicePath, Name: "Intel · " + entry.Name(), Type: "intel"} + case "/dev": + if !ascendPattern.MatchString(entry.Name()) { + continue + } + devicePath = filepath.Join(directory, entry.Name()) + device = dto.ModelGPUDevice{ID: "ascend:" + strings.TrimPrefix(entry.Name(), "davinci"), Name: "Ascend · " + entry.Name(), Type: "ascend"} + } + info, err := os.Stat(devicePath) + if err != nil { + result.Warnings = append(result.Warnings, err.Error()) + } else { + device.Available = info.Mode()&os.ModeCharDevice != 0 + } + result.Devices = append(result.Devices, device) + } + } + return result +} + +func ApplyModelGPU(service map[string]interface{}, config *dto.ModelGPUConfig, params map[string]interface{}, imageType string) error { + if config == nil { + return nil + } + if err := validateModelGPU(config, imageType); err != nil { + return err + } + resolved, err := resolveModelGPUService(service, params, false) + if err != nil { + return err + } + devicePaths := []string{} + ids := []string{} + logicalIDs := []string{} + if config.Enabled && imageType != "nvidia" { + for _, deviceID := range config.DeviceIDs { + id := strings.TrimPrefix(deviceID, imageType+":") + if !slices.Contains(ids, id) { + ids = append(ids, id) + } + } + if imageType == "ascend" { + slices.SortFunc(ids, func(a, b string) int { + left, _ := strconv.Atoi(a) + right, _ := strconv.Atoi(b) + return left - right + }) + visibleIDs := ids + if config.Mode == "all" || resolved["privileged"] == true || resolved["privileged"] == "true" { + visibleIDs = loadModelAscendDeviceIDs() + if len(visibleIDs) == 0 { + return fmt.Errorf("no available Ascend devices") + } + } + if config.Mode == "all" { + ids = visibleIDs + } else { + for _, id := range ids { + index := slices.Index(visibleIDs, id) + if index < 0 { + return fmt.Errorf("Ascend GPU is unavailable: %s", id) + } + logicalIDs = append(logicalIDs, strconv.Itoa(index)) + } + } + for _, id := range ids { + devicePaths = append(devicePaths, "/dev/davinci"+id) + } + devicePaths = append(devicePaths, "/dev/davinci_manager", "/dev/devmm_svm", "/dev/hisi_hdc") + } else if config.Mode == "all" { + devicePaths = append(devicePaths, "/dev/dri") + } else { + devicePaths = append(devicePaths, ids...) + } + } + if imageType == "nvidia" { + delete(service, "gpus") + } + if imageType == "nvidia" && resolved["runtime"] == "nvidia" && !config.Enabled { + delete(service, "runtime") + } + deploy, _ := service["deploy"].(map[string]interface{}) + resources, _ := deploy["resources"].(map[string]interface{}) + reservations, _ := resources["reservations"].(map[string]interface{}) + devices, _ := reservations["devices"].([]interface{}) + resolvedDevices, _ := resolved["reservations"].([]interface{}) + kept := []interface{}{} + var gpuDevice map[string]interface{} + for index, item := range devices { + device, _ := resolvedDevices[index].(map[string]interface{}) + capabilities, _ := device["capabilities"].([]interface{}) + driver, _ := device["driver"].(string) + isNvidiaGPU := driver == "nvidia" || (driver == "" && slices.Contains(capabilities, interface{}("gpu"))) + if imageType != "nvidia" || !isNvidiaGPU { + kept = append(kept, item) + } else if gpuDevice == nil { + gpuDevice, _ = item.(map[string]interface{}) + } + } + if config.Enabled && imageType == "nvidia" { + device := gpuDevice + if device == nil { + device = map[string]interface{}{} + } + device["driver"] = "nvidia" + device["capabilities"] = config.Capabilities + delete(device, "count") + delete(device, "device_ids") + switch config.Mode { + case "all": + device["count"] = "all" + case "count": + device["count"] = config.Count + default: + ids := []string{} + for _, id := range config.DeviceIDs { + ids = append(ids, strings.TrimPrefix(id, "nvidia:")) + } + device["device_ids"] = ids + } + kept = append(kept, device) + } + + if len(kept) > 0 { + if deploy == nil { + deploy = map[string]interface{}{} + service["deploy"] = deploy + } + if resources == nil { + resources = map[string]interface{}{} + deploy["resources"] = resources + } + if reservations == nil { + reservations = map[string]interface{}{} + resources["reservations"] = reservations + } + reservations["devices"] = kept + } else { + delete(reservations, "devices") + if len(reservations) == 0 { + delete(resources, "reservations") + } + if len(resources) == 0 { + delete(deploy, "resources") + } + if len(deploy) == 0 { + delete(service, "deploy") + } + } + mappedPaths := []string{} + for _, key := range []string{"devices", "volumes"} { + mappings, _ := service[key].([]interface{}) + resolvedMappings, _ := resolved[key].([]interface{}) + kept := []interface{}{} + for index, item := range mappings { + source := modelGPUDeviceSource(resolvedMappings[index]) + managed := false + switch imageType { + case "nvidia": + managed = strings.HasPrefix(source, "/dev/nvidia") || strings.HasPrefix(source, "nvidia.com/gpu=") + case "intel": + managed = source == "/dev/dri" || strings.HasPrefix(source, "/dev/dri/") + case "ascend": + managed = strings.HasPrefix(source, "/dev/davinci") || source == "/dev/devmm_svm" || source == "/dev/hisi_hdc" + } + if managed { + selected := slices.Contains(devicePaths, source) + if config.Enabled && config.Mode == "all" && imageType == "intel" { + selected = true + } + if selected { + kept = append(kept, item) + mappedPaths = append(mappedPaths, source) + } + continue + } + kept = append(kept, item) + } + if len(kept) == 0 { + delete(service, key) + } else { + service[key] = kept + } + } + if config.Enabled && imageType != "nvidia" { + devices, _ := service["devices"].([]interface{}) + for _, path := range devicePaths { + if !slices.Contains(mappedPaths, path) { + devices = append(devices, path+":"+path) + } + } + if len(devices) > 0 { + service["devices"] = devices + } + } + gpuEnvironment := map[string]string{} + if config.Enabled && imageType == "ascend" && config.Mode != "all" { + gpuEnvironment["ASCEND_RT_VISIBLE_DEVICES"] = strings.Join(logicalIDs, ",") + } + var environmentKeys []string + switch imageType { + case "nvidia": + environmentKeys = []string{"NVIDIA_VISIBLE_DEVICES", "NVIDIA_DRIVER_CAPABILITIES"} + case "ascend": + environmentKeys = []string{"ASCEND_RT_VISIBLE_DEVICES"} + if params != nil { + params["ASCEND_RT_VISIBLE_DEVICES"] = strings.Join(logicalIDs, ",") + } + default: + return nil + } + switch environment := service["environment"].(type) { + case map[string]interface{}: + for _, key := range environmentKeys { + delete(environment, key) + } + for key, value := range gpuEnvironment { + environment[key] = value + } + case []interface{}: + kept := []interface{}{} + for _, item := range environment { + value, ok := item.(string) + if !ok { + return fmt.Errorf("invalid Compose environment") + } + key, _, _ := strings.Cut(value, "=") + if !slices.Contains(environmentKeys, key) { + kept = append(kept, item) + } + } + for key, value := range gpuEnvironment { + kept = append(kept, key+"="+value) + } + service["environment"] = kept + case nil: + if len(gpuEnvironment) > 0 { + environment := map[string]interface{}{} + for key, value := range gpuEnvironment { + environment[key] = value + } + service["environment"] = environment + } + default: + return fmt.Errorf("invalid Compose environment") + } + return nil +} + +func ReadModelGPU(service map[string]interface{}, params map[string]interface{}, imageType string) *dto.ModelGPUConfig { + resolved, err := resolveModelGPUService(service, params, true) + if err != nil { + return &dto.ModelGPUConfig{Mode: "devices", DeviceIDs: []string{}, Capabilities: []string{"gpu"}, Error: err.Error()} + } + if imageType == "nvidia" { + config, err := readModelGPU(service, resolved) + if err != nil { + return &dto.ModelGPUConfig{Mode: "devices", DeviceIDs: []string{}, Capabilities: []string{"gpu"}, Error: err.Error()} + } + return config + } + config := &dto.ModelGPUConfig{Mode: "devices", DeviceIDs: []string{}, Capabilities: []string{"gpu"}} + privileged := resolved["privileged"] == true || resolved["privileged"] == "true" + all := false + for _, key := range []string{"devices", "volumes"} { + mappings, _ := resolved[key].([]interface{}) + for _, item := range mappings { + source := modelGPUDeviceSource(item) + id := "" + if imageType == "intel" { + switch { + case source == "/dev/dri": + all = true + case re.GetRegex(re.IntelGPUDevicePathPattern).MatchString(source): + id = imageType + ":" + source + case strings.HasPrefix(source, "/dev/dri/"): + config.Error = "unsupported Intel GPU mapping requires an explicit selection: " + source + return config + } + } + if imageType == "ascend" && re.GetRegex(re.AscendGPUDevicePathPattern).MatchString(source) { + id = imageType + ":" + strings.TrimPrefix(source, "/dev/davinci") + } + if id != "" && !slices.Contains(config.DeviceIDs, id) { + config.DeviceIDs = append(config.DeviceIDs, id) + } + } + } + config.Enabled = all || len(config.DeviceIDs) > 0 + config.Mode = "devices" + if all { + config.Mode = "all" + config.DeviceIDs = []string{} + } + if imageType == "ascend" { + const key = "ASCEND_RT_VISIBLE_DEVICES" + visible := "" + switch environment := service["environment"].(type) { + case map[string]interface{}: + if value, exists := environment[key]; exists { + if value == nil { + visible = "${" + key + "}" + } else { + visible = fmt.Sprint(value) + } + } + case []interface{}: + for _, item := range environment { + value, _ := item.(string) + if value == key { + visible = "${" + key + "}" + } else if strings.HasPrefix(value, key+"=") { + visible = strings.TrimPrefix(value, key+"=") + } + } + } + visible, err := resolveModelGPUValue(visible, params) + if err != nil { + config.Error = err.Error() + return config + } + visible = strings.TrimSpace(visible) + if visible == "all" || (visible == "" && config.Enabled) { + config.Enabled = true + config.Mode = "all" + config.DeviceIDs = []string{} + } else if visible != "" { + physicalIDs := []string{} + for _, id := range config.DeviceIDs { + physicalIDs = append(physicalIDs, strings.TrimPrefix(id, "ascend:")) + } + slices.SortFunc(physicalIDs, func(a, b string) int { + left, _ := strconv.Atoi(a) + right, _ := strconv.Atoi(b) + return left - right + }) + if privileged { + physicalIDs = loadModelAscendDeviceIDs() + } + selected := []string{} + for _, id := range strings.Split(visible, ",") { + id = strings.TrimSpace(id) + if !re.GetRegex(re.AscendGPUDeviceIDPattern).MatchString(id) { + config.Error = "invalid Ascend GPU device: " + id + return config + } + index, err := strconv.Atoi(id) + if err != nil || index < 0 || index >= len(physicalIDs) { + config.Error = "Ascend logical GPU is not available in the container: " + id + return config + } + selected = append(selected, "ascend:"+physicalIDs[index]) + } + config.Enabled = true + config.Mode = "devices" + config.DeviceIDs = selected + } + } + return config +} + +func readModelGPU(service, resolved map[string]interface{}) (*dto.ModelGPUConfig, error) { + config := &dto.ModelGPUConfig{Mode: "all", DeviceIDs: []string{}, Capabilities: []string{"gpu"}} + devices, _ := resolved["reservations"].([]interface{}) + if service["gpus"] != nil { + return nil, fmt.Errorf("custom GPU declarations require an explicit selection") + } + for _, item := range devices { + device, ok := item.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid GPU reservation") + } + driver, _ := device["driver"].(string) + capabilities, _ := device["capabilities"].([]interface{}) + isGPU := driver == "nvidia" || (driver == "" && slices.Contains(capabilities, interface{}("gpu"))) + if !isGPU { + continue + } + if config.Enabled { + return nil, fmt.Errorf("multiple GPU declarations require an explicit selection") + } + config.Enabled = true + ids, _ := device["device_ids"].([]interface{}) + if len(ids) > 0 { + if device["count"] != nil { + return nil, fmt.Errorf("GPU count and device_ids cannot be combined") + } + config.Mode = "devices" + for _, id := range ids { + config.DeviceIDs = append(config.DeviceIDs, "nvidia:"+fmt.Sprint(id)) + } + } else if count := device["count"]; count != nil && fmt.Sprint(count) != "all" && fmt.Sprint(count) != "-1" { + config.Mode = "count" + countValue, err := strconv.Atoi(fmt.Sprint(count)) + config.Count = countValue + if err != nil { + return nil, fmt.Errorf("invalid GPU count: %v", count) + } + } + for _, capability := range capabilities { + value, ok := capability.(string) + if !ok { + return nil, fmt.Errorf("unsupported GPU capabilities: %v", capabilities) + } + if !slices.Contains(config.Capabilities, value) { + config.Capabilities = append(config.Capabilities, value) + } + } + } + for _, key := range []string{"devices", "volumes"} { + mappings, _ := resolved[key].([]interface{}) + for _, item := range mappings { + source := modelGPUDeviceSource(item) + if strings.HasPrefix(source, "/dev/nvidia") || strings.HasPrefix(source, "nvidia.com/gpu=") { + return nil, fmt.Errorf("direct NVIDIA device mappings require an explicit selection") + } + } + } + if !config.Enabled && resolved["runtime"] == "nvidia" { + return nil, fmt.Errorf("custom NVIDIA runtime configuration requires an explicit selection") + } + if err := validateModelGPU(config, "nvidia"); err != nil { + return nil, err + } + return config, nil +} + +func validateModelGPU(config *dto.ModelGPUConfig, imageType string) error { + if config.Error != "" { + return fmt.Errorf("GPU configuration must be selected explicitly: %s", config.Error) + } + if !config.Enabled { + return nil + } + patterns := map[string]string{ + "nvidia": re.NvidiaGPUDeviceIDPattern, + "intel": re.IntelGPUDevicePathPattern, + "ascend": re.AscendGPUDeviceIDPattern, + } + pattern, supported := patterns[imageType] + if !supported { + return fmt.Errorf("unsupported GPU type: %s", imageType) + } + switch config.Mode { + case "all": + if config.Count != 0 || len(config.DeviceIDs) != 0 { + return fmt.Errorf("all GPUs cannot be combined with count or device IDs") + } + case "count": + if imageType != "nvidia" || config.Count < 1 || len(config.DeviceIDs) != 0 { + return fmt.Errorf("GPU count requires a positive number and runtime device allocation") + } + case "devices", "": + if config.Count != 0 || len(config.DeviceIDs) == 0 { + return fmt.Errorf("select at least one GPU device") + } + for _, deviceID := range config.DeviceIDs { + id := strings.TrimPrefix(deviceID, imageType+":") + if !re.GetRegex(pattern).MatchString(id) { + return fmt.Errorf("invalid %s GPU device: %s", imageType, deviceID) + } + } + default: + return fmt.Errorf("unsupported GPU selection mode: %s", config.Mode) + } + + if imageType != "nvidia" { + return nil + } + if !slices.Contains(config.Capabilities, "gpu") { + return fmt.Errorf("NVIDIA capabilities must include gpu") + } + for _, capability := range config.Capabilities { + if capability != "gpu" && capability != "utility" && capability != "compute" { + return fmt.Errorf("unsupported NVIDIA capability: %s", capability) + } + } + return nil +} + +func resolveModelGPUService(service map[string]interface{}, params map[string]interface{}, resolveSelection bool) (map[string]interface{}, error) { + fields := map[string]interface{}{} + fields["runtime"] = service["runtime"] + deploy, _ := service["deploy"].(map[string]interface{}) + resources, _ := deploy["resources"].(map[string]interface{}) + reservations, _ := resources["reservations"].(map[string]interface{}) + fields["reservations"] = reservations["devices"] + if devices, ok := reservations["devices"].([]interface{}); ok && !resolveSelection { + declarations := make([]interface{}, 0, len(devices)) + for _, item := range devices { + device, _ := item.(map[string]interface{}) + declarations = append(declarations, map[string]interface{}{"driver": device["driver"], "capabilities": device["capabilities"]}) + } + fields["reservations"] = declarations + } + resolved, err := interpolation.Interpolate(fields, interpolation.Options{ + Substitute: func(value string, _ template.Mapping) (string, error) { return resolveModelGPUValue(value, params) }, + }) + if err != nil { + return nil, err + } + for _, key := range []string{"devices", "volumes"} { + mappings, _ := service[key].([]interface{}) + sources := make([]interface{}, 0, len(mappings)) + for _, item := range mappings { + source, err := template.SubstituteWithOptions(modelGPUDeviceSource(item), func(name string) (string, bool) { + value, ok := params[name] + if !ok || value == nil { + return "", false + } + return fmt.Sprint(value), true + }, template.WithoutLogging) + if err != nil { + return nil, err + } + if _, short := item.(string); short { + source = modelGPUDeviceSource(source) + } + sources = append(sources, map[string]interface{}{"source": source}) + } + resolved[key] = sources + } + resolved["privileged"] = service["privileged"] + if value, ok := service["privileged"].(string); ok { + if value, err := resolveModelGPUValue(value, params); err == nil { + resolved["privileged"] = value + } + } + return resolved, nil +} + +func modelGPUDeviceSource(item interface{}) string { + switch mapping := item.(type) { + case string: + if strings.HasPrefix(mapping, "nvidia.com/gpu=") { + return mapping + } + depth := 0 + for index, ch := range mapping { + switch ch { + case '{': + depth++ + case '}': + depth-- + case ':': + if depth == 0 { + return mapping[:index] + } + } + } + return mapping + case map[string]interface{}: + source, _ := mapping["source"].(string) + return source + } + return "" +} + +func resolveModelGPUValue(value string, params map[string]interface{}) (string, error) { + return template.SubstituteWithOptions(value, func(name string) (string, bool) { + value, ok := params[name] + if !ok || value == nil { + return "", false + } + return fmt.Sprint(value), true + }, template.WithoutLogging, template.WithReplacementFunction(func(value string, mapping template.Mapping, config *template.Config) (string, error) { + resolved, applied, err := template.DefaultReplacementAppliedFunc(value, mapping, config) + if err != nil { + return "", err + } + if !applied { + return "", fmt.Errorf("unresolved GPU variable: %s", value) + } + return resolved, nil + })) +} + +func loadModelAscendDeviceIDs() []string { + ids := []string{} + for _, device := range LoadModelGPUDevices().Devices { + if device.Type == "ascend" && device.Available { + ids = append(ids, strings.TrimPrefix(device.ID, "ascend:")) + } + } + slices.SortFunc(ids, func(a, b string) int { + left, _ := strconv.Atoi(a) + right, _ := strconv.Atoi(b) + return left - right + }) + return ids +} diff --git a/agent/app/service/tensorrt_llm.go b/agent/app/service/tensorrt_llm.go index 5fbf68cb0..e12e392f4 100644 --- a/agent/app/service/tensorrt_llm.go +++ b/agent/app/service/tensorrt_llm.go @@ -57,12 +57,31 @@ func (t TensorRTLLMService) Page(req request.TensorRTLLMSearch) response.TensorR serverDTO.Dir = path.Join(global.Dir.TensorRTLLMDir, item.Name) serverDTO.Image = envs["IMAGE"] serverDTO.Command = getCommand(item.Env) + var project map[string]interface{} + if yaml.Unmarshal([]byte(item.DockerCompose), &project) == nil { + services, _ := project["services"].(map[string]interface{}) + if service, ok := services[item.Name].(map[string]interface{}); ok { + params := make(map[string]interface{}, len(envs)) + for key, value := range envs { + params[key] = value + } + serverDTO.GPU = ReadModelGPU(service, params, "nvidia") + } + } serverDTO.ExposedPorts, _ = loadComposeExposedPortsFromEnv(envs, "0.0.0.0", false) composeByte, err := files.NewFileOp().GetContent(path.Join(global.Dir.TensorRTLLMDir, item.Name, "docker-compose.yml")) if err == nil { serverDTO.Environments, _ = getDockerComposeEnvironments(composeByte) + environments := make([]request.Environment, 0, len(serverDTO.Environments)) + for _, item := range serverDTO.Environments { + if item.Key == "MODEL_PATH" || (serverDTO.GPU != nil && (item.Key == "NVIDIA_VISIBLE_DEVICES" || item.Key == "NVIDIA_DRIVER_CAPABILITIES")) { + continue + } + environments = append(environments, item) + } + serverDTO.Environments = environments } volumes, err := getDockerComposeVolumes(composeByte) if err == nil { @@ -133,6 +152,9 @@ func handleLLMParams(llm *model.TensorRTLLM, create request.TensorRTLLMCreate) e var environments []interface{} environments = append(environments, fmt.Sprintf("MODEL_PATH=%s", create.ModelDir)) for _, e := range create.Environments { + if e.Key == "MODEL_PATH" { + continue + } environments = append(environments, fmt.Sprintf("%s=%s", e.Key, e.Value)) } serviceValue["environment"] = environments @@ -148,6 +170,18 @@ func handleLLMParams(llm *model.TensorRTLLM, create request.TensorRTLLMCreate) e volumes = append(volumes, formatComposeVolume(volume.Source, volume.Target, volume.Mode)) } serviceValue["volumes"] = volumes + gpuParams := map[string]interface{}{} + existingEnv, err := gotenv.Unmarshal(llm.Env) + if err != nil { + return err + } + for key, value := range existingEnv { + gpuParams[key] = value + } + gpuParams["MODEL_PATH"] = create.ModelDir + if err := ApplyModelGPU(serviceValue, create.GPU, gpuParams, "nvidia"); err != nil { + return err + } services[llm.Name] = serviceValue composeByte, err := yaml.Marshal(composeMap) diff --git a/agent/router/ro_ai.go b/agent/router/ro_ai.go index f66485d48..bcdc3e268 100644 --- a/agent/router/ro_ai.go +++ b/agent/router/ro_ai.go @@ -26,6 +26,7 @@ func (a *AIToolsRouter) InitRouter(Router *gin.RouterGroup) { aiToolsRouter.POST("/domain/update", baseApi.UpdateBindDomain) aiToolsRouter.GET("/gpu/load", baseApi.LoadGpuInfo) + aiToolsRouter.GET("/model/gpu/devices", baseApi.LoadModelGPUDevices) aiToolsRouter.POST("/gpu/search", baseApi.LoadGPUMonitor) aiToolsRouter.GET("/gpu/options", baseApi.GetCPUOptions) aiToolsRouter.POST("/vllm/monitor/search", baseApi.LoadVLLMMonitor) diff --git a/agent/utils/re/re.go b/agent/utils/re/re.go index 3b143175d..c82b59099 100644 --- a/agent/utils/re/re.go +++ b/agent/utils/re/re.go @@ -61,6 +61,13 @@ const ( AcceleratorMetricValuePattern = `^\s*([+-]?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+))\s*(.*?)\s*$` AscendVersionPattern = `(?i)\bVersion:\s*([^\s|]+)` AscendMemoryPattern = `([0-9]+(?:\.[0-9]+)?)\s*/\s*([0-9]+(?:\.[0-9]+)?)` + NvidiaGPUUUIDPattern = `^GPU-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$` + NvidiaGPUDeviceIDPattern = `^([0-9]+|GPU-[a-zA-Z0-9-]+|MIG-[a-zA-Z0-9/-]+)$` + IntelGPURenderNamePattern = `^renderD[0-9]+$` + IntelGPUDevicePathPattern = `^/dev/dri/renderD[0-9]+$` + AscendGPUDeviceNamePattern = `^davinci[0-9]+$` + AscendGPUDevicePathPattern = `^/dev/davinci[0-9]+$` + AscendGPUDeviceIDPattern = `^[0-9]+$` ) var regexMap = make(map[string]*regexp.Regexp) @@ -122,6 +129,13 @@ func Init() { AcceleratorMetricValuePattern, AscendVersionPattern, AscendMemoryPattern, + NvidiaGPUUUIDPattern, + NvidiaGPUDeviceIDPattern, + IntelGPURenderNamePattern, + IntelGPUDevicePathPattern, + AscendGPUDeviceNamePattern, + AscendGPUDevicePathPattern, + AscendGPUDeviceIDPattern, } for _, pattern := range patterns { diff --git a/frontend/src/api/interface/ai.ts b/frontend/src/api/interface/ai.ts index 49f6a4e11..75cd34b9e 100644 --- a/frontend/src/api/interface/ai.ts +++ b/frontend/src/api/interface/ai.ts @@ -1,6 +1,21 @@ import { ReqPage } from '.'; export namespace AI { + export interface ModelGPUDevice { + id: string; + name: string; + type: string; + uuid?: string; + available: boolean; + } + export interface ModelGPUConfig { + mode: 'all' | 'count' | 'devices'; + count?: number; + error?: string; + enabled: boolean; + deviceIDs: string[]; + capabilities: string[]; + } export interface ModelAccountSync { id: number; modelAccountBaseURLType: 'container' | 'localhost' | 'systemIP' | 'custom'; @@ -392,6 +407,7 @@ export namespace AI { } export interface TensorRTLLM { + gpu?: ModelGPUConfig; id?: number; name: string; containerName: string; diff --git a/frontend/src/api/interface/app.ts b/frontend/src/api/interface/app.ts index 03de17f32..fd4cc8a24 100644 --- a/frontend/src/api/interface/app.ts +++ b/frontend/src/api/interface/app.ts @@ -137,6 +137,7 @@ export namespace App { } export interface AppInstall { + gpu?: AI.ModelGPUConfig; syncModelAccount?: boolean; modelAccountBaseURLType?: string; modelAccountBaseURL?: string; @@ -310,6 +311,7 @@ export namespace App { } export interface AppConfig { + gpu?: AI.ModelGPUConfig; appKey: string; name: string; httpPort: number; diff --git a/frontend/src/api/modules/ai.ts b/frontend/src/api/modules/ai.ts index 3b550661f..80afefe44 100644 --- a/frontend/src/api/modules/ai.ts +++ b/frontend/src/api/modules/ai.ts @@ -43,6 +43,10 @@ export const updateBindDomain = (req: AI.BindDomain) => { return http.post(`/ai/domain/update`, req); }; +export const loadModelGPUDevices = () => { + return http.get<{ devices: AI.ModelGPUDevice[]; warnings: string[] }>('/ai/model/gpu/devices'); +}; + export const loadGPUInfo = () => { return http.get(`/ai/gpu/load`); }; diff --git a/frontend/src/components/advanced-setting/index.vue b/frontend/src/components/advanced-setting/index.vue index 0db052796..a241607b5 100644 --- a/frontend/src/components/advanced-setting/index.vue +++ b/frontend/src/components/advanced-setting/index.vue @@ -1,5 +1,5 @@