mirror of
https://github.com/1Panel-dev/1Panel.git
synced 2026-11-20 00:01:06 +00:00
* feat(ai): add GPU groups and NUMA binding * fix(ai): normalize NVIDIA GPU index aliases before validation
405 lines
12 KiB
Go
405 lines
12 KiB
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/1Panel-dev/1Panel/agent/app/dto"
|
|
"github.com/1Panel-dev/1Panel/agent/utils/docker"
|
|
"github.com/compose-spec/compose-go/v2/types"
|
|
)
|
|
|
|
const modelNUMAKey = "x-1panel-numa"
|
|
|
|
type modelNUMAState struct {
|
|
Config dto.ModelNUMAConfig `json:"config"`
|
|
Original map[string]interface{} `json:"original"`
|
|
Generated map[string]interface{} `json:"generated"`
|
|
}
|
|
|
|
func loadModelNUMANodes(sysRoot string) ([]dto.ModelNUMANode, error) {
|
|
nodes := []dto.ModelNUMANode{}
|
|
content, err := os.ReadFile(filepath.Join(sysRoot, "devices/system/node/online"))
|
|
if err != nil {
|
|
return nodes, fmt.Errorf("NUMA topology unavailable: %w", err)
|
|
}
|
|
online, err := parseModelCPUList(strings.TrimSpace(string(content)))
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
cpuContent, err := os.ReadFile(filepath.Join(sysRoot, "devices/system/cpu/online"))
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
onlineCPUs, err := parseModelCPUList(strings.TrimSpace(string(cpuContent)))
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
for _, id := range online {
|
|
directory := filepath.Join(sysRoot, "devices/system/node", fmt.Sprintf("node%d", id))
|
|
cpus, err := os.ReadFile(filepath.Join(directory, "cpulist"))
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
meminfo, err := os.ReadFile(filepath.Join(directory, "meminfo"))
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
node := dto.ModelNUMANode{ID: id, CPUs: strings.TrimSpace(string(cpus))}
|
|
nodeCPUs, err := parseModelCPUList(node.CPUs)
|
|
if err != nil {
|
|
return nodes, err
|
|
}
|
|
activeCPUs := []string{}
|
|
for _, cpu := range nodeCPUs {
|
|
if slices.Contains(onlineCPUs, cpu) {
|
|
activeCPUs = append(activeCPUs, strconv.Itoa(cpu))
|
|
}
|
|
}
|
|
if len(activeCPUs) != len(nodeCPUs) {
|
|
node.CPUs = strings.Join(activeCPUs, ",")
|
|
}
|
|
for _, line := range strings.Split(string(meminfo), "\n") {
|
|
_, value, found := strings.Cut(line, "MemTotal:")
|
|
if found {
|
|
fields := strings.Fields(value)
|
|
if len(fields) > 0 {
|
|
total, _ := strconv.ParseUint(fields[0], 10, 64)
|
|
node.Memory = total > 0
|
|
}
|
|
}
|
|
}
|
|
nodes = append(nodes, node)
|
|
}
|
|
return nodes, nil
|
|
}
|
|
|
|
func loadModelGPUNode(sysRoot, busID string) *int {
|
|
var domain, bus, device, function uint
|
|
if n, err := fmt.Sscanf(strings.ToLower(busID), "%x:%x:%x.%x", &domain, &bus, &device, &function); err != nil || n != 4 || domain > 0xffff || bus > 0xff || device > 0x1f || function > 7 {
|
|
return nil
|
|
}
|
|
address := fmt.Sprintf("%04x:%02x:%02x.%x", domain, bus, device, function)
|
|
content, err := os.ReadFile(filepath.Join(sysRoot, "bus/pci/devices", address, "numa_node"))
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
node, err := strconv.Atoi(strings.TrimSpace(string(content)))
|
|
if err != nil || node < 0 {
|
|
return nil
|
|
}
|
|
return &node
|
|
}
|
|
|
|
func parseModelCPUList(value string) ([]int, error) {
|
|
values := []int{}
|
|
if value == "" {
|
|
return values, nil
|
|
}
|
|
for _, item := range strings.Split(value, ",") {
|
|
startText, endText, ranged := strings.Cut(item, "-")
|
|
start, err := strconv.Atoi(startText)
|
|
if err != nil || start < 0 || start > 1048576 {
|
|
return nil, fmt.Errorf("invalid CPU/node list: %s", value)
|
|
}
|
|
end := start
|
|
if ranged {
|
|
end, err = strconv.Atoi(endText)
|
|
}
|
|
if err != nil || end < start || end > 1048576 {
|
|
return nil, fmt.Errorf("invalid CPU/node list: %s", value)
|
|
}
|
|
for id := start; id <= end; id++ {
|
|
values = append(values, id)
|
|
}
|
|
}
|
|
slices.Sort(values)
|
|
return slices.Compact(values), nil
|
|
}
|
|
|
|
func resolveModelNUMANodes(config *dto.ModelGPUConfig, topology dto.ModelGPUDevices, imageType, cpuset string) ([]int, error) {
|
|
nodes := slices.Clone(config.NUMA.Nodes)
|
|
switch config.NUMA.Mode {
|
|
case "auto":
|
|
if config.Mode == "count" {
|
|
return nil, fmt.Errorf("automatic NUMA binding requires specific GPU devices or all GPUs")
|
|
}
|
|
nodes = nil
|
|
selected := 0
|
|
for _, device := range topology.Devices {
|
|
if device.Type != imageType || (config.Mode != "all" && !slices.Contains(config.DeviceIDs, device.ID)) {
|
|
continue
|
|
}
|
|
selected++
|
|
if !device.Available || device.NUMANode == nil {
|
|
return nil, fmt.Errorf("NUMA topology is unknown for GPU %s; select NUMA nodes manually", device.Name)
|
|
}
|
|
nodes = append(nodes, *device.NUMANode)
|
|
}
|
|
if selected == 0 || (config.Mode != "all" && selected != len(config.DeviceIDs)) {
|
|
return nil, fmt.Errorf("selected GPUs are unavailable for automatic NUMA binding")
|
|
}
|
|
slices.Sort(nodes)
|
|
nodes = slices.Compact(nodes)
|
|
if len(nodes) != 1 {
|
|
return nil, fmt.Errorf("selected GPUs span NUMA nodes; select nodes manually")
|
|
}
|
|
case "manual":
|
|
default:
|
|
return nil, fmt.Errorf("unsupported NUMA binding mode: %s", config.NUMA.Mode)
|
|
}
|
|
if len(nodes) == 0 {
|
|
return nil, fmt.Errorf("select at least one NUMA node")
|
|
}
|
|
allowed, err := parseModelCPUList(cpuset)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for _, id := range nodes {
|
|
index := slices.IndexFunc(topology.NUMANodes, func(node dto.ModelNUMANode) bool { return node.ID == id })
|
|
if index < 0 {
|
|
return nil, fmt.Errorf("NUMA node %d is unavailable", id)
|
|
}
|
|
node := topology.NUMANodes[index]
|
|
cpus, err := parseModelCPUList(node.CPUs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(cpus) == 0 || !node.Memory {
|
|
return nil, fmt.Errorf("NUMA node %d has no CPUs or memory", id)
|
|
}
|
|
if cpuset != "" && !slices.ContainsFunc(cpus, func(cpu int) bool { return slices.Contains(allowed, cpu) }) {
|
|
return nil, fmt.Errorf("NUMA node %d conflicts with the container CPU set", id)
|
|
}
|
|
}
|
|
slices.Sort(nodes)
|
|
return slices.Compact(nodes), nil
|
|
}
|
|
|
|
func readModelNUMAState(service map[string]interface{}) (*modelNUMAState, error) {
|
|
value, exists := service[modelNUMAKey]
|
|
if !exists {
|
|
return nil, nil
|
|
}
|
|
content, ok := value.(string)
|
|
if !ok {
|
|
return nil, fmt.Errorf("invalid managed NUMA configuration")
|
|
}
|
|
var state modelNUMAState
|
|
if err := json.Unmarshal([]byte(content), &state); err != nil {
|
|
return nil, fmt.Errorf("invalid managed NUMA configuration: %w", err)
|
|
}
|
|
if state.Original == nil || state.Generated["entrypoint"] == nil {
|
|
return nil, fmt.Errorf("invalid managed NUMA startup configuration")
|
|
}
|
|
return &state, nil
|
|
}
|
|
|
|
func restoreModelNUMA(service map[string]interface{}) error {
|
|
state, err := readModelNUMAState(service)
|
|
if err != nil || state == nil {
|
|
return err
|
|
}
|
|
for key, generated := range state.Generated {
|
|
currentJSON, _ := json.Marshal(service[key])
|
|
generatedJSON, _ := json.Marshal(generated)
|
|
if string(currentJSON) != string(generatedJSON) {
|
|
if key == "entrypoint" {
|
|
return fmt.Errorf("managed NUMA entrypoint was edited; restore it before changing NUMA binding")
|
|
}
|
|
if key == "cap_add" {
|
|
var capabilities []string
|
|
if err := json.Unmarshal(currentJSON, &capabilities); err != nil {
|
|
return fmt.Errorf("invalid Compose cap_add: %w", err)
|
|
}
|
|
capabilities = slices.DeleteFunc(capabilities, func(value string) bool { return value == "SYS_NICE" })
|
|
if len(capabilities) == 0 {
|
|
delete(service, key)
|
|
} else {
|
|
service[key] = capabilities
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if original, exists := state.Original[key]; exists {
|
|
service[key] = original
|
|
} else {
|
|
delete(service, key)
|
|
}
|
|
}
|
|
delete(service, modelNUMAKey)
|
|
return nil
|
|
}
|
|
|
|
func applyModelNUMA(service map[string]interface{}, config *dto.ModelGPUConfig, params map[string]interface{}, imageType string) error {
|
|
if err := restoreModelNUMA(service); err != nil {
|
|
return err
|
|
}
|
|
if !config.Enabled || config.NUMA == nil || config.NUMA.Mode == "" || config.NUMA.Mode == "off" {
|
|
return nil
|
|
}
|
|
var topology dto.ModelGPUDevices
|
|
if config.NUMA.Mode == "auto" {
|
|
topology = LoadModelGPUDevices()
|
|
} else {
|
|
nodes, err := loadModelNUMANodes("/sys")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
topology.NUMANodes = nodes
|
|
}
|
|
cpuset := ""
|
|
if value, exists := service["cpuset"]; exists {
|
|
var err error
|
|
cpuset, err = resolveModelGPUValue(fmt.Sprint(value), params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
nodes, err := resolveModelNUMANodes(config, topology, imageType, cpuset)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var imageEntrypoint, imageCommand []string
|
|
if service["entrypoint"] == nil {
|
|
imageName, err := resolveModelGPUValue(fmt.Sprint(service["image"]), params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cli, err := docker.NewDockerClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer cli.Close()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
info, err := cli.ImageInspect(ctx, imageName)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot preserve image startup command for NUMA binding; pull image %s first: %w", imageName, err)
|
|
}
|
|
if info.Config != nil {
|
|
imageEntrypoint = info.Config.Entrypoint
|
|
imageCommand = info.Config.Cmd
|
|
}
|
|
}
|
|
return wrapModelNUMA(service, *config.NUMA, nodes, params, imageEntrypoint, imageCommand)
|
|
}
|
|
|
|
func wrapModelNUMA(service map[string]interface{}, config dto.ModelNUMAConfig, nodes []int, params map[string]interface{}, imageEntrypoint, imageCommand []string) error {
|
|
original := map[string]interface{}{}
|
|
for _, key := range []string{"entrypoint", "command", "cap_add"} {
|
|
if value, exists := service[key]; exists {
|
|
original[key] = value
|
|
}
|
|
}
|
|
entrypoint := imageEntrypoint
|
|
if value := service["entrypoint"]; value != nil {
|
|
var err error
|
|
entrypoint, err = modelNUMACommand(value, params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
ids := make([]string, len(nodes))
|
|
for i, node := range nodes {
|
|
ids[i] = strconv.Itoa(node)
|
|
}
|
|
binding := strings.Join(ids, ",")
|
|
wrapped := []string{"numactl", "--cpunodebind=" + binding, "--membind=" + binding, "--"}
|
|
for _, arg := range entrypoint {
|
|
wrapped = append(wrapped, strings.ReplaceAll(arg, "$", "$$"))
|
|
}
|
|
generated := map[string]interface{}{"entrypoint": wrapped}
|
|
capabilityJSON, err := json.Marshal(service["cap_add"])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var capabilities []string
|
|
if err := json.Unmarshal(capabilityJSON, &capabilities); err != nil {
|
|
return fmt.Errorf("invalid Compose cap_add: %w", err)
|
|
}
|
|
hasMemoryBindingCapability := false
|
|
for _, capability := range capabilities {
|
|
resolved, err := resolveModelGPUValue(capability, params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
resolved = strings.TrimPrefix(strings.ToUpper(resolved), "CAP_")
|
|
if resolved == "SYS_NICE" || resolved == "ALL" {
|
|
hasMemoryBindingCapability = true
|
|
}
|
|
}
|
|
if !hasMemoryBindingCapability {
|
|
generated["cap_add"] = append(capabilities, "SYS_NICE")
|
|
}
|
|
if service["entrypoint"] == nil && service["command"] == nil {
|
|
args := make([]string, len(imageCommand))
|
|
for i, arg := range imageCommand {
|
|
args[i] = strings.ReplaceAll(arg, "$", "$$")
|
|
}
|
|
generated["command"] = args
|
|
}
|
|
effectiveCommand := service["command"]
|
|
if value, exists := generated["command"]; exists {
|
|
effectiveCommand = value
|
|
}
|
|
if len(entrypoint) == 0 {
|
|
command, err := modelNUMACommand(effectiveCommand, params)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(command) == 0 {
|
|
return fmt.Errorf("NUMA binding requires an executable startup command")
|
|
}
|
|
}
|
|
config.Nodes = nodes
|
|
state, err := json.Marshal(modelNUMAState{Config: config, Original: original, Generated: generated})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for key, value := range generated {
|
|
service[key] = value
|
|
}
|
|
service[modelNUMAKey] = string(state)
|
|
return nil
|
|
}
|
|
|
|
func modelNUMACommand(value interface{}, params map[string]interface{}) ([]string, error) {
|
|
if value == nil {
|
|
return nil, nil
|
|
}
|
|
if text, ok := value.(string); ok {
|
|
resolved, err := resolveModelGPUValue(text, params)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var command types.ShellCommand
|
|
if err := command.DecodeMapstructure(resolved); err != nil {
|
|
return nil, err
|
|
}
|
|
return command, nil
|
|
}
|
|
raw, err := json.Marshal(value)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var args []string
|
|
if err := json.Unmarshal(raw, &args); err != nil {
|
|
return nil, fmt.Errorf("invalid NUMA startup command: %w", err)
|
|
}
|
|
for i, arg := range args {
|
|
args[i], err = resolveModelGPUValue(arg, params)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return args, nil
|
|
}
|