feat(container): support batch upgrades with bounded concurrency and recovery (#14231)

2026-10-28 16:00:18 +00:00 · 2026-10-28 18:45:13 +08:00
parent d7c42e3e8b
commit ecedaf8c21
31 changed files with 802 additions and 228 deletions
@@ -1660,6 +1660,10 @@ func calculateNetwork(network map[string]container.NetworkStats) (float64, float
 }

 func checkImageExist(client *client.Client, imageItem string) bool {
+	return checkImageExistContext(context.Background(), client, imageItem)
+}
+
+func checkImageExistContext(ctx context.Context, client *client.Client, imageItem string) bool {
 	if client == nil {
 		var err error
 		client, err = docker.NewDockerClient()
@@ -1667,7 +1671,7 @@ func checkImageExist(client *client.Client, imageItem string) bool {
 			return false
 		}
 	}
-	images, err := client.ImageList(context.Background(), image.ListOptions{})
+	images, err := client.ImageList(ctx, image.ListOptions{})
 	if err != nil {
 		return false
 	}
@@ -1709,6 +1713,10 @@ func checkImageLike(client *client.Client, imageName string) bool {
 }

 func pullImages(task *task.Task, client *client.Client, imageName string) error {
+	return pullImagesContext(context.Background(), task, client, imageName)
+}
+
+func pullImagesContext(ctx context.Context, task *task.Task, client *client.Client, imageName string) error {
 	dockerCli := docker.NewClientWithExist(client)
 	repos, err := imageRepoRepo.List()
 	if err != nil {
@@ -1716,7 +1724,7 @@ func pullImages(task *task.Task, client *client.Client, imageName string) error
 	}
 	imageRepo := selectImageRepo(imageName, repos)
 	if imageRepo == nil || !imageRepo.Auth {
-		return dockerCli.PullImageWithProcess(task, imageName)
+		return dockerCli.PullImageWithProcessContext(ctx, task, imageName)
 	}

 	options := image.PullOptions{}
@@ -1729,7 +1737,7 @@ func pullImages(task *task.Task, client *client.Client, imageName string) error
 		return err
 	}
 	options.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
-	return dockerCli.PullImageWithProcessAndOptions(task, imageName, options)
+	return dockerCli.PullImageWithProcessAndOptionsContext(ctx, task, imageName, options)
 }

 func selectImageRepo(imageName string, repos []model.ImageRepo) *model.ImageRepo {
@@ -91,81 +91,95 @@ func (u *ContainerService) ContainerUpdate(req dto.ContainerOperate) error {
 	return nil
 }

+const containerUpgradeConcurrency = 2
+
+// Shared by all upgrade requests handled by this agent.
+var containerUpgradeSlots = make(chan struct{}, containerUpgradeConcurrency)
+
 func (u *ContainerService) ContainerUpgrade(req dto.ContainerUpgrade) error {
 	client, err := docker.NewDockerClient()
 	if err != nil {
 		return err
 	}
-	ctx := context.Background()
 	taskItem, err := task.NewTaskWithOps(req.Image, task.TaskUpgrade, task.TaskScopeImage, req.TaskID, 1)
 	if err != nil {
 		_ = client.Close()
 		global.LOG.Errorf("new task for create container failed, err: %v", err)
 		return err
 	}
-	go func() {
-		defer client.Close()
-		taskItem.AddSubTask(i18n.GetWithName("ContainerImagePull", req.Image), func(t *task.Task) error {
-			taskItem.LogStart(i18n.GetWithName("ContainerImagePull", req.Image))
-			if !checkImageExist(client, req.Image) || req.ForcePull {
-				if err := pullImages(taskItem, client, req.Image); err != nil {
-					if !req.ForcePull {
-						return err
-					}
-					return fmt.Errorf("pull image %s failed, err: %v", req.Image, err)
-				}
+	taskItem.AddSubTask(i18n.GetWithName("ContainerImagePull", req.Image), func(t *task.Task) error {
+		ctx, cancel := context.WithTimeout(context.Background(), containerUpgradePullTimeout)
+		defer cancel()
+		taskItem.LogStart(i18n.GetWithName("ContainerImagePull", req.Image))
+		if req.ForcePull || !checkImageExistContext(ctx, client, req.Image) {
+			if err := pullImagesContext(ctx, taskItem, client, req.Image); err != nil {
+				return fmt.Errorf("pull image %s failed: %w", req.Image, err)
+			}
+		}
+		return nil
+	}, nil)
+	var upgradeErrors []error
+	for _, item := range req.Names {
+		item := item
+		taskItem.AddSubTaskWithIgnoreErr(i18n.GetWithName("ContainerUpgradeItem", item), func(t *task.Task) error {
+			t.Logf("----------------- %s -----------------", item)
+			t.LogStart(i18n.GetWithName("ContainerAcquireLock", item))
+			ctx, cancel := context.WithTimeout(context.Background(), containerUpgradeReplaceTimeout)
+			defer cancel()
+			unlock, lockErr := containerOperationLock.lockContext(ctx, item)
+			if lockErr != nil {
+				upgradeErrors = append(upgradeErrors, fmt.Errorf("wait for container %s lock: %w", item, lockErr))
+				return lockErr
+			}
+			defer unlock()
+			t.LogWithStatus(i18n.GetWithName("ContainerAcquireLock", item), nil)
+
+			oldContainer, inspectErr := client.ContainerInspect(ctx, item)
+			t.LogWithStatus(i18n.GetWithName("ContainerLoadInfo", item), inspectErr)
+			if inspectErr != nil {
+				err := fmt.Errorf("reload container %s failed: %w", item, inspectErr)
+				upgradeErrors = append(upgradeErrors, err)
+				return err
+			}
+			config := cloneContainerConfig(oldContainer.Config)
+			config.Image = req.Image
+			if config.Labels == nil {
+				config.Labels = make(map[string]string)
+			}
+			config.Labels[containerUpgradeOwnerLabel] = oldContainer.ID
+			hostConf := cloneContainerHostConfig(oldContainer.HostConfig)
+			preserveContainerVolumeMounts(hostConf, oldContainer.Mounts)
+			networkConf, extraNetworks := buildContainerRecoverNetworkConfig(oldContainer.NetworkSettings, hostConf)
+			if err := normalizeContainerEndpointSettings(ctx, client, networkConf, extraNetworks); err != nil {
+				upgradeErr := fmt.Errorf("prepare networks for container %s failed: %w", item, err)
+				upgradeErrors = append(upgradeErrors, upgradeErr)
+				return upgradeErr
+			}
+			cleanupErr, err := switchContainerWithRecovery(ctx, client, item, oldContainer, func() (container.CreateResponse, error) {
+				return createContainerWithNetworks(ctx, client, config, hostConf, networkConf, extraNetworks, item)
+			}, config.Tty, t, true)
+			if err != nil {
+				upgradeErr := fmt.Errorf("upgrade container %s failed: %w", item, err)
+				upgradeErrors = append(upgradeErrors, upgradeErr)
+				return upgradeErr
+			}
+			if cleanupErr != nil {
+				t.Log(i18n.GetWithNameAndErr("ContainerCleanupWarning", containerSwitchBackupName(oldContainer.ID), cleanupErr))
 			}
 			return nil
-		}, nil)
-		var upgradeErrors []error
-		for _, item := range req.Names {
-			item := item
-			taskItem.AddSubTaskWithIgnoreErr(i18n.GetWithName("ContainerUpgradeItem", item), func(t *task.Task) error {
-				t.Logf("----------------- %s -----------------", item)
-				t.LogStart(i18n.GetWithName("ContainerAcquireLock", item))
-				unlock := containerOperationLock.lock(item)
-				defer unlock()
-				t.LogWithStatus(i18n.GetWithName("ContainerAcquireLock", item), nil)
-
-				oldContainer, inspectErr := client.ContainerInspect(ctx, item)
-				t.LogWithStatus(i18n.GetWithName("ContainerLoadInfo", item), inspectErr)
-				if inspectErr != nil {
-					err := fmt.Errorf("reload container %s failed: %w", item, inspectErr)
-					upgradeErrors = append(upgradeErrors, err)
-					return err
-				}
-				config := cloneContainerConfig(oldContainer.Config)
-				config.Image = req.Image
-				hostConf := cloneContainerHostConfig(oldContainer.HostConfig)
-				preserveContainerVolumeMounts(hostConf, oldContainer.Mounts)
-				networkConf, extraNetworks := buildContainerRecoverNetworkConfig(oldContainer.NetworkSettings, hostConf)
-				if err := normalizeContainerEndpointSettings(ctx, client, networkConf, extraNetworks); err != nil {
-					upgradeErr := fmt.Errorf("prepare networks for container %s failed: %w", item, err)
-					upgradeErrors = append(upgradeErrors, upgradeErr)
-					return upgradeErr
-				}
-				cleanupErr, err := switchContainer(ctx, client, item, oldContainer, func() (container.CreateResponse, error) {
-					return createContainerWithNetworks(ctx, client, config, hostConf, networkConf, extraNetworks, item)
-				}, config.Tty, t)
-				if err != nil {
-					upgradeErr := fmt.Errorf("upgrade container %s failed: %w", item, err)
-					upgradeErrors = append(upgradeErrors, upgradeErr)
-					return upgradeErr
-				}
-				if cleanupErr != nil {
-					t.Log(i18n.GetWithNameAndErr("ContainerCleanupWarning", containerSwitchBackupName(oldContainer.ID), cleanupErr))
-				}
-				return nil
-			})
-		}
-		taskItem.AddSubTask(i18n.GetMsgByKey("ContainerUpgradeSummary"), func(t *task.Task) error {
-			return errors.Join(upgradeErrors...)
-		}, nil)
-		if err := taskItem.Execute(); err != nil {
-			global.LOG.Error(err.Error())
-		}
-	}()
-
+		})
+	}
+	taskItem.AddSubTask(i18n.GetMsgByKey("ContainerUpgradeSummary"), func(t *task.Task) error {
+		return errors.Join(upgradeErrors...)
+	}, nil)
+	// Stage contexts own the deadlines, including a separate recovery budget.
+	for _, subTask := range taskItem.SubTasks {
+		subTask.Timeout = 0
+	}
+	if err := taskItem.ExecuteQueued(containerUpgradeSlots, "ContainerUpgradeWaiting", func() { _ = client.Close() }); err != nil {
+		_ = client.Close()
+		return err
+	}
 	return nil
 }

@@ -192,11 +206,16 @@ type containerOperationMutex struct {
 var containerOperationLock containerOperationMutex

 type containerOperationLockEntry struct {
-	mutex      sync.Mutex
+	token      chan struct{}
 	references int
 }

 func (l *containerOperationMutex) lock(names ...string) func() {
+	unlock, _ := l.lockContext(context.Background(), names...)
+	return unlock
+}
+
+func (l *containerOperationMutex) lockContext(ctx context.Context, names ...string) (func(), error) {
 	nameSet := make(map[string]struct{}, len(names))
 	for _, name := range names {
 		if name != "" {
@@ -209,7 +228,7 @@ func (l *containerOperationMutex) lock(names ...string) func() {
 	}
 	sort.Strings(orderedNames)
 	if len(orderedNames) == 0 {
-		return func() {}
+		return func() {}, nil
 	}

 	l.mutex.Lock()
@@ -220,7 +239,7 @@ func (l *containerOperationMutex) lock(names ...string) func() {
 	for _, name := range orderedNames {
 		entry := l.locks[name]
 		if entry == nil {
-			entry = &containerOperationLockEntry{}
+			entry = &containerOperationLockEntry{token: make(chan struct{}, 1)}
 			l.locks[name] = entry
 		}
 		entry.references++
@@ -228,12 +247,10 @@ func (l *containerOperationMutex) lock(names ...string) func() {
 	}
 	l.mutex.Unlock()

-	for _, entry := range entries {
-		entry.mutex.Lock()
-	}
-	return func() {
-		for index := len(entries) - 1; index >= 0; index-- {
-			entries[index].mutex.Unlock()
+	acquired := 0
+	release := func() {
+		for index := acquired - 1; index >= 0; index-- {
+			<-entries[index].token
 		}
 		l.mutex.Lock()
 		defer l.mutex.Unlock()
@@ -245,6 +262,20 @@ func (l *containerOperationMutex) lock(names ...string) func() {
 			}
 		}
 	}
+	for _, entry := range entries {
+		select {
+		case entry.token <- struct{}{}:
+			acquired++
+		case <-ctx.Done():
+			release()
+			return nil, ctx.Err()
+		}
+	}
+	if err := ctx.Err(); err != nil {
+		release()
+		return nil, err
+	}
+	return release, nil
 }

 type containerNetworkAttachment struct {
@@ -272,6 +303,131 @@ func switchContainer(
 	createNew func() (container.CreateResponse, error),
 	tty bool,
 	logger containerSwitchLogger,
+) (cleanupErr error, err error) {
+	return switchContainerWithRecovery(ctx, cli, name, oldContainer, createNew, tty, logger, false)
+}
+
+
+const (
+	containerUpgradePullTimeout     = 30 * time.Minute
+	containerUpgradeReplaceTimeout  = 15 * time.Minute
+	containerUpgradeRecoveryTimeout = 5 * time.Minute
+	containerUpgradeOwnerLabel      = "io.1panel.upgrade.original-id"
+)
+
+// Reconcile against Docker's actual state: a failed request may already have
+// changed names, created a container, or disconnected a network on the daemon.
+func recoverContainerUpgrade(ctx context.Context, cli containerSwitchClient, name string, old container.InspectResponse, newID string, logger containerSwitchLogger) error {
+	original, err := cli.ContainerInspect(ctx, old.ID)
+	if err != nil {
+		return fmt.Errorf("inspect original %s: %w", old.ID, err)
+	}
+	if original.State == nil {
+		return fmt.Errorf("original container state unavailable")
+	}
+
+	replacement, inspectErr := cli.ContainerInspect(ctx, name)
+	if inspectErr != nil && !client.IsErrNotFound(inspectErr) {
+		return fmt.Errorf("inspect replacement by name: %w", inspectErr)
+	}
+	if inspectErr == nil && replacement.ID != old.ID {
+		if replacement.Config == nil || replacement.Config.Labels[containerUpgradeOwnerLabel] != old.ID {
+			return fmt.Errorf("container %s belongs to another operation; refusing to remove it", name)
+		}
+		if newID != "" && replacement.ID != newID {
+			return fmt.Errorf("replacement ID changed; refusing to remove it")
+		}
+		newID = replacement.ID
+	}
+	if newID != "" {
+		replacement, inspectErr = cli.ContainerInspect(ctx, newID)
+		if inspectErr != nil && !client.IsErrNotFound(inspectErr) {
+			return fmt.Errorf("inspect replacement %s: %w", newID, inspectErr)
+		}
+		if inspectErr == nil {
+			if replacement.ID == old.ID || replacement.Config == nil || replacement.Config.Labels[containerUpgradeOwnerLabel] != old.ID {
+				return fmt.Errorf("replacement identity could not be confirmed")
+			}
+			err = cli.ContainerRemove(ctx, newID, container.RemoveOptions{Force: true, RemoveVolumes: false})
+			logContainerSwitchStep(logger, "ContainerRollbackRemoveReplacement", name, err)
+			if err != nil && !client.IsErrNotFound(err) {
+				return fmt.Errorf("remove replacement: %w", err)
+			}
+		}
+	}
+	if strings.TrimPrefix(original.Name, "/") != name {
+		err = cli.ContainerRename(ctx, old.ID, name)
+		logContainerSwitchStep(logger, "ContainerRollbackRenameOld", name, err)
+		if err != nil {
+			return fmt.Errorf("restore original name: %w", err)
+		}
+	}
+	// Reload after mutations; connect only networks which are actually missing.
+	original, err = cli.ContainerInspect(ctx, old.ID)
+	if err != nil {
+		return err
+	}
+	primary, extras := buildContainerRecoverNetworkConfig(old.NetworkSettings, old.HostConfig)
+	wanted := make(map[string]*network.EndpointSettings)
+	if primary != nil {
+		for n, endpoint := range primary.EndpointsConfig {
+			if n != "bridge" {
+				wanted[n] = endpoint
+			}
+		}
+	}
+	for n, endpoint := range extras {
+		if n != "bridge" {
+			wanted[n] = endpoint
+		}
+	}
+	var reconnectErr error
+	for n, endpoint := range wanted {
+		if original.NetworkSettings != nil && original.NetworkSettings.Networks[n] != nil {
+			continue
+		}
+		err = cli.NetworkConnect(ctx, n, old.ID, endpoint)
+		logContainerSwitchStep(logger, "ContainerRollbackReconnectOld", n, err)
+		if err != nil {
+			reconnectErr = errors.Join(reconnectErr, fmt.Errorf("reconnect network %s: %w", n, err))
+		}
+	}
+	if reconnectErr != nil {
+		return reconnectErr
+	}
+	if old.State != nil && old.State.Running {
+		if original.State == nil {
+			return fmt.Errorf("original state unavailable after recovery")
+		}
+		if !original.State.Running {
+			err = restartOriginalContainer(ctx, cli, old.ID)
+			logContainerSwitchStep(logger, "ContainerRollbackRestartOld", name, err)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	verified, err := cli.ContainerInspect(ctx, old.ID)
+	if err != nil {
+		return fmt.Errorf("verify recovery: %w", err)
+	}
+	if strings.TrimPrefix(verified.Name, "/") != name {
+		return fmt.Errorf("original name was not restored")
+	}
+	if old.State != nil && old.State.Running && (verified.State == nil || !verified.State.Running) {
+		return fmt.Errorf("original container is not running after recovery")
+	}
+	for n := range wanted {
+		if verified.NetworkSettings == nil || verified.NetworkSettings.Networks[n] == nil {
+			return fmt.Errorf("network %s was not restored", n)
+		}
+	}
+	return nil
+}
+
+func switchContainerWithRecovery(ctx context.Context, cli containerSwitchClient, name string,
+	oldContainer container.InspectResponse, createNew func() (container.CreateResponse, error),
+	tty bool, logger containerSwitchLogger, boundedRecovery bool,
 ) (cleanupErr error, err error) {
 	if oldContainer.ID == "" {
 		return nil, fmt.Errorf("original container ID is empty")
@@ -281,9 +437,32 @@ func switchContainer(
 		return nil, fmt.Errorf("cannot safely replace container %s with auto-remove enabled", name)
 	}

+	recoverUpgrade := func(newID string, cause error) error {
+		recoveryCtx, cancel := context.WithTimeout(context.Background(), containerUpgradeRecoveryTimeout)
+		defer cancel()
+		if newID != "" {
+			logContainerStartupLogs(recoveryCtx, cli, newID, name, tty, logger)
+		}
+		recoveryErr := recoverContainerUpgrade(recoveryCtx, cli, name, oldContainer, newID, logger)
+		if ctx.Err() != nil {
+			recoveryErr = errors.Join(recoveryErr, fmt.Errorf("Docker request timed out or was canceled; daemon-side completion cannot be confirmed: %w", ctx.Err()))
+		}
+		if recoveryErr != nil {
+			message := fmt.Sprintf("Recovery unconfirmed for container %s (original ID %s, backup %s): %v", name, oldContainer.ID, containerSwitchBackupName(oldContainer.ID), recoveryErr)
+			if logger != nil {
+				logger.Log(message)
+			}
+			return errors.Join(cause, errors.New(message))
+		}
+		return cause
+	}
+
 	if wasRunning {
 		if err := cli.ContainerStop(ctx, oldContainer.ID, container.StopOptions{}); err != nil {
 			logContainerSwitchStep(logger, "ContainerStopOld", name, err)
+			if boundedRecovery {
+				return nil, recoverUpgrade("", fmt.Errorf("stop original container failed: %w", err))
+			}
 			current, inspectErr := cli.ContainerInspect(ctx, oldContainer.ID)
 			if inspectErr == nil && current.State != nil && !current.State.Running {
 				restartErr := restartOriginalContainer(ctx, cli, oldContainer.ID)
@@ -298,6 +477,9 @@ func switchContainer(
 	backupName := containerSwitchBackupName(oldContainer.ID)
 	if err := cli.ContainerRename(ctx, oldContainer.ID, backupName); err != nil {
 		logContainerSwitchStep(logger, "ContainerRenameOld", name, err)
+		if boundedRecovery {
+			return nil, recoverUpgrade("", fmt.Errorf("rename original container failed: %w", err))
+		}
 		if wasRunning {
 			restartErr := restartOriginalContainer(ctx, cli, oldContainer.ID)
 			logContainerSwitchStep(logger, "ContainerRollbackRestartOld", name, restartErr)
@@ -309,6 +491,9 @@ func switchContainer(
 	disconnectedNetworks, disconnectErr := disconnectOriginalContainerNetworks(ctx, cli, oldContainer)
 	logContainerSwitchStep(logger, "ContainerDisconnectOld", backupName, disconnectErr)
 	if disconnectErr != nil {
+		if boundedRecovery {
+			return nil, recoverUpgrade("", disconnectErr)
+		}
 		rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, "", disconnectedNetworks, logger)
 		return nil, errors.Join(disconnectErr, rollbackErr)
 	}
@@ -316,11 +501,17 @@ func switchContainer(
 	created, createErr := createNew()
 	logContainerSwitchStep(logger, "ContainerCreateReplacement", name, createErr)
 	if createErr != nil {
+		if boundedRecovery {
+			return nil, recoverUpgrade(created.ID, createErr)
+		}
 		rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, created.ID, disconnectedNetworks, logger)
 		return nil, errors.Join(createErr, rollbackErr)
 	}
 	if err := cli.ContainerStart(ctx, created.ID, container.StartOptions{}); err != nil {
 		logContainerSwitchStep(logger, "ContainerStartReplacement", name, err)
+		if boundedRecovery {
+			return nil, recoverUpgrade(created.ID, fmt.Errorf("start new container failed: %w", err))
+		}
 		logContainerStartupLogs(ctx, cli, created.ID, name, tty, logger)
 		rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, created.ID, disconnectedNetworks, logger)
 		return nil, errors.Join(fmt.Errorf("start new container failed: %w", err), rollbackErr)
@@ -329,6 +520,9 @@ func switchContainer(
 	if wasRunning {
 		if err := waitContainerReady(ctx, cli, created.ID); err != nil {
 			logContainerSwitchStep(logger, "ContainerWaitReplacement", name, err)
+			if boundedRecovery {
+				return nil, recoverUpgrade(created.ID, fmt.Errorf("new container readiness check failed: %w", err))
+			}
 			logContainerStartupLogs(ctx, cli, created.ID, name, tty, logger)
 			rollbackErr := restoreOriginalContainer(ctx, cli, oldContainer.ID, name, wasRunning, created.ID, disconnectedNetworks, logger)
 			return nil, errors.Join(fmt.Errorf("new container readiness check failed: %w", err), rollbackErr)
@@ -336,6 +530,11 @@ func switchContainer(
 		logContainerSwitchStep(logger, "ContainerWaitReplacement", name, nil)
 	}

+	if boundedRecovery {
+		cleanupCtx, cancel := context.WithTimeout(context.Background(), containerUpgradeRecoveryTimeout)
+		defer cancel()
+		ctx = cleanupCtx
+	}
 	cleanupErr = cli.ContainerRemove(ctx, oldContainer.ID, container.RemoveOptions{Force: true, RemoveVolumes: false})
 	logContainerSwitchStep(logger, "ContainerRemoveOld", backupName, cleanupErr)
 	return cleanupErr, nil
@@ -292,10 +292,36 @@ func (t *Task) updateTask(task *model.Task) {
 	_ = t.taskRepo.Update(context.Background(), task)
 }

+// ExecuteQueued registers the task before returning and limits concurrent execution.
+// Subtasks must use Timeout = 0 and manage their own operation and recovery deadlines
+// so that a slot is held until the action and its cleanup have actually returned.
+func (t *Task) ExecuteQueued(slots chan struct{}, waitingStep string, onFinish func()) error {
+	t.Task.CurrentStep = waitingStep
+	if err := t.taskRepo.Save(context.Background(), t.Task); err != nil {
+		_ = t.logFile.Close()
+		global.RemoveTaskCancel(t.TaskID)
+		return err
+	}
+	t.Log(i18n.GetMsgByKey(waitingStep))
+	go func() {
+		slots <- struct{}{}
+		defer func() { <-slots }()
+		defer onFinish()
+		if err := t.executeRegistered(); err != nil {
+			global.LOG.Error(err.Error())
+		}
+	}()
+	return nil
+}
+
 func (t *Task) Execute() error {
 	if err := t.taskRepo.Save(context.Background(), t.Task); err != nil {
 		return err
 	}
+	return t.executeRegistered()
+}
+
+func (t *Task) executeRegistered() error {
 	var err error
 	t.Log(i18n.GetWithName("TaskStart", t.Name))
 	for i := 0; i < len(t.SubTasks); i++ {
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "A full application backup is required for the Feishu co
 ErrFeishuUpgradeMounts: "These mounts cannot be fully backed up or restored. Use standard data mounts inside the application directory."

 AgentFeishuRecoveryDisabled: "Gateway recovered with Feishu disabled; the original configuration backup is retained."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "La actualización de compatibilidad de Feishu requiere
 ErrFeishuUpgradeMounts: "Estos montajes no permiten una copia y restauración completas. Use los montajes de datos estándar dentro del directorio de la aplicación."

 AgentFeishuRecoveryDisabled: "Puerta de enlace recuperada con Feishu desactivado; se conserva la copia de configuración original."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -801,3 +801,5 @@ ErrFeishuUpgradeBackup: "ارتقای سازگاری Feishu به پشتیبان
 ErrFeishuUpgradeMounts: "این اتصال‌های ذخیره‌سازی کاملاً قابل پشتیبان‌گیری و بازیابی نیستند. از مسیرهای استاندارد داده در پوشه برنامه استفاده کنید."

 AgentFeishuRecoveryDisabled: "دروازه بازیابی شد و Feishu غیرفعال است؛ پشتیبان تنظیمات اصلی حفظ شد."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "Feishu 互換更新にはアプリ全体のバックア
 ErrFeishuUpgradeMounts: "現在のマウントは完全なバックアップ・復元に対応しません。アプリディレクトリ内の標準データマウントを使用してください。"

 AgentFeishuRecoveryDisabled: "ゲートウェイを復旧しました。Feishu は無効のままで、元の設定のバックアップは保持しています。"
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "Feishu 호환 업그레이드에는 전체 앱 백업
 ErrFeishuUpgradeMounts: "현재 마운트는 전체 백업 및 복원을 지원하지 않습니다. 앱 디렉터리 내부의 표준 데이터 마운트를 사용하세요."

 AgentFeishuRecoveryDisabled: "게이트웨이가 복구되었습니다. Feishu는 비활성 상태이며 원본 설정 백업은 유지됩니다."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -791,3 +791,5 @@ ErrFeishuUpgradeBackup: "ການອັບເກຣດ Feishu ຕ້ອງສ
 ErrFeishuUpgradeMounts: "ຈຸດເຊື່ອມຂໍ້ມູນນີ້ບໍ່ສາມາດສຳຮອງແລະກູ້ຄືນທັງໝົດໄດ້. ໃຊ້ຈຸດເຊື່ອມມາດຕະຖານພາຍໃນໂຟນເດີແອັບ."

 AgentFeishuRecoveryDisabled: "ເກດເວກູ້ຄືນແລ້ວ ແຕ່ Feishu ຍັງປິດ; ສຳຮອງການຕັ້ງຄ່າເດີມຍັງຖືກເກັບໄວ້."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "Naik taraf keserasian Feishu memerlukan sandaran aplika
 ErrFeishuUpgradeMounts: "Lekapan ini tidak boleh disandar dan dipulihkan sepenuhnya. Gunakan lekapan data standard dalam direktori aplikasi."

 AgentFeishuRecoveryDisabled: "Gerbang dipulihkan dengan Feishu dinyahaktifkan; sandaran konfigurasi asal dikekalkan."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "A atualização de compatibilidade do Feishu exige back
 ErrFeishuUpgradeMounts: "Estas montagens não permitem backup e restauração completos. Use montagens de dados padrão dentro do diretório do aplicativo."

 AgentFeishuRecoveryDisabled: "Gateway recuperado com Feishu desativado; o backup da configuração original foi mantido."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "Для обновления совместимости
 ErrFeishuUpgradeMounts: "Эти монтирования не поддерживают полное резервное копирование и восстановление. Используйте стандартные данные внутри каталога приложения."

 AgentFeishuRecoveryDisabled: "Шлюз восстановлен, Feishu отключён; исходная резервная копия конфигурации сохранена."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "Feishu uyumluluk yükseltmesi tam uygulama yedeği gere
 ErrFeishuUpgradeMounts: "Bu bağlamalar tam yedeklenip geri yüklenemiyor. Uygulama dizinindeki standart veri bağlamalarını kullanın."

 AgentFeishuRecoveryDisabled: "Ağ geçidi kurtarıldı, Feishu kapalı; özgün yapılandırma yedeği korundu."
+
+ContainerUpgradeWaiting: "Waiting for a container upgrade execution slot"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "飛書相容升級必須開啟完整應用程式備份
 ErrFeishuUpgradeMounts: "目前掛載無法完整備份或還原，請使用應用程式目錄內的標準資料掛載。"

 AgentFeishuRecoveryDisabled: "閘道已恢復，飛書保持停用；原設定備份已保留。"
+
+ContainerUpgradeWaiting: "等待容器升級任務執行名額"
@@ -800,3 +800,5 @@ ErrFeishuUpgradeBackup: "飞书兼容升级必须开启完整应用备份。"
 ErrFeishuUpgradeMounts: "当前挂载无法完整备份或恢复，请使用应用目录内的标准数据挂载后重试。"

 AgentFeishuRecoveryDisabled: "网关已恢复，飞书保持停用；原配置备份已保留。"
+
+ContainerUpgradeWaiting: "等待容器升级任务执行名额"
@@ -198,12 +198,22 @@ func setLog(id, newLastLine string, task *task.Task) error {
 }

 func (c Client) PullImageWithProcessAndOptions(task *task.Task, imageName string, options image.PullOptions) error {
-	out, err := c.cli.ImagePull(context.Background(), imageName, options)
+	return c.PullImageWithProcessAndOptionsContext(context.Background(), task, imageName, options)
+}
+
+func (c Client) PullImageWithProcessAndOptionsContext(ctx context.Context, task *task.Task, imageName string, options image.PullOptions) error {
+	out, err := c.cli.ImagePull(ctx, imageName, options)
 	if err != nil {
 		return err
 	}
 	defer out.Close()
-	return handlePullImageProcess(out, task)
+	stopClose := context.AfterFunc(ctx, func() { _ = out.Close() })
+	defer stopClose()
+	err = handlePullImageProcess(out, task)
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+	return err
 }

 func handlePullImageProcess(out io.Reader, task *task.Task) error {
@@ -320,11 +330,15 @@ func (c Client) BuildImageWithProcessAndOptions(task *task.Task, tar io.ReadClos
 }

 func (c Client) PullImageWithProcess(task *task.Task, imageName string) error {
+	return c.PullImageWithProcessContext(context.Background(), task, imageName)
+}
+
+func (c Client) PullImageWithProcessContext(ctx context.Context, task *task.Task, imageName string) error {
 	options := image.PullOptions{}
 	if authStr, ok := RegistryAuthForImage(imageName); ok {
 		options.RegistryAuth = authStr
 	}
-	return c.PullImageWithProcessAndOptions(task, imageName, options)
+	return c.PullImageWithProcessAndOptionsContext(ctx, task, imageName, options)
 }

 func logProcess(progress map[string]interface{}, task *task.Task) {
@@ -14,8 +14,13 @@ export const searchContainer = (params: Container.ContainerSearch, currentNode?:
 export const listContainer = () => {
    return http.post<Array<Container.ContainerOption>>(`/containers/list`, {});
 };
-export const listContainerByImage = (image: string) => {
-    return http.post<Array<Container.ContainerOption>>(`/containers/list/byimage`, { name: image });
+export const listContainerByImage = (image: string, node?: string) => {
+    return http.post<Array<Container.ContainerOption>>(
+        `/containers/list/byimage`,
+        { name: image },
+        undefined,
+        node ? { CurrentNode: node } : undefined,
+    );
 };
 export const loadContainerUsers = (name: string) => {
    return http.post<Array<string>>(`/containers/users`, { name: name });
@@ -60,8 +65,8 @@ export const createContainer = (params: Container.ContainerHelper) => {
 export const updateContainer = (params: Container.ContainerHelper) => {
    return http.post(`/containers/update`, params, TimeoutEnum.T_10M);
 };
-export const upgradeContainer = (params: Container.ContainerUpgrade) => {
-    return http.post(`/containers/upgrade`, params, TimeoutEnum.T_10M);
+export const upgradeContainer = (params: Container.ContainerUpgrade, node?: string) => {
+    return http.post(`/containers/upgrade`, params, TimeoutEnum.T_10M, node ? { CurrentNode: node } : undefined);
 };
 export const commitContainer = (params: Container.ContainerCommit) => {
    return http.post(`/containers/commit`, params);
@@ -2105,6 +2105,17 @@ const message = {
        missing: 'Missing',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Create by command',
        importContainerBackupTip: 'Import container backup file, only .tar.gz is supported',
        importComposeBackupTip: 'Import compose backup file, only .tar.gz is supported',
@@ -2124,6 +2124,17 @@ const message = {
        missing: 'No existe',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Crear por comando',
        importContainerBackupTip: 'Importar archivo de copia de seguridad de contenedor, solo se admite .tar.gz',
        importComposeBackupTip: 'Importar archivo de copia de seguridad de compose, solo se admite .tar.gz',
@@ -2082,6 +2082,17 @@ const message = {
        missing: 'ناموجود',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'ایجاد با دستور',
        importContainerBackupTip: 'واردات فایل پشتیبان کانتینر، فقط .tar.gz پشتیبانی می‌شود',
        importComposeBackupTip: 'واردات فایل پشتیبان compose، فقط .tar.gz پشتیبانی می‌شود',
@@ -2089,6 +2089,17 @@ const message = {
        missing: '存在しない',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'コマンドで作成',
        importContainerBackupTip: 'コンテナバックアップファイルをインポートします。.tar.gz のみ対応です',
        importComposeBackupTip: 'Compose バックアップファイルをインポートします。.tar.gz のみ対応です',
@@ -2064,6 +2064,17 @@ const message = {
        missing: '없음',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: '명령으로 생성',
        importContainerBackupTip: '컨테이너 백업 파일 가져오기, .tar.gz 형식만 지원합니다',
        importComposeBackupTip: 'Compose 백업 파일 가져오기, .tar.gz 형식만 지원합니다',
@@ -2049,6 +2049,17 @@ const message = {
        missing: 'ບໍ່ມີ',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'ສ້າງດ້ວຍຄຳສັ່ງ',
        importContainerBackupTip: 'ນຳເຂົ້າໄຟລ໌ສຳຮອງຄອນເທນເນີ, ຮອງຮັບສະເພາະ .tar.gz',
        importComposeBackupTip: 'ນຳເຂົ້າໄຟລ໌ສຳຮອງ compose, ຮອງຮັບສະເພາະ .tar.gz',
@@ -2120,6 +2120,17 @@ const message = {
        missing: 'Tiada',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Cipta melalui perintah',
        importContainerBackupTip: 'Import fail sandaran kontena, hanya format .tar.gz disokong',
        importComposeBackupTip: 'Import fail sandaran compose, hanya format .tar.gz disokong',
@@ -2123,6 +2123,17 @@ const message = {
        missing: 'Não existe',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Criar por comando',
        importContainerBackupTip: 'Importar arquivo de backup de contêiner, apenas .tar.gz é suportado',
        importComposeBackupTip: 'Importar arquivo de backup de compose, apenas .tar.gz é suportado',
@@ -2116,6 +2116,17 @@ const message = {
        missing: 'Отсутствует',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Создать с помощью команды',
        importContainerBackupTip: 'Импорт файла резервной копии контейнера, поддерживается только .tar.gz',
        importComposeBackupTip: 'Импорт файла резервной копии compose, поддерживается только .tar.gz',
@@ -2121,6 +2121,17 @@ const message = {
        missing: 'Mevcut değil',
    },
    container: {
+        batchUpgrade: 'Batch upgrade',
+        batchUpgradeLog: 'View log',
+        batchUpgradeCheck: 'Check task',
+        batchUpgradeRetry: 'Retry failed submissions',
+        batchUpgradeSubmitting: 'Submitting',
+        batchUpgradeSubmitFailed: 'Submission failed',
+        batchUpgradeUnknown: 'Submission unconfirmed; check the task before resubmitting',
+        batchUpgradeCanceled: 'Canceled',
+        batchUpgradeWaiting: 'Queued',
+        batchUpgradeRunning: 'Upgrading',
+        batchUpgradeSubmitted: 'Upgrade task submitted',
        createByCommand: 'Komutla oluştur',
        importContainerBackupTip: 'Konteyner yedek dosyasını içe aktar, yalnızca .tar.gz desteklenir',
        importComposeBackupTip: 'Compose yedek dosyasını içe aktar, yalnızca .tar.gz desteklenir',
@@ -1986,6 +1986,17 @@ const message = {
        missing: '不存在',
    },
    container: {
+        batchUpgrade: '批次升級',
+        batchUpgradeLog: '查看日誌',
+        batchUpgradeCheck: '核對任務',
+        batchUpgradeRetry: '重試提交失敗的組',
+        batchUpgradeSubmitting: '正在提交',
+        batchUpgradeSubmitFailed: '提交失敗',
+        batchUpgradeUnknown: '提交結果待確認，請核對任務，勿重複提交',
+        batchUpgradeCanceled: '已取消',
+        batchUpgradeWaiting: '等待執行',
+        batchUpgradeRunning: '正在升級',
+        batchUpgradeSubmitted: '已提交升級任務',
        createByCommand: '指令建立',
        importContainerBackupTip: '匯入容器備份檔案，僅支援 .tar.gz 格式',
        importComposeBackupTip: '匯入編排備份檔案，僅支援 .tar.gz 格式',
@@ -1995,6 +1995,17 @@ const message = {
        missing: '不存在',
    },
    container: {
+        batchUpgrade: '批量升级',
+        batchUpgradeLog: '查看日志',
+        batchUpgradeCheck: '核对任务',
+        batchUpgradeRetry: '重试提交失败的组',
+        batchUpgradeSubmitting: '正在提交',
+        batchUpgradeSubmitFailed: '提交失败',
+        batchUpgradeUnknown: '提交结果待确认，请核对任务，勿重复提交',
+        batchUpgradeCanceled: '已取消',
+        batchUpgradeWaiting: '等待执行',
+        batchUpgradeRunning: '正在升级',
+        batchUpgradeSubmitted: '已提交升级任务',
        createByCommand: '命令创建',
        importContainerBackupTip: '导入容器备份文件，仅支持 .tar.gz 格式',
        importComposeBackupTip: '导入编排备份文件，仅支持 .tar.gz 格式',
@@ -356,6 +356,7 @@
                                        @change="toggleSelection"
                                    ></el-checkbox>
                                </template>
+                                <el-option :label="$t('commons.button.upgrade')" value="upgrade" />
                                <el-option
                                    :label="$t('container.start')"
                                    value="start"
@@ -910,6 +911,10 @@ const checkStatus = (operation: string, row: Container.ContainerInfo | null) =>
 };

 const onOperate = async (op: string, row: Container.ContainerInfo | null) => {
+    if (op === 'upgrade') {
+        dialogUpgradeRef.value.acceptParams(selects.value);
+        return;
+    }
    let opList = row ? [row] : selects.value;
    let msg = i18n.global.t('container.operatorHelper', [i18n.global.t('container.' + op)]);
    batchNames.value = [];
@@ -1,185 +1,346 @@
 <template>
-    <DrawerPro v-model="drawerVisible" :header="$t('commons.button.upgrade')" @close="handleClose" size="large">
-        <el-alert
-            :title="$t('container.appHelper')"
-            v-if="form.fromApp"
-            class="common-prompt"
-            :closable="false"
-            type="warning"
-        />
-        <el-form @submit.prevent ref="formRef" :model="form" label-position="top">
-            <el-form-item :label="$t('container.oldImage')" prop="oldImage">
-                <el-tooltip placement="top-start" :content="form.oldImageName" v-if="form.oldImageName.length > 50">
-                    <el-tag>{{ form.oldImageName.substring(0, 50) }}...</el-tag>
-                </el-tooltip>
-                <el-tag v-else>{{ form.oldImageName }}</el-tag>
-            </el-form-item>
-            <el-form-item :label="$t('container.sameImageContainer')" v-if="containerOptions.length > 1">
-                <div class="w-full">
-                    <el-checkbox v-model="checkAll" :indeterminate="isIndeterminate" @change="handleCheckAllChange">
-                        {{ $t('commons.table.all') }}
-                    </el-checkbox>
-                </div>
-                <el-checkbox-group v-model="form.names" @change="handleCheckedChange">
-                    <el-checkbox
-                        v-for="item in containerOptions"
-                        :key="item.name"
-                        :label="item.name"
-                        :value="item.name"
-                    >
-                        {{ item.name }}
-                    </el-checkbox>
-                </el-checkbox-group>
-                <span class="input-help">{{ $t('container.sameImageHelper') }}</span>
-            </el-form-item>
-            <el-form-item prop="newImageName" :rules="Rules.imageName">
-                <template #label>
-                    {{ $t('container.targetImage') }}
-                    <span v-if="!form.hasName">
-                        {{ ' (' + $t('container.imageLoadErr') + ')' }}
+    <DrawerPro
+        v-model="visible"
+        :header="$t(isBatch ? 'container.batchUpgrade' : 'commons.button.upgrade')"
+        size="large"
+        :confirm-before-close="true"
+        @before-close="beforeClose"
+        @close="handleClose"
+    >
+        <el-form ref="formRef" :model="form" label-position="top" @submit.prevent>
+            <section v-for="(group, index) in form.groups" :key="group.currentImage" class="upgrade-group">
+                <div v-if="isBatch" class="mb-3">
+                    <span v-for="name in group.names" :key="name" class="group-name mr-2 mb-2">
+                        {{ name }}
                    </span>
-                </template>
-                <el-input v-model="form.newImageName" />
-                <span class="input-help">{{ $t('container.upgradeHelper') }}</span>
-            </el-form-item>
-            <el-form-item prop="forcePull">
-                <el-checkbox v-model="form.forcePull">
+                </div>
+                <el-alert
+                    v-if="group.appNames.some((name) => group.names.includes(name))"
+                    :title="$t('container.appHelper')"
+                    type="warning"
+                    :closable="false"
+                    class="common-prompt app-warning"
+                />
+                <el-form-item :label="$t('container.oldImage')">
+                    <el-tag class="image-name">{{ group.currentImage }}</el-tag>
+                </el-form-item>
+                <el-form-item v-if="group.options.length > 1" :label="$t('container.sameImageContainer')">
+                    <div class="w-full">
+                        <el-checkbox
+                            :model-value="isGroupChecked(group)"
+                            :indeterminate="group.names.length > 0 && !isGroupChecked(group)"
+                            :disabled="loading || !isGroupEditable(group)"
+                            @change="(value) => handleCheckAllChange(group, value)"
+                        >
+                            {{ $t('commons.table.all') }}
+                        </el-checkbox>
+                    </div>
+                    <el-checkbox-group v-model="group.names" :disabled="loading || !isGroupEditable(group)">
+                        <el-checkbox v-for="item in group.options" :key="item.name" :value="item.name">
+                            {{ item.name }}
+                        </el-checkbox>
+                    </el-checkbox-group>
+                    <span class="input-help">{{ $t('container.sameImageHelper') }}</span>
+                </el-form-item>
+                <el-form-item :prop="`groups.${index}.targetImage`" :rules="group.names.length ? Rules.imageName : []">
+                    <template #label>
+                        {{ $t('container.targetImage') }}
+                        <span v-if="group.currentImage.includes('sha256:')">
+                            {{ ' (' + $t('container.imageLoadErr') + ')' }}
+                        </span>
+                    </template>
+                    <el-input
+                        v-model="group.targetImage"
+                        :disabled="loading || !group.names.length || !isGroupEditable(group)"
+                    />
+                    <span class="input-help">{{ $t('container.upgradeHelper') }}</span>
+                </el-form-item>
+                <div v-if="group.state !== 'draft'" class="mb-3">
+                    <el-tag :type="statusType(group)">{{ statusLabel(group) }}</el-tag>
+                    <el-button v-if="group.state === 'submitted'" link type="primary" @click="openLog(group)">
+                        {{ $t('container.batchUpgradeLog') }}
+                    </el-button>
+                    <el-button v-if="group.state === 'unknown'" link type="primary" @click="refreshGroup(group)">
+                        {{ $t('container.batchUpgradeCheck') }}
+                    </el-button>
+                </div>
+            </section>
+            <el-form-item>
+                <el-checkbox v-model="form.forcePull" :disabled="started || loading">
                    {{ $t('container.forcePull') }}
                </el-checkbox>
                <span class="input-help">{{ $t('container.forcePullHelper') }}</span>
            </el-form-item>
        </el-form>
        <template #footer>
-            <span class="dialog-footer">
-                <el-button :disabled="loading" @click="drawerVisible = false">
-                    {{ $t('commons.button.cancel') }}
-                </el-button>
-                <el-button :disabled="loading || form.names.length === 0" type="primary" @click="onSubmit(formRef)">
-                    {{ $t('commons.button.confirm') }}
-                </el-button>
-            </span>
+            <el-button :disabled="loading" @click="visible = false">
+                {{ $t('commons.button.close') }}
+            </el-button>
+            <el-button v-if="canSubmit" type="primary" :loading="loading" @click="submit">
+                {{ started ? $t('container.batchUpgradeRetry') : $t('commons.button.confirm') }}
+            </el-button>
        </template>
    </DrawerPro>
-    <TaskLog ref="taskLogRef" width="70%" @close="closeTaskLog()" />
+    <TaskLog ref="taskLogRef" width="70%" @close="emit('search')" />
 </template>

 <script lang="ts" setup>
+import { computed, onBeforeUnmount, reactive, ref } from 'vue';
+import { ElMessageBox, type CheckboxValueType, type FormInstance } from 'element-plus';
+import { Container } from '@/api/interface/container';
 import { listContainerByImage, upgradeContainer } from '@/api/modules/container';
+import { searchTasks } from '@/api/modules/log';
 import { Rules } from '@/global/form-rules';
-import TaskLog from '@/components/log/task/index.vue';
-import i18n from '@/lang';
-import { MsgSuccess } from '@/utils/message';
 import { newUUID } from '@/utils/id';
-import { CheckboxValueType, ElForm } from 'element-plus';
-import { reactive, ref } from 'vue';
-
-const loading = ref(false);
-const taskLogRef = ref();
-
-const form = reactive({
-    names: [],
-    oldImageName: '',
-    newImageName: '',
-    hasName: true,
-
-    fromApp: false,
-    forcePull: false,
-});
-
-const formRef = ref<FormInstance>();
-
-const drawerVisible = ref<boolean>(false);
-type FormInstance = InstanceType<typeof ElForm>;
-
-const containerOptions = ref([]);
-const isIndeterminate = ref();
-const checkAll = ref();
+import i18n from '@/lang';
+import TaskLog from '@/components/log/task/index.vue';
+import bus from '@/global/bus';
+import { GlobalStore } from '@/store';

+interface UpgradeGroup {
+    currentImage: string;
+    targetImage: string;
+    names: string[];
+    appNames: string[];
+    options: Container.ContainerOption[];
+    taskID: string;
+    state: 'draft' | 'submitting' | 'submitted' | 'failed' | 'unknown';
+    taskStatus: string;
+    currentStep: string;
+}
 interface DialogProps {
    container: string;
    image: string;
    fromApp: boolean;
 }
-const acceptParams = (props: DialogProps): void => {
-    form.names = [props.container];
-    isIndeterminate.value = true;
-    form.oldImageName = props.image;
-    form.fromApp = props.fromApp;
-    form.hasName = props.image.indexOf('sha256:') === -1;
-    if (form.hasName) {
-        form.newImageName = props.image;
-    } else {
-        form.newImageName = '';
-    }
-    loadContainers();
-    drawerVisible.value = true;
+const isBatch = ref(false);
+const isGroupEditable = (group: UpgradeGroup) => ['draft', 'failed'].includes(group.state);
+const isGroupChecked = (group: UpgradeGroup) => group.options.length > 0 && group.names.length === group.options.length;
+const handleCheckAllChange = (group: UpgradeGroup, value: CheckboxValueType) => {
+    if (loading.value || !isGroupEditable(group)) return;
+    group.names = value ? group.options.map((item) => item.name) : [];
 };
+const visible = ref(false);
+const loading = ref(false);
+const started = ref(false);
+const formRef = ref<FormInstance>();
+const taskLogRef = ref<InstanceType<typeof TaskLog>>();
+const form = reactive({ groups: [] as UpgradeGroup[], forcePull: false });
 const emit = defineEmits<{ (e: 'search'): void }>();
+let timer: ReturnType<typeof setTimeout> | undefined;
+let generation = 0;
+let operateNode = '';
+const canSubmit = computed(() => form.groups.some((group) => group.names.length > 0 && isGroupEditable(group)));

-const loadContainers = async () => {
-    const res = await listContainerByImage(form.oldImageName);
-    containerOptions.value = res.data || [];
+const acceptParams = async (props: DialogProps | Container.ContainerInfo[]) => {
+    if (loading.value) return;
+    isBatch.value = Array.isArray(props);
+    const containers = Array.isArray(props)
+        ? props
+        : [{ name: props.container, imageName: props.image, isFromApp: props.fromApp }];
+    operateNode = GlobalStore().currentNode;
+    clearTimeout(timer);
+    generation++;
+    const groups = new Map<string, UpgradeGroup>();
+    for (const item of containers) {
+        let group = groups.get(item.imageName);
+        if (!group) {
+            group = {
+                currentImage: item.imageName,
+                targetImage: item.imageName.includes('sha256:') ? '' : item.imageName,
+                names: [],
+                appNames: [],
+                options: [],
+                taskID: newUUID(),
+                state: 'draft',
+                taskStatus: '',
+                currentStep: '',
+            };
+            groups.set(item.imageName, group);
+        }
+        if (!group.names.includes(item.name)) {
+            group.names.push(item.name);
+            group.options.push({ name: item.name, state: '' });
+        }
+        if (item.isFromApp && !group.appNames.includes(item.name)) group.appNames.push(item.name);
+    }
+    form.groups = [...groups.values()];
+    form.forcePull = false;
+    started.value = false;
+    visible.value = true;
+    formRef.value?.clearValidate();
+    const version = generation;
+    const node = operateNode;
+    const pending = [...form.groups];
+    for (let index = 0; index < pending.length; index += 3) {
+        if (version !== generation || !visible.value || started.value) return;
+        await Promise.all(
+            pending.slice(index, index + 3).map(async (group) => {
+                try {
+                    const res = await listContainerByImage(group.currentImage, node);
+                    if (version !== generation || !visible.value || started.value) return;
+                    const options = new Map(group.options.map((item) => [item.name, item]));
+                    for (const item of res.data || []) options.set(item.name, item);
+                    group.options = [...options.values()];
+                } catch {
+                    // Keep the original selection available when loading peers fails.
+                }
+            }),
+        );
+    }
 };

-const handleCheckAllChange = (val: CheckboxValueType) => {
-    form.names = [];
-    if (!val) {
-        isIndeterminate.value = false;
+const refreshGroup = async (group: UpgradeGroup, version = generation) => {
+    try {
+        const res = await searchTasks(
+            { page: 1, pageSize: 1, type: 'Image', status: '', taskID: group.taskID },
+            operateNode,
+        );
+        if (!visible.value || version !== generation) return false;
+        const task = res.data.items?.[0];
+        if (!task) return false;
+        group.state = 'submitted';
+        group.taskStatus = task.status;
+        group.currentStep = task.currentStep;
+        return true;
+    } catch {
+        // A failed status request does not imply that submitting the upgrade failed.
+        return false;
+    }
+};
+const needsRefresh = (group: UpgradeGroup) =>
+    group.state === 'unknown' ||
+    (group.state === 'submitted' && !['Success', 'Failed', 'Canceled'].includes(group.taskStatus));
+const poll = async (version: number) => {
+    if (!visible.value || version !== generation) return;
+    const pending = form.groups.filter(needsRefresh);
+    // Bound request concurrency without adding another API or polling abstraction.
+    for (let index = 0; index < pending.length; index += 3) {
+        await Promise.all(pending.slice(index, index + 3).map((group) => refreshGroup(group, version)));
+        if (!visible.value || version !== generation) return;
+    }
+    if (form.groups.some(needsRefresh)) {
+        timer = setTimeout(() => poll(version), 3000);
+    }
+};
+const submit = async () => {
+    if (loading.value || !(await formRef.value?.validate().catch(() => false))) return;
+    if (loading.value || !canSubmit.value) return;
+    loading.value = true;
+    try {
+        await ElMessageBox.confirm(
+            i18n.global.t('container.upgradeWarning2'),
+            i18n.global.t(isBatch.value ? 'container.batchUpgrade' : 'commons.button.upgrade'),
+            {
+                confirmButtonText: i18n.global.t('commons.button.confirm'),
+                cancelButtonText: i18n.global.t('commons.button.cancel'),
+            },
+        );
+    } catch {
+        loading.value = false;
        return;
    }
-    for (const item of containerOptions.value) {
-        form.names.push(item.name);
+    started.value = true;
+    clearTimeout(timer);
+    const version = ++generation;
+    const forcePull = form.forcePull;
+    const pending = form.groups.filter((group) => group.names.length > 0 && isGroupEditable(group));
+    try {
+        for (const group of pending) {
+            group.state = 'submitting';
+            try {
+                await upgradeContainer(
+                    {
+                        taskID: group.taskID,
+                        names: [...group.names],
+                        image: group.targetImage.trim(),
+                        forcePull,
+                    },
+                    operateNode,
+                );
+                group.state = 'submitted';
+            } catch (error) {
+                if (!(await refreshGroup(group))) {
+                    const failure = error as { code?: number; response?: { status?: number } };
+                    const rejected =
+                        typeof failure.code === 'number' ||
+                        ((failure.response?.status || 0) >= 400 && (failure.response?.status || 0) < 500);
+                    group.state = rejected ? 'failed' : 'unknown';
+                }
+            }
+        }
+    } finally {
+        loading.value = false;
+        bus.emit('refreshTask', true);
+        if (!isBatch.value && form.groups[0]?.state === 'submitted') {
+            openLog(form.groups[0]);
+            visible.value = false;
+        } else {
+            void poll(version);
+        }
    }
 };
-const handleCheckedChange = (value: CheckboxValueType[]) => {
-    const checkedCount = value.length;
-    checkAll.value = checkedCount === containerOptions.value.length;
-    isIndeterminate.value = checkedCount > 0 && checkedCount < containerOptions.value.length;
+const statusLabel = (group: UpgradeGroup) => {
+    const t = i18n.global.t;
+    if (group.state === 'submitting') return t('container.batchUpgradeSubmitting');
+    if (group.state === 'failed') return t('container.batchUpgradeSubmitFailed');
+    if (group.state === 'unknown') return t('container.batchUpgradeUnknown');
+    if (group.taskStatus === 'Success') return t('commons.status.success');
+    if (group.taskStatus === 'Failed') return t('commons.status.failed');
+    if (group.taskStatus === 'Canceled') return t('container.batchUpgradeCanceled');
+    if (group.currentStep === 'ContainerUpgradeWaiting') return t('container.batchUpgradeWaiting');
+    if (group.taskStatus === 'Executing') return t('container.batchUpgradeRunning');
+    return t('container.batchUpgradeSubmitted');
 };
-
-const onSubmit = async (formEl: FormInstance | undefined) => {
-    if (!formEl) return;
-    formEl.validate(async (valid) => {
-        if (!valid) return;
-        ElMessageBox.confirm(i18n.global.t('container.upgradeWarning2'), i18n.global.t('commons.button.upgrade'), {
-            confirmButtonText: i18n.global.t('commons.button.confirm'),
-            cancelButtonText: i18n.global.t('commons.button.cancel'),
-        }).then(async () => {
-            let taskID = newUUID();
-            let param = {
-                taskID: taskID,
-                names: form.names,
-                image: form.newImageName,
-                forcePull: form.forcePull,
-            };
-            loading.value = true;
-            await upgradeContainer(param)
-                .then(() => {
-                    loading.value = false;
-                    openTaskLog(taskID);
-                    drawerVisible.value = false;
-                    MsgSuccess(i18n.global.t('commons.msg.operationSuccess'));
-                })
-                .catch(() => {
-                    loading.value = false;
-                });
-        });
-    });
+const statusType = (group: UpgradeGroup) => {
+    if (group.state === 'failed' || group.taskStatus === 'Failed') return 'danger';
+    if (group.taskStatus === 'Success') return 'success';
+    if (group.state === 'unknown') return 'warning';
+    return 'info';
 };
-const openTaskLog = (taskID: string) => {
-    taskLogRef.value.openWithTaskID(taskID);
+const openLog = (group: UpgradeGroup) => taskLogRef.value?.openWithTaskID(group.taskID, true, operateNode);
+const beforeClose = (done: () => void) => {
+    if (!loading.value) done();
 };
-
-const handleClose = async () => {
-    drawerVisible.value = false;
+const handleClose = () => {
+    clearTimeout(timer);
+    generation++;
    emit('search');
 };
-
-const closeTaskLog = () => {
-    emit('search');
-};
-
-defineExpose({
-    acceptParams,
+onBeforeUnmount(() => {
+    clearTimeout(timer);
+    generation++;
 });
+defineExpose({ acceptParams });
 </script>
+
+<style scoped>
+.upgrade-group + .upgrade-group {
+    border-top: 1px solid var(--el-border-color);
+    padding-top: 20px;
+}
+.group-name {
+    display: inline-block;
+    font-weight: 600;
+    color: var(--el-text-color-primary);
+    font-size: var(--el-font-size-base);
+    overflow-wrap: anywhere;
+}
+:global(html.dark) .group-name {
+    color: #fff;
+}
+.app-warning {
+    --el-alert-title-font-size: var(--el-font-size-base);
+    --el-alert-title-with-description-font-size: var(--el-font-size-base);
+    --el-alert-description-font-size: var(--el-font-size-base);
+}
+.image-name {
+    max-width: 100%;
+    height: auto;
+    min-height: 24px;
+    font-size: var(--el-font-size-base);
+    line-height: 22px;
+    white-space: normal;
+    overflow-wrap: anywhere;
+}
+</style>