fix: Fix large file/slow network upload timeout in file management (#13584)

agent/app/api/v2/file.go

@@ -35,6 +35,81 @@ var cancelledChunkUploads = struct {
 	ids map[string]struct{}
 }{ids: make(map[string]struct{})}
 
+type chunkUploadLock struct {
+	mutex sync.Mutex
+	refs  int
+}
+
+var chunkUploadLocks = struct {
+	sync.Mutex
+	items map[string]*chunkUploadLock
+}{items: make(map[string]*chunkUploadLock)}
+
+type completedChunkUpload struct {
+	dstDir   string
+	filename string
+	fileSize int64
+}
+
+var completedChunkUploads = struct {
+	sync.RWMutex
+	items map[string]completedChunkUpload
+}{items: make(map[string]completedChunkUpload)}
+
+var activeChunkUploadTTL = 24 * time.Hour
+
+var (
+	errChunkUploadCancelled = errors.New("upload cancelled")
+	errInvalidChunkUpload   = errors.New("invalid chunk upload")
+)
+
+type activeChunkUpload struct {
+	upload    completedChunkUpload
+	expiresAt time.Time
+	timer     *time.Timer
+}
+
+var activeChunkUploads = struct {
+	sync.RWMutex
+	items map[string]activeChunkUpload
+}{items: make(map[string]activeChunkUpload)}
+
+type resumableUploadChunk struct {
+	UploadID   string
+	Filename   string
+	DstDir     string
+	ChunkIndex int
+	ChunkCount int
+	Offset     int64
+	FileSize   int64
+	Overwrite  bool
+}
+
+func invalidChunkUploadError(message string) error {
+	return fmt.Errorf("%w: %s", errInvalidChunkUpload, message)
+}
+
+func isRetryableChunkUploadError(err error) bool {
+	if err == nil {
+		return false
+	}
+	if errors.Is(err, errChunkUploadCancelled) ||
+		errors.Is(err, errInvalidChunkUpload) ||
+		errors.Is(err, os.ErrExist) ||
+		errors.Is(err, os.ErrPermission) ||
+		errors.Is(err, os.ErrInvalid) ||
+		errors.Is(err, syscall.ENOSPC) ||
+		errors.Is(err, syscall.EDQUOT) ||
+		errors.Is(err, syscall.EROFS) ||
+		errors.Is(err, syscall.EFBIG) ||
+		errors.Is(err, syscall.ENAMETOOLONG) ||
+		errors.Is(err, syscall.ENOTDIR) ||
+		errors.Is(err, syscall.EISDIR) {
+		return false
+	}
+	return true
+}
+
 // @Tags File
 // @Summary List files
 // @Accept json
@@ -474,11 +549,7 @@ func (b *BaseApi) UploadFiles(c *gin.Context) {
 			continue
 		}
 		dstInfo, statErr := os.Stat(dstFilename)
-		if overwrite {
-			_ = os.Remove(dstFilename)
-		}
-
-		err = os.Rename(tmpFilename, dstFilename)
+		err = finalizeUploadedFile(tmpFilename, dstFilename, overwrite)
 		if err != nil {
 			_ = os.Remove(tmpFilename)
 			e := fmt.Errorf("upload [%s] file failed, err: %v", file.Filename, err)
@@ -812,6 +883,289 @@ func (b *BaseApi) DepthDirSize(c *gin.Context) {
 	helper.SuccessWithData(c, res)
 }
 
+func lockChunkUpload(uploadID string) func() {
+	chunkUploadLocks.Lock()
+	lock, ok := chunkUploadLocks.items[uploadID]
+	if !ok {
+		lock = &chunkUploadLock{}
+		chunkUploadLocks.items[uploadID] = lock
+	}
+	lock.refs++
+	chunkUploadLocks.Unlock()
+
+	lock.mutex.Lock()
+	return func() {
+		lock.mutex.Unlock()
+		chunkUploadLocks.Lock()
+		lock.refs--
+		if lock.refs == 0 {
+			delete(chunkUploadLocks.items, uploadID)
+		}
+		chunkUploadLocks.Unlock()
+	}
+}
+
+func resumableUploadPartPath(dstDir, uploadID string) string {
+	return filepath.Join(dstDir, fmt.Sprintf(".1panel-upload-%s.part", uploadID))
+}
+
+func finalizeUploadedFile(tmpFile, dstFile string, overwrite bool) error {
+	if overwrite {
+		return os.Rename(tmpFile, dstFile)
+	}
+	if err := os.Link(tmpFile, dstFile); err != nil {
+		return err
+	}
+	if err := os.Remove(tmpFile); err != nil {
+		if rollbackErr := os.Remove(dstFile); rollbackErr != nil {
+			return fmt.Errorf("remove upload temporary file failed: %v, rollback destination failed: %w", err, rollbackErr)
+		}
+		return err
+	}
+	return nil
+}
+
+func registerActiveChunkUpload(uploadID string, upload completedChunkUpload) error {
+	activeChunkUploads.Lock()
+	defer activeChunkUploads.Unlock()
+	if active, ok := activeChunkUploads.items[uploadID]; ok {
+		if active.upload != upload {
+			return invalidChunkUploadError("upload ID is already used by another file")
+		}
+		active.timer.Stop()
+	}
+	expiresAt := time.Now().Add(activeChunkUploadTTL)
+	timer := time.AfterFunc(activeChunkUploadTTL, func() {
+		expireActiveChunkUpload(uploadID, expiresAt)
+	})
+	activeChunkUploads.items[uploadID] = activeChunkUpload{
+		upload:    upload,
+		expiresAt: expiresAt,
+		timer:     timer,
+	}
+	return nil
+}
+
+func loadActiveChunkUpload(uploadID string) (completedChunkUpload, bool) {
+	activeChunkUploads.RLock()
+	active, ok := activeChunkUploads.items[uploadID]
+	activeChunkUploads.RUnlock()
+	return active.upload, ok
+}
+
+func deleteActiveChunkUpload(uploadID string) {
+	activeChunkUploads.Lock()
+	if active, ok := activeChunkUploads.items[uploadID]; ok {
+		active.timer.Stop()
+	}
+	delete(activeChunkUploads.items, uploadID)
+	activeChunkUploads.Unlock()
+}
+
+func discardActiveChunkUpload(uploadID, partFile string) error {
+	deleteActiveChunkUpload(uploadID)
+	if err := os.Remove(partFile); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("remove upload temporary file failed: %w", err)
+	}
+	return nil
+}
+
+func finalizeActiveChunkUpload(uploadID, partFile, dstFile string, overwrite bool) error {
+	if err := finalizeUploadedFile(partFile, dstFile, overwrite); err != nil {
+		if removeErr := discardActiveChunkUpload(uploadID, partFile); removeErr != nil {
+			return errors.Join(err, removeErr)
+		}
+		return err
+	}
+	return nil
+}
+
+func expireActiveChunkUpload(uploadID string, expiresAt time.Time) {
+	unlock := lockChunkUpload(uploadID)
+	defer unlock()
+	activeChunkUploads.Lock()
+	active, ok := activeChunkUploads.items[uploadID]
+	if !ok || !active.expiresAt.Equal(expiresAt) {
+		activeChunkUploads.Unlock()
+		return
+	}
+	delete(activeChunkUploads.items, uploadID)
+	activeChunkUploads.Unlock()
+	partFile := resumableUploadPartPath(active.upload.dstDir, uploadID)
+	if err := os.Remove(partFile); err != nil && !os.IsNotExist(err) {
+		global.LOG.Warnf("remove inactive upload part [%s] failed: %v", partFile, err)
+	}
+}
+
+func removeActiveResumableUploadPart(uploadID string) error {
+	unlock := lockChunkUpload(uploadID)
+	defer unlock()
+	upload, ok := loadActiveChunkUpload(uploadID)
+	if !ok {
+		return nil
+	}
+	err := os.Remove(resumableUploadPartPath(upload.dstDir, uploadID))
+	if err == nil || os.IsNotExist(err) {
+		deleteActiveChunkUpload(uploadID)
+		return nil
+	}
+	return err
+}
+
+func loadCompletedChunkUpload(uploadID string) (completedChunkUpload, bool) {
+	completedChunkUploads.RLock()
+	completed, ok := completedChunkUploads.items[uploadID]
+	completedChunkUploads.RUnlock()
+	return completed, ok
+}
+
+func markChunkUploadCompleted(uploadID string, completed completedChunkUpload) {
+	completedChunkUploads.Lock()
+	completedChunkUploads.items[uploadID] = completed
+	completedChunkUploads.Unlock()
+	time.AfterFunc(10*time.Minute, func() {
+		completedChunkUploads.Lock()
+		delete(completedChunkUploads.items, uploadID)
+		completedChunkUploads.Unlock()
+	})
+}
+
+func writeResumableUploadChunk(chunk resumableUploadChunk, chunkData []byte) error {
+	unlock := lockChunkUpload(chunk.UploadID)
+	defer unlock()
+	if chunkUploadCancelled(chunk.UploadID) {
+		return errChunkUploadCancelled
+	}
+	if chunk.UploadID == "" || filepath.Base(chunk.UploadID) != chunk.UploadID || strings.ContainsAny(chunk.UploadID, `/\`) {
+		return invalidChunkUploadError("invalid upload ID")
+	}
+	if chunk.Filename == "" || filepath.Base(chunk.Filename) != chunk.Filename || strings.ContainsAny(chunk.Filename, `/\`) {
+		return invalidChunkUploadError("invalid filename")
+	}
+	if strings.TrimSpace(chunk.DstDir) == "" {
+		return invalidChunkUploadError("upload destination is required")
+	}
+	dstDir := filepath.Clean(strings.TrimSpace(chunk.DstDir))
+
+	if chunk.ChunkCount <= 0 || chunk.ChunkIndex < 0 || chunk.ChunkIndex >= chunk.ChunkCount {
+		return invalidChunkUploadError("invalid chunk index")
+	}
+	if chunk.FileSize <= 0 || chunk.Offset < 0 || chunk.Offset > chunk.FileSize {
+		return invalidChunkUploadError("invalid upload offset")
+	}
+	chunkEnd := chunk.Offset + int64(len(chunkData))
+	if chunkEnd > chunk.FileSize {
+		return invalidChunkUploadError("chunk exceeds file size")
+	}
+	if chunk.ChunkIndex+1 == chunk.ChunkCount {
+		if chunkEnd != chunk.FileSize {
+			return invalidChunkUploadError("final chunk does not match file size")
+		}
+	} else if chunkEnd >= chunk.FileSize {
+		return invalidChunkUploadError("non-final chunk reaches file size")
+	}
+	if completed, ok := loadCompletedChunkUpload(chunk.UploadID); ok {
+		if completed.dstDir == dstDir && completed.filename == chunk.Filename && completed.fileSize == chunk.FileSize {
+			return nil
+		}
+		return invalidChunkUploadError("upload ID has already completed another file")
+	}
+	upload := completedChunkUpload{dstDir: dstDir, filename: chunk.Filename, fileSize: chunk.FileSize}
+	if err := registerActiveChunkUpload(chunk.UploadID, upload); err != nil {
+		return err
+	}
+
+	mode, err := files.GetParentMode(dstDir)
+	if err != nil {
+		return err
+	}
+	if err = os.MkdirAll(dstDir, mode); err != nil {
+		return err
+	}
+	dstDirInfo, err := os.Stat(dstDir)
+	if err != nil {
+		return err
+	}
+	if !dstDirInfo.IsDir() {
+		return invalidChunkUploadError(fmt.Sprintf("upload destination [%s] is not a directory", dstDir))
+	}
+
+	dstFile := filepath.Join(dstDir, chunk.Filename)
+	partFile := resumableUploadPartPath(dstDir, chunk.UploadID)
+	if dstFile == partFile {
+		return invalidChunkUploadError("filename conflicts with upload temporary file")
+	}
+	fileMode := dstDirInfo.Mode().Perm()
+	ownerInfo := dstDirInfo
+	if dstInfo, statErr := os.Stat(dstFile); statErr == nil {
+		if !chunk.Overwrite {
+			if err := discardActiveChunkUpload(chunk.UploadID, partFile); err != nil {
+				return errors.Join(os.ErrExist, err)
+			}
+			return os.ErrExist
+		}
+		fileMode = dstInfo.Mode().Perm()
+		ownerInfo = dstInfo
+	} else if !os.IsNotExist(statErr) {
+		return statErr
+	}
+
+	part, err := os.OpenFile(partFile, os.O_CREATE|os.O_RDWR, fileMode)
+	if err != nil {
+		return err
+	}
+	partClosed := false
+	defer func() {
+		if !partClosed {
+			_ = part.Close()
+		}
+	}()
+	if stat, statErr := part.Stat(); statErr != nil {
+		return statErr
+	} else if chunk.Offset > stat.Size() {
+		return invalidChunkUploadError(fmt.Sprintf("unexpected upload offset %d, current size is %d", chunk.Offset, stat.Size()))
+	} else if chunk.Offset < stat.Size() && chunkEnd > stat.Size() {
+		if err = part.Truncate(chunk.Offset); err != nil {
+			return err
+		}
+	}
+	if _, err = part.WriteAt(chunkData, chunk.Offset); err != nil {
+		return err
+	}
+
+	if chunk.ChunkIndex+1 != chunk.ChunkCount {
+		return nil
+	}
+	partInfo, err := part.Stat()
+	if err != nil {
+		return err
+	}
+	if partInfo.Size() != chunk.FileSize {
+		return invalidChunkUploadError(fmt.Sprintf("uploaded file size mismatch: expected %d, got %d", chunk.FileSize, partInfo.Size()))
+	}
+	if err = part.Close(); err != nil {
+		return err
+	}
+	partClosed = true
+	if err = os.Chmod(partFile, fileMode); err != nil {
+		return err
+	}
+	if stat, ok := ownerInfo.Sys().(*syscall.Stat_t); ok {
+		if err = os.Chown(partFile, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+	}
+	if chunkUploadCancelled(chunk.UploadID) {
+		return errChunkUploadCancelled
+	}
+	if err = finalizeActiveChunkUpload(chunk.UploadID, partFile, dstFile, chunk.Overwrite); err != nil {
+		return err
+	}
+	markChunkUploadCompleted(chunk.UploadID, upload)
+	deleteActiveChunkUpload(chunk.UploadID)
+	return nil
+}
+
 func mergeChunks(fileName string, fileDir string, dstDir string, chunkCount int, overwrite bool) error {
 	defer func() {
 		_ = os.RemoveAll(fileDir)
@@ -891,6 +1245,10 @@ func (b *BaseApi) UploadChunkFiles(c *gin.Context) {
 		helper.BadRequest(c, err)
 		return
 	}
+	if chunkCount <= 0 || chunkIndex < 0 || chunkIndex >= chunkCount {
+		helper.BadRequest(c, errors.New("invalid chunk index"))
+		return
+	}
 	fileOp := files.NewFileOp()
 	tmpDir := path.Join(global.Dir.TmpDir, "upload")
 	if !fileOp.Stat(tmpDir) {
@@ -905,20 +1263,25 @@ func (b *BaseApi) UploadChunkFiles(c *gin.Context) {
 		return
 	}
 	uploadID := strings.TrimSpace(c.PostForm("uploadID"))
+	resumable := c.PostForm("fileSize") != "" || c.PostForm("offset") != ""
 	cancellable := uploadID != ""
 	if cancellable && (filepath.Base(uploadID) != uploadID || strings.ContainsAny(uploadID, `/\\`)) {
 		helper.BadRequest(c, errors.New("invalid upload ID"))
 		return
 	}
+	if resumable && !cancellable {
+		helper.BadRequest(c, errors.New("upload ID is required"))
+		return
+	}
 	if !cancellable {
 		uploadID = filename
 	}
 	fileDir := filepath.Join(tmpDir, uploadID)
 	if cancellable && chunkUploadCancelled(uploadID) {
-		helper.BadRequest(c, errors.New("upload cancelled"))
+		helper.BadRequest(c, errChunkUploadCancelled)
 		return
 	}
-	if chunkIndex == 0 {
+	if !resumable && chunkIndex == 0 {
 		if fileOp.Stat(fileDir) {
 			_ = fileOp.DeleteDir(fileDir)
 		}
@@ -927,32 +1290,67 @@ func (b *BaseApi) UploadChunkFiles(c *gin.Context) {
 	filePath := filepath.Join(fileDir, filename)
 
 	defer func() {
-		if err != nil {
+		if !resumable && err != nil {
 			_ = os.RemoveAll(fileDir)
 		}
 	}()
-	var (
-		emptyFile *os.File
-		chunkData []byte
-	)
-
-	emptyFile, err = os.Create(filePath)
-	if err != nil {
-		helper.BadRequest(c, err)
-		return
-	}
-	defer emptyFile.Close()
-
-	chunkData, err = io.ReadAll(uploadFile)
+	chunkData, err := io.ReadAll(uploadFile)
 	if err != nil {
 		helper.InternalServer(c, buserr.WithMap("ErrFileUpload", map[string]interface{}{"name": filename, "detail": err.Error()}, err))
 		return
 	}
 	if cancellable && chunkUploadCancelled(uploadID) {
-		err = errors.New("upload cancelled")
+		err = errChunkUploadCancelled
 		helper.BadRequest(c, err)
 		return
 	}
+	if resumable {
+		offset, parseErr := strconv.ParseInt(c.PostForm("offset"), 10, 64)
+		if parseErr != nil {
+			helper.BadRequest(c, parseErr)
+			return
+		}
+		fileSize, parseErr := strconv.ParseInt(c.PostForm("fileSize"), 10, 64)
+		if parseErr != nil {
+			helper.BadRequest(c, parseErr)
+			return
+		}
+		overwrite := true
+		if ow := c.PostForm("overwrite"); ow != "" {
+			overwrite, _ = strconv.ParseBool(ow)
+		}
+		err = writeResumableUploadChunk(resumableUploadChunk{
+			UploadID:   uploadID,
+			Filename:   filename,
+			DstDir:     c.PostForm("path"),
+			ChunkIndex: chunkIndex,
+			ChunkCount: chunkCount,
+			Offset:     offset,
+			FileSize:   fileSize,
+			Overwrite:  overwrite,
+		}, chunkData)
+		if err != nil {
+			uploadErr := buserr.WithMap("ErrFileUpload", map[string]interface{}{"name": filename, "detail": err.Error()}, err)
+			helper.ErrorWithDetailAndData(c, http.StatusInternalServerError, "ErrInternalServer", uploadErr, gin.H{
+				"retryable": isRetryableChunkUploadError(err),
+			})
+			return
+		}
+		if chunkIndex+1 == chunkCount {
+			cancelledChunkUploads.Lock()
+			delete(cancelledChunkUploads.ids, uploadID)
+			cancelledChunkUploads.Unlock()
+		}
+		helper.SuccessWithData(c, true)
+		return
+	}
+
+	emptyFile, err := os.Create(filePath)
+	if err != nil {
+		helper.BadRequest(c, err)
+		return
+	}
+	defer emptyFile.Close()
 
 	chunkPath := filepath.Join(fileDir, fmt.Sprintf("%s.%d", filename, chunkIndex))
 	err = os.WriteFile(chunkPath, chunkData, constant.DirPerm)
@@ -1005,6 +1403,10 @@ func (b *BaseApi) StopChunkUpload(c *gin.Context) {
 		helper.InternalServer(c, err)
 		return
 	}
+	if err := removeActiveResumableUploadPart(uploadID); err != nil {
+		helper.InternalServer(c, err)
+		return
+	}
 	helper.Success(c)
 }
 

agent/app/api/v2/helper/helper.go

@@ -30,6 +30,16 @@ func ErrorWithDetail(ctx *gin.Context, code int, msgKey string, err error) {
 	ctx.Abort()
 }
 
+func ErrorWithDetailAndData(ctx *gin.Context, code int, msgKey string, err error, data interface{}) {
+	res := dto.Response{
+		Code: code,
+		Data: data,
+	}
+	res.Message = i18n.GetMsgWithDetail(msgKey, err.Error())
+	ctx.JSON(http.StatusOK, res)
+	ctx.Abort()
+}
+
 func InternalServer(ctx *gin.Context, err error) {
 	ErrorWithDetail(ctx, http.StatusInternalServerError, "ErrInternalServer", err)
 }

frontend/src/api/modules/files.ts

@@ -6,6 +6,10 @@ import { TimeoutEnum } from '@/enums/http-enum';
 import { ReqPage } from '@/api/interface';
 import { Dashboard } from '@/api/interface/dashboard';
 
+export type FileUploadRequestConfig = AxiosRequestConfig & {
+    skipErrorMessage?: boolean;
+};
+
 export const getFilesList = (params: File.ReqFile) => {
     return http.post<File.File>('files/search', params, TimeoutEnum.T_5M);
 };
@@ -105,7 +109,7 @@ export const checkFile = (path: string, withInit: boolean) => {
     return http.post<boolean>('files/check', { path: path, withInit: withInit });
 };
 
-export const uploadFileData = (params: FormData, config: AxiosRequestConfig) => {
+export const uploadFileData = (params: FormData, config: FileUploadRequestConfig) => {
     return http.upload<File.File>('files/upload', params, config);
 };
 
@@ -121,12 +125,17 @@ export const setFileRemark = (params: File.FileRemarkUpdate) => {
     return http.post('files/remark', params);
 };
 
-export const chunkUploadFileData = (params: FormData, config: AxiosRequestConfig) => {
+export const chunkUploadFileData = (params: FormData, config: FileUploadRequestConfig) => {
     return http.upload<File.File>('files/chunkupload', params, config);
 };
 
-export const stopChunkUpload = (key: string) => {
-    return http.post('files/chunkupload/stop', { key });
+export const stopChunkUpload = (key: string, currentNode?: string) => {
+    return http.post(
+        'files/chunkupload/stop',
+        { key },
+        undefined,
+        currentNode ? { CurrentNode: currentNode } : undefined,
+    );
 };
 
 export const renameRile = (params: File.FileRename) => {

frontend/src/views/host/file-management/upload/index.vue

@@ -117,12 +117,20 @@
 <script setup lang="ts">
 import { computed, nextTick, reactive, ref } from 'vue';
 import { UploadFile, UploadFiles, UploadInstance, UploadProps, UploadRawFile } from 'element-plus';
-import { batchCheckFiles, chunkUploadFileData, uploadFileData } from '@/api/modules/files';
+import {
+    batchCheckFiles,
+    chunkUploadFileData,
+    stopChunkUpload,
+    uploadFileData,
+    type FileUploadRequestConfig,
+} from '@/api/modules/files';
 import i18n from '@/lang';
 import { MsgError, MsgSuccess, MsgWarning } from '@/utils/message';
 import { Close, Document, UploadFilled } from '@element-plus/icons-vue';
-import { TimeoutEnum } from '@/enums/http-enum';
 import ExistFileDialog from '@/components/exist-file/index.vue';
+import { newUUID } from '@/utils/id';
+import { getErrorMessage } from '@/utils/misc';
+import { useGlobalStore } from '@/composables/useGlobalStore';
 
 interface UploadFileProps {
     path: string;
@@ -149,6 +157,8 @@ const path = ref();
 let uploadHelper = ref('');
 const dialogExistFileRef = ref();
 const abortController = ref<AbortController | null>(null);
+const activeChunkUpload = ref<{ uploadID: string; node: string } | null>(null);
+const { currentNode } = useGlobalStore();
 
 const em = defineEmits(['close']);
 const handleClose = (done) => {
@@ -158,9 +168,13 @@ const handleClose = (done) => {
             cancelButtonText: i18n.global.t('commons.button.cancel'),
             type: 'info',
         })
-            .then(() => {
-                abortController.value.abort();
-                abortController.value = null;
+            .then(async () => {
+                const controller = abortController.value;
+                controller?.abort();
+                await cleanupActiveChunkUpload();
+                if (abortController.value === controller) {
+                    abortController.value = null;
+                }
                 closePage();
                 done();
             })
@@ -186,7 +200,7 @@ const hoverIndex = ref<number | null>(null);
 const tmpFiles = ref<UploadFiles>([]);
 const breakFlag = ref(false);
 const CHUNK_SIZE = 1024 * 1024 * 5;
-const MAX_SINGLE_FILE_SIZE = 1024 * 1024 * 10;
+const MAX_CHUNK_RETRIES = 3;
 
 const upload = (command: string) => {
     state.uploadEle.webkitdirectory = command == 'dir';
@@ -342,7 +356,7 @@ const submit = async () => {
             onConfirm: handleFileUpload,
         });
     } else {
-        await uploadFile(files);
+        await uploadFile(files, false);
     }
 };
 
@@ -353,109 +367,219 @@ const handleFileUpload = (action: 'skip' | 'overwrite', skippedPaths: string[] =
             (file) => !skippedPaths.includes(`${path.value}/${file.raw.webkitRelativePath || file.name}`),
         );
         uploaderFiles.value = filteredFiles;
-        uploadFile(filteredFiles);
+        uploadFile(filteredFiles, false);
     } else if (action === 'overwrite') {
-        uploadFile(files);
+        uploadFile(files, true);
     }
 };
 
-const uploadFile = async (files: any[]) => {
+const uploadFile = async (files: any[], overwrite: boolean) => {
     if (files.length == 0) {
         clearFiles();
-    } else {
-        loading.value = true;
-        upLoading.value = true;
-        uploadTotalCount.value = files.length;
-        abortController.value = new AbortController();
-        let successCount = 0;
+        return;
+    }
+
+    loading.value = true;
+    upLoading.value = true;
+    uploadTotalCount.value = files.length;
+    const controller = new AbortController();
+    const uploadNode = currentNode.value;
+    abortController.value = controller;
+    let successCount = 0;
+    try {
         for (let i = 0; i < files.length; i++) {
             const file = files[i];
             uploadCurrentIndex.value = i;
             uploadPercent.value = 0;
             uploadHelper.value = i18n.global.t('file.fileUploadStart', [file.name]);
 
-            if (abortController.value.signal.aborted) {
+            if (controller.signal.aborted) {
                 break;
             }
 
-            let isSuccess =
-                file.size <= MAX_SINGLE_FILE_SIZE ? await uploadSingleFile(file) : await uploadLargeFile(file);
+            let isSuccess = false;
+            try {
+                isSuccess =
+                    Number(file.size) === 0
+                        ? await uploadEmptyFile(file, controller, uploadNode, overwrite)
+                        : await uploadChunkedFile(file, controller, uploadNode, overwrite);
+            } catch (error) {
+                if (!controller.signal.aborted) {
+                    MsgError(getErrorMessage(error));
+                }
+            }
 
             if (isSuccess) {
                 successCount++;
-                uploaderFiles.value[i].status = 'success';
+                file.status = 'success';
             } else {
-                uploaderFiles.value[i].status = 'fail';
+                file.status = 'fail';
             }
         }
 
+        if (successCount === files.length && !controller.signal.aborted) {
+            clearFiles();
+            MsgSuccess(i18n.global.t('file.uploadSuccess'));
+        }
+    } finally {
         loading.value = false;
         upLoading.value = false;
         uploadTotalCount.value = 0;
         uploadCurrentIndex.value = 0;
         uploadHelper.value = '';
-
-        if (successCount === files.length && !abortController.value.signal.aborted) {
-            clearFiles();
-            MsgSuccess(i18n.global.t('file.uploadSuccess'));
+        if (abortController.value === controller) {
+            abortController.value = null;
         }
     }
 };
 
-const uploadSingleFile = async (file: { raw: string | Blob }) => {
+const uploadEmptyFile = async (
+    file: { raw: string | Blob },
+    controller: AbortController,
+    node: string,
+    overwrite: boolean,
+) => {
     const formData = new FormData();
     formData.append('file', file.raw);
     formData.append('path', getUploadPath(file));
-    formData.append('overwrite', 'True');
+    formData.append('overwrite', overwrite.toString());
     uploadPercent.value = 0;
     await uploadFileData(formData, {
         onUploadProgress: (progressEvent) => {
-            uploadPercent.value = Math.round((progressEvent.loaded / progressEvent.total) * 100);
+            uploadPercent.value = progressEvent.total
+                ? Math.round((progressEvent.loaded / progressEvent.total) * 100)
+                : 0;
         },
-        timeout: 40000,
-        signal: abortController.value?.signal,
+        headers: { CurrentNode: node },
+        skipErrorMessage: true,
+        timeout: 0,
+        signal: controller.signal,
     });
     return true;
 };
 
-const uploadLargeFile = async (file: { size: any; raw: string | Blob; name: string }) => {
-    const fileSize = file.size;
-    const chunkCount = Math.ceil(fileSize / CHUNK_SIZE);
-    let uploadedChunkCount = 0;
-    for (let c = 0; c < chunkCount; c++) {
-        if (abortController.value?.signal.aborted) {
-            return false;
-        }
-        const start = c * CHUNK_SIZE;
-        const end = Math.min(start + CHUNK_SIZE, fileSize);
-        const chunk = file.raw.slice(start, end);
-        const formData = new FormData();
-        formData.append('filename', getFilenameFromPath(file.name));
-        formData.append('path', getUploadPath(file));
-        formData.append('chunk', chunk);
-        formData.append('chunkIndex', c.toString());
-        formData.append('chunkCount', chunkCount.toString());
+const cleanupActiveChunkUpload = async () => {
+    const active = activeChunkUpload.value;
+    if (!active) {
+        return;
+    }
+    activeChunkUpload.value = null;
+    try {
+        await stopChunkUpload(active.uploadID, active.node);
+    } catch (error) {
+        console.error(error);
+    }
+};
 
+const shouldRetryChunkUpload = (error: unknown) => {
+    const item = error as {
+        code?: string | number;
+        data?: { retryable?: boolean };
+        response?: { status?: number };
+    };
+    if (item?.code === 'ERR_CANCELED') {
+        return false;
+    }
+    if (typeof item?.data?.retryable === 'boolean') {
+        return item.data.retryable;
+    }
+    const status = item?.response?.status ?? (typeof item?.code === 'number' ? item.code : undefined);
+    return !status || status === 408 || status === 429 || status >= 500;
+};
+
+const waitForChunkRetry = (attempt: number, signal: AbortSignal) => {
+    return new Promise<void>((resolve) => {
+        if (signal.aborted) {
+            resolve();
+            return;
+        }
+        const timer = window.setTimeout(done, 500 * 2 ** attempt);
+        function done() {
+            window.clearTimeout(timer);
+            signal.removeEventListener('abort', done);
+            resolve();
+        }
+        signal.addEventListener('abort', done, { once: true });
+    });
+};
+
+const uploadChunkWithRetry = async (formData: FormData, config: FileUploadRequestConfig, signal: AbortSignal) => {
+    let retryCount = 0;
+    while (true) {
         try {
-            await chunkUploadFileData(formData, {
-                onUploadProgress: (progressEvent) => {
-                    uploadPercent.value = Math.round(
-                        ((uploadedChunkCount + progressEvent.loaded / progressEvent.total) * 100) / chunkCount,
-                    );
-                },
-                timeout: TimeoutEnum.T_60S,
-                signal: abortController.value?.signal,
-            });
-            uploadedChunkCount++;
+            await chunkUploadFileData(formData, config);
+            return;
         } catch (error) {
-            if (abortController.value?.signal.aborted) {
-                return false;
+            if (signal.aborted || retryCount >= MAX_CHUNK_RETRIES || !shouldRetryChunkUpload(error)) {
+                throw error;
             }
-            return false;
+            await waitForChunkRetry(retryCount, signal);
+            retryCount++;
         }
     }
+};
 
-    return uploadedChunkCount === chunkCount;
+const uploadChunkedFile = async (
+    file: { size: number; raw: Blob; name: string },
+    controller: AbortController,
+    node: string,
+    overwrite: boolean,
+) => {
+    const fileSize = file.size;
+    const chunkCount = Math.ceil(fileSize / CHUNK_SIZE);
+    const uploadID = newUUID();
+    const uploadPath = getUploadPath(file);
+    const filename = getFilenameFromPath(file.name);
+    let uploadedChunkCount = 0;
+    activeChunkUpload.value = { uploadID, node };
+    try {
+        for (let c = 0; c < chunkCount; c++) {
+            if (controller.signal.aborted) {
+                return false;
+            }
+            const start = c * CHUNK_SIZE;
+            const end = Math.min(start + CHUNK_SIZE, fileSize);
+            const chunk = file.raw.slice(start, end);
+            const formData = new FormData();
+            formData.append('uploadID', uploadID);
+            formData.append('filename', filename);
+            formData.append('path', uploadPath);
+            formData.append('chunk', chunk);
+            formData.append('chunkIndex', c.toString());
+            formData.append('chunkCount', chunkCount.toString());
+            formData.append('offset', start.toString());
+            formData.append('fileSize', fileSize.toString());
+            formData.append('overwrite', overwrite.toString());
+
+            await uploadChunkWithRetry(
+                formData,
+                {
+                    headers: { CurrentNode: node },
+                    skipErrorMessage: true,
+                    timeout: 0,
+                    signal: controller.signal,
+                    onUploadProgress: (progressEvent) => {
+                        const chunkProgress = progressEvent.total ? progressEvent.loaded / progressEvent.total : 0;
+                        uploadPercent.value = Math.round(((uploadedChunkCount + chunkProgress) * 100) / chunkCount);
+                    },
+                },
+                controller.signal,
+            );
+            uploadedChunkCount++;
+        }
+        return uploadedChunkCount === chunkCount;
+    } catch (error) {
+        await cleanupActiveChunkUpload();
+        throw error;
+    } finally {
+        if (activeChunkUpload.value?.uploadID === uploadID) {
+            if (controller.signal.aborted) {
+                await cleanupActiveChunkUpload();
+            } else {
+                activeChunkUpload.value = null;
+            }
+        }
+    }
 };
 
 const getUploadPath = (file) => {

frontend/src/views/setting/panel/hidemenu/index.vue

@@ -25,9 +25,11 @@
                 <div class="menu-setting-card__label mb-3">{{ $t('setting.menuHide') }}</div>
                 <el-alert :closable="false" :title="$t('setting.menuSettingHelper')" type="warning" />
                 <el-tree
+                    ref="menuTreeRef"
                     :data="treeData.hideMenu"
                     :allow-drag="allowDrag"
                     :allow-drop="allowDrop"
+                    :filter-node-method="filterMenu"
                     draggable
                     node-key="id"
                     class="mt-3 menu-hide-tree"
@@ -75,8 +77,8 @@
 </template>
 
 <script setup lang="ts">
-import { reactive, ref } from 'vue';
-import { AllowDropType, ElMessageBox, RenderContentContext } from 'element-plus';
+import { nextTick, reactive, ref } from 'vue';
+import { AllowDropType, ElMessageBox, ElTree, RenderContentContext } from 'element-plus';
 import i18n from '@/lang';
 import { defaultMenu, updateMenu, updateSetting } from '@/api/modules/setting';
 import { MsgSuccess } from '@/utils/message';
@@ -87,6 +89,7 @@ const { isEE, isIntl, isAdmin, menuAccordion } = useGlobalStore();
 
 const drawerVisible = ref();
 const loading = ref();
+const menuTreeRef = ref<InstanceType<typeof ElTree>>();
 const em = defineEmits(['search']);
 interface DialogProps {
     hideMenu: string;
@@ -103,9 +106,7 @@ const acceptParams = (params: DialogProps): void => {
     let hideMenu = JSON.parse(params.hideMenu);
     sortMenu(hideMenu);
     treeData.hideMenu = hideMenu;
-    if (isIntl.value || (isEE.value && !isAdmin.value)) {
-        treeData.hideMenu = removeUpage(treeData.hideMenu);
-    }
+    nextTick(() => menuTreeRef.value?.filter(true));
 };
 type Node = RenderContentContext['node'];
 
@@ -148,25 +149,25 @@ const allowDrop = (draggingNode: Node, dropNode: Node, type: AllowDropType) => {
 
 const handleDrop = (draggingNode: Node, dropNode: Node) => {
     const siblingNodes = dropNode.level == 2 ? dropNode.parent.parent.data : dropNode.parent.data;
-    siblingNodes.forEach((node, index) => {
-        node.sort = (index + 1) * 100;
-    });
-
-    const updateChildSort = (nodes) => {
-        nodes.forEach((node, index) => {
-            node.sort = (index + 1) * 100;
+    const updateSort = (nodes) => {
+        const reservedSorts = new Set(nodes.filter((node) => !isMenuVisible(node)).map((node) => node.sort));
+        let nextSort = 100;
+        nodes.forEach((node) => {
+            if (isMenuVisible(node)) {
+                while (reservedSorts.has(nextSort)) {
+                    nextSort += 100;
+                }
+                node.sort = nextSort;
+                nextSort += 100;
+            }
             if (node.children && node.children.length) {
-                updateChildSort(node.children);
+                updateSort(node.children);
             }
         });
     };
 
     if (siblingNodes.length) {
-        siblingNodes.forEach((node) => {
-            if (node.children && node.children.length) {
-                updateChildSort(node.children);
-            }
-        });
+        updateSort(siblingNodes);
     }
 };
 
@@ -175,21 +176,23 @@ const treeData = reactive({
     checkedData: [],
 });
 
-const removeUpage = (data: any): any => {
-    return data
-        .filter((item: { label: string }) => item.label !== 'Upage' && item.label !== 'XApp')
-        .map((item: { children: any }) => {
-            if (Array.isArray(item.children)) {
-                item.children = removeUpage(item.children);
-            }
-            return item;
-        });
+const isMenuVisible = (data: { label: string }) => {
+    if (data.label === 'Upage') {
+        return !(isIntl.value || (isEE.value && !isAdmin.value));
+    }
+    if (data.label === 'XApp') {
+        return !isIntl.value;
+    }
+    return true;
 };
+const filterMenu = (_value: boolean, data: { label: string }) => isMenuVisible(data);
 
 const onChangeShow = async (row: any) => {
     if (row.children) {
         for (const item of row.children) {
-            item.isShow = row.isShow;
+            if (isMenuVisible(item)) {
+                item.isShow = row.isShow;
+            }
         }
         return;
     }
@@ -199,6 +202,9 @@ const onChangeShow = async (row: any) => {
         }
         let allHide = true;
         for (const item2 of item.children) {
+            if (!isMenuVisible(item2)) {
+                continue;
+            }
             if (item2.isShow) {
                 allHide = false;
             }