1Panel/agent/utils/files/file_op.go

package files

import (
	"archive/zip"
	"bufio"
	"compress/gzip"
	"context"
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/fs"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/1Panel-dev/1Panel/agent/buserr"

	"github.com/1Panel-dev/1Panel/agent/constant"
	"github.com/1Panel-dev/1Panel/agent/utils/cmd"
	"github.com/1Panel-dev/1Panel/agent/utils/req_helper"
	cZip "github.com/klauspost/compress/zip"
	"golang.org/x/text/encoding/simplifiedchinese"
	"golang.org/x/text/transform"

	"github.com/1Panel-dev/1Panel/agent/global"
	"github.com/mholt/archiver/v4"
	"github.com/spf13/afero"
	"golang.org/x/sync/singleflight"
)

const (
	cmdDefaultTimeout   = 10 * time.Second
	cmdRecursiveTimeout = 5 * time.Minute
)

var protectedPaths = []string{
	"/",
	"/bin",
	"/sbin",
	"/etc",
	"/boot",
	"/usr",
	"/lib",
	"/lib64",
	"/dev",
	"/proc",
	"/sys",
	"/root",
}

var (
	dirSizeGroup   singleflight.Group
	dirSizeLimiter = make(chan struct{}, 2)
)

func IsProtected(path string) bool {
	real, err := filepath.EvalSymlinks(path)
	if err == nil {
		path = real
	}

	abs, err := filepath.Abs(path)
	if err == nil {
		path = abs
	}

	for _, p := range protectedPaths {
		if path == p {
			return true
		}
	}
	return false
}

type FileOp struct {
	Fs afero.Fs
}

func NewFileOp() FileOp {
	return FileOp{
		Fs: afero.NewOsFs(),
	}
}

func (f FileOp) OpenFile(dst string) (fs.File, error) {
	return f.Fs.Open(dst)
}

func (f FileOp) GetContent(dst string) ([]byte, error) {
	afs := &afero.Afero{Fs: f.Fs}
	cByte, err := afs.ReadFile(dst)
	if err != nil {
		return nil, err
	}
	return cByte, nil
}

func (f FileOp) CreateDir(dst string, mode fs.FileMode) error {
	return f.Fs.MkdirAll(dst, mode)
}

func (f FileOp) CreateDirWithMode(dst string, mode fs.FileMode) error {
	if err := f.Fs.MkdirAll(dst, mode); err != nil {
		return err
	}
	return f.ChmodRWithMode(dst, mode, true)
}
func (f FileOp) CreateDirWithPath(isDir bool, pathItem string) (string, error) {
	checkPath := pathItem
	if !isDir {
		checkPath = path.Dir(pathItem)
	}
	if !f.Stat(checkPath) {
		if err := f.CreateDir(checkPath, os.ModePerm); err != nil {
			return pathItem, err
		}
	}
	return pathItem, nil
}

func (f FileOp) CreateFile(dst string) error {
	file, err := f.Fs.Create(dst)
	if err != nil {
		return err
	}
	return file.Close()
}

func (f FileOp) CreateFileWithMode(dst string, mode fs.FileMode) error {
	file, err := f.Fs.OpenFile(dst, os.O_CREATE, mode)
	if err != nil {
		return err
	}
	return file.Close()
}

func (f FileOp) LinkFile(source string, dst string, isSymlink bool) error {
	if isSymlink {
		osFs := afero.OsFs{}
		return osFs.SymlinkIfPossible(source, dst)
	} else {
		return os.Link(source, dst)
	}
}

func (f FileOp) DeleteDir(dst string) error {
	if IsProtected(dst) {
		return buserr.New("ErrPathNotDelete")
	}
	return f.Fs.RemoveAll(dst)
}

func (f FileOp) Stat(dst string) bool {
	info, _ := f.Fs.Stat(dst)
	return info != nil
}

func (f FileOp) DeleteFile(dst string) error {
	if IsProtected(dst) {
		return buserr.New("ErrPathNotDelete")
	}
	return f.Fs.Remove(dst)
}

func (f FileOp) CleanDir(dst string) error {
	if IsProtected(dst) {
		return buserr.New("ErrPathNotDelete")
	}
	items, err := afero.ReadDir(f.Fs, dst)
	if err != nil {
		return err
	}
	for _, item := range items {
		if err := f.Fs.RemoveAll(filepath.Join(dst, item.Name())); err != nil {
			return err
		}
	}
	return nil
}

func (f FileOp) RmRf(dst string) error {
	if IsProtected(dst) {
		return buserr.New("ErrPathNotDelete")
	}
	return f.Fs.RemoveAll(dst)
}

func (f FileOp) WriteFile(dst string, in io.Reader, mode fs.FileMode) error {
	file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
	if err != nil {
		return err
	}
	defer file.Close()

	if _, err = io.Copy(file, in); err != nil {
		return err
	}

	if _, err = file.Stat(); err != nil {
		return err
	}
	return nil
}

func (f FileOp) SaveFile(dst string, content string, mode fs.FileMode) error {
	if !f.Stat(path.Dir(dst)) {
		_ = f.CreateDir(path.Dir(dst), mode.Perm())
	}
	file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
	if err != nil {
		return err
	}
	defer file.Close()
	write := bufio.NewWriter(file)
	_, _ = write.WriteString(content)
	write.Flush()
	return nil
}

func (f FileOp) SaveFileWithByte(dst string, content []byte, mode fs.FileMode) error {
	if !f.Stat(path.Dir(dst)) {
		_ = f.CreateDir(path.Dir(dst), mode.Perm())
	}
	file, err := f.Fs.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
	if err != nil {
		return err
	}
	defer file.Close()
	write := bufio.NewWriter(file)
	_, _ = write.Write(content)
	write.Flush()
	return nil
}

func (f FileOp) ChownR(dst string, uid string, gid string, sub bool) error {
	args := []string{uid + ":" + gid, dst}
	if sub {
		args = append([]string{"-R", uid + ":" + gid}, dst)
	}
	timeout := cmdDefaultTimeout
	if sub {
		timeout = cmdRecursiveTimeout
	}
	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(timeout))
	if err := cmdMgr.RunWithOptionalSudo("chown", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) ChmodR(dst string, mode int64, sub bool) error {
	args := []string{fmt.Sprintf("%04o", mode), dst}
	if sub {
		args = append([]string{"-R", fmt.Sprintf("%04o", mode)}, dst)
	}
	timeout := cmdDefaultTimeout
	if sub {
		timeout = cmdRecursiveTimeout
	}
	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(timeout))
	if err := cmdMgr.RunWithOptionalSudo("chmod", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) ChmodRWithMode(dst string, mode fs.FileMode, sub bool) error {
	args := []string{fmt.Sprintf("%o", mode.Perm()), dst}
	if sub {
		args = append([]string{"-R", fmt.Sprintf("%o", mode.Perm())}, dst)
	}
	timeout := cmdDefaultTimeout
	if sub {
		timeout = cmdRecursiveTimeout
	}
	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(timeout))
	if err := cmdMgr.RunWithOptionalSudo("chmod", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) ChownRPaths(paths []string, uid string, gid string, sub bool) error {
	if len(paths) == 0 {
		return nil
	}
	if len(paths) == 1 {
		return f.ChownR(paths[0], uid, gid, sub)
	}
	args := []string{uid + ":" + gid}
	if sub {
		args = append([]string{"-R", uid + ":" + gid}, paths...)
	} else {
		args = append(args, paths...)
	}
	timeout := cmdDefaultTimeout
	if sub {
		timeout = cmdRecursiveTimeout
	}
	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(timeout))
	if err := cmdMgr.RunWithOptionalSudo("chown", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) ChmodRPaths(paths []string, mode int64, sub bool) error {
	if len(paths) == 0 {
		return nil
	}
	if len(paths) == 1 {
		return f.ChmodR(paths[0], mode, sub)
	}
	modeStr := fmt.Sprintf("%04o", mode)
	args := []string{modeStr}
	if sub {
		args = append([]string{"-R", modeStr}, paths...)
	} else {
		args = append(args, paths...)
	}
	timeout := cmdDefaultTimeout
	if sub {
		timeout = cmdRecursiveTimeout
	}
	cmdMgr := cmd.NewCommandMgr(cmd.WithTimeout(timeout))
	if err := cmdMgr.RunWithOptionalSudo("chmod", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) Rename(oldName string, newName string) error {
	return f.Fs.Rename(oldName, newName)
}

type downloadTask struct {
	resp *http.Response
	file *os.File
	dst  string
}

var (
	downloadMu    sync.Mutex
	downloadTasks = make(map[string]*downloadTask)
)

type WriteCounter struct {
	Total   uint64
	Written uint64
	Key     string
	Name    string
}

type Process struct {
	Total   uint64  `json:"total"`
	Written uint64  `json:"written"`
	Percent float64 `json:"percent"`
	Name    string  `json:"name"`
}

type DownloadProxyConfig struct {
	Type     string
	URL      string
	Port     string
	User     string
	Password string
}

type DownloadOptions struct {
	IgnoreCertificate bool
	Proxy             *DownloadProxyConfig
}

func (w *WriteCounter) Write(p []byte) (n int, err error) {
	n = len(p)
	w.Written += uint64(n)
	w.SaveProcess()
	return n, nil
}

func (w *WriteCounter) SaveProcess() {
	percentValue := 0.0
	if w.Total > 0 {
		percent := float64(w.Written) / float64(w.Total) * 100
		percentValue, _ = strconv.ParseFloat(fmt.Sprintf("%.2f", percent), 64)
	}
	process := Process{
		Total:   w.Total,
		Written: w.Written,
		Percent: percentValue,
		Name:    w.Name,
	}
	by, _ := json.Marshal(process)
	if percentValue < 100 {
		global.CACHE.Set(w.Key, string(by))
	} else {
		global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10))
	}
}

func buildDownloadProxyURL(proxy DownloadProxyConfig) (*url.URL, error) {
	proxyType := strings.TrimSpace(proxy.Type)
	proxyHost := strings.TrimSpace(proxy.URL)
	if proxyType == "" || proxyHost == "" {
		return nil, buserr.New("ErrWgetProxyNotConfigured")
	}
	if !strings.Contains(proxyHost, "://") {
		proxyHost = fmt.Sprintf("%s://%s", proxyType, proxyHost)
	}
	parsedURL, err := url.Parse(proxyHost)
	if err != nil {
		return nil, buserr.WithDetail("ErrWgetProxyInvalid", err.Error(), err)
	}
	if parsedURL.Scheme == "" {
		parsedURL.Scheme = proxyType
	}
	if parsedURL.Host == "" && parsedURL.Path != "" {
		parsedURL.Host = parsedURL.Path
		parsedURL.Path = ""
	}
	if parsedURL.Host == "" {
		return nil, buserr.New("ErrWgetProxyNotConfigured")
	}
	if strings.TrimSpace(proxy.Port) != "" && parsedURL.Port() == "" {
		parsedURL.Host = net.JoinHostPort(parsedURL.Hostname(), strings.TrimSpace(proxy.Port))
	}
	if proxy.User != "" && proxy.Password != "" {
		parsedURL.User = url.UserPassword(proxy.User, proxy.Password)
	} else if proxy.User != "" {
		parsedURL.User = url.User(proxy.User)
	}
	return parsedURL, nil
}

func newDownloadHTTPClient(options DownloadOptions) (*http.Client, error) {
	if !options.IgnoreCertificate && options.Proxy == nil {
		return &http.Client{}, nil
	}
	transport := &http.Transport{}
	if options.IgnoreCertificate {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}
	if options.Proxy != nil {
		proxyURL, err := buildDownloadProxyURL(*options.Proxy)
		if err != nil {
			return nil, err
		}
		transport.Proxy = http.ProxyURL(proxyURL)
	}
	return &http.Client{Transport: transport}, nil
}

func (f FileOp) DownloadFileWithProcess(url, dst, key string, options DownloadOptions) error {
	client, err := newDownloadHTTPClient(options)
	if err != nil {
		return err
	}
	defer client.CloseIdleConnections()

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return buserr.WithDetail("ErrWgetRemoteFailed", err.Error(), err)
	}
	request.Header.Set("Accept-Encoding", "identity")

	resp, err := client.Do(request)
	if err != nil {
		global.LOG.Errorf("get download file [%s] error, err %s", dst, err.Error())
		return buserr.WithDetail("ErrWgetRemoteFailed", err.Error(), err)
	}

	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024))
		_ = resp.Body.Close()
		global.LOG.Errorf("wget remote returned non-success status %s for url %s", resp.Status, url)
		return buserr.WithDetail("ErrWgetRemoteFailed", resp.StatusCode, nil)
	}

	ct := strings.ToLower(resp.Header.Get("Content-Type"))
	dstExt := strings.ToLower(filepath.Ext(dst))
	if (strings.Contains(ct, "text/html") || strings.Contains(ct, "text/xml")) &&
		dstExt != ".html" && dstExt != ".htm" && dstExt != ".xml" && dstExt != ".svg" {
		_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64*1024))
		_ = resp.Body.Close()
		detail := fmt.Sprintf("Content-Type: %s", ct)
		global.LOG.Errorf("wget got html/xml response for non-html file %s, url %s, %s", dst, url, detail)
		return buserr.WithDetail("ErrWgetInvalidContentType", detail, nil)
	}

	out, err := os.Create(dst)
	if err != nil {
		global.LOG.Errorf("create download file [%s] error, err %s", dst, err.Error())
		resp.Body.Close()
		return err
	}

	downloadMu.Lock()
	downloadTasks[key] = &downloadTask{
		resp: resp,
		file: out,
		dst:  dst,
	}
	downloadMu.Unlock()

	go func() {
		defer func() {
			out.Close()
			resp.Body.Close()

			downloadMu.Lock()
			delete(downloadTasks, key)
			downloadMu.Unlock()
		}()

		counter := &WriteCounter{}
		counter.Key = key
		if resp.ContentLength > 0 {
			counter.Total = uint64(resp.ContentLength)
		}
		counter.Name = filepath.Base(dst)

		if _, err := io.Copy(out, io.TeeReader(resp.Body, counter)); err != nil {
			global.LOG.Errorf("save download file [%s] error, err %s", dst, err.Error())
			global.CACHE.Del(counter.Key)
			return
		}

		value := global.CACHE.Get(counter.Key)
		if value == "" {
			return
		}
		process := &Process{}
		if err := json.Unmarshal([]byte(value), process); err != nil {
			return
		}
		process.Percent = 100
		process.Name = counter.Name
		process.Total = process.Written
		by, _ := json.Marshal(process)
		global.CACHE.Set(counter.Key, string(by))
	}()
	return nil
}

func CancelDownload(key string) {
	downloadMu.Lock()
	task, ok := downloadTasks[key]
	if !ok {
		downloadMu.Unlock()
		return
	}
	dst := task.dst
	downloadMu.Unlock()

	_ = task.file.Close()
	_ = task.resp.Body.Close()

	if dst != "" {
		_ = os.Remove(dst)
	}
	global.CACHE.Del(key)
}

func (f FileOp) DownloadFile(url, dst string) error {
	resp, err := req_helper.HandleGet(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	out, err := os.Create(dst)
	if err != nil {
		return fmt.Errorf("create download file [%s] error, err %s", dst, err.Error())
	}
	defer out.Close()

	if _, err = io.Copy(out, resp.Body); err != nil {
		return fmt.Errorf("save download file [%s] error, err %s", dst, err.Error())
	}
	return nil
}

func (f FileOp) Cut(oldPaths []string, dst, name string, cover bool) error {
	if len(oldPaths) == 0 {
		return nil
	}
	var dstPath string
	coverFlag := ""
	if name != "" {
		dstPath = filepath.Join(dst, name)
		if f.Stat(dstPath) {
			dstPath = dst
		}
		if cover {
			coverFlag = "-f"
		}
	} else {
		dstPath = dst
		coverFlag = "-f"
	}
	args := []string{}
	if coverFlag != "" {
		args = append(args, coverFlag)
	}
	args = append(args, oldPaths...)
	args = append(args, dstPath)
	if err := cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("mv", args...); err != nil {
		return err
	}
	return nil
}

func (f FileOp) Mv(oldPath, dstPath string) error {
	if err := cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("mv", oldPath, dstPath); err != nil {
		return err
	}
	return nil
}

func (f FileOp) Copy(src, dst string) error {
	if src = path.Clean("/" + src); src == "" {
		return os.ErrNotExist
	}
	if dst = path.Clean("/" + dst); dst == "" {
		return os.ErrNotExist
	}
	if src == "/" || dst == "/" {
		return os.ErrInvalid
	}
	if dst == src {
		return os.ErrInvalid
	}
	info, err := f.Fs.Stat(src)
	if err != nil {
		return err
	}
	if info.IsDir() {
		return f.CopyDir(src, dst)
	}
	return f.CopyFile(src, dst)
}

func (f FileOp) CopyAndReName(src, dst, name string, cover bool) error {
	if src = path.Clean("/" + src); src == "" {
		return os.ErrNotExist
	}
	if dst = path.Clean("/" + dst); dst == "" {
		return os.ErrNotExist
	}
	if src == "/" || dst == src {
		return os.ErrInvalid
	}

	srcInfo, err := f.Fs.Stat(src)
	if err != nil {
		return err
	}

	if name != "" && !cover {
		if f.Stat(filepath.Join(dst, name)) {
			return buserr.New("ErrFileIsExist")
		}
	}

	if srcInfo.IsDir() {
		dstPath := dst
		if name != "" && !cover {
			dstPath = filepath.Join(dst, name)
		}
		return cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("cp", "-rfp", src, dstPath)
	} else {
		dstPath := filepath.Join(dst, name)
		if cover {
			dstPath = dst
		}
		return cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("cp", "-fp", src, dstPath)
	}
}

func (f FileOp) CopyDirWithNewName(src, dst, newName string) error {
	if newName == "." || newName == "" {
		return cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("cp", "-rfp", filepath.Clean(src)+"/.", dst)
	}
	dstDir := filepath.Join(dst, newName)
	return cmd.NewCommandMgr(cmd.WithTimeout(cmdRecursiveTimeout)).Run("cp", "-rfp", src, dstDir)
}

func (f FileOp) CopyDir(src, dst string) error {
	srcInfo, err := f.Fs.Stat(src)
	if err != nil {
		return err
	}
	dstDir := filepath.Join(dst, srcInfo.Name())
	if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil {
		return err
	}
	return cmd.NewCommandMgr(cmd.WithIgnoreExist1()).Run("cp", "-rfp", src, dst+"/")
}

func (f FileOp) CopyDirWithExclude(src, dst string, excludeNames []string) error {
	srcInfo, err := f.Fs.Stat(src)
	if err != nil {
		return err
	}
	dstDir := filepath.Join(dst, srcInfo.Name())
	if err = f.Fs.MkdirAll(dstDir, srcInfo.Mode()); err != nil {
		return err
	}
	if len(excludeNames) == 0 {
		return cmd.NewCommandMgr(cmd.WithIgnoreExist1()).Run("cp", "-rfp", src, dst+"/")
	}
	tmpFiles, err := os.ReadDir(src)
	if err != nil {
		return err
	}
	for _, item := range tmpFiles {
		isExclude := false
		for _, name := range excludeNames {
			if item.Name() == name {
				isExclude = true
				break
			}
		}
		if isExclude {
			continue
		}
		if item.IsDir() {
			if err := f.CopyDir(path.Join(src, item.Name()), dstDir); err != nil {
				return err
			}
			continue
		}
		if err := f.CopyFile(path.Join(src, item.Name()), dstDir); err != nil {
			return err
		}
	}

	return nil
}

func (f FileOp) CopyFile(src, dst string) error {
	dst = filepath.Clean(dst) + string(filepath.Separator)
	return cmd.NewCommandMgr(cmd.WithIgnoreExist1()).Run("cp", "-fp", src, dst+"/")
}

func (f FileOp) GetDirSize(path string) (int64, error) {
	cleanPath := filepath.Clean(path)
	result, err, _ := dirSizeGroup.Do("single:"+cleanPath, func() (interface{}, error) {
		dirSizeLimiter <- struct{}{}
		defer func() {
			<-dirSizeLimiter
		}()
		return f.getDirSize(cleanPath)
	})
	if err != nil {
		return 0, err
	}
	return result.(int64), nil
}

func (f FileOp) getDirSize(path string) (int64, error) {
	ctx, cancel := context.WithTimeout(context.Background(), cmdRecursiveTimeout)
	defer cancel()
	duCmd := exec.CommandContext(ctx, "du", "-s", path)
	output, err := duCmd.Output()
	if err == nil {
		fields := strings.Fields(string(output))
		if len(fields) == 2 {
			var cmdSize int64
			_, err = fmt.Sscanf(fields[0], "%d", &cmdSize)
			if err == nil {
				return cmdSize * 1024, nil
			}
		}
	}
	if ctx.Err() != nil {
		return 0, ctx.Err()
	}

	var size int64
	err = filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() {
			size += info.Size()
		}
		return nil
	})
	if err != nil {
		return 0, err
	}
	return size, nil
}

type DirSize struct {
	Path string `json:"path"`
	Size int64  `json:"size"`
}

func (f FileOp) GetDepthDirSize(path string) ([]DirSize, error) {
	cleanPath := filepath.Clean(path)
	result, err, _ := dirSizeGroup.Do("depth:"+cleanPath, func() (interface{}, error) {
		dirSizeLimiter <- struct{}{}
		defer func() {
			<-dirSizeLimiter
		}()
		return f.getDepthDirSize(cleanPath)
	})
	if err != nil {
		return nil, err
	}
	return result.([]DirSize), nil
}

func (f FileOp) getDepthDirSize(path string) ([]DirSize, error) {
	var result []DirSize
	sizeMap := make(map[string]int64)
	ctx, cancel := context.WithTimeout(context.Background(), cmdRecursiveTimeout)
	defer cancel()
	duCmd := exec.CommandContext(ctx, "du", "-k", "--max-depth=1", "--exclude=proc", path)
	output, err := duCmd.Output()
	if err == nil {
		parseDUOutput(output, sizeMap)
	} else if ctx.Err() != nil {
		return nil, ctx.Err()
	} else {
		calculateDirSizeFallback(path, sizeMap)
	}

	for dir, size := range sizeMap {
		result = append(result, DirSize{
			Path: dir,
			Size: size,
		})
	}

	return result, nil
}

func parseDUOutput(output []byte, sizeMap map[string]int64) {
	lines := strings.Split(string(output), "\n")
	for _, line := range lines {
		if strings.TrimSpace(line) == "" {
			continue
		}
		sizeText, dir, ok := strings.Cut(strings.TrimSpace(line), "\t")
		if !ok {
			fields := strings.Fields(line)
			if len(fields) < 2 {
				continue
			}
			sizeText = fields[0]
			dir = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), sizeText))
		}
		if sizeKB, err := strconv.ParseInt(strings.TrimSpace(sizeText), 10, 64); err == nil {
			sizeMap[strings.TrimSpace(dir)] = sizeKB * 1024
		}
	}
}

func calculateDirSizeFallback(path string, sizeMap map[string]int64) {
	_ = filepath.Walk(path, func(p string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}
		if !info.IsDir() {
			rel, err := filepath.Rel(path, p)
			if err != nil {
				return nil
			}
			parts := strings.Split(rel, string(os.PathSeparator))
			var topLevel string
			if len(parts) == 0 || parts[0] == "." {
				topLevel = path
			} else {
				topLevel = filepath.Join(path, parts[0])
			}
			sizeMap[topLevel] += info.Size()
		}
		return nil
	})
}

func getFormat(cType CompressType) archiver.CompressedArchive {
	format := archiver.CompressedArchive{}
	switch cType {
	case Tar:
		format.Archival = archiver.Tar{}
	case TarGz, Gz, Tgz:
		format.Compression = archiver.Gz{}
		format.Archival = archiver.Tar{}
	case SdkTarGz:
		format.Compression = archiver.Gz{}
		format.Archival = archiver.Tar{}
	case SdkZip, Zip:
		format.Archival = archiver.Zip{
			Compression: zip.Deflate,
		}
	case Bz2, TarBz2:
		format.Compression = archiver.Bz2{}
		format.Archival = archiver.Tar{}
	case Xz, TarXz:
		format.Compression = archiver.Xz{}
		format.Archival = archiver.Tar{}
	}
	return format
}

func (f FileOp) Compress(ctx context.Context, srcRiles []string, dst string, name string, cType CompressType, secret string, progress func(current, total int, message string)) error {
	format := getFormat(cType)

	fileMaps := make(map[string]string, len(srcRiles))
	for _, s := range srcRiles {
		base := filepath.Base(s)
		fileMaps[s] = base
	}

	if !f.Stat(dst) {
		_ = f.CreateDir(dst, constant.DirPerm)
	}

	files, err := archiver.FilesFromDisk(nil, fileMaps)
	if err != nil {
		return err
	}
	dstFile := filepath.Join(dst, name)

	switch cType {
	case Zip, SdkZip:
		out, err := f.Fs.Create(dstFile)
		if err != nil {
			return err
		}
		defer out.Close()
		if err := ZipFile(ctx, files, out, progress); err == nil {
			return nil
		}
		_ = f.DeleteFile(dstFile)
		return NewZipArchiver().Compress(ctx, srcRiles, dstFile, "")
	case TarGz:
		err = NewTarGzArchiver().Compress(ctx, srcRiles, dstFile, secret)
		if err != nil {
			_ = f.DeleteFile(dstFile)
			return err
		}
	case Rar:
		if err := checkCmdAvailability("rar"); err != nil {
			return err
		}
		err = NewRarArchiver().Compress(ctx, srcRiles, dstFile, secret)
		if err != nil {
			_ = f.DeleteFile(dstFile)
			return err
		}
	case X7z:
		if err := checkCmdAvailability("7z"); err != nil {
			return err
		}
		err = NewX7zArchiver().Compress(ctx, srcRiles, dstFile, secret)
		if err != nil {
			_ = f.DeleteFile(dstFile)
			return err
		}
	default:
		tmpFile, err := os.CreateTemp(dst, fmt.Sprintf("temp_*%s", filepath.Ext(name)))
		if err != nil {
			return err
		}
		success := false
		defer func() {
			_ = tmpFile.Close()
			if !success {
				_ = os.Remove(tmpFile.Name())
				_ = f.DeleteFile(dstFile)
			}
		}()

		err = format.Archive(ctx, &contextAwareWriter{ctx: ctx, writer: tmpFile}, files)
		if err != nil {
			return err
		}
		if err = tmpFile.Close(); err != nil {
			return err
		}
		if err = os.Rename(tmpFile.Name(), dstFile); err != nil {
			return err
		}
		success = true
	}
	return nil
}

type contextAwareWriter struct {
	ctx    context.Context
	writer io.Writer
}

func (w *contextAwareWriter) Write(p []byte) (int, error) {
	if err := w.ctx.Err(); err != nil {
		return 0, err
	}
	return w.writer.Write(p)
}

func isIgnoreFile(name string) bool {
	return strings.HasPrefix(name, "__MACOSX") || strings.HasSuffix(name, ".DS_Store") || strings.HasPrefix(name, "._")
}

func decodeGBK(input string) (string, error) {
	decoder := simplifiedchinese.GBK.NewDecoder()
	decoded, _, err := transform.String(decoder, input)
	if err != nil {
		return "", err
	}
	return decoded, nil
}

func (f FileOp) decompressWithSDK(ctx context.Context, srcFile string, dst string, cType CompressType) error {
	format := getFormat(cType)
	if cType == Gz {
		if err := f.tryDecompressTarGz(ctx, srcFile, dst, format); err == nil {
			return nil
		}
		return f.DecompressGzFile(ctx, srcFile, dst)
	}

	type dirEntry struct {
		path    string
		modTime time.Time
	}
	var dirs []dirEntry

	handler := func(ctx context.Context, archFile archiver.File) error {
		info := archFile.FileInfo
		if isIgnoreFile(archFile.Name()) {
			return nil
		}
		fileName := archFile.NameInArchive
		var err error
		if header, ok := archFile.Header.(cZip.FileHeader); ok {
			if header.NonUTF8 && header.Flags == 0 {
				fileName, err = decodeGBK(fileName)
				if err != nil {
					return err
				}
			}
		}
		filePath := filepath.Join(dst, fileName)
		if archFile.FileInfo.IsDir() {
			if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil {
				return err
			}
			dirs = append(dirs, dirEntry{path: filePath, modTime: info.ModTime()})
			return nil
		} else {
			parentDir := path.Dir(filePath)
			if !f.Stat(parentDir) {
				if err := f.Fs.MkdirAll(parentDir, constant.DirPerm); err != nil {
					return err
				}
			}
		}
		fr, err := archFile.Open()
		if err != nil {
			return err
		}
		defer fr.Close()
		fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode())
		if err != nil {
			return err
		}
		defer fw.Close()
		if _, err := io.Copy(fw, fr); err != nil {
			return err
		}
		_ = os.Chtimes(filePath, info.ModTime(), info.ModTime())
		return nil
	}
	input, err := f.Fs.Open(srcFile)
	if err != nil {
		return err
	}
	defer input.Close()
	if err := format.Extract(ctx, input, nil, handler); err != nil {
		return err
	}
	for i := len(dirs) - 1; i >= 0; i-- {
		_ = os.Chtimes(dirs[i].path, dirs[i].modTime, dirs[i].modTime)
	}
	return nil
}

func (f FileOp) Decompress(ctx context.Context, srcFile string, dst string, cType CompressType, secret string) error {
	if cType == Tar || cType == Zip || cType == TarGz || cType == Rar || cType == X7z {
		shellArchiver, err := NewExtractShellArchiver(cType)
		if !f.Stat(dst) {
			_ = f.CreateDir(dst, 0755)
		}
		if err == nil {
			if err = shellArchiver.Extract(ctx, srcFile, dst, secret); err == nil {
				return nil
			}
			if cType == TarGz {
				if strings.Contains(err.Error(), "bad decrypt") {
					return buserr.New("ErrBadDecrypt")
				}
				if err := shellArchiver.Extract(ctx, srcFile, dst, "-"); strings.Contains(err.Error(), "bad decrypt") {
					return buserr.New("ErrBadDecrypt")
				}
			}
		} else {
			if cType == Rar || cType == X7z {
				return err
			}
		}
	}
	return f.decompressWithSDK(ctx, srcFile, dst, cType)
}

func ZipFile(ctx context.Context, files []archiver.File, dst afero.File, progress func(current, total int, message string)) error {
	zw := zip.NewWriter(dst)
	defer zw.Close()

	total := len(files)
	for i, file := range files {
		if ctx != nil {
			select {
			case <-ctx.Done():
				return ctx.Err()
			default:
			}
		}
		hdr, err := zip.FileInfoHeader(file)
		if err != nil {
			return err
		}
		hdr.Method = zip.Deflate
		hdr.Name = file.NameInArchive
		if file.IsDir() {
			if !strings.HasSuffix(hdr.Name, "/") {
				hdr.Name += "/"
			}
		}
		w, err := zw.CreateHeader(hdr)
		if err != nil {
			return err
		}
		if file.IsDir() {
			continue
		}

		if file.LinkTarget != "" {
			_, err = w.Write([]byte(filepath.ToSlash(file.LinkTarget)))
			if err != nil {
				return err
			}
		} else {
			fileReader, err := file.Open()
			if err != nil {
				return err
			}
			_, err = io.Copy(w, newContextReader(ctx, fileReader))
			fileReader.Close()
			if err != nil {
				return err
			}
		}
		if progress != nil {
			progress(i+1, total, file.NameInArchive)
		}
	}
	return nil
}

type contextReader struct {
	ctx context.Context
	r   io.Reader
}

func newContextReader(ctx context.Context, r io.Reader) io.Reader {
	if ctx == nil {
		return r
	}
	return &contextReader{ctx: ctx, r: r}
}

func (r *contextReader) Read(p []byte) (int, error) {
	select {
	case <-r.ctx.Done():
		return 0, r.ctx.Err()
	default:
		return r.r.Read(p)
	}
}

func (f FileOp) tryDecompressTarGz(ctx context.Context, srcFile string, dst string, format archiver.CompressedArchive) error {
	input, err := f.Fs.Open(srcFile)
	if err != nil {
		return err
	}
	defer input.Close()

	type dirEntry struct {
		path    string
		modTime time.Time
	}
	var dirs []dirEntry
	extracted := false

	handler := func(ctx context.Context, archFile archiver.File) error {
		info := archFile.FileInfo
		if isIgnoreFile(archFile.Name()) {
			return nil
		}
		filePath := filepath.Join(dst, archFile.NameInArchive)
		if info.IsDir() {
			if err := f.Fs.MkdirAll(filePath, info.Mode()); err != nil {
				return err
			}
			dirs = append(dirs, dirEntry{path: filePath, modTime: info.ModTime()})
		} else {
			parentDir := filepath.Dir(filePath)
			if !f.Stat(parentDir) {
				if err := f.Fs.MkdirAll(parentDir, constant.DirPerm); err != nil {
					return err
				}
			}
			fr, err := archFile.Open()
			if err != nil {
				return err
			}
			defer fr.Close()
			fw, err := f.Fs.OpenFile(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode())
			if err != nil {
				return err
			}
			defer fw.Close()
			if _, err := io.Copy(fw, fr); err != nil {
				return err
			}
			_ = os.Chtimes(filePath, info.ModTime(), info.ModTime())
		}
		extracted = true
		return nil
	}

	if err := format.Extract(ctx, input, nil, handler); err != nil {
		return err
	}
	if !extracted {
		return fmt.Errorf("no files extracted as tar.gz")
	}
	for i := len(dirs) - 1; i >= 0; i-- {
		_ = os.Chtimes(dirs[i].path, dirs[i].modTime, dirs[i].modTime)
	}
	return nil
}

func (f FileOp) DecompressGzFile(ctx context.Context, srcFile, dst string) error {
	var archiveModTime time.Time
	if st, err := f.Fs.Stat(srcFile); err == nil {
		archiveModTime = st.ModTime()
	}

	in, err := f.Fs.Open(srcFile)
	if err != nil {
		return fmt.Errorf("open source file failed: %w", err)
	}
	defer in.Close()

	gr, err := gzip.NewReader(&contextReader{ctx: ctx, r: in})
	if err != nil {
		return fmt.Errorf("gzip reader creation failed: %w", err)
	}
	defer gr.Close()

	outName := ""
	if gr.Name != "" {
		outName = filepath.Base(gr.Name)
	}
	if outName == "" || outName == "." {
		outName = strings.TrimSuffix(filepath.Base(srcFile), ".gz")
	}
	outPath := filepath.Join(dst, outName)
	parentDir := filepath.Dir(outPath)
	if !f.Stat(parentDir) {
		if err := f.Fs.MkdirAll(parentDir, 0755); err != nil {
			return err
		}
	}

	fw, err := f.Fs.OpenFile(outPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0644)
	if err != nil {
		return fmt.Errorf("create output file failed: %w", err)
	}
	defer fw.Close()

	if _, err := io.Copy(fw, gr); err != nil {
		return fmt.Errorf("copy content failed: %w", err)
	}

	if !archiveModTime.IsZero() {
		_ = os.Chtimes(outPath, archiveModTime, archiveModTime)
	}

	return nil
}

func (f FileOp) TarGzCompressPro(withDir bool, src, dst, secret, exclusionRules string) error {
	if !f.Stat(path.Dir(dst)) {
		if err := f.Fs.MkdirAll(path.Dir(dst), constant.FilePerm); err != nil {
			return err
		}
	}
	workdir := src
	srcItem := "."
	if withDir {
		workdir = path.Dir(src)
		srcItem = path.Base(src)
	}
	exMap := make(map[string]struct{})
	excludeArgs := []string{}
	excludes := strings.Split(exclusionRules, ",")
	for _, exclude := range excludes {
		if len(exclude) == 0 {
			continue
		}
		if strings.HasPrefix(exclude, "/") {
			exclude, _ = filepath.Rel(src, exclude)
		}
		if _, ok := exMap[exclude]; ok {
			continue
		}
		excludeArgs = append(excludeArgs, "--exclude", exclude)
		exMap[exclude] = struct{}{}
	}

	tarArgs := append([]string{}, excludeArgs...)
	if len(secret) != 0 {
		cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(workdir), cmd.WithIgnoreExist1())
		return runTarGzEncryptToFile(cmdMgr, dst, secret, append(tarArgs, srcItem)...)
	} else {
		cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(workdir), cmd.WithIgnoreExist1())
		return runTarGzToFile(cmdMgr, dst, append(tarArgs, srcItem)...)
	}
}

func (f FileOp) TarGzFilesWithCompressPro(list []string, dst, secret string) error {
	if !f.Stat(path.Dir(dst)) {
		if err := f.Fs.MkdirAll(path.Dir(dst), constant.FilePerm); err != nil {
			return err
		}
	}

	var tarArgs []string
	for _, item := range list {
		tarArgs = append(tarArgs, "-C", path.Dir(item), path.Base(item))
	}
	if len(secret) != 0 {
		cmdMgr := cmd.NewCommandMgr(cmd.WithIgnoreExist1())
		return runTarGzEncryptToFile(cmdMgr, dst, secret, tarArgs...)
	} else {
		cmdMgr := cmd.NewCommandMgr(cmd.WithIgnoreExist1())
		return runTarGzToFile(cmdMgr, dst, tarArgs...)
	}
}

func (f FileOp) TarGzExtractPro(src, dst string, secret string) error {
	if _, err := os.Stat(path.Dir(dst)); err != nil && os.IsNotExist(err) {
		if err = os.MkdirAll(path.Dir(dst), os.ModePerm); err != nil {
			return err
		}
	}

	if len(secret) != 0 {
		cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(dst), cmd.WithIgnoreExist1())
		return runTarGzDecryptToDir(cmdMgr, src, dst, secret, true)
	} else {
		cmdMgr := cmd.NewCommandMgr(cmd.WithWorkDir(dst), cmd.WithIgnoreExist1())
		return runTarGzExtractToDir(cmdMgr, src, dst)
	}
}
func CopyCustomAppFile(srcPath, dstPath string) error {
	if _, err := os.Stat(srcPath); os.IsNotExist(err) {
		return fmt.Errorf("source file does not exist: %s", srcPath)
	}

	destDir := path.Dir(dstPath)
	if err := os.MkdirAll(destDir, 0755); err != nil {
		return fmt.Errorf("failed to create destination directory %s: %v", destDir, err)
	}

	source, err := os.Open(srcPath)
	if err != nil {
		return fmt.Errorf("failed to open source file %s: %v", srcPath, err)
	}
	defer source.Close()

	tempFile, err := os.CreateTemp(destDir, "temp_*.tar.gz")
	if err != nil {
		return fmt.Errorf("failed to create temporary file in %s: %v", destDir, err)
	}
	defer os.Remove(tempFile.Name())
	defer tempFile.Close()

	if _, err = io.Copy(tempFile, source); err != nil {
		return fmt.Errorf("failed to copy file contents: %v", err)
	}

	tempFile.Close()
	source.Close()

	if err = os.Rename(tempFile.Name(), dstPath); err != nil {
		return fmt.Errorf("failed to rename temporary file to %s: %v", dstPath, err)
	}
	return nil
}

func OpensslEncrypt(filePath, secret string) error {
	tmpName := path.Join(path.Dir(filePath), "tmp_"+path.Base(filePath))
	if err := cmd.NewCommandMgr(cmd.WithEnv("MY_PASS="+secret)).Run("openssl", "enc", "-aes-256-cbc", "-salt", "-pass", "env:MY_PASS", "-in", filePath, "-out", tmpName); err != nil {
		_ = os.Remove(tmpName)
		return err
	}
	return os.Rename(tmpName, filePath)
}

func OpensslDecrypt(filePath, secret string) error {
	tmpName := path.Join(path.Dir(filePath), "tmp_"+path.Base(filePath))
	if err := cmd.NewCommandMgr(cmd.WithEnv("MY_PASS="+secret)).Run("openssl", "enc", "-aes-256-cbc", "-d", "-salt", "-pass", "env:MY_PASS", "-in", filePath, "-out", tmpName); err != nil {
		if strings.Contains(err.Error(), "bad decrypt") || strings.Contains(err.Error(), "bad magic number") {
			return buserr.New("ErrBadDecrypt")
		}
		return err
	}
	return nil
}