Files
1Panel/agent/app/service/file.go

1973 lines
54 KiB
Go

package service
import (
"bufio"
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/fs"
"os"
"os/user"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"time"
"unicode/utf8"
"github.com/1Panel-dev/1Panel/agent/app/task"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/utils/convert"
"github.com/1Panel-dev/1Panel/agent/utils/ini_conf"
"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/jinzhu/copier"
"golang.org/x/text/encoding"
"golang.org/x/text/encoding/simplifiedchinese"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/app/dto/request"
"github.com/1Panel-dev/1Panel/agent/app/dto/response"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/1Panel-dev/1Panel/agent/constant"
"golang.org/x/net/html/charset"
"golang.org/x/sys/unix"
"golang.org/x/text/transform"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/common"
"github.com/1Panel-dev/1Panel/agent/utils/files"
terminalai "github.com/1Panel-dev/1Panel/agent/utils/terminal/ai"
"github.com/pkg/errors"
)
type FileService struct {
}
const fileHistorySnapshotMaxSize = 10 * 1024 * 1024
var fileTransferLocks = newFileTransferLocks()
type IFileService interface {
GetFileList(op request.FileOption) (response.FileInfo, error)
SearchUploadWithPage(req request.SearchUploadWithPage) (int64, interface{}, error)
GetFileTree(op request.FileOption) ([]response.FileTree, error)
Create(op request.FileCreate) error
Delete(op request.FileDelete) error
BatchDelete(op request.FileBatchDelete) error
Compress(c request.FileCompress) error
StopCompress(taskID string) error
DeCompress(c request.FileDeCompress) error
StopDeCompress(taskID string) error
GetContent(op request.FileContentReq) (response.FileInfo, error)
GetPreviewContent(op request.FileContentReq) (response.FileInfo, error)
SaveContent(edit request.FileEdit) error
FileDownload(d request.FileDownload) (string, error)
DirSize(req request.DirSizeReq) (response.DirSizeRes, error)
DepthDirSize(req request.DirSizeReq) ([]response.DepthDirSizeRes, error)
ChangeName(req request.FileRename) error
Wget(w request.FileWget) (string, error)
MvFile(m request.FileMove) error
StopMvFile(taskID string) error
ChangeOwner(req request.FileRoleUpdate) error
ChangeMode(op request.FileCreate) error
BatchChangeModeAndOwner(op request.FileRoleReq) error
ReadLogByLine(req request.FileReadByLineReq) (*response.FileLineContent, error)
BatchCheckFiles(req request.FilePathsCheck) []response.ExistFileInfo
GetHostMount() []dto.DiskInfo
GetUsersAndGroups() (*response.UserGroupResponse, error)
Convert(req request.FileConvertRequest)
ConvertLog(req dto.PageInfo) (int64, []response.FileConvertLog, error)
BatchGetRemarks(req request.FileRemarkBatch) map[string]string
SetRemark(req request.FileRemarkUpdate) error
AISearch(req request.FileAISearch) (*response.FileAISearchResult, error)
}
const (
fileRemarkXattr = "user.1panel.remark"
fileRemarkEncodedMaxLen = 256
)
func NewIFileService() IFileService {
return &FileService{}
}
func (f *FileService) GetFileList(op request.FileOption) (response.FileInfo, error) {
var fileInfo response.FileInfo
data, err := os.Stat(op.Path)
if err != nil && os.IsNotExist(err) {
return fileInfo, nil
}
if !data.IsDir() {
op.FileOption.Path = filepath.Dir(op.FileOption.Path)
}
info, err := files.NewFileInfo(op.FileOption)
if err != nil {
return fileInfo, err
}
shareMap, err := NewIFileShareService().SharePathCodeMap()
if err != nil {
return fileInfo, err
}
applyFileShares(info, shareMap)
fileInfo.FileInfo = *info
return fileInfo, nil
}
func applyFileShares(info *files.FileInfo, shareMap map[string]string) {
if info == nil {
return
}
if code, ok := shareMap[info.Path]; ok {
info.ShareCode = code
} else {
info.ShareCode = ""
}
for _, item := range info.Items {
applyFileShares(item, shareMap)
}
}
func (f *FileService) SearchUploadWithPage(req request.SearchUploadWithPage) (int64, interface{}, error) {
var (
files []response.UploadInfo
backData []response.UploadInfo
)
fileList, err := os.ReadDir(req.Path)
if err != nil {
return 0, files, nil
}
for _, item := range fileList {
if item.IsDir() {
continue
}
fileItem, err := item.Info()
if err != nil {
continue
}
files = append(files, response.UploadInfo{
CreatedAt: fileItem.ModTime().Format(constant.DateTimeLayout),
Size: int(fileItem.Size()),
Name: item.Name(),
})
}
sort.SliceStable(files, func(i, j int) bool {
return files[i].CreatedAt > files[j].CreatedAt
})
total, start, end := len(files), (req.Page-1)*req.PageSize, req.Page*req.PageSize
if start > total {
backData = make([]response.UploadInfo, 0)
} else {
if end >= total {
end = total
}
backData = files[start:end]
}
return int64(total), backData, nil
}
func (f *FileService) GetFileTree(op request.FileOption) ([]response.FileTree, error) {
var treeArray []response.FileTree
if _, err := os.Stat(op.Path); err != nil && os.IsNotExist(err) {
return treeArray, nil
}
info, err := files.NewFileInfo(op.FileOption)
if err != nil {
return nil, err
}
node := response.FileTree{
ID: common.GetUuid(),
Name: info.Name,
Path: info.Path,
IsDir: info.IsDir,
Extension: info.Extension,
}
err = f.buildFileTree(&node, info.Items, op, 2)
if err != nil {
return nil, err
}
return append(treeArray, node), nil
}
func shouldFilterPath(path string) bool {
return files.ShouldFilterSensitivePath(path)
}
func (f *FileService) buildFileTree(node *response.FileTree, items []*files.FileInfo, op request.FileOption, level int) error {
for _, v := range items {
if shouldFilterPath(v.Path) {
global.LOG.Infof("File Tree: Skipping %s due to filter\n", v.Path)
continue
}
childNode := response.FileTree{
ID: common.GetUuid(),
Name: v.Name,
Path: v.Path,
IsDir: v.IsDir,
Extension: v.Extension,
}
if level > 1 && v.IsDir {
if err := f.buildChildNode(&childNode, v, op, level); err != nil {
return err
}
}
node.Children = append(node.Children, childNode)
}
return nil
}
func (f *FileService) buildChildNode(childNode *response.FileTree, fileInfo *files.FileInfo, op request.FileOption, level int) error {
op.Path = fileInfo.Path
subInfo, err := files.NewFileInfo(op.FileOption)
if err != nil {
if os.IsPermission(err) || errors.Is(err, unix.EACCES) {
global.LOG.Infof("File Tree: Skipping %s due to permission denied\n", fileInfo.Path)
return nil
}
global.LOG.Errorf("File Tree: Skipping %s due to error: %s\n", fileInfo.Path, err.Error())
return nil
}
return f.buildFileTree(childNode, subInfo.Items, op, level-1)
}
func hasInvalidFileName(fullPath string) bool {
return files.IsInvalidChar(filepath.Base(fullPath))
}
func (f *FileService) Create(op request.FileCreate) error {
if hasInvalidFileName(op.Path) {
return buserr.New("ErrInvalidChar")
}
fo := files.NewFileOp()
if fo.Stat(op.Path) {
return buserr.New("ErrFileIsExist")
}
mode := op.Mode
if mode == 0 {
fileInfo, err := os.Stat(filepath.Dir(op.Path))
if err == nil {
mode = int64(fileInfo.Mode().Perm())
} else {
mode = constant.DirPerm
}
}
if op.IsDir {
if err := fo.CreateDirWithMode(op.Path, fs.FileMode(mode)); err != nil {
return err
}
handleDefaultOwn(op.Path)
return nil
}
if op.IsLink {
if !fo.Stat(op.LinkPath) {
return buserr.New("ErrLinkPathNotFound")
}
if err := fo.LinkFile(op.LinkPath, op.Path, op.IsSymlink); err != nil {
return err
}
handleDefaultOwn(op.Path)
return nil
}
if err := fo.CreateFileWithMode(op.Path, fs.FileMode(mode)); err != nil {
return err
}
handleDefaultOwn(op.Path)
return nil
}
func (f *FileService) Delete(op request.FileDelete) error {
if err := files.CheckFileDeletePaths(op.Path); err != nil {
return err
}
fo := files.NewFileOp()
recycleBinStatus, _ := settingRepo.Get(settingRepo.WithByKey("FileRecycleBin"))
if recycleBinStatus.Value == "Disable" {
op.ForceDelete = true
}
var historyTargets []string
if op.ForceDelete {
var err error
historyTargets, err = f.collectPermanentDeleteTargets(op.Path, op.IsDir)
if err != nil {
return err
}
}
if op.ForceDelete {
if err := files.CheckFileDeletePaths(op.Path); err != nil {
return err
}
var err error
if op.IsDir {
err = fo.DeleteDir(op.Path)
} else {
err = fo.DeleteFile(op.Path)
}
if err != nil {
return err
}
if err := cleanupTrashInfoByEntryPath(op.Path); err != nil {
global.LOG.Warnf("cleanup trashinfo failed for %s: %v", op.Path, err)
}
f.cleanupPermanentDeleteHistory(historyTargets)
return nil
}
info, _ := fo.Fs.Stat(op.Path)
if info == nil || files.IsSymlink(info.Mode()) {
if err := files.CheckFileDeletePaths(op.Path); err != nil {
return err
}
if err := os.Remove(op.Path); err != nil {
return err
}
f.cleanupPermanentDeleteHistory([]string{op.Path})
return nil
}
if err := files.CheckFileDeletePaths(op.Path); err != nil {
return err
}
if err := NewIRecycleBinService().Create(request.RecycleBinCreate{SourcePath: op.Path}); err != nil {
return err
}
return favoriteRepo.Delete(favoriteRepo.WithByPath(op.Path))
}
func (f *FileService) BatchDelete(op request.FileBatchDelete) error {
plans, err := f.prepareBatchFileDelete(op)
if err != nil {
return err
}
fo := files.NewFileOp()
for _, plan := range plans {
if err := files.CheckFileDeletePaths(plan.path); err != nil {
return err
}
if op.IsDir {
err = fo.DeleteDir(plan.path)
} else {
err = fo.DeleteFile(plan.path)
}
if err != nil {
return err
}
if err := cleanupTrashInfoByEntryPath(plan.path); err != nil {
global.LOG.Warnf("cleanup trashinfo failed for %s: %v", plan.path, err)
}
f.cleanupPermanentDeleteHistory(plan.historyTargets)
}
return nil
}
type batchFileDeletePlan struct {
path string
historyTargets []string
absolute string
physical string
info os.FileInfo
}
func (f *FileService) prepareBatchFileDelete(op request.FileBatchDelete) ([]batchFileDeletePlan, error) {
if err := files.CheckFileDeletePaths(op.Paths...); err != nil {
return nil, err
}
plans := make([]batchFileDeletePlan, 0, len(op.Paths))
for _, target := range op.Paths {
info, err := os.Lstat(target)
if err != nil {
return nil, err
}
absolute, err := filepath.Abs(target)
if err != nil {
return nil, err
}
physicalInput := target
if !filepath.IsAbs(physicalInput) {
cwd, err := os.Getwd()
if err != nil {
return nil, err
}
physicalInput = cwd + string(filepath.Separator) + physicalInput
}
physical, resolveErr := filepath.EvalSymlinks(physicalInput)
if resolveErr != nil && !os.IsNotExist(resolveErr) {
return nil, resolveErr
}
if resolveErr == nil {
physical, err = filepath.Abs(physical)
if err != nil {
return nil, err
}
}
plan := batchFileDeletePlan{path: target, absolute: absolute, physical: physical, info: info}
for _, previous := range plans {
if fileDeletePlansOverlap(previous, plan) {
return nil, buserr.New("ErrInvalidParams")
}
}
// Keep isDir=false as Remove, rather than silently upgrading to recursion.
if !op.IsDir && info.IsDir() {
entries, err := os.ReadDir(target)
if err != nil {
return nil, err
}
if len(entries) > 0 {
return nil, &os.PathError{Op: "remove", Path: target, Err: syscall.ENOTEMPTY}
}
}
historyTargets, err := f.collectPermanentDeleteTargets(target, op.IsDir)
if err != nil {
return nil, err
}
plan.historyTargets = historyTargets
plans = append(plans, plan)
}
return plans, nil
}
func fileDeletePlansOverlap(a, b batchFileDeletePlan) bool {
contains := func(parent, child string) bool {
if parent == "" || child == "" {
return false
}
relative, err := filepath.Rel(parent, child)
return err == nil && relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}
if contains(a.absolute, b.absolute) || contains(b.absolute, a.absolute) {
return true
}
if a.info.IsDir() && contains(a.physical, b.physical) || b.info.IsDir() && contains(b.physical, a.physical) {
return true
}
if os.SameFile(a.info, b.info) && strings.EqualFold(filepath.Base(a.absolute), filepath.Base(b.absolute)) {
parentA, errA := os.Stat(filepath.Dir(a.absolute))
parentB, errB := os.Stat(filepath.Dir(b.absolute))
return errA == nil && errB == nil && os.SameFile(parentA, parentB)
}
return false
}
func (f *FileService) ChangeMode(op request.FileCreate) error {
fo := files.NewFileOp()
if err := fo.ChmodR(op.Path, op.Mode, op.Sub); err != nil {
return err
}
return nil
}
func (f *FileService) BatchChangeModeAndOwner(op request.FileRoleReq) error {
fo := files.NewFileOp()
for _, p := range op.Paths {
if !fo.Stat(p) {
return buserr.New("ErrPathNotFound")
}
}
if err := fo.ChownRPaths(op.Paths, op.User, op.Group, op.Sub); err != nil {
return err
}
if err := fo.ChmodRPaths(op.Paths, op.Mode, op.Sub); err != nil {
return err
}
return nil
}
func (f *FileService) collectPermanentDeleteTargets(targetPath string, isDir bool) ([]string, error) {
if !isDir {
return []string{targetPath}, nil
}
var targets []string
if err := filepath.WalkDir(targetPath, func(currentPath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d == nil || d.IsDir() {
return nil
}
targets = append(targets, currentPath)
return nil
}); err != nil {
return nil, err
}
return targets, nil
}
func (f *FileService) cleanupPermanentDeleteHistory(targets []string) {
if len(targets) == 0 {
return
}
for _, target := range targets {
if err := historyService.DeleteRelatedHistory(target); err != nil {
global.LOG.Warnf("cleanup file history failed for %s: %v", target, err)
}
}
}
func (f *FileService) ChangeOwner(req request.FileRoleUpdate) error {
fo := files.NewFileOp()
if err := fo.ChownR(req.Path, req.User, req.Group, req.Sub); err != nil {
return err
}
return nil
}
func (f *FileService) Compress(c request.FileCompress) error {
fo := files.NewFileOp()
if !c.Replace && fo.Stat(filepath.Join(c.Dst, c.Name)) {
return buserr.New("ErrFileIsExist")
}
if err := preflightCompressTool(files.CompressType(c.Type)); err != nil {
return err
}
taskName := i18n.GetMsgWithMap("FileTaskCompress", map[string]interface{}{"dst": strconv.Quote(filepath.Join(c.Dst, c.Name))})
taskItem, err := task.NewTask(taskName, task.TaskExec, task.TaskScopeTask, c.TaskID, 1)
if err != nil {
return err
}
go func() {
taskItem.AddSubTask(taskName, func(t *task.Task) error {
logFileTaskSources(t, c.Files)
t.Log(i18n.GetMsgWithMap("FileTaskFormat", map[string]interface{}{"format": strconv.Quote(c.Type)}))
compressType := files.CompressType(c.Type)
dstFile := filepath.Join(c.Dst, c.Name)
success := false
defer func() {
if !success {
_ = os.Remove(dstFile)
}
}()
if err := fo.Compress(t.TaskCtx, c.Files, c.Dst, c.Name, compressType, c.Secret, nil); err != nil {
return err
}
info, err := os.Stat(dstFile)
if err != nil {
return err
}
if info.Size() == 0 {
return fmt.Errorf("compressed file not generated: %s", dstFile)
}
success = true
return nil
}, nil)
_ = taskItem.Execute()
}()
return nil
}
func preflightCompressTool(compressType files.CompressType) error {
switch compressType {
case files.Tar, files.Gz, files.Bz2, files.TarBz2, files.Tgz, files.TarGz, files.Xz, files.TarXz, files.Rar, files.X7z:
_, err := files.NewShellArchiver(compressType)
return err
default:
return nil
}
}
func preflightDecompressTool(decompressType files.CompressType) error {
switch decompressType {
case files.Rar:
_, err := files.NewExtractShellArchiver(decompressType)
return err
default:
return nil
}
}
func (f *FileService) StopCompress(taskID string) error {
if cancel, ok := global.LoadTaskCancel(taskID); ok {
cancel()
return nil
}
return buserr.New("TaskNotFound")
}
func (f *FileService) StopDeCompress(taskID string) error {
if cancel, ok := global.LoadTaskCancel(taskID); ok {
cancel()
return nil
}
return buserr.New("TaskNotFound")
}
func (f *FileService) DeCompress(c request.FileDeCompress) error {
fo := files.NewFileOp()
if c.Type == "tar" && len(c.Secret) != 0 {
c.Type = "tar.gz"
}
if err := preflightDecompressTool(files.CompressType(c.Type)); err != nil {
return err
}
taskName := i18n.GetMsgWithMap("FileTaskDecompress", map[string]interface{}{"dst": strconv.Quote(c.Dst)})
taskItem, err := task.NewTask(taskName, task.TaskExec, task.TaskScopeTask, c.TaskID, 1)
if err != nil {
return err
}
go func() {
taskItem.AddSubTask(taskName, func(t *task.Task) error {
logFileTaskSources(t, []string{c.Path})
t.Log(i18n.GetMsgWithMap("FileTaskFormat", map[string]interface{}{"format": strconv.Quote(c.Type)}))
dstExisted := fo.Stat(c.Dst)
parentDir := filepath.Dir(c.Dst)
if !fo.Stat(parentDir) {
if err := fo.CreateDir(parentDir, constant.DirPerm); err != nil {
return err
}
}
tempDst, err := os.MkdirTemp(parentDir, ".decompress-*")
if err != nil {
return err
}
success := false
defer func() {
_ = os.RemoveAll(tempDst)
if !success && !dstExisted {
_ = os.RemoveAll(c.Dst)
}
}()
if err := fo.DecompressWithOptions(t.TaskCtx, c.Path, tempDst, files.CompressType(c.Type), c.Secret, files.DecompressOptions{
PreserveOwner: true,
AllowCLIReextract: true,
}); err != nil {
return err
}
if err := fo.CreateDir(c.Dst, constant.DirPerm); err != nil {
return err
}
if err := copyDecompressTree(t.TaskCtx, tempDst, c.Dst); err != nil {
return err
}
success = true
return nil
}, nil)
_ = taskItem.Execute()
}()
return nil
}
func copyDecompressTree(ctx context.Context, srcDir, dstDir string) error {
state := decompressCopyState{hardlinks: make(map[decompressFileIdentity]string)}
entries, err := os.ReadDir(srcDir)
if err != nil {
return err
}
for _, entry := range entries {
if err := copyDecompressEntryWithState(ctx, filepath.Join(srcDir, entry.Name()), filepath.Join(dstDir, entry.Name()), &state); err != nil {
return err
}
}
return nil
}
type decompressFileIdentity struct {
device uint64
inode uint64
}
type decompressCopyState struct {
hardlinks map[decompressFileIdentity]string
}
func decompressHardlinkIdentity(info os.FileInfo) (decompressFileIdentity, bool) {
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok || stat.Nlink < 2 {
return decompressFileIdentity{}, false
}
return decompressFileIdentity{device: uint64(stat.Dev), inode: uint64(stat.Ino)}, true
}
func copyDecompressEntry(ctx context.Context, srcPath, dstPath string) (retErr error) {
state := decompressCopyState{hardlinks: make(map[decompressFileIdentity]string)}
return copyDecompressEntryWithState(ctx, srcPath, dstPath, &state)
}
func copyDecompressEntryWithState(ctx context.Context, srcPath, dstPath string, state *decompressCopyState) (retErr error) {
if err := ctx.Err(); err != nil {
return err
}
info, err := os.Lstat(srcPath)
if err != nil {
return err
}
if info.Mode()&os.ModeSymlink != 0 {
if err := os.RemoveAll(dstPath); err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(dstPath), constant.DirPerm); err != nil {
return err
}
target, err := os.Readlink(srcPath)
if err != nil {
return err
}
if err := os.Symlink(target, dstPath); err != nil {
return err
}
return applyDecompressOwnership(srcPath, dstPath)
}
if info.IsDir() {
dstInfo, err := os.Lstat(dstPath)
keepExistingDir := err == nil && dstInfo.IsDir() && dstInfo.Mode()&os.ModeSymlink == 0
if err != nil && !os.IsNotExist(err) {
return err
}
if !keepExistingDir {
if err := os.RemoveAll(dstPath); err != nil {
return err
}
if err := os.MkdirAll(dstPath, info.Mode().Perm()); err != nil {
return err
}
if err := applyDecompressOwnership(srcPath, dstPath); err != nil {
return err
}
if err := os.Chmod(dstPath, info.Mode().Perm()); err != nil {
return err
}
}
entries, err := os.ReadDir(srcPath)
if err != nil {
return err
}
for _, entry := range entries {
if err := copyDecompressEntryWithState(ctx, filepath.Join(srcPath, entry.Name()), filepath.Join(dstPath, entry.Name()), state); err != nil {
return err
}
}
if keepExistingDir {
return nil
}
return os.Chtimes(dstPath, info.ModTime(), info.ModTime())
}
dstInfo, err := os.Lstat(dstPath)
keepExistingFile := err == nil && dstInfo.Mode().IsRegular()
if err != nil && !os.IsNotExist(err) {
return err
}
if !keepExistingFile {
if err := os.RemoveAll(dstPath); err != nil {
return err
}
}
if err := os.MkdirAll(filepath.Dir(dstPath), constant.DirPerm); err != nil {
return err
}
identity, isHardlink := decompressHardlinkIdentity(info)
if !keepExistingFile && isHardlink {
if existingPath, ok := state.hardlinks[identity]; ok {
if err := os.Link(existingPath, dstPath); err != nil {
return err
}
return os.Chtimes(dstPath, info.ModTime(), info.ModTime())
}
}
srcFile, err := os.Open(srcPath)
if err != nil {
return err
}
defer srcFile.Close()
dstFile, err := os.OpenFile(dstPath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, info.Mode().Perm())
if err != nil {
return err
}
defer func() {
if cerr := dstFile.Close(); cerr != nil && retErr == nil {
retErr = cerr
}
}()
if _, err := io.Copy(dstFile, srcFile); err != nil {
return err
}
if !keepExistingFile {
if err := applyDecompressOwnership(srcPath, dstPath); err != nil {
return err
}
if err := os.Chmod(dstPath, info.Mode().Perm()); err != nil {
return err
}
if isHardlink {
state.hardlinks[identity] = dstPath
}
}
return os.Chtimes(dstPath, info.ModTime(), info.ModTime())
}
func applyDecompressOwnership(srcPath, dstPath string) error {
info, err := os.Lstat(srcPath)
if err != nil {
return err
}
stat, ok := info.Sys().(*syscall.Stat_t)
if !ok {
return nil
}
if info.Mode()&os.ModeSymlink != 0 {
return os.Lchown(dstPath, int(stat.Uid), int(stat.Gid))
}
return os.Chown(dstPath, int(stat.Uid), int(stat.Gid))
}
func (f *FileService) GetContent(op request.FileContentReq) (response.FileInfo, error) {
if files.ShouldDenySensitiveFileRead(op.Path) {
return response.FileInfo{}, buserr.New("ErrSensitiveFileRead")
}
info, err := files.NewFileInfo(files.FileOption{
Path: op.Path,
Expand: true,
IsDetail: op.IsDetail,
})
if err != nil {
return response.FileInfo{}, err
}
content := []byte(info.Content)
if len(content) > 1024 {
content = content[:1024]
}
if !utf8.Valid(content) {
_, decodeName, _ := charset.DetermineEncoding(content, "")
decoder := files.GetDecoderByName(decodeName)
if decoder != nil {
reader := strings.NewReader(info.Content)
var dec *encoding.Decoder
if decodeName == "windows-1252" {
dec = simplifiedchinese.GBK.NewDecoder()
} else {
dec = decoder.NewDecoder()
}
decodedReader := transform.NewReader(reader, dec)
contents, err := io.ReadAll(decodedReader)
if err != nil {
return response.FileInfo{}, err
}
info.Content = string(contents)
}
}
return response.FileInfo{FileInfo: *info}, nil
}
func (f *FileService) GetPreviewContent(op request.FileContentReq) (response.FileInfo, error) {
if files.ShouldDenySensitiveFileRead(op.Path) {
return response.FileInfo{}, buserr.New("ErrSensitiveFileRead")
}
info, err := files.NewFileInfo(files.FileOption{
Path: op.Path,
Expand: false,
IsDetail: op.IsDetail,
})
if err != nil {
return response.FileInfo{}, err
}
if files.IsBinaryPreviewFile(info.MimeType, info.Extension) {
return response.FileInfo{}, buserr.New("ErrFileCanNotRead")
}
if files.IsBlockDevice(info.FileMode) {
return response.FileInfo{FileInfo: *info}, nil
}
file, err := os.Open(op.Path)
if err != nil {
return response.FileInfo{}, err
}
defer file.Close()
headBuf := make([]byte, 1024)
n, err := file.Read(headBuf)
if err != nil && err != io.EOF {
return response.FileInfo{}, err
}
headBuf = headBuf[:n]
if len(headBuf) > 0 && files.DetectBinary(headBuf) {
return response.FileInfo{FileInfo: *info}, nil
}
const maxSize = 10 * 1024 * 1024
if info.Size <= maxSize {
if _, err := file.Seek(0, 0); err != nil {
return response.FileInfo{}, err
}
content, err := io.ReadAll(file)
if err != nil {
return response.FileInfo{}, err
}
info.Content = string(content)
} else {
lines, err := files.TailFromEnd(op.Path, 300)
if err != nil {
return response.FileInfo{}, err
}
info.Content = strings.Join(lines, "\n")
}
content := []byte(info.Content)
if len(content) > 1024 {
content = content[:1024]
}
if !utf8.Valid(content) {
_, decodeName, _ := charset.DetermineEncoding(content, "")
decoder := files.GetDecoderByName(decodeName)
if decoder != nil {
reader := strings.NewReader(info.Content)
var dec *encoding.Decoder
if decodeName == "windows-1252" {
dec = simplifiedchinese.GBK.NewDecoder()
} else {
dec = decoder.NewDecoder()
}
decodedReader := transform.NewReader(reader, dec)
contents, err := io.ReadAll(decodedReader)
if err != nil {
return response.FileInfo{}, err
}
info.Content = string(contents)
}
}
return response.FileInfo{FileInfo: *info}, nil
}
func (f *FileService) SaveContent(edit request.FileEdit) error {
info, err := files.NewFileInfo(files.FileOption{
Path: edit.Path,
Expand: false,
})
if err != nil {
return err
}
fo := files.NewFileOp()
oldContent, _ := os.ReadFile(edit.Path)
if bytes.Equal(oldContent, []byte(edit.Content)) {
return nil
}
if err := fo.WriteFile(edit.Path, strings.NewReader(edit.Content), info.FileMode); err != nil {
return err
}
if err := historyService.RecordSave(edit.Path, oldContent, info.FileMode); err != nil {
global.LOG.Warnf("record file save history failed for %s: %v", edit.Path, err)
}
return nil
}
func (f *FileService) ChangeName(req request.FileRename) error {
if err := files.CheckFileDeletePaths(req.OldName, req.NewName); err != nil {
return err
}
if hasInvalidFileName(req.NewName) {
return buserr.New("ErrInvalidChar")
}
if req.OldName == req.NewName {
return nil
}
// os.Rename silently replaces an existing target on Linux, so reject the
// rename here to avoid losing the target file without confirmation.
if _, err := os.Lstat(req.NewName); err == nil {
return buserr.New("ErrFileIsExist")
} else if !os.IsNotExist(err) {
return err
}
fo := files.NewFileOp()
content, mode, shouldRecordHistory := readEditableFileHistoryContent(req.OldName)
if err := files.CheckFileDeletePaths(req.OldName, req.NewName); err != nil {
return err
}
if err := fo.Rename(req.OldName, req.NewName); err != nil {
return err
}
if shouldRecordHistory {
if histErr := historyService.RecordOperation(fileHistoryOpRename, req.OldName, content, mode, req.OldName, req.NewName); histErr != nil {
global.LOG.Warnf("record file rename history failed for %s: %v", req.OldName, histErr)
}
}
return nil
}
func (f *FileService) Wget(w request.FileWget) (string, error) {
fo := files.NewFileOp()
destination, err := files.ResolveDownloadDestination(w.Path, w.Name)
if err != nil {
return "", err
}
key := "file-wget-" + common.GetUuid()
options := files.DownloadOptions{
IgnoreCertificate: w.IgnoreCertificate,
UseServerFilename: w.UseServerFilename,
}
if w.UseProxy {
systemProxy, err := NewISettingService().GetSystemProxy()
if err != nil {
return "", err
}
options.Proxy = &files.DownloadProxyConfig{
Type: systemProxy.Type,
URL: systemProxy.URL,
Port: systemProxy.Port,
User: systemProxy.User,
Password: systemProxy.Password,
}
}
return key, fo.DownloadFileWithProcess(w.Url, destination, key, options)
}
func logFileTaskSources(t *task.Task, sources []string) {
for _, source := range sources {
t.Log(i18n.GetMsgWithMap("FileTaskSource", map[string]interface{}{"path": strconv.Quote(source)}))
}
}
func (f *FileService) MvFile(m request.FileMove) error {
fo := files.NewFileOp()
if err := validateFileMove(fo, m); err != nil {
return err
}
if m.TaskID == "" {
m.TaskID = common.GetUuid()
}
if !fileTransferLocks.Acquire(m.TaskID, getFileTransferPaths(m)) {
return buserr.New("TaskIsExecuting")
}
nameKey := "FileTaskCopy"
if m.Type == "cut" {
nameKey = "FileTaskMove"
}
taskName := i18n.GetMsgWithMap(nameKey, map[string]interface{}{"dst": strconv.Quote(m.NewPath)})
taskItem, err := task.NewTask(taskName, task.TaskExec, task.TaskScopeTask, m.TaskID, 1)
if err != nil {
fileTransferLocks.Release(m.TaskID)
return err
}
go func() {
defer fileTransferLocks.Release(m.TaskID)
taskItem.AddSubTaskWithOps(taskName, func(t *task.Task) error {
logFileTaskSources(t, m.OldPaths)
logFileTaskSources(t, m.CoverPaths)
if m.Name != "" {
t.Log(i18n.GetMsgWithMap("FileTaskRename", map[string]interface{}{"name": strconv.Quote(m.Name)}))
}
err := f.moveFileWithContext(t.TaskCtx, m)
if err != nil && t.TaskCtx.Err() != nil {
return t.TaskCtx.Err()
}
return err
}, nil, 0, 0)
_ = taskItem.Execute()
}()
return nil
}
func (f *FileService) StopMvFile(taskID string) error {
if cancel, ok := global.LoadTaskCancel(taskID); ok {
cancel()
return nil
}
return buserr.New("TaskNotFound")
}
func validateFileMove(fo files.FileOp, m request.FileMove) error {
if err := checkFileMoveDeletePaths(m); err != nil {
return err
}
if !fo.Stat(m.NewPath) {
return buserr.New("ErrPathNotFound")
}
for _, oldPath := range append(append([]string{}, m.OldPaths...), m.CoverPaths...) {
if !fo.Stat(oldPath) {
return buserr.WithName("ErrFileNotFound", oldPath)
}
oldPath = filepath.Clean(oldPath)
newPath := filepath.Clean(m.NewPath)
if oldPath == newPath || strings.HasPrefix(newPath, oldPath+string(filepath.Separator)) {
return buserr.New("ErrMovePathFailed")
}
}
return nil
}
func (f *FileService) moveFileWithContext(ctx context.Context, m request.FileMove) error {
if err := checkFileMoveDeletePaths(m); err != nil {
return err
}
fo := files.NewFileOp()
type moveSnapshot struct {
path string
content []byte
mode os.FileMode
record bool
}
var errs []error
if m.Type == "cut" {
snapshots := make([]moveSnapshot, 0, len(m.OldPaths))
for _, oldPath := range m.OldPaths {
content, mode, record := readEditableFileHistoryContent(oldPath)
snapshots = append(snapshots, moveSnapshot{path: oldPath, content: content, mode: mode, record: record})
}
if len(m.CoverPaths) > 0 {
for _, src := range m.CoverPaths {
if err := ctx.Err(); err != nil {
return err
}
if err := checkFileMoveDeletePaths(request.FileMove{Type: "cut", CoverPaths: []string{src}, NewPath: m.NewPath}); err != nil {
return err
}
if err := fo.CopyAndReNameWithContext(ctx, src, m.NewPath, "", true); err != nil {
errs = append(errs, err)
global.LOG.Errorf("cut copy file [%s] to [%s] failed, err: %s", src, m.NewPath, err.Error())
continue
}
if err := ctx.Err(); err != nil {
return err
}
if err := files.CheckFileDeletePaths(src); err != nil {
return err
}
if err := fo.DeleteDir(src); err != nil {
removeErr := fmt.Errorf("remove merged source [%s] failed: %w", src, err)
errs = append(errs, removeErr)
global.LOG.Errorf("%s", removeErr.Error())
}
}
}
cutSources, cutDestination, err := absoluteFileCutOperands(m.OldPaths, m.NewPath)
if err != nil {
return err
}
if err := checkFileMoveDeletePaths(request.FileMove{Type: "cut", OldPaths: cutSources, NewPath: cutDestination, Name: m.Name, Cover: m.Cover}); err != nil {
return err
}
if err := fo.CutWithContext(ctx, cutSources, cutDestination, m.Name, m.Cover); err != nil {
return err
}
for _, snapshot := range snapshots {
if snapshot.record {
targetPath := buildHistoryMoveTargetPath(m.NewPath, m.Name, snapshot.path, len(m.OldPaths))
if histErr := historyService.RecordOperation(fileHistoryOpMove, snapshot.path, snapshot.content, snapshot.mode, snapshot.path, targetPath); histErr != nil {
global.LOG.Warnf("record file move history failed for %s: %v", snapshot.path, histErr)
}
}
}
return aggregateFileMoveErrors(errs)
}
if m.Type == "copy" {
for _, src := range m.OldPaths {
if err := checkFileMoveDeletePaths(request.FileMove{Type: "copy", OldPaths: []string{src}, NewPath: m.NewPath, Name: m.Name, Cover: m.Cover}); err != nil {
return err
}
if err := fo.CopyAndReNameWithContext(ctx, src, m.NewPath, m.Name, m.Cover); err != nil {
errs = append(errs, err)
global.LOG.Errorf("copy file [%s] to [%s] failed, err: %s", src, m.NewPath, err.Error())
}
}
if len(m.CoverPaths) > 0 {
for _, src := range m.CoverPaths {
if err := checkFileMoveDeletePaths(request.FileMove{Type: "copy", CoverPaths: []string{src}, NewPath: m.NewPath}); err != nil {
return err
}
if err := fo.CopyAndReNameWithContext(ctx, src, m.NewPath, "", true); err != nil {
errs = append(errs, err)
global.LOG.Errorf("copy file [%s] to [%s] failed, err: %s", src, m.NewPath, err.Error())
}
}
}
}
return aggregateFileMoveErrors(errs)
}
func absoluteFileCutOperands(sources []string, destination string) ([]string, string, error) {
cwd, err := os.Getwd()
if err != nil {
return nil, "", err
}
absolute := func(value string) string {
if filepath.IsAbs(value) {
return value
}
return cwd + string(filepath.Separator) + value
}
result := make([]string, len(sources))
for i, source := range sources {
result[i] = absolute(source)
}
return result, absolute(destination), nil
}
func checkFileMoveDeletePaths(m request.FileMove) error {
if m.Type != "cut" && m.Type != "copy" {
return nil
}
if m.Type == "cut" {
if err := files.CheckFileDeletePaths(append(append([]string{}, m.OldPaths...), m.CoverPaths...)...); err != nil {
return err
}
}
var destinations []string
commandTarget := func(operand, source string) (string, error) {
info, err := os.Stat(operand)
if err != nil && !os.IsNotExist(err) {
return "", err
}
if err == nil && info.IsDir() {
return operand + string(filepath.Separator) + filepath.Base(filepath.Clean(source)), nil
}
return operand, nil
}
copyTarget := func(source, name string, cover bool) (string, error) {
source = path.Clean("/" + source)
destination := path.Clean("/" + m.NewPath)
info, err := os.Stat(source)
if err != nil {
return "", err
}
operand := destination
if info.IsDir() {
if name != "" && !cover {
operand = filepath.Join(destination, name)
}
} else {
operand = filepath.Join(destination, name)
if cover {
operand = destination
}
}
return commandTarget(operand, source)
}
if m.Type == "cut" && len(m.OldPaths) > 0 {
operand := m.NewPath
if m.Name != "" {
operand = filepath.Join(m.NewPath, m.Name)
if _, err := os.Stat(operand); err == nil {
operand = m.NewPath
} else if !os.IsNotExist(err) {
return err
}
}
for _, source := range m.OldPaths {
target, err := commandTarget(operand, source)
if err != nil {
return err
}
destinations = append(destinations, target)
}
} else if m.Type == "copy" {
for _, source := range m.OldPaths {
target, err := copyTarget(source, m.Name, m.Cover)
if err != nil {
return err
}
destinations = append(destinations, target)
}
}
for _, source := range m.CoverPaths {
target, err := copyTarget(source, "", true)
if err != nil {
return err
}
destinations = append(destinations, target)
}
return files.CheckFileDeletePaths(destinations...)
}
func readEditableFileHistoryContent(filePath string) ([]byte, os.FileMode, bool) {
info, err := os.Lstat(filePath)
if err != nil {
return nil, 0640, false
}
mode := info.Mode()
if mode.IsDir() || mode&os.ModeSymlink != 0 || !mode.IsRegular() || files.IsBlockDevice(mode) || info.Size() > fileHistorySnapshotMaxSize {
return nil, mode, false
}
if files.IsBinaryPreviewFile(files.GetMimeType(filePath), filepath.Ext(info.Name())) {
return nil, mode, false
}
file, err := os.Open(filePath)
if err != nil {
return nil, mode, false
}
defer file.Close()
headBuf := make([]byte, 1024)
n, err := file.Read(headBuf)
if err != nil && err != io.EOF {
return nil, mode, false
}
if n > 0 && files.DetectBinary(headBuf[:n]) {
return nil, mode, false
}
if _, err := file.Seek(0, 0); err != nil {
return nil, mode, false
}
content, err := io.ReadAll(io.LimitReader(file, fileHistorySnapshotMaxSize+1))
if err != nil || int64(len(content)) > fileHistorySnapshotMaxSize {
return nil, mode, false
}
return content, mode, true
}
func buildHistoryMoveTargetPath(dst, name, sourcePath string, sourceCount int) string {
if strings.TrimSpace(dst) == "" {
return sourcePath
}
if strings.TrimSpace(name) != "" && sourceCount == 1 {
return filepath.Join(dst, name)
}
return filepath.Join(dst, filepath.Base(sourcePath))
}
func (f *FileService) FileDownload(d request.FileDownload) (string, error) {
for _, p := range d.Paths {
if files.ShouldDenySensitiveFileRead(p) {
return "", buserr.New("ErrSensitiveFileRead")
}
}
filePath := d.Paths[0]
if d.Compress {
tempPath := filepath.Join(os.TempDir(), fmt.Sprintf("%d", time.Now().UnixNano()))
if err := os.MkdirAll(tempPath, os.ModePerm); err != nil {
return "", err
}
fo := files.NewFileOp()
if err := fo.Compress(context.Background(), d.Paths, tempPath, d.Name, files.CompressType(d.Type), "", nil); err != nil {
return "", err
}
filePath = filepath.Join(tempPath, d.Name)
}
return filePath, nil
}
func (f *FileService) DirSize(req request.DirSizeReq) (response.DirSizeRes, error) {
var (
res response.DirSizeRes
)
if req.Path == "/proc" {
return res, nil
}
fo := files.NewFileOp()
size, err := fo.GetDirSize(req.Path)
if err != nil {
return res, err
}
res.Size = size
return res, nil
}
func (f *FileService) DepthDirSize(req request.DirSizeReq) ([]response.DepthDirSizeRes, error) {
var (
res []response.DepthDirSizeRes
)
if req.Path == "/proc" {
return res, nil
}
fo := files.NewFileOp()
dirSizes, err := fo.GetDepthDirSize(req.Path)
_ = copier.Copy(&res, &dirSizes)
if err != nil {
return res, err
}
return res, nil
}
func (f *FileService) ReadLogByLine(req request.FileReadByLineReq) (*response.FileLineContent, error) {
logFilePath := ""
taskStatus := ""
if len(req.Name) != 0 {
safeName := path.Base(req.Name)
if safeName != req.Name || strings.Contains(safeName, "..") {
return nil, buserr.New("ErrInvalidParams")
}
}
switch req.Type {
case constant.TypeWebsite:
website, err := websiteRepo.GetFirst(repo.WithByID(req.ID))
if err != nil {
return nil, err
}
logFilePath = GetSitePath(website, req.Name)
case constant.TypePhp:
php, err := runtimeRepo.GetFirst(context.Background(), repo.WithByID(req.ID))
if err != nil {
return nil, err
}
logFilePath = php.GetLogPath()
case constant.TypeSSL:
ssl, err := websiteSSLRepo.GetFirst(repo.WithByID(req.ID))
if err != nil {
return nil, err
}
logFilePath = ssl.GetLogPath()
case constant.TypeSystem:
fileName := ""
if len(req.Name) == 0 {
fileName = "1Panel.log"
} else {
if strings.HasSuffix(req.Name, time.Now().Format("2006-01-02")) {
fileName = "1Panel.log"
if strings.HasPrefix(req.Name, "Core-") {
fileName = "1Panel-Core.log"
}
} else {
fileName = "1Panel-" + req.Name + ".log"
}
}
logFilePath = path.Join(global.Dir.DataDir, "log", fileName)
if _, err := os.Stat(logFilePath); err != nil {
fileGzPath := path.Join(global.Dir.DataDir, "log", fileName+".gz")
if _, err := os.Stat(fileGzPath); err != nil {
return nil, buserr.New("ErrHttpReqNotFound")
}
if err := handleGunzip(fileGzPath); err != nil {
return nil, fmt.Errorf("handle ungzip file %s failed, err: %v", fileGzPath, err)
}
}
case constant.TypeTask:
var opts []repo.DBOption
if req.TaskID != "" {
opts = append(opts, taskRepo.WithByID(req.TaskID))
} else {
opts = append(opts, repo.WithOrderRuleBy("created_at", "desc"), repo.WithByType(req.TaskType), taskRepo.WithOperate(req.TaskOperate), taskRepo.WithResourceID(req.ResourceID))
}
taskModel, err := taskRepo.GetFirst(opts...)
if err != nil {
return nil, err
}
logFilePath = taskModel.LogFile
taskStatus = taskModel.Status
case "mysql-slow-logs":
logFilePath = path.Join(global.Dir.DataDir, "apps", "mysql", req.Name, "data", "1Panel-slow.log")
case "mariadb-slow-logs":
logFilePath = path.Join(global.Dir.DataDir, "apps", "mariadb", req.Name, "db", "data", "1Panel-slow.log")
case "php-fpm-slow-logs":
php, err := runtimeRepo.GetFirst(context.Background(), repo.WithByID(req.ID))
if err != nil {
return nil, err
}
logFilePath = php.GetSlowLogPath()
case constant.Supervisord:
configPath := "/etc/supervisord.conf"
pathSet, _ := settingRepo.Get(settingRepo.WithByKey(constant.SupervisorConfigPath))
if pathSet.ID != 0 || pathSet.Value != "" {
configPath = pathSet.Value
}
logFilePath, _ = ini_conf.GetIniValue(configPath, "supervisord", "logfile")
case constant.Supervisor:
logFilePath = path.Join(global.Dir.DataDir, "tools", "supervisord", "log", req.Name)
case "ai-proxy":
safeName := path.Base(req.Name)
if safeName != req.Name || strings.Contains(safeName, "..") {
return nil, buserr.New("ErrInvalidParams")
}
logFilePath = path.Join(global.Dir.LogDir, "ai", safeName)
default:
return nil, buserr.WithName("ErrNotSupportType", req.Type)
}
file, err := os.Open(logFilePath)
if err != nil {
return nil, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return nil, err
}
var (
lines []string
isEndOfFile bool
scope string
logFileRes *dto.LogFileRes
)
if stat.Size() > files.MaxReadFileSize {
lines, _ = files.TailFromEnd(logFilePath, req.PageSize)
isEndOfFile = true
scope = "tail"
} else {
logFileRes, err = files.ReadFileByLine(logFilePath, req.Page, req.PageSize, req.Latest)
if err != nil {
return nil, err
}
scope = "page"
lines = logFileRes.Lines
}
res := &response.FileLineContent{
End: isEndOfFile,
Path: logFilePath,
TaskStatus: taskStatus,
Lines: lines,
Scope: scope,
}
if logFileRes != nil {
res.TotalLines = logFileRes.TotalLines
res.Total = logFileRes.TotalPages
res.End = logFileRes.IsEndOfFile
}
return res, nil
}
func (f *FileService) BatchCheckFiles(req request.FilePathsCheck) []response.ExistFileInfo {
fileList := make([]response.ExistFileInfo, 0, len(req.Paths))
for _, filePath := range req.Paths {
if info, err := os.Stat(filePath); err == nil {
fileList = append(fileList, response.ExistFileInfo{
Size: info.Size(),
Name: info.Name(),
Path: filePath,
ModTime: info.ModTime(),
IsDir: info.IsDir(),
})
}
}
return fileList
}
func (f *FileService) GetHostMount() []dto.DiskInfo {
return loadDiskInfo(false)
}
func (f *FileService) GetUsersAndGroups() (*response.UserGroupResponse, error) {
groupMap, err := getValidGroups()
if err != nil {
return nil, err
}
users, groupSet, err := getValidUsers(groupMap)
if err != nil {
return nil, err
}
var groups []string
for group := range groupSet {
groups = append(groups, group)
}
sort.Strings(groups)
return &response.UserGroupResponse{
Users: users,
Groups: groups,
}, nil
}
func (f *FileService) BatchGetRemarks(req request.FileRemarkBatch) map[string]string {
remarks := make(map[string]string)
for _, filePath := range req.Paths {
remark, err := getFileRemark(filePath)
if err != nil {
if isXattrNotSupported(err) {
return map[string]string{}
}
continue
}
if remark == "" {
continue
}
remarks[filePath] = remark
}
return remarks
}
func (f *FileService) SetRemark(req request.FileRemarkUpdate) error {
if req.Remark == "" {
if err := unix.Lremovexattr(req.Path, fileRemarkXattr); err != nil {
if isXattrNotFound(err) {
return nil
}
if isXattrNotSupported(err) {
return buserr.WithDetail("ErrInvalidParams", "xattr not supported", err)
}
return err
}
return nil
}
encoded := base64.StdEncoding.EncodeToString([]byte(req.Remark))
if len(encoded) >= fileRemarkEncodedMaxLen {
return buserr.WithDetail("ErrInvalidParams", "remark length must be less than 256", nil)
}
if err := unix.Lsetxattr(req.Path, fileRemarkXattr, []byte(encoded), 0); err != nil {
if isXattrNotSupported(err) {
return buserr.WithDetail("ErrInvalidParams", "xattr not supported", err)
}
return err
}
return nil
}
func getValidGroups() (map[string]bool, error) {
groupFile, err := os.Open("/etc/group")
if err != nil {
return nil, fmt.Errorf("failed to open /etc/group: %w", err)
}
defer groupFile.Close()
groupMap := make(map[string]bool)
scanner := bufio.NewScanner(groupFile)
for scanner.Scan() {
parts := strings.Split(scanner.Text(), ":")
if len(parts) < 3 {
continue
}
groupName := parts[0]
gid, _ := strconv.Atoi(parts[2])
if groupName == "root" || gid >= 1000 {
groupMap[groupName] = true
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to scan /etc/group: %w", err)
}
return groupMap, nil
}
func getFileRemark(filePath string) (string, error) {
size, err := unix.Lgetxattr(filePath, fileRemarkXattr, nil)
if err != nil {
if isXattrNotFound(err) {
return "", nil
}
return "", err
}
if size == 0 {
return "", nil
}
buf := make([]byte, size)
n, err := unix.Lgetxattr(filePath, fileRemarkXattr, buf)
if err != nil {
return "", err
}
decoded, err := base64.StdEncoding.DecodeString(string(buf[:n]))
if err != nil {
return "", err
}
return string(decoded), nil
}
func isXattrNotSupported(err error) bool {
return errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EOPNOTSUPP)
}
func isXattrNotFound(err error) bool {
return errors.Is(err, unix.ENODATA)
}
func getValidUsers(validGroups map[string]bool) ([]response.UserInfo, map[string]struct{}, error) {
passwdFile, err := os.Open("/etc/passwd")
if err != nil {
return nil, nil, fmt.Errorf("failed to open /etc/passwd: %w", err)
}
defer passwdFile.Close()
var users []response.UserInfo
groupSet := make(map[string]struct{})
scanner := bufio.NewScanner(passwdFile)
for scanner.Scan() {
parts := strings.Split(scanner.Text(), ":")
if len(parts) < 4 {
continue
}
username := parts[0]
uid, _ := strconv.Atoi(parts[2])
gid := parts[3]
if username != "root" && uid < 1000 {
continue
}
groupName := gid
if g, err := user.LookupGroupId(gid); err == nil {
groupName = g.Name
}
if !validGroups[groupName] {
continue
}
users = append(users, response.UserInfo{
Username: username,
Group: groupName,
})
groupSet[groupName] = struct{}{}
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("failed to scan /etc/passwd: %w", err)
}
return users, groupSet, nil
}
func (f *FileService) Convert(req request.FileConvertRequest) {
convertTask, err := task.NewTaskWithOps(i18n.GetMsgByKey("FileConvert"), task.TaskExec, task.TaskScopeFileConvert, req.TaskID, 1)
if err != nil {
global.LOG.Errorf("Create convert task failed %v", err)
return
}
convertTask.AddSubTask(task.GetTaskName(i18n.GetMsgByKey("FileConvert"), task.TaskExec, task.TaskScopeFileConvert), func(t *task.Task) (err error) {
for _, file := range req.Files {
input := filepath.Join(file.Path, file.InputFile)
nameOnly := file.InputFile[0 : len(file.InputFile)-len(file.Extension)]
output := filepath.Join(req.OutputPath, nameOnly+"."+file.OutputFormat)
status, errMsg := convert.MediaFile(input, output, file.OutputFormat, req.DeleteSource)
if status == "FAILED" {
convertTask.Log(fmt.Sprintf("%s -> %s [%s]: %s\n",
input, output, status, errMsg))
} else {
convertTask.Log(fmt.Sprintf("%s -> %s [%s]: %s\n",
input, output, status, "SUCCESS"))
}
}
return nil
}, nil)
go func() {
_ = convertTask.Execute()
}()
}
func (f *FileService) ConvertLog(req dto.PageInfo) (total int64, data []response.FileConvertLog, err error) {
logFilePath := filepath.Join(global.Dir.ConvertLogDir, "convert.log")
file, err := os.Open(logFilePath)
if err != nil {
return 0, nil, err
}
defer file.Close()
const chunkSize = 64 * 1024
stat, err := file.Stat()
if err != nil {
return 0, nil, err
}
fileSize := stat.Size()
var (
buf []byte
remainder []byte
offset = fileSize
lines []string
)
pageStart := int64((req.Page - 1) * req.PageSize)
pageEnd := pageStart + int64(req.PageSize)
for offset > 0 {
readSize := chunkSize
if offset < int64(readSize) {
readSize = int(offset)
}
offset -= int64(readSize)
tmp := make([]byte, readSize)
if _, err := file.ReadAt(tmp, offset); err != nil && err != io.EOF {
return 0, nil, err
}
buf = append(tmp, remainder...)
linesSplit := strings.Split(string(buf), "\n")
if offset > 0 {
remainder = []byte(linesSplit[0])
linesSplit = linesSplit[1:]
}
for i := len(linesSplit) - 1; i >= 0; i-- {
line := strings.TrimSpace(linesSplit[i])
if line == "" {
continue
}
total++
if total > pageStart && total <= pageEnd {
lines = append(lines, line)
}
if total >= pageEnd {
break
}
}
if total >= pageEnd {
break
}
}
for _, line := range lines {
var entry response.FileConvertLog
if err := json.Unmarshal([]byte(line), &entry); err == nil {
data = append(data, entry)
}
}
return total, data, nil
}
func (f *FileService) AISearch(req request.FileAISearch) (*response.FileAISearchResult, error) {
root := filepath.Clean(strings.TrimSpace(req.Path))
if root == "" {
return nil, buserr.WithDetail("ErrInvalidParams", "path is required", nil)
}
query := strings.TrimSpace(req.Query)
if query == "" {
return nil, buserr.WithDetail("ErrInvalidParams", "query is required", nil)
}
st, err := os.Stat(root)
if err != nil {
if os.IsNotExist(err) {
return nil, buserr.New("ErrPathNotFound")
}
return nil, err
}
if !st.IsDir() {
return nil, buserr.New("ErrPathNotFound")
}
maxItems := req.MaxItems
if maxItems <= 0 {
maxItems = files.DefaultFileAIMaxItems
}
if maxItems > 2000 {
maxItems = 2000
}
containSub := true
if req.ContainSub != nil {
containSub = *req.ContainSub
}
searchOpts, err := files.MergeContentSearchOptions(
req.MatchCase, req.WholeWord, req.UseRegex,
req.Extensions,
req.MinSize, req.MaxSize,
req.ModifiedAfter, req.ModifiedBefore,
req.MaxScanFiles,
req.MaxFileBytes,
req.MaxHitsPerFile, req.MaxTotalHits,
req.ContentHitsPromptMaxBytes,
req.LlmMaxOutputTokens,
)
if err != nil {
return nil, buserr.WithDetail("ErrInvalidParams", err.Error(), nil)
}
matchFn, err := files.NewContentLineMatcher(query, searchOpts)
if err != nil {
return nil, buserr.WithDetail("ErrFileAISearchBadPattern", err.Error(), nil)
}
cfg, timeout, err := terminalai.LoadFileAIRuntimeConfig()
aiEnabled := err == nil
if err != nil && !errors.Is(err, os.ErrNotExist) {
return nil, err
}
items, truncated, err := files.CollectDirInventory(root, containSub, maxItems)
if err != nil {
return nil, err
}
preFiltered := false
llmItems := items
qLower := strings.ToLower(query)
if len(llmItems) > 0 && query != "" {
filtered := make([]files.AISearchInventoryItem, 0, len(llmItems))
for _, it := range llmItems {
rel := strings.TrimSpace(it.RelPath)
if rel == "" {
continue
}
if !req.UseRegex && !req.MatchCase && !req.WholeWord && strings.Contains(strings.ToLower(rel), qLower) {
filtered = append(filtered, it)
}
}
if len(filtered) >= 8 {
llmItems = filtered
preFiltered = true
}
}
start := time.Now()
contentHits, scannedFiles, hitsTrunc := files.SearchFileAIContentHits(root, llmItems, searchOpts, matchFn)
hitsDTO := make([]response.FileAIContentHit, 0, len(contentHits))
for _, h := range contentHits {
hitsDTO = append(hitsDTO, response.FileAIContentHit{Path: h.Path, Line: h.Line, Text: h.Text})
}
matchDesc := searchOpts.ContentMatchDescription()
result := &response.FileAISearchResult{
Hits: hitsDTO,
ContentScannedFiles: scannedFiles,
ContentHitsTruncated: hitsTrunc,
Truncated: truncated,
PreFiltered: preFiltered,
ItemCount: len(llmItems),
}
if len(llmItems) == 0 {
if aiEnabled {
result.Mode = "ai"
result.Summary = i18n.GetMsgByKey("FileAISearchEmptyDir")
if result.Summary == "" || result.Summary == "FileAISearchEmptyDir" {
result.Summary = "No files or directories found under this path (or all entries were filtered)."
}
} else {
result.Mode = "grep"
result.Summary = ""
}
result.Duration = time.Since(start).Round(time.Millisecond).String()
return result, nil
}
if !aiEnabled {
result.Mode = "grep"
result.Summary = ""
result.Duration = time.Since(start).Round(time.Millisecond).String()
return result, nil
}
result.Mode = "ai"
clientTimeout := timeout
if clientTimeout < 30*time.Second {
clientTimeout = 90 * time.Second
}
if clientTimeout > 5*time.Minute {
clientTimeout = 5 * time.Minute
}
runCtx, cancel := context.WithTimeout(context.Background(), timeout+time.Minute)
defer cancel()
llmMaxOut := searchOpts.LlmMaxOutputTokens
summary, usage, err := files.RunFileAISearchLLM(runCtx, cfg, clientTimeout, root, query, req.ResponseLanguage, llmItems, truncated, preFiltered, contentHits, scannedFiles, hitsTrunc, matchDesc, searchOpts.ContentHitsPromptMaxBytes, llmMaxOut)
if err != nil {
result.Mode = "grep"
result.Summary = ""
result.Duration = time.Since(start).Round(time.Millisecond).String()
if errors.Is(err, context.DeadlineExceeded) || strings.Contains(strings.ToLower(err.Error()), "timeout") {
return result, nil
}
return result, nil
}
result.Summary = summary
result.PromptTokens = usage.PromptTokens
result.CompletionTokens = usage.CompletionTokens
result.TotalTokens = usage.TotalTokens
if result.TotalTokens == 0 {
result.TotalTokens = usage.PromptTokens + usage.CompletionTokens
}
result.Duration = time.Since(start).Round(time.Millisecond).String()
return result, nil
}