Files
1Panel/agent/utils/files/fileinfo.go

692 lines
17 KiB
Go

package files
import (
"bufio"
"context"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
"time"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/buserr"
"github.com/spf13/afero"
)
var binaryPreviewMimeTypes = map[string]struct{}{
"application/pdf": {},
"application/zip": {},
"application/gzip": {},
"application/x-gzip": {},
"application/x-7z-compressed": {},
"application/x-rar-compressed": {},
"application/x-bzip2": {},
"application/x-xz": {},
"application/x-tar": {},
"application/java-archive": {},
"application/msword": {},
"application/vnd.ms-excel": {},
"application/vnd.ms-powerpoint": {},
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": {},
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": {},
"application/vnd.openxmlformats-officedocument.presentationml.presentation": {},
}
var binaryPreviewMimePrefixes = []string{
"image/",
"audio/",
"video/",
"application/vnd.ms-",
"application/vnd.openxmlformats-officedocument.",
"application/vnd.oasis.opendocument.",
}
var binaryPreviewExtensions = map[string]struct{}{
".pdf": {},
".zip": {},
".gz": {},
".bz2": {},
".xz": {},
".tar": {},
".tgz": {},
".rar": {},
".7z": {},
".war": {},
".doc": {},
".docx": {},
".xls": {},
".xlsx": {},
".ppt": {},
".pptx": {},
".jpg": {},
".jpeg": {},
".png": {},
".bmp": {},
".gif": {},
".tiff": {},
".ico": {},
".webp": {},
".svg": {},
".mp3": {},
".wav": {},
".wma": {},
".ape": {},
".acc": {},
".ogg": {},
".flac": {},
".mp4": {},
".webm": {},
".mov": {},
".wmv": {},
".mkv": {},
".avi": {},
".flv": {},
}
type FileInfo struct {
Fs afero.Fs `json:"-"`
Path string `json:"path"`
Name string `json:"name"`
User string `json:"user"`
Group string `json:"group"`
Uid string `json:"uid"`
Gid string `json:"gid"`
Extension string `json:"extension"`
Content string `json:"content"`
Size int64 `json:"size"`
IsDir bool `json:"isDir"`
IsSymlink bool `json:"isSymlink"`
IsHidden bool `json:"isHidden"`
LinkPath string `json:"linkPath"`
Type string `json:"type"`
Mode string `json:"mode"`
MimeType string `json:"mimeType"`
UpdateTime time.Time `json:"updateTime"`
ModTime time.Time `json:"modTime"`
FileMode os.FileMode `json:"-"`
Items []*FileInfo `json:"items"`
ItemTotal int `json:"itemTotal"`
FavoriteID uint `json:"favoriteID"`
ShareCode string `json:"shareCode"`
IsDetail bool `json:"isDetail"`
IsAppendOnly bool `json:"isAppendOnly"`
IsImmutable bool `json:"isImmutable"`
}
type FileOption struct {
Path string `json:"path"`
Search string `json:"search"`
ContainSub bool `json:"containSub"`
Expand bool `json:"expand"`
Dir bool `json:"dir"`
ShowHidden bool `json:"showHidden"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
SortBy string `json:"sortBy"`
SortOrder string `json:"sortOrder"`
IsDetail bool `json:"isDetail"`
}
type FileSearchInfo struct {
Path string `json:"path"`
fs.FileInfo
}
func NewFileInfo(op FileOption) (*FileInfo, error) {
return NewFileInfoWithContext(context.Background(), op)
}
func NewFileInfoWithContext(ctx context.Context, op FileOption) (*FileInfo, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
var appFs = afero.NewOsFs()
info, err := appFs.Stat(op.Path)
if err != nil {
if os.IsNotExist(err) {
return nil, buserr.New("ErrLinkPathNotFound")
}
return nil, err
}
file := &FileInfo{
Fs: appFs,
Path: op.Path,
Name: info.Name(),
IsDir: info.IsDir(),
FileMode: info.Mode(),
ModTime: info.ModTime(),
Size: info.Size(),
IsSymlink: IsSymlink(info.Mode()),
Extension: filepath.Ext(info.Name()),
IsHidden: IsHidden(op.Path),
Mode: fmt.Sprintf("%04o", info.Mode().Perm()),
User: GetUsername(info.Sys().(*syscall.Stat_t).Uid),
Uid: strconv.FormatUint(uint64(info.Sys().(*syscall.Stat_t).Uid), 10),
Gid: strconv.FormatUint(uint64(info.Sys().(*syscall.Stat_t).Gid), 10),
Group: GetGroup(info.Sys().(*syscall.Stat_t).Gid),
IsDetail: op.IsDetail,
}
if info.Mode().IsRegular() {
file.MimeType = GetMimeType(op.Path)
}
file.IsAppendOnly, file.IsImmutable = getFileAttributes(ctx, op.Path)
if file.IsSymlink {
linkPath := GetSymlink(op.Path)
if !filepath.IsAbs(linkPath) {
dir := filepath.Dir(op.Path)
var err error
linkPath, err = filepath.Abs(filepath.Join(dir, linkPath))
if err != nil {
return nil, err
}
}
file.LinkPath = linkPath
targetInfo, err := appFs.Stat(linkPath)
if err != nil {
file.IsDir = false
file.Mode = "-"
file.User = "-"
file.Group = "-"
} else {
file.IsDir = targetInfo.IsDir()
}
file.Extension = filepath.Ext(file.LinkPath)
}
if op.Expand {
if err := handleExpansion(ctx, file, op); err != nil {
return nil, err
}
}
if !op.Expand || !file.IsDir {
if err := file.loadFavorites(ctx); err != nil {
return nil, err
}
}
return file, nil
}
func getFileAttributes(ctx context.Context, filePath string) (bool, bool) {
output, err := exec.CommandContext(ctx, "lsattr", "-d", "--", filePath).Output()
if err != nil {
return false, false
}
fields := strings.Fields(string(output))
if len(fields) == 0 {
return false, false
}
return strings.Contains(fields[0], "a"), strings.Contains(fields[0], "i")
}
func handleExpansion(ctx context.Context, file *FileInfo, op FileOption) error {
if file.IsDir {
return file.listChildrenWithContext(ctx, op)
}
if !file.IsDetail {
return file.getContent()
}
return nil
}
func (f *FileInfo) search(ctx context.Context, search string, count int) (files []FileSearchInfo, total int, err error) {
searchCtx, cancel := context.WithCancel(ctx)
defer cancel()
cmd := exec.CommandContext(searchCtx, "find", f.Path, "-name", fmt.Sprintf("*%s*", search), "-print0")
output, err := cmd.StdoutPipe()
if err != nil {
return nil, 0, err
}
if err := cmd.Start(); err != nil {
return nil, 0, err
}
reader := bufio.NewReader(output)
for {
if err = ctx.Err(); err != nil {
break
}
var entry string
entry, err = reader.ReadString(0)
if err != nil {
break
}
entry = strings.TrimSuffix(entry, "\x00")
info, statErr := os.Stat(entry)
if statErr != nil {
continue
}
total++
if total <= count {
files = append(files, FileSearchInfo{Path: entry, FileInfo: info})
}
}
if err != nil && err != io.EOF {
cancel()
}
_ = cmd.Wait()
if contextErr := ctx.Err(); contextErr != nil {
return nil, 0, contextErr
}
if err == io.EOF {
err = nil
}
return
}
func sortFileList(list []FileSearchInfo, sortBy, sortOrder string) {
_ = sortFileListWithContext(context.Background(), list, sortBy, sortOrder)
}
func sortFileListWithContext(ctx context.Context, list []FileSearchInfo, sortBy, sortOrder string) error {
var less func(i, j int) bool
switch sortBy {
case "name":
less = func(i, j int) bool { return list[i].Name() < list[j].Name() }
case "size":
less = func(i, j int) bool { return list[i].Size() < list[j].Size() }
case "modTime":
less = func(i, j int) bool { return list[i].ModTime().Before(list[j].ModTime()) }
default:
return ctx.Err()
}
return sortSliceWithContext(ctx, list, func(i, j int) bool {
if sortOrder != "ascending" {
i, j = j, i
}
return less(i, j)
})
}
func sortSliceWithContext(ctx context.Context, items interface{}, less func(i, j int) bool) (err error) {
if err := ctx.Err(); err != nil {
return err
}
if ctx.Done() == nil {
sort.Slice(items, less)
return nil
}
canceled := &struct{ marker byte }{}
defer func() {
if recovered := recover(); recovered != nil {
if recovered != canceled {
panic(recovered)
}
err = ctx.Err()
}
}()
comparisons := 0
sort.Slice(items, func(i, j int) bool {
if comparisons%256 == 0 && ctx.Err() != nil {
panic(canceled)
}
comparisons++
return less(i, j)
})
return ctx.Err()
}
func (f *FileInfo) listChildren(option FileOption) error {
return f.listChildrenWithContext(context.Background(), option)
}
func (f *FileInfo) listChildrenWithContext(ctx context.Context, option FileOption) error {
if err := ctx.Err(); err != nil {
return err
}
maxInt := int(^uint(0) >> 1)
if option.Page < 1 || option.PageSize < 1 || option.Page > maxInt/option.PageSize {
return fmt.Errorf("invalid file pagination")
}
var (
entries []FileSearchInfo
total int
err error
)
if option.Search != "" && option.ContainSub {
entries, total, err = f.search(ctx, option.Search, option.Page*option.PageSize)
} else {
entries, err = f.getFiles(ctx, option)
}
if err != nil {
return err
}
candidates := entries[:0]
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return err
}
if shouldSkipFile(entry, option) {
continue
}
name, _ := f.getFilePathAndName(option, entry)
if !option.ShowHidden && IsHidden(name) {
continue
}
candidates = append(candidates, entry)
}
f.ItemTotal = len(candidates)
if option.ContainSub {
f.ItemTotal = total
}
page := candidates
if !option.ContainSub || option.Search != "" {
start := (option.Page - 1) * option.PageSize
end := option.Page * option.PageSize
if start >= len(candidates) {
page = nil
} else {
if end > len(candidates) {
end = len(candidates)
}
page = candidates[start:end]
}
}
f.Items, err = f.processFiles(ctx, page, option)
if err != nil {
return err
}
return f.loadFavorites(ctx)
}
func readDirWithContext(ctx context.Context, dirPath string) ([]os.DirEntry, error) {
dir, err := os.Open(dirPath)
if err != nil {
return nil, err
}
defer dir.Close()
var entries []os.DirEntry
for {
if err := ctx.Err(); err != nil {
return nil, err
}
batch, err := dir.ReadDir(256)
entries = append(entries, batch...)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
}
if err := sortSliceWithContext(ctx, entries, func(i, j int) bool { return entries[i].Name() < entries[j].Name() }); err != nil {
return nil, err
}
return entries, nil
}
func (f *FileInfo) getFiles(ctx context.Context, option FileOption) ([]FileSearchInfo, error) {
entries, err := readDirWithContext(ctx, f.Path)
if err != nil {
return nil, err
}
var dirs, fileList []FileSearchInfo
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return nil, err
}
info, err := entry.Info()
if err != nil {
continue
}
item := FileSearchInfo{Path: f.Path, FileInfo: info}
if entry.IsDir() {
dirs = append(dirs, item)
} else {
fileList = append(fileList, item)
}
}
if err := sortFileListWithContext(ctx, dirs, option.SortBy, option.SortOrder); err != nil {
return nil, err
}
if err := sortFileListWithContext(ctx, fileList, option.SortBy, option.SortOrder); err != nil {
return nil, err
}
return append(dirs, fileList...), ctx.Err()
}
func (f *FileInfo) processFiles(ctx context.Context, entries []FileSearchInfo, option FileOption) ([]*FileInfo, error) {
var items []*FileInfo
users := make(map[uint32]string)
groups := make(map[uint32]string)
for _, entry := range entries {
if err := ctx.Err(); err != nil {
return nil, err
}
name, filePath := f.getFilePathAndName(option, entry)
stat := entry.Sys().(*syscall.Stat_t)
username, ok := users[stat.Uid]
if !ok {
username = GetUsername(stat.Uid)
users[stat.Uid] = username
}
group, ok := groups[stat.Gid]
if !ok {
group = GetGroup(stat.Gid)
groups[stat.Gid] = group
}
isSymlink, isInvalidLink := f.checkSymlink(entry)
file := &FileInfo{
Fs: f.Fs, Name: name, Path: filePath,
Size: entry.Size(), ModTime: entry.ModTime(), FileMode: entry.Mode(),
IsDir: entry.IsDir(), IsSymlink: isSymlink, IsHidden: IsHidden(filePath),
Extension: filepath.Ext(name), Mode: fmt.Sprintf("%04o", entry.Mode().Perm()),
User: username, Group: group,
Uid: strconv.FormatUint(uint64(stat.Uid), 10),
Gid: strconv.FormatUint(uint64(stat.Gid), 10),
}
mimeInfo := entry.FileInfo
if isSymlink {
linkPath := GetSymlink(filePath)
if !filepath.IsAbs(linkPath) {
resolved, err := filepath.Abs(filepath.Join(filepath.Dir(filePath), linkPath))
if err != nil {
return nil, err
}
linkPath = resolved
}
file.LinkPath = linkPath
targetInfo, err := file.Fs.Stat(linkPath)
mimeInfo = targetInfo
if err != nil {
file.IsDir = false
file.Mode, file.User, file.Group = "-", "-", "-"
} else {
file.IsDir = targetInfo.IsDir()
}
file.Extension = filepath.Ext(file.LinkPath)
}
if entry.Size() > 0 && mimeInfo != nil && mimeInfo.Mode().IsRegular() {
file.MimeType = getMimeTypeFromFS(f.Fs, filePath)
}
if isInvalidLink {
file.Type = "invalid_link"
}
items = append(items, file)
}
return items, ctx.Err()
}
func (f *FileInfo) loadFavorites(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
paths := make([]string, 0, len(f.Items)+1)
paths = append(paths, f.Path)
for _, item := range f.Items {
paths = append(paths, item.Path)
}
favorites, err := repo.NewIFavoriteRepo().ListByPaths(ctx, paths)
if ctx.Err() != nil {
return ctx.Err()
}
if err != nil {
return nil
}
ids := make(map[string]uint, len(favorites))
for _, favorite := range favorites {
if id, ok := ids[favorite.Path]; !ok || favorite.ID < id {
ids[favorite.Path] = favorite.ID
}
}
f.FavoriteID = ids[f.Path]
for _, item := range f.Items {
item.FavoriteID = ids[item.Path]
}
return nil
}
func shouldSkipFile(df FileSearchInfo, option FileOption) bool {
if option.Dir && !df.IsDir() {
return true
}
if option.Search != "" && !option.ContainSub {
lowerName := strings.ToLower(df.Name())
lowerSearch := strings.ToLower(option.Search)
if !strings.Contains(lowerName, lowerSearch) {
return true
}
}
return false
}
func (f *FileInfo) getFilePathAndName(option FileOption, df FileSearchInfo) (string, string) {
name := df.Name()
fPath := path.Join(df.Path, df.Name())
if option.Search != "" && option.ContainSub {
fPath = df.Path
name = strings.TrimPrefix(strings.TrimPrefix(fPath, f.Path), "/")
}
return name, fPath
}
func (f *FileInfo) checkSymlink(df FileSearchInfo) (bool, bool) {
isSymlink := false
isInvalidLink := false
if IsSymlink(df.Mode()) {
isSymlink = true
info, err := f.Fs.Stat(path.Join(df.Path, df.Name()))
if err == nil {
df.FileInfo = info
} else {
isInvalidLink = true
}
}
return isSymlink, isInvalidLink
}
func (f *FileInfo) getContent() error {
if IsBlockDevice(f.FileMode) {
return buserr.New("ErrFileCanNotRead")
}
if f.Size > 10*1024*1024 {
return buserr.New("ErrFileToLarge")
}
if IsBinaryPreviewFile(f.MimeType, f.Extension) {
return buserr.New("ErrFileCanNotRead")
}
afs := &afero.Afero{Fs: f.Fs}
cByte, err := afs.ReadFile(f.Path)
if err != nil {
return nil
}
if len(cByte) > 0 && DetectBinary(cByte) {
return buserr.New("ErrFileCanNotRead")
}
f.Content = string(cByte)
return nil
}
func IsBinaryPreviewFile(mimeType, extension string) bool {
mimeType = strings.ToLower(strings.TrimSpace(mimeType))
extension = strings.ToLower(strings.TrimSpace(extension))
if mimeType == "" || mimeType == "application/octet-stream" {
_, ok := binaryPreviewExtensions[extension]
return ok
}
if strings.HasPrefix(mimeType, "text/") {
return false
}
for _, prefix := range binaryPreviewMimePrefixes {
if strings.HasPrefix(mimeType, prefix) {
return true
}
}
_, ok := binaryPreviewMimeTypes[mimeType]
if ok {
return true
}
_, ok = binaryPreviewExtensions[extension]
return ok
}
func DetectBinary(buf []byte) bool {
mimeType := http.DetectContentType(buf)
if IsBinaryPreviewFile(mimeType, "") {
return true
}
if !strings.HasPrefix(mimeType, "text/") {
whiteByte := 0
n := min(1024, len(buf))
for i := 0; i < n; i++ {
if (buf[i] >= 0x20) || buf[i] == 9 || buf[i] == 10 || buf[i] == 13 {
whiteByte++
} else if buf[i] <= 6 || (buf[i] >= 14 && buf[i] <= 31) {
return true
}
}
return whiteByte < 1
}
return false
}
func min(x, y int) int {
if x < y {
return x
}
return y
}
type CompressType string
const (
Zip CompressType = "zip"
Gz CompressType = "gz"
Bz2 CompressType = "bz2"
TarBz2 CompressType = "tar.bz2"
Tar CompressType = "tar"
TarGz CompressType = "tar.gz"
Tgz CompressType = "tgz"
Xz CompressType = "xz"
TarXz CompressType = "tar.xz"
SdkZip CompressType = "sdkZip"
SdkTarGz CompressType = "sdkTarGz"
Rar CompressType = "rar"
X7z CompressType = "7z"
)