starred/1Panel
1
0
Fork 0
mirror of https://github.com/1Panel-dev/1Panel.git synced 2026-09-22 08:00:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: 修改备份账号同步方式 (#6120)

Browse Source
This commit is contained in:
ssongliu
2024-08-14 02:15:55 +00:00
committed by GitHub
parent f0274701cf
commit 6da0445509
48 changed files with 730 additions and 333 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent/app/api/v2/backup.go
+1 -1
View File
@@ -186,7 +186,7 @@ func (b *BaseApi) Recover(c *gin.Context) {
}
downloadPath, err := backupService.DownloadRecord(dto.DownloadRecord{
DownloadAccountID: req.BackupAccountID,
DownloadAccountID: req.DownloadAccountID,
FileDir: path.Dir(req.File),
FileName: path.Base(req.File),
})
agent/app/api/v2/health.go
+19
View File
@@ -0,0 +1,19 @@
package v2
import (
"net/http"
"github.com/1Panel-dev/1Panel/agent/app/api/v2/helper"
"github.com/1Panel-dev/1Panel/agent/constant"
httpUtils "github.com/1Panel-dev/1Panel/agent/utils/http"
"github.com/gin-gonic/gin"
)
func (b *BaseApi) CheckHealth(c *gin.Context) {
_, err := httpUtils.RequestToMaster("/api/v2/agent/health", http.MethodGet, nil)
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithOutData(c)
}
agent/app/dto/backup.go
+14 -24
View File
@@ -4,17 +4,6 @@ import (
"time"
)
type BackupInfo struct {
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type" validate:"required"`
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
Credential string `json:"credential"`
BackupPath string `json:"backupPath"`
Vars string `json:"vars" validate:"required"`
}
type CommonBackup struct {
Type string `json:"type" validate:"required,oneof=app mysql mariadb redis website postgresql"`
Name string `json:"name"`
@@ -22,12 +11,12 @@ type CommonBackup struct {
Secret string `json:"secret"`
}
type CommonRecover struct {
BackupAccountID uint `json:"backupAccountID" validate:"required"`
Type string `json:"type" validate:"required,oneof=app mysql mariadb redis website postgresql"`
Name string `json:"name"`
DetailName string `json:"detailName"`
File string `json:"file"`
Secret string `json:"secret"`
DownloadAccountID uint `json:"downloadAccountID" validate:"required"`
Type string `json:"type" validate:"required,oneof=app mysql mariadb redis website postgresql"`
Name string `json:"name"`
DetailName string `json:"detailName"`
File string `json:"file"`
Secret string `json:"secret"`
}
type RecordSearch struct {
@@ -43,13 +32,14 @@ type RecordSearchByCronjob struct {
}
type BackupRecords struct {
ID uint `json:"id"`
CreatedAt time.Time `json:"createdAt"`
Source string `json:"source"`
BackupType string `json:"backupType"`
FileDir string `json:"fileDir"`
FileName string `json:"fileName"`
Size int64 `json:"size"`
ID uint `json:"id"`
CreatedAt time.Time `json:"createdAt"`
AccountType string `json:"accountType"`
AccountName string `json:"accountName"`
DownloadAccountID uint `json:"downloadAccountID"`
FileDir string `json:"fileDir"`
FileName string `json:"fileName"`
Size int64 `json:"size"`
}
type DownloadRecord struct {
agent/app/dto/cronjob.go
+21 -21
View File
@@ -27,10 +27,10 @@ type CronjobCreate struct {
URL string `json:"url"`
SourceDir string `json:"sourceDir"`
BackupAccounts string `json:"backupAccounts"`
DefaultDownload string `json:"defaultDownload"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
}
type CronjobUpdate struct {
@@ -49,10 +49,10 @@ type CronjobUpdate struct {
URL string `json:"url"`
SourceDir string `json:"sourceDir"`
BackupAccounts string `json:"backupAccounts"`
DefaultDownload string `json:"defaultDownload"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
RetainCopies int `json:"retainCopies" validate:"number,min=1"`
Secret string `json:"secret"`
}
type CronjobUpdateStatus struct {
@@ -82,19 +82,19 @@ type CronjobInfo struct {
Type string `json:"type"`
Spec string `json:"spec"`
Script string `json:"script"`
Command string `json:"command"`
ContainerName string `json:"containerName"`
AppID string `json:"appID"`
Website string `json:"website"`
ExclusionRules string `json:"exclusionRules"`
DBType string `json:"dbType"`
DBName string `json:"dbName"`
URL string `json:"url"`
SourceDir string `json:"sourceDir"`
BackupAccounts string `json:"backupAccounts"`
DefaultDownload string `json:"defaultDownload"`
RetainCopies int `json:"retainCopies"`
Script string `json:"script"`
Command string `json:"command"`
ContainerName string `json:"containerName"`
AppID string `json:"appID"`
Website string `json:"website"`
ExclusionRules string `json:"exclusionRules"`
DBType string `json:"dbType"`
DBName string `json:"dbName"`
URL string `json:"url"`
SourceDir string `json:"sourceDir"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
RetainCopies int `json:"retainCopies"`
LastRecordTime string `json:"lastRecordTime"`
Status string `json:"status"`
agent/app/dto/setting.go
+1 -1
View File
@@ -49,7 +49,7 @@ type SnapshotStatus struct {
type SnapshotCreate struct {
ID uint `json:"id"`
SourceAccountIDs string `json:"sourceAccountsIDs" validate:"required"`
SourceAccountIDs string `json:"sourceAccountIDs" validate:"required"`
DownloadAccountID uint `json:"downloadAccountID" validate:"required"`
Description string `json:"description" validate:"max=256"`
Secret string `json:"secret"`
agent/app/model/backup.go
+14 -1
View File
@@ -1,10 +1,23 @@
package model
type BackupAccount struct {
BaseModel
Name string `json:"name"`
Type string `json:"type"`
Bucket string `json:"bucket"`
AccessKey string `json:"accessKey"`
Credential string `json:"credential"`
BackupPath string `json:"backupPath"`
Vars string `json:"vars"`
}
// Source ---> SourceAccountIDs
// BackupType ---> DownloadAccountID
type BackupRecord struct {
BaseModel
From string `json:"from"`
CronjobID uint `json:"cronjobID"`
SourceAccountIDs string `json:"sourceAccountsIDs"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
Type string `gorm:"not null" json:"type"`
agent/app/model/cronjob.go
+3 -1
View File
@@ -4,6 +4,8 @@ import (
"time"
)
// BackupAccounts ---> SourceAccountIDs
// BackupAccounts ---> DownloadAccountID
type Cronjob struct {
BaseModel
@@ -22,7 +24,7 @@ type Cronjob struct {
SourceDir string `json:"sourceDir"`
ExclusionRules string `json:"exclusionRules"`
SourceAccountIDs string `json:"sourceAccountsIDs"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
RetainCopies uint64 `json:"retainCopies"`
agent/app/model/snapshot.go
+1 -1
View File
@@ -4,7 +4,7 @@ type Snapshot struct {
BaseModel
Name string `json:"name" gorm:"not null;unique"`
Description string `json:"description"`
SourceAccountIDs string `json:"sourceAccountsIDs"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
Status string `json:"status"`
Message string `json:"message"`
agent/app/service/app_utils.go
+1 -1
View File
@@ -705,7 +705,7 @@ func upgradeInstall(req request.AppInstallUpgrade) error {
rollBackApp := func(t *task.Task) {
if req.Backup {
t.Log(i18n.GetWithName("AppRecover", install.Name))
if err := NewIBackupService().AppRecover(dto.CommonRecover{Name: install.App.Key, DetailName: install.Name, Type: "app", BackupAccountID: 1, File: backupFile}); err != nil {
if err := NewIBackupService().AppRecover(dto.CommonRecover{Name: install.App.Key, DetailName: install.Name, Type: "app", DownloadAccountID: 1, File: backupFile}); err != nil {
t.LogFailedWithErr(i18n.GetWithName("AppRecover", install.Name), err)
return
}
agent/app/service/backup.go
+108 -48
View File
@@ -17,6 +17,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage"
"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
httpUtils "github.com/1Panel-dev/1Panel/agent/utils/http"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
@@ -66,6 +67,9 @@ func (u *BackupService) SearchRecordsWithPage(search dto.RecordSearch) (int64, [
return 0, nil, err
}
if total == 0 {
return 0, nil, nil
}
datas, err := u.loadRecordSize(records)
sort.Slice(datas, func(i, j int) bool {
return datas[i].CreatedAt.After(datas[j].CreatedAt)
@@ -83,6 +87,9 @@ func (u *BackupService) SearchRecordsByCronjobWithPage(search dto.RecordSearchBy
return 0, nil, err
}
if total == 0 {
return 0, nil, nil
}
datas, err := u.loadRecordSize(records)
sort.Slice(datas, func(i, j int) bool {
return datas[i].CreatedAt.After(datas[j].CreatedAt)
@@ -189,32 +196,35 @@ func (u *BackupService) ListFiles(req dto.OperateByID) []string {
}
func (u *BackupService) loadRecordSize(records []model.BackupRecord) ([]dto.BackupRecords, error) {
recordMap := make(map[uint]struct{})
var recordIds []string
for _, record := range records {
if _, ok := recordMap[record.DownloadAccountID]; !ok {
recordMap[record.DownloadAccountID] = struct{}{}
recordIds = append(recordIds, fmt.Sprintf("%v", record.DownloadAccountID))
}
}
clientMap, err := NewBackupClientMap(recordIds)
if err != nil {
return nil, err
}
var datas []dto.BackupRecords
clientMap := make(map[uint]loadSizeHelper)
var wg sync.WaitGroup
for i := 0; i < len(records); i++ {
var item dto.BackupRecords
if err := copier.Copy(&item, &records[i]); err != nil {
return nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
}
itemPath := path.Join(records[i].FileDir, records[i].FileName)
if _, ok := clientMap[records[i].DownloadAccountID]; !ok {
account, client, err := NewBackupClientWithID(records[i].DownloadAccountID)
if err != nil {
global.LOG.Errorf("load backup client from db failed, err: %v", err)
clientMap[records[i].DownloadAccountID] = loadSizeHelper{}
datas = append(datas, item)
continue
}
item.Size, _ = client.Size(path.Join(strings.TrimLeft(account.BackupPath, "/"), itemPath))
datas = append(datas, item)
clientMap[records[i].DownloadAccountID] = loadSizeHelper{backupPath: strings.TrimLeft(account.BackupPath, "/"), client: client, isOk: true}
continue
}
if clientMap[records[i].DownloadAccountID].isOk {
if val, ok := clientMap[fmt.Sprintf("%v", records[i].DownloadAccountID)]; ok {
item.AccountName = val.name
item.AccountType = val.accountType
item.DownloadAccountID = val.id
wg.Add(1)
go func(index int) {
item.Size, _ = clientMap[records[index].DownloadAccountID].client.Size(path.Join(clientMap[records[index].DownloadAccountID].backupPath, itemPath))
item.Size, _ = val.client.Size(path.Join(strings.TrimLeft(val.backupPath, "/"), itemPath))
datas = append(datas, item)
wg.Done()
}(i)
@@ -226,25 +236,46 @@ func (u *BackupService) loadRecordSize(records []model.BackupRecord) ([]dto.Back
return datas, nil
}
func NewBackupClientWithID(id uint) (*dto.BackupInfo, cloud_storage.CloudStorageClient, error) {
data, err := httpUtils.RequestToMaster(fmt.Sprintf("/api/v2/backup/%v", id), http.MethodGet, nil)
if err != nil {
return nil, nil, err
}
global.LOG.Debug("我走到了这里11")
account, ok := data.(dto.BackupInfo)
if !ok {
return nil, nil, fmt.Errorf("err response from master: %v", data)
}
global.LOG.Debug("我走到了这里22")
if account.Type == constant.Local {
localDir, err := LoadLocalDirByStr(account.Vars)
func NewBackupClientWithID(id uint) (*model.BackupAccount, cloud_storage.CloudStorageClient, error) {
var account model.BackupAccount
if global.IsMaster {
var setting model.Setting
if err := global.CoreDB.Where("key = ?", "EncryptKey").First(&setting).Error; err != nil {
return nil, nil, err
}
if err := global.CoreDB.Where("id = ?", id).First(&account).Error; err != nil {
return nil, nil, err
}
if account.ID == 0 {
return nil, nil, constant.ErrRecordNotFound
}
account.AccessKey, _ = encrypt.StringDecryptWithKey(account.AccessKey, setting.Value)
account.Credential, _ = encrypt.StringDecryptWithKey(account.Credential, setting.Value)
} else {
bodyItem, err := json.Marshal(dto.OperateByID{ID: id})
if err != nil {
return nil, nil, err
}
global.CONF.System.Backup = localDir
data, err := httpUtils.RequestToMaster("/api/v2/agent/backup", http.MethodPost, bytes.NewReader(bodyItem))
if err != nil {
return nil, nil, err
}
item, err := json.Marshal(data)
if err != nil {
return nil, nil, err
}
if err := json.Unmarshal(item, &account); err != nil {
return nil, nil, fmt.Errorf("err response from master: %v", data)
}
if account.Type == constant.Local {
localDir, err := LoadLocalDirByStr(account.Vars)
if err != nil {
return nil, nil, err
}
global.CONF.System.Backup = localDir
}
}
global.LOG.Debug("我走到了这里33")
backClient, err := newClient(&account)
if err != nil {
return nil, nil, err
@@ -253,24 +284,46 @@ func NewBackupClientWithID(id uint) (*dto.BackupInfo, cloud_storage.CloudStorage
}
type backupClientHelper struct {
id uint
name string
backupPath string
client cloud_storage.CloudStorageClient
id uint
accountType string
name string
backupPath string
client cloud_storage.CloudStorageClient
}
func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
bodyItem, err := json.Marshal(ids)
if err != nil {
return nil, err
}
data, err := httpUtils.RequestToMaster("/api/v2/backup/list", http.MethodPost, bytes.NewReader(bodyItem))
if err != nil {
return nil, err
}
accounts, ok := data.([]dto.BackupInfo)
if !ok {
return nil, fmt.Errorf("err response from master: %v", data)
var accounts []model.BackupAccount
if global.IsMaster {
var setting model.Setting
if err := global.CoreDB.Where("key = ?", "EncryptKey").First(&setting).Error; err != nil {
return nil, err
}
if err := global.CoreDB.Where("id in (?)", ids).Find(&accounts).Error; err != nil {
return nil, err
}
if len(accounts) == 0 {
return nil, constant.ErrRecordNotFound
}
for i := 0; i < len(accounts); i++ {
accounts[i].AccessKey, _ = encrypt.StringDecryptWithKey(accounts[i].AccessKey, setting.Value)
accounts[i].Credential, _ = encrypt.StringDecryptWithKey(accounts[i].Credential, setting.Value)
}
} else {
bodyItem, err := json.Marshal(ids)
if err != nil {
return nil, err
}
data, err := httpUtils.RequestToMaster("/api/v2/agent/backup/list", http.MethodPost, bytes.NewReader(bodyItem))
if err != nil {
return nil, err
}
item, err := json.Marshal(data)
if err != nil {
return nil, err
}
if err := json.Unmarshal(item, &accounts); err != nil {
return nil, fmt.Errorf("err response from master: %v", data)
}
}
clientMap := make(map[string]backupClientHelper)
for _, item := range accounts {
@@ -282,12 +335,18 @@ func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
if item.BackupPath != "/" {
pathItem = strings.TrimPrefix(item.BackupPath, "/")
}
clientMap[item.Name] = backupClientHelper{client: backClient, backupPath: pathItem, name: item.Name}
clientMap[fmt.Sprintf("%v", item.ID)] = backupClientHelper{
client: backClient,
backupPath: pathItem,
name: item.Name,
accountType: item.Type,
id: item.ID,
}
}
return clientMap, nil
}
func newClient(account *dto.BackupInfo) (cloud_storage.CloudStorageClient, error) {
func newClient(account *model.BackupAccount) (cloud_storage.CloudStorageClient, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(account.Vars), &varMap); err != nil {
return nil, err
@@ -323,6 +382,7 @@ func LoadLocalDirByStr(vars string) (string, error) {
if err = os.MkdirAll(baseDir, os.ModePerm); err != nil {
return "", fmt.Errorf("mkdir %s failed, err: %v", baseDir, err)
}
return baseDir, nil
}
}
return "", fmt.Errorf("error type dir: %T", varMap["dir"])
agent/app/service/cronjob.go
+2 -2
View File
@@ -273,8 +273,8 @@ func (u *CronjobService) Update(id uint, req dto.CronjobUpdate) error {
upMap["url"] = req.URL
upMap["source_dir"] = req.SourceDir
upMap["backup_accounts"] = req.BackupAccounts
upMap["default_download"] = req.DefaultDownload
upMap["source_account_ids"] = req.SourceAccountIDs
upMap["download_account_id"] = req.DownloadAccountID
upMap["retain_copies"] = req.RetainCopies
upMap["secret"] = req.Secret
return cronjobRepo.Update(id, upMap)
agent/app/service/cronjob_backup.go
+7 -23
View File
@@ -38,7 +38,7 @@ func (u *CronjobService) handleApp(cronjob model.Cronjob, startTime time.Time) e
record.CronjobID = cronjob.ID
record.Name = app.App.Key
record.DetailName = app.Name
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("app/%s/%s", app.App.Key, app.Name))
record.FileName = fmt.Sprintf("app_%s_%s.tar.gz", app.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleAppBackup(&app, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret); err != nil {
@@ -71,7 +71,7 @@ func (u *CronjobService) handleWebsite(cronjob model.Cronjob, startTime time.Tim
record.CronjobID = cronjob.ID
record.Name = web.PrimaryDomain
record.DetailName = web.Alias
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("website/%s", web.PrimaryDomain))
record.FileName = fmt.Sprintf("website_%s_%s.tar.gz", web.PrimaryDomain, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
if err := handleWebsiteBackup(&web, backupDir, record.FileName, cronjob.ExclusionRules, cronjob.Secret); err != nil {
@@ -104,7 +104,7 @@ func (u *CronjobService) handleDatabase(cronjob model.Cronjob, startTime time.Ti
record.CronjobID = cronjob.ID
record.Name = dbInfo.Database
record.DetailName = dbInfo.Name
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
backupDir := path.Join(global.CONF.System.TmpDir, fmt.Sprintf("database/%s/%s/%s", dbInfo.DBType, record.Name, dbInfo.Name))
record.FileName = fmt.Sprintf("db_%s_%s.sql.gz", dbInfo.Name, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5))
@@ -146,7 +146,7 @@ func (u *CronjobService) handleDirectory(cronjob model.Cronjob, startTime time.T
record.Type = "directory"
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(backupDir, fileName))
if err != nil {
return err
@@ -177,7 +177,7 @@ func (u *CronjobService) handleSystemLog(cronjob model.Cronjob, startTime time.T
record.Type = "log"
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
downloadPath, err := u.uploadCronjobBackFile(cronjob, accountMap, path.Join(path.Dir(backupDir), fileName))
if err != nil {
return err
@@ -203,11 +203,11 @@ func (u *CronjobService) handleSnapshot(cronjob model.Cronjob, startTime time.Ti
record.Type = "directory"
record.CronjobID = cronjob.ID
record.Name = cronjob.Name
record.DownloadAccountID, record.SourceAccountIDs = loadRecordPath(cronjob, accountMap)
record.DownloadAccountID, record.SourceAccountIDs = cronjob.DownloadAccountID, cronjob.SourceAccountIDs
record.FileDir = "system_snapshot"
req := dto.SnapshotCreate{
SourceAccountIDs: record.SourceAccountIDs,
SourceAccountIDs: record.SourceAccountIDs,
DownloadAccountID: cronjob.DownloadAccountID,
}
name, err := NewISnapshotService().HandleSnapshot(true, logPath, req, startTime.Format(constant.DateTimeSlimLayout)+common.RandStrAndNum(5), cronjob.Secret)
@@ -287,22 +287,6 @@ func loadWebsForJob(cronjob model.Cronjob) []model.Website {
return weblist
}
func loadRecordPath(cronjob model.Cronjob, accountMap map[string]backupClientHelper) (uint, string) {
download := accountMap[fmt.Sprintf("%v", cronjob.DownloadAccountID)].id
sources := strings.Split(cronjob.SourceAccountIDs, ",")
var itemAccounts []string
for _, target := range sources {
if len(target) == 0 {
continue
}
if accountMap[target].id != 0 {
itemAccounts = append(itemAccounts, fmt.Sprintf("%v", accountMap[target].id))
}
}
backupType := strings.Join(itemAccounts, ",")
return download, backupType
}
func handleBackupLogs(targetDir, fileName string, secret string) error {
websites, err := websiteRepo.List()
if err != nil {
agent/configs/system.go
+16 -16
View File
@@ -1,20 +1,20 @@
package configs
type System struct {
MasterRequestAddr string `mapstructure:"master_request_addr"`
MasterRequestToken string `mapstructure:"master_request_token"`
DbFile string `mapstructure:"db_agent_file"`
DbPath string `mapstructure:"db_path"`
LogPath string `mapstructure:"log_path"`
DataDir string `mapstructure:"data_dir"`
TmpDir string `mapstructure:"tmp_dir"`
Cache string `mapstructure:"cache"`
Backup string `mapstructure:"backup"`
EncryptKey string `mapstructure:"encrypt_key"`
BaseDir string `mapstructure:"base_dir"`
Mode string `mapstructure:"mode"`
RepoUrl string `mapstructure:"repo_url"`
Version string `mapstructure:"version"`
IsDemo bool `mapstructure:"is_demo"`
AppRepo string `mapstructure:"app_repo"`
MasterAddr string `mapstructure:"master_addr"`
MasterToken string `mapstructure:"master_token"`
DbFile string `mapstructure:"db_agent_file"`
DbPath string `mapstructure:"db_path"`
LogPath string `mapstructure:"log_path"`
DataDir string `mapstructure:"data_dir"`
TmpDir string `mapstructure:"tmp_dir"`
Cache string `mapstructure:"cache"`
Backup string `mapstructure:"backup"`
EncryptKey string `mapstructure:"encrypt_key"`
BaseDir string `mapstructure:"base_dir"`
Mode string `mapstructure:"mode"`
RepoUrl string `mapstructure:"repo_url"`
Version string `mapstructure:"version"`
IsDemo bool `mapstructure:"is_demo"`
AppRepo string `mapstructure:"app_repo"`
}
agent/global/global.go
+2 -1
View File
@@ -15,6 +15,7 @@ import (
var (
DB *gorm.DB
MonitorDB *gorm.DB
CoreDB *gorm.DB
LOG *logrus.Logger
CONF configs.ServerConfig
VALID *validator.Validate
@@ -26,7 +27,7 @@ var (
MonitorCronID cron.EntryID
OneDriveCronID cron.EntryID
CurrentNode string
IsMaster bool
I18n *i18n.Localizer
)
agent/init/db/db.go
+46 -18
View File
@@ -28,20 +28,11 @@ func Init() {
_ = f.Close()
}
newLogger := logger.New(
log.New(os.Stdout, "\r\n", log.LstdFlags),
logger.Config{
SlowThreshold: time.Second,
LogLevel: logger.Silent,
IgnoreRecordNotFoundError: true,
Colorful: false,
},
)
initMonitorDB(newLogger)
initMonitorDB()
db, err := gorm.Open(sqlite.Open(fullPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: newLogger,
Logger: newLogger(),
})
if err != nil {
panic(err)
@@ -56,14 +47,10 @@ func Init() {
global.DB = db
global.LOG.Info("init db successfully")
}
func initMonitorDB(newLogger logger.Interface) {
if _, err := os.Stat(global.CONF.System.DbPath); err != nil {
if err := os.MkdirAll(global.CONF.System.DbPath, os.ModePerm); err != nil {
panic(fmt.Errorf("init db dir failed, err: %v", err))
}
}
func initMonitorDB() {
fullPath := path.Join(global.CONF.System.DbPath, "monitor.db")
if _, err := os.Stat(fullPath); err != nil {
f, err := os.Create(fullPath)
@@ -75,7 +62,7 @@ func initMonitorDB(newLogger logger.Interface) {
db, err := gorm.Open(sqlite.Open(fullPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: newLogger,
Logger: newLogger(),
})
if err != nil {
panic(err)
@@ -91,3 +78,44 @@ func initMonitorDB(newLogger logger.Interface) {
global.MonitorDB = db
global.LOG.Info("init monitor db successfully")
}
func InitCoreDB() {
fullPath := path.Join(global.CONF.System.DbPath, "core.db")
if _, err := os.Stat(fullPath); err != nil {
f, err := os.Create(fullPath)
if err != nil {
panic(fmt.Errorf("init db file failed, err: %v", err))
}
_ = f.Close()
}
db, err := gorm.Open(sqlite.Open(fullPath), &gorm.Config{
DisableForeignKeyConstraintWhenMigrating: true,
Logger: newLogger(),
})
if err != nil {
panic(err)
}
sqlDB, dbError := db.DB()
if dbError != nil {
panic(dbError)
}
sqlDB.SetConnMaxIdleTime(10)
sqlDB.SetMaxOpenConns(100)
sqlDB.SetConnMaxLifetime(time.Hour)
global.CoreDB = db
global.LOG.Info("init local core db successfully")
}
func newLogger() logger.Interface {
return logger.New(
log.New(os.Stdout, "\r\n", log.LstdFlags),
logger.Config{
SlowThreshold: time.Second,
LogLevel: logger.Silent,
IgnoreRecordNotFoundError: true,
Colorful: false,
},
)
}
agent/init/hook/hook.go
+35 -6
View File
@@ -10,6 +10,7 @@ import (
"github.com/1Panel-dev/1Panel/agent/app/service"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/init/db"
"github.com/1Panel-dev/1Panel/agent/utils/encrypt"
)
@@ -25,7 +26,21 @@ func Init() {
if err != nil {
global.LOG.Fatalf("load current node before start failed, err: %v", err)
}
global.CurrentNode = node.Value
global.IsMaster = node.Value == "127.0.0.1" || len(node.Value) == 0
if global.IsMaster {
db.InitCoreDB()
} else {
masterAddr, err := settingRepo.Get(settingRepo.WithByKey("MasterAddr"))
if err != nil {
global.LOG.Fatalf("load master addr before start failed, err: %v", err)
}
global.CONF.System.MasterAddr = masterAddr.Value
token, err := settingRepo.Get(settingRepo.WithByKey("Token"))
if err != nil {
global.LOG.Fatalf("load token before start failed, err: %v", err)
}
global.CONF.System.MasterToken, _ = encrypt.StringDecrypt(token.Value)
}
handleCronjobStatus()
handleSnapStatus()
@@ -101,14 +116,28 @@ func handleCronjobStatus() {
}
func loadLocalDir() {
account, _, err := service.NewBackupClientWithID(1)
if err != nil {
global.LOG.Errorf("load local backup account info failed, err: %v", err)
var vars string
if global.IsMaster {
var account model.BackupAccount
if err := global.CoreDB.Where("id = 1").First(&account).Error; err != nil {
global.LOG.Errorf("load local backup account info failed, err: %v", err)
return
}
vars = account.Vars
} else {
account, _, err := service.NewBackupClientWithID(1)
if err != nil {
global.LOG.Errorf("load local backup account info failed, err: %v", err)
return
}
vars = account.Vars
}
global.CONF.System.Backup, err = service.LoadLocalDirByStr(account.Vars)
localDir, err := service.LoadLocalDirByStr(vars)
if err != nil {
global.LOG.Errorf("load local backup dir failed, err: %v", err)
return
}
global.CONF.System.Backup = localDir
}
func initDir() {
@@ -150,7 +179,7 @@ func initSSL() error {
if err := settingRepo.Update("CurrentNode", node.CurrentNode); err != nil {
return err
}
global.CurrentNode = node.CurrentNode
global.IsMaster = node.CurrentNode == "127.0.0.1" || len(node.CurrentNode) == 0
_ = os.Remove(("/opt/1panel/nodeJson"))
return nil
}
agent/init/migration/migrate.go
+1
View File
@@ -10,6 +10,7 @@ import (
func Init() {
m := gormigrate.New(global.DB, gormigrate.DefaultOptions, []*gormigrate.Migration{
migrations.AddTable,
migrations.AddMonitorTable,
migrations.InitHost,
migrations.InitSetting,
migrations.InitImageRepo,
agent/init/migration/migrations/init.go
+26 -10
View File
@@ -63,6 +63,17 @@ var AddTable = &gormigrate.Migration{
},
}
var AddMonitorTable = &gormigrate.Migration{
ID: "20240813-add-monitor-table",
Migrate: func(tx *gorm.DB) error {
return global.MonitorDB.AutoMigrate(
&model.MonitorBase{},
&model.MonitorIO{},
&model.MonitorNetwork{},
)
},
}
var InitHost = &gormigrate.Migration{
ID: "20240722-init-host",
Migrate: func(tx *gorm.DB) error {
@@ -90,13 +101,14 @@ var InitSetting = &gormigrate.Migration{
if err := tx.Create(&model.Setting{Key: "EncryptKey", Value: encryptKey}).Error; err != nil {
return err
}
currentNode := ""
if _, err := os.Stat("/opt/1panel/nodeJson"); err == nil {
type nodeInfo struct {
MasterRequestAddr string `json:"masterRequestAddr"`
Token string `json:"token"`
ServerCrt string `json:"serverCrt"`
ServerKey string `json:"serverKey"`
CurrentNode string `json:"currentNode"`
MasterAddr string `json:"masterAddr"`
Token string `json:"token"`
ServerCrt string `json:"serverCrt"`
ServerKey string `json:"serverKey"`
CurrentNode string `json:"currentNode"`
}
nodeJson, err := os.ReadFile("/opt/1panel/nodeJson")
if err != nil {
@@ -118,13 +130,17 @@ var InitSetting = &gormigrate.Migration{
if err := tx.Create(&model.Setting{Key: "Token", Value: itemToken}).Error; err != nil {
return err
}
global.CONF.System.MasterRequestAddr = node.MasterRequestAddr
global.CONF.System.MasterRequestToken = itemToken
global.CurrentNode = node.CurrentNode
if err := tx.Create(&model.Setting{Key: "MasterAddr", Value: node.MasterAddr}).Error; err != nil {
return err
}
global.CONF.System.MasterAddr = node.MasterAddr
global.CONF.System.MasterToken = itemToken
global.IsMaster = false
currentNode = node.CurrentNode
} else {
global.CurrentNode = "127.0.0.1"
global.IsMaster = true
}
if err := tx.Create(&model.Setting{Key: "CurrentNode", Value: global.CurrentNode}).Error; err != nil {
if err := tx.Create(&model.Setting{Key: "CurrentNode", Value: currentNode}).Error; err != nil {
return err
}
agent/init/router/router.go
+3 -4
View File
@@ -1,6 +1,7 @@
package router
import (
v2 "github.com/1Panel-dev/1Panel/agent/app/api/v2"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/i18n"
"github.com/1Panel-dev/1Panel/agent/middleware"
@@ -19,19 +20,17 @@ func Routers() *gin.Engine {
PublicGroup := Router.Group("")
{
PublicGroup.GET("/health", func(c *gin.Context) {
c.JSON(200, "ok")
})
PublicGroup.Use(gzip.Gzip(gzip.DefaultCompression))
PublicGroup.Static("/api/v2/images", "./uploads")
}
PrivateGroup := Router.Group("/api/v2")
if global.CurrentNode != "127.0.0.1" {
if !global.IsMaster {
PrivateGroup.Use(middleware.Certificate())
}
for _, router := range rou.RouterGroupApp {
router.InitRouter(PrivateGroup)
}
PrivateGroup.GET("/health/check", v2.ApiGroupApp.BaseApi.CheckHealth)
return Router
}
agent/middleware/certificate.go
+1 -1
View File
@@ -12,7 +12,7 @@ import (
func Certificate() gin.HandlerFunc {
return func(c *gin.Context) {
if global.CurrentNode == "127.0.0.1" || len(global.CurrentNode) == 0 {
if global.IsMaster {
c.Next()
return
}
agent/server/server.go
+1 -1
View File
@@ -44,7 +44,7 @@ func Start() {
server := &http.Server{
Handler: rootRouter,
}
if len(global.CurrentNode) == 0 || global.CurrentNode == "127.0.0.1" {
if global.IsMaster {
_ = os.Remove("/tmp/agent.sock")
listener, err := net.Listen("unix", "/tmp/agent.sock")
if err != nil {
agent/utils/encrypt/encrypt.go
+18 -11
View File
@@ -46,6 +46,23 @@ func StringEncrypt(text string) (string, error) {
return "", err
}
func StringDecryptWithKey(text, key string) (string, error) {
if len(text) == 0 {
return "", nil
}
bytesPass, err := base64.StdEncoding.DecodeString(text)
if err != nil {
return "", err
}
var tpass []byte
tpass, err = aesDecryptWithSalt([]byte(key), bytesPass)
if err == nil {
result := string(tpass[:])
return result, err
}
return "", err
}
func StringDecrypt(text string) (string, error) {
if len(text) == 0 {
return "", nil
@@ -58,17 +75,7 @@ func StringDecrypt(text string) (string, error) {
global.CONF.System.EncryptKey = encryptSetting.Value
}
key := global.CONF.System.EncryptKey
bytesPass, err := base64.StdEncoding.DecodeString(text)
if err != nil {
return "", err
}
var tpass []byte
tpass, err = aesDecryptWithSalt([]byte(key), bytesPass)
if err == nil {
result := string(tpass[:])
return result, err
}
return "", err
return StringDecryptWithKey(text, key)
}
func padding(plaintext []byte, blockSize int) []byte {
agent/utils/http/master.go
+5 -4
View File
@@ -17,20 +17,21 @@ func RequestToMaster(reqUrl, reqMethod string, reqBody io.Reader) (interface{},
client := &http.Client{
Timeout: time.Second * 5,
}
parsedURL, err := url.Parse(global.CONF.System.MasterRequestAddr)
parsedURL, err := url.Parse(global.CONF.System.MasterAddr)
if err != nil {
return nil, fmt.Errorf("handle url Parse failed, err: %v \n", err)
}
rURL := &url.URL{
Path: reqUrl,
Host: parsedURL.Host,
Scheme: "http",
Path: reqUrl,
Host: parsedURL.Host,
}
req, err := http.NewRequest(reqMethod, rURL.String(), reqBody)
if err != nil {
return nil, fmt.Errorf("handle request failed, err: %v \n", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set(constant.JWTHeaderName, global.CONF.System.MasterRequestToken)
req.Header.Set(constant.JWTHeaderName, global.CONF.System.MasterToken)
resp, err := client.Do(req)
if err != nil {
core/app/api/v2/backup.go
+32
View File
@@ -78,6 +78,22 @@ func (b *BaseApi) LoadOneDriveInfo(c *gin.Context) {
helper.SuccessWithData(c, data)
}
// @Tags Backup Account
// @Summary Load backup account options
// @Description 获取备份账号选项
// @Accept json
// @Success 200 {array} dto.BackupOption
// @Security ApiKeyAuth
// @Router /core/backup/options [get]
func (b *BaseApi) LoadBackupOptions(c *gin.Context) {
list, err := backupService.LoadBackupOptions()
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, list)
}
// @Tags Backup Account
// @Summary Delete backup account
// @Description 删除备份账号
@@ -148,6 +164,22 @@ func (b *BaseApi) SearchBackup(c *gin.Context) {
})
}
// @Tags Backup Account
// @Summary get local backup dir
// @Description 获取本地备份目录
// @Success 200
// @Security ApiKeyAuth
// @Router /core/backup/local [get]
func (b *BaseApi) GetLocalDir(c *gin.Context) {
dir, err := backupService.GetLocalDir()
if err != nil {
helper.ErrorWithDetail(c, constant.CodeErrInternalServer, constant.ErrTypeInternalServer, err)
return
}
helper.SuccessWithData(c, dir)
}
func (b *BaseApi) GetBackup(c *gin.Context) {
var req dto.OperateByID
if err := helper.CheckBindAndValidate(&req, c); err != nil {
core/app/dto/backup.go
+6
View File
@@ -15,6 +15,12 @@ type BackupOperate struct {
RememberAuth bool `json:"rememberAuth"`
}
type BackupOption struct {
ID uint `json:"id"`
Name string `json:"name"`
Type string `json:"type"`
}
type BackupInfo struct {
ID uint `json:"id"`
Name string `json:"name"`
core/app/dto/setting.go
+1
View File
@@ -6,6 +6,7 @@ type SettingInfo struct {
UserName string `json:"userName"`
SystemVersion string `json:"systemVersion"`
DeveloperMode string `json:"developerMode"`
MasterAddr string `json:"masterAddr"`
SessionTimeout string `json:"sessionTimeout"`
Port string `json:"port"`
core/app/service/backup.go
+34 -4
View File
@@ -30,6 +30,8 @@ type IBackupService interface {
Get(req dto.OperateByID) (dto.BackupInfo, error)
List(req dto.OperateByIDs) ([]dto.BackupInfo, error)
GetLocalDir() (string, error)
LoadBackupOptions() ([]dto.BackupOption, error)
SearchWithPage(search dto.SearchPageWithType) (int64, interface{}, error)
LoadOneDriveInfo() (dto.OneDriveInfo, error)
Create(backupDto dto.BackupOperate) error
@@ -47,7 +49,7 @@ func NewIBackupService() IBackupService {
func (u *BackupService) Get(req dto.OperateByID) (dto.BackupInfo, error) {
var data dto.BackupInfo
account, err := backupRepo.List(commonRepo.WithByID(req.ID))
account, err := backupRepo.Get(commonRepo.WithByID(req.ID))
if err != nil {
return data, err
}
@@ -89,6 +91,34 @@ func (u *BackupService) List(req dto.OperateByIDs) ([]dto.BackupInfo, error) {
return data, nil
}
func (u *BackupService) GetLocalDir() (string, error) {
account, err := backupRepo.Get(commonRepo.WithByType(constant.Local))
if err != nil {
return "", err
}
dir, err := LoadLocalDirByStr(account.Vars)
if err != nil {
return "", err
}
return dir, nil
}
func (u *BackupService) LoadBackupOptions() ([]dto.BackupOption, error) {
accounts, err := backupRepo.List(commonRepo.WithOrderBy("created_at desc"))
if err != nil {
return nil, err
}
var data []dto.BackupOption
for _, account := range accounts {
var item dto.BackupOption
if err := copier.Copy(&item, &account); err != nil {
global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
}
data = append(data, item)
}
return data, nil
}
func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, interface{}, error) {
count, accounts, err := backupRepo.Page(
req.Page,
@@ -272,11 +302,11 @@ func (u *BackupService) Update(req dto.BackupOperate) error {
newBackup.Credential = string(itemCredential)
if backup.Type == constant.Local {
if newBackup.Vars != backup.Vars {
oldPath, err := loadLocalDirByStr(backup.Vars)
oldPath, err := LoadLocalDirByStr(backup.Vars)
if err != nil {
return err
}
newPath, err := loadLocalDirByStr(newBackup.Vars)
newPath, err := LoadLocalDirByStr(newBackup.Vars)
if err != nil {
return err
}
@@ -362,7 +392,7 @@ func (u *BackupService) loadAccessToken(backup *model.BackupAccount) error {
return nil
}
func loadLocalDirByStr(vars string) (string, error) {
func LoadLocalDirByStr(vars string) (string, error) {
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(vars), &varMap); err != nil {
return "", err
core/app/service/setting.go
+2
View File
@@ -76,6 +76,8 @@ func (u *SettingService) Update(key, value string) error {
_ = settingRepo.Create("AppStoreLastModified", value)
return nil
}
case "MasterAddr":
global.CONF.System.MasterAddr = value
}
if err := settingRepo.Update(key, value); err != nil {
core/configs/system.go
+17 -18
View File
@@ -1,22 +1,21 @@
package configs
type System struct {
MasterRequestAddr string `mapstructure:"masterRequestAddr"`
Port string `mapstructure:"port"`
Ipv6 string `mapstructure:"ipv6"`
BindAddress string `mapstructure:"bindAddress"`
SSL string `mapstructure:"ssl"`
DbCoreFile string `mapstructure:"db_core_file"`
EncryptKey string `mapstructure:"encrypt_key"`
BaseDir string `mapstructure:"base_dir"`
BackupDir string `mapstructure:"backup_dir"`
Mode string `mapstructure:"mode"`
RepoUrl string `mapstructure:"repo_url"`
Version string `mapstructure:"version"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Entrance string `mapstructure:"entrance"`
IsDemo bool `mapstructure:"is_demo"`
ChangeUserInfo string `mapstructure:"change_user_info"`
DbPath string `mapstructure:"db_path"`
Port string `mapstructure:"port"`
Ipv6 string `mapstructure:"ipv6"`
BindAddress string `mapstructure:"bindAddress"`
SSL string `mapstructure:"ssl"`
MasterAddr string `mapstructure:"masterAddr"`
DbCoreFile string `mapstructure:"db_core_file"`
EncryptKey string `mapstructure:"encrypt_key"`
BaseDir string `mapstructure:"base_dir"`
BackupDir string `mapstructure:"backup_dir"`
Mode string `mapstructure:"mode"`
RepoUrl string `mapstructure:"repo_url"`
Version string `mapstructure:"version"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
Entrance string `mapstructure:"entrance"`
IsDemo bool `mapstructure:"is_demo"`
ChangeUserInfo string `mapstructure:"change_user_info"`
}
core/constant/errs.go
+1
View File
@@ -55,4 +55,5 @@ var (
var (
ErrBackupInUsed = "ErrBackupInUsed"
ErrBackupLocalDelete = "ErrBackupLocalDelete"
ErrMasterAddr = "ErrMasterAddr"
)
core/i18n/lang/zh.yaml
+1
View File
@@ -16,6 +16,7 @@ ErrCmdTimeout: "命令执行超时！"
ErrEntrance: "安全入口信息错误，请检查后重试！"
ErrGroupIsUsed: "分组正在使用中，无法删除"
ErrLocalDelete: "无法删除本地节点！"
ErrMasterAddr: "当前未设置主节点地址，无法添加节点！"
#app
ErrPortInUsed: "{{ .detail }} 端口已被占用！"
core/init/hook/hook.go
+8 -23
View File
@@ -1,12 +1,11 @@
package hook
import (
"encoding/json"
"os"
"strings"
"github.com/1Panel-dev/1Panel/core/app/model"
"github.com/1Panel-dev/1Panel/core/app/repo"
"github.com/1Panel-dev/1Panel/core/app/service"
"github.com/1Panel-dev/1Panel/core/global"
"github.com/1Panel-dev/1Panel/core/utils/cmd"
"github.com/1Panel-dev/1Panel/core/utils/common"
@@ -15,11 +14,11 @@ import (
func Init() {
settingRepo := repo.NewISettingRepo()
masterSetting, err := settingRepo.Get(settingRepo.WithByKey("MasterRequestAddr"))
masterSetting, err := settingRepo.Get(settingRepo.WithByKey("MasterAddr"))
if err != nil {
global.LOG.Errorf("load master request addr from setting failed, err: %v", err)
global.LOG.Errorf("load master addr from setting failed, err: %v", err)
}
global.CONF.System.MasterRequestAddr = masterSetting.Value
global.CONF.System.MasterAddr = masterSetting.Value
portSetting, err := settingRepo.Get(settingRepo.WithByKey("ServerPort"))
if err != nil {
global.LOG.Errorf("load service port from setting failed, err: %v", err)
@@ -97,24 +96,10 @@ func loadLocalDir() {
global.LOG.Errorf("no such backup account `%s` in db", "LOCAL")
return
}
varMap := make(map[string]interface{})
if err := json.Unmarshal([]byte(backup.Vars), &varMap); err != nil {
global.LOG.Errorf("json unmarshal backup.Vars: %v failed, err: %v", backup.Vars, err)
dir, err := service.LoadLocalDirByStr(backup.Vars)
if err != nil {
global.LOG.Errorf("load local backup dir failed,err: %v", err)
return
}
if _, ok := varMap["dir"]; !ok {
global.LOG.Error("load local backup dir failed")
return
}
baseDir, ok := varMap["dir"].(string)
if ok {
if _, err := os.Stat(baseDir); err != nil && os.IsNotExist(err) {
if err = os.MkdirAll(baseDir, os.ModePerm); err != nil {
global.LOG.Errorf("mkdir %s failed, err: %v", baseDir, err)
return
}
}
}
global.CONF.System.BackupDir = baseDir
global.LOG.Errorf("error type dir: %T", varMap["dir"])
global.CONF.System.BackupDir = dir
}
core/init/migration/migrate.go
+1
View File
@@ -12,6 +12,7 @@ func Init() {
migrations.AddTable,
migrations.InitSetting,
migrations.InitOneDrive,
migrations.InitMasterAddr,
})
if err := m.Migrate(); err != nil {
global.LOG.Error(err)
core/init/migration/migrations/init.go
+10
View File
@@ -163,3 +163,13 @@ var InitOneDrive = &gormigrate.Migration{
return nil
},
}
var InitMasterAddr = &gormigrate.Migration{
ID: "20240814-init-master-addr",
Migrate: func(tx *gorm.DB) error {
if err := tx.Create(&model.Setting{Key: "MasterAddr", Value: ""}).Error; err != nil {
return err
}
return nil
},
}
core/init/router/router.go
+6 -4
View File
@@ -6,6 +6,8 @@ import (
"github.com/1Panel-dev/1Panel/cmd/server/docs"
"github.com/1Panel-dev/1Panel/cmd/server/web"
"github.com/1Panel-dev/1Panel/core/app/dto"
"github.com/1Panel-dev/1Panel/core/constant"
"github.com/1Panel-dev/1Panel/core/global"
"github.com/1Panel-dev/1Panel/core/i18n"
"github.com/1Panel-dev/1Panel/core/middleware"
@@ -47,17 +49,17 @@ func Routers() *gin.Engine {
swaggerRouter.Use(middleware.JwtAuth()).Use(middleware.SessionAuth()).GET("/swagger/*any", ginSwagger.WrapHandler(swaggerfiles.Handler))
PublicGroup := Router.Group("")
{
PublicGroup.GET("/health", func(c *gin.Context) {
c.JSON(200, "ok")
})
PublicGroup.Use(gzip.Gzip(gzip.DefaultCompression))
setWebStatic(PublicGroup)
}
agentRouter := Router.Group("agent")
agentRouter := Router.Group("/api/v2/agent")
agentRouter.Use(middleware.JwtAuth())
var agent router.AgentRouter
agent.InitRouter(agentRouter)
agentRouter.GET("/health", func(c *gin.Context) {
c.JSON(200, dto.Response{Code: constant.CodeSuccess, Data: "ok"})
})
Router.Use(middleware.OperationLog())
if global.CONF.System.IsDemo {
core/middleware/jwt.go
+16
View File
@@ -1,6 +1,7 @@
package middleware
import (
"fmt"
"strings"
"github.com/1Panel-dev/1Panel/core/app/api/v2/helper"
@@ -27,6 +28,21 @@ func JwtAuth() gin.HandlerFunc {
helper.ErrorWithDetail(c, constant.CodeErrUnauthorized, constant.ErrTypeInternalServer, err)
return
}
if claims.BaseClaims.IsAgent {
if strings.HasPrefix(c.Request.URL.Path, "/api/v2/agent/") {
c.Set("claims", claims)
c.Set("authMethod", constant.AuthMethodJWT)
c.Next()
return
} else {
helper.ErrorWithDetail(c, constant.CodeErrUnauthorized, constant.ErrTypeInternalServer, fmt.Errorf("err token from request"))
return
}
}
if strings.HasPrefix(c.Request.URL.Path, "/api/v2/agent/") {
helper.ErrorWithDetail(c, constant.CodeErrUnauthorized, constant.ErrTypeInternalServer, fmt.Errorf("err token from request"))
return
}
c.Set("claims", claims)
c.Set("authMethod", constant.AuthMethodJWT)
c.Next()
core/router/ro_agent.go
+1 -1
View File
@@ -10,7 +10,7 @@ type AgentRouter struct{}
func (s *AgentRouter) InitRouter(Router *gin.RouterGroup) {
baseApi := v2.ApiGroupApp.BaseApi
{
Router.GET("/backup/:id", baseApi.GetBackup)
Router.POST("/backup", baseApi.GetBackup)
Router.POST("/backup/list", baseApi.ListBackup)
}
}
core/router/ro_backup.go
+2
View File
@@ -11,7 +11,9 @@ func (s *BackupRouter) InitRouter(Router *gin.RouterGroup) {
backupRouter := Router.Group("backup")
baseApi := v2.ApiGroupApp.BaseApi
{
backupRouter.GET("/local", baseApi.GetLocalDir)
backupRouter.GET("/onedrive", baseApi.LoadOneDriveInfo)
backupRouter.GET("/options", baseApi.LoadBackupOptions)
backupRouter.POST("/search", baseApi.SearchBackup)
backupRouter.POST("/refresh/onedrive", baseApi.RefreshOneDriveToken)
backupRouter.POST("/buckets", baseApi.ListBuckets)
core/utils/xpack/xpack.go
-7
View File
@@ -3,16 +3,9 @@
package xpack
import (
"github.com/1Panel-dev/1Panel/core/app/model"
"github.com/gin-gonic/gin"
)
func Proxy(c *gin.Context, currentNode string) error {
return nil
}
type Node struct{}
func SyncBackupOperation(operate string, accounts []model.BackupAccount) error {
return nil
}
frontend/src/api/interface/backup.ts
+10 -4
View File
@@ -5,6 +5,11 @@ export namespace Backup {
type: string;
name: string;
}
export interface BackupOption {
id: number;
name: string;
type: string;
}
export interface BackupInfo {
id: number;
name: string;
@@ -33,15 +38,16 @@ export namespace Backup {
vars: string;
}
export interface RecordDownload {
source: string;
downloadAccountID: number;
fileDir: string;
fileName: string;
}
export interface RecordInfo {
id: number;
createdAt: Date;
source: string;
backupType: string;
accountType: string;
accountName: string;
downloadAccountID: number;
fileDir: string;
fileName: string;
}
@@ -66,7 +72,7 @@ export namespace Backup {
secret: string;
}
export interface Recover {
source: string;
downloadAccountID: number;
type: string;
name: string;
detailName: string;
frontend/src/api/interface/cronjob.ts
+7 -7
View File
@@ -21,9 +21,9 @@ export namespace Cronjob {
url: string;
sourceDir: string;
backupAccounts: string;
defaultDownload: string;
backupAccountList: Array<string>;
sourceAccountIDs: string;
downloadAccountID: number;
sourceAccounts: Array<number>;
retainCopies: number;
status: string;
secret: string;
@@ -42,8 +42,8 @@ export namespace Cronjob {
url: string;
sourceDir: string;
backupAccounts: string;
defaultDownload: string;
sourceAccountIDs: string;
downloadAccountID: number;
retainCopies: number;
secret: string;
}
@@ -67,8 +67,8 @@ export namespace Cronjob {
url: string;
sourceDir: string;
backupAccounts: string;
defaultDownload: string;
sourceAccountIDs: string;
downloadAccountID: number;
retainCopies: number;
secret: string;
}
frontend/src/api/interface/setting.ts
+1
View File
@@ -6,6 +6,7 @@ export namespace Setting {
password: string;
email: string;
systemIP: string;
masterAddr: string;
systemVersion: string;
dockerSockPath: string;
developerMode: string;
frontend/src/api/modules/setting.ts
+4 -1
View File
@@ -114,7 +114,10 @@ export const refreshOneDrive = () => {
return http.post(`/core/backup/refresh/onedrive`, {});
};
export const getBackupList = () => {
return http.post<Array<Backup.BackupInfo>>(`/core/backup/list`);
return http.get<Array<Backup.BackupOption>>(`/core/backup/options`);
};
export const getLocalBackupDir = () => {
return http.get<string>(`/core/backup/local`);
};
export const searchBackup = (params: Backup.SearchWithType) => {
return http.post<ResPage<Backup.BackupInfo>>(`/core/backup/search`, params);
frontend/src/api/modules/website.ts
+2 -2
View File
@@ -38,8 +38,8 @@ export const GetWebsite = (id: number) => {
return http.get<Website.WebsiteDTO>(`/websites/${id}`);
};
export const GetWebsiteOptions = (req: Website.OptionReq) => {
return http.post<any>(`/websites/options`, req);
export const GetWebsiteOptions = () => {
return http.get<any>(`/websites/options`);
};
export const GetWebsiteConfig = (id: number, type: string) => {
frontend/src/components/backup/index.vue
+116 -35
View File
@@ -46,9 +46,13 @@
</el-table-column>
<el-table-column :label="$t('database.source')" prop="backupType">
<template #default="{ row }">
<span v-if="row.source">
{{ $t('setting.' + row.source) }}
<span v-if="row.accountType === 'LOCAL'">
{{ $t('setting.LOCAL') }}
</span>
<span v-if="row.accountType && row.accountType !== 'LOCAL'">
{{ $t('setting.' + row.accountType) + ' - ' + row.accountName }}
</span>
<span v-if="!row.accountType">-</span>
</template>
</el-table-column>
<el-table-column
@@ -62,27 +66,51 @@
</ComplexTable>
</template>
</DrawerPro>
<el-dialog
v-model="open"
:title="isBackup ? $t('commons.button.backup') : $t('commons.button.recover') + ' - ' + name"
width="40%"
:close-on-click-modal="false"
:before-close="handleBackupClose"
>
<el-form ref="backupForm" label-position="left" v-loading="loading">
<el-form-item
:label="$t('setting.compressPassword')"
style="margin-top: 10px"
v-if="type === 'app' || type === 'website'"
>
<el-input v-model="secret" :placeholder="$t('setting.backupRecoverMessage')" />
</el-form-item>
</el-form>
<template #footer>
<span class="dialog-footer">
<el-button @click="handleClose" :disabled="loading">
{{ $t('commons.button.cancel') }}
</el-button>
<el-button type="primary" @click="onSubmit" :disabled="loading">
{{ $t('commons.button.confirm') }}
</el-button>
</span>
</template>
</el-dialog>
<OpDialog ref="opRef" @search="search" />
<AppBackUp ref="backupRef" @close="search" />
<AppRecover ref="recoverRef" />
</template>
<script lang="ts" setup>
import { reactive, ref } from 'vue';
import { computeSize, dateFormat, downloadFile } from '@/utils/util';
import { getBackupList } from '@/api/modules/setting';
import { getLocalBackupDir, handleBackup, handleRecover } from '@/api/modules/setting';
import i18n from '@/lang';
import { deleteBackupRecord, downloadBackupRecord, searchBackupRecords } from '@/api/modules/setting';
import { Backup } from '@/api/interface/backup';
import router from '@/routers';
import AppBackUp from '@/views/app-store/installed/backup/index.vue';
import AppRecover from '@/views/app-store/installed/recover/index.vue';
import { MsgSuccess } from '@/utils/message';
const selects = ref<any>([]);
const loading = ref();
const opRef = ref();
const backupRef = ref();
const recoverRef = ref();
const data = ref();
const paginationConfig = reactive({
@@ -100,6 +128,9 @@ const backupPath = ref();
const status = ref();
const secret = ref();
const open = ref();
const isBackup = ref();
interface DialogProps {
type: string;
name: string;
@@ -120,21 +151,13 @@ const acceptParams = (params: DialogProps): void => {
const handleClose = () => {
backupVisible.value = false;
};
const handleBackupClose = () => {
open.value = false;
};
const loadBackupDir = () => {
getBackupList().then((res) => {
let backupList = res.data || [];
for (const bac of backupList) {
if (bac.type !== 'LOCAL') {
continue;
}
if (bac.id !== 0) {
bac.varsJson = JSON.parse(bac.vars);
}
backupPath.value = bac.varsJson['dir'];
break;
}
});
const loadBackupDir = async () => {
const res = await getLocalBackupDir();
backupPath.value = res.data;
};
const goFile = async () => {
@@ -161,31 +184,89 @@ const search = async () => {
});
};
const onBackup = async () => {
const onSubmit = async (row?: any) => {
if (isBackup.value) {
let params = {
type: type.value,
name: name.value,
detailName: detailName.value,
secret: secret.value,
};
loading.value = true;
await handleBackup(params)
.then(() => {
loading.value = false;
handleClose();
handleBackupClose();
MsgSuccess(i18n.global.t('commons.msg.operationSuccess'));
search();
})
.catch(() => {
loading.value = false;
});
return;
}
let params = {
downloadAccountID: row.downloadAccountID,
type: type.value,
name: name.value,
detailName: detailName.value,
file: row.file,
secret: secret.value,
};
backupRef.value.acceptParams(params);
loading.value = true;
await handleRecover(params)
.then(() => {
loading.value = false;
handleClose();
handleBackupClose();
MsgSuccess(i18n.global.t('commons.msg.operationSuccess'));
search();
})
.catch(() => {
loading.value = false;
});
};
const onBackup = async () => {
isBackup.value = true;
if (type.value !== 'app' && type.value !== 'website') {
ElMessageBox.confirm(
i18n.global.t('commons.msg.backupHelper', [name.value + '( ' + detailName.value + ' )']),
i18n.global.t('commons.button.backup'),
{
confirmButtonText: i18n.global.t('commons.button.confirm'),
cancelButtonText: i18n.global.t('commons.button.cancel'),
},
).then(async () => {
onSubmit();
});
return;
}
open.value = true;
};
const onRecover = async (row: Backup.RecordInfo) => {
let params = {
source: row.source,
type: type.value,
name: name.value,
detailName: detailName.value,
file: row.fileDir + '/' + row.fileName,
secret: secret.value,
};
recoverRef.value.acceptParams(params);
isBackup.value = false;
if (type.value !== 'app' && type.value !== 'website') {
ElMessageBox.confirm(
i18n.global.t('commons.msg.backupHelper', [name.value + '( ' + detailName.value + ' )']),
i18n.global.t('commons.button.backup'),
{
confirmButtonText: i18n.global.t('commons.button.confirm'),
cancelButtonText: i18n.global.t('commons.button.cancel'),
},
).then(async () => {
onSubmit(row);
});
return;
}
open.value = true;
};
const onDownload = async (row: Backup.RecordInfo) => {
let params = {
source: row.source,
downloadAccountID: row.downloadAccountID,
fileDir: row.fileDir,
fileName: row.fileName,
};
frontend/src/components/upload/index.vue
+66 -8
View File
@@ -95,9 +95,37 @@
</div>
</template>
</DrawerPro>
<el-dialog
v-model="open"
:title="$t('commons.button.recover') + ' - ' + name"
width="40%"
:close-on-click-modal="false"
:before-close="handleBackupClose"
>
<el-form ref="backupForm" label-position="left" v-loading="loading">
<el-form-item
:label="$t('setting.compressPassword')"
style="margin-top: 10px"
v-if="type === 'app' || type === 'website'"
>
<el-input v-model="secret" :placeholder="$t('setting.backupRecoverMessage')" />
</el-form-item>
</el-form>
<template #footer>
<span class="dialog-footer">
<el-button @click="handleClose" :disabled="loading">
{{ $t('commons.button.cancel') }}
</el-button>
<el-button type="primary" @click="onHandleRecover" :disabled="loading">
{{ $t('commons.button.confirm') }}
</el-button>
</span>
</template>
</el-dialog>
<OpDialog ref="opRef" @search="search" />
</div>
<AppRecover ref="recoverRef" />
</template>
<script lang="ts" setup>
@@ -107,9 +135,8 @@ import i18n from '@/lang';
import { UploadFile, UploadFiles, UploadInstance } from 'element-plus';
import { File } from '@/api/interface/file';
import { BatchDeleteFile, CheckFile, ChunkUploadFileData, GetUploadList } from '@/api/modules/files';
import { loadBaseDir } from '@/api/modules/setting';
import { handleRecoverByUpload, loadBaseDir } from '@/api/modules/setting';
import { MsgError, MsgSuccess } from '@/utils/message';
import AppRecover from '@/views/app-store/installed/recover/index.vue';
const loading = ref();
const isUpload = ref();
@@ -117,7 +144,8 @@ const uploadPercent = ref<number>(0);
const selects = ref<any>([]);
const baseDir = ref();
const opRef = ref();
const recoverRef = ref();
const open = ref();
const data = ref();
const title = ref();
@@ -181,17 +209,44 @@ const search = async () => {
paginationConfig.total = res.data.total;
};
const onRecover = async (row: File.File) => {
const onHandleRecover = async (row?: any) => {
let params = {
source: 'LOCAL',
downloadAccountID: 1,
type: type.value,
name: name.value,
detailName: detailName.value,
file: baseDir.value + row.name,
secret: secret.value,
recoverType: 'upload',
};
recoverRef.value.acceptParams(params);
loading.value = true;
await handleRecoverByUpload(params)
.then(() => {
loading.value = false;
handleClose();
handleBackupClose();
MsgSuccess(i18n.global.t('commons.msg.operationSuccess'));
search();
})
.catch(() => {
loading.value = false;
});
};
const onRecover = async (row: File.File) => {
if (type.value !== 'app' && type.value !== 'website') {
ElMessageBox.confirm(
i18n.global.t('commons.msg.backupHelper', [name.value + '( ' + detailName.value + ' )']),
i18n.global.t('commons.button.backup'),
{
confirmButtonText: i18n.global.t('commons.button.confirm'),
cancelButtonText: i18n.global.t('commons.button.cancel'),
},
).then(async () => {
onHandleRecover(row);
});
return;
}
open.value = true;
};
const uploaderFiles = ref<UploadFiles>([]);
@@ -221,6 +276,9 @@ const handleClose = () => {
uploadRef.value!.clearFiles();
upVisible.value = false;
};
const handleBackupClose = () => {
open.value = false;
};
const onSubmit = async () => {
if (uploaderFiles.value.length !== 1) {
frontend/src/views/cronjob/backup/index.vue
+8 -4
View File
@@ -23,11 +23,15 @@
<span v-else>-</span>
</template>
</el-table-column>
<el-table-column :label="$t('database.source')" prop="backupType">
<el-table-column :label="$t('database.source')" prop="accountType" show-overflow-tooltip>
<template #default="{ row }">
<span v-if="row.source">
{{ $t('setting.' + row.source) }}
<span v-if="row.accountType === 'LOCAL'">
{{ $t('setting.LOCAL') }}
</span>
<span v-if="row.accountType && row.accountType !== 'LOCAL'">
{{ $t('setting.' + row.accountType) + ' - ' + row.accountName }}
</span>
<span v-if="!row.accountType">-</span>
</template>
</el-table-column>
<el-table-column
@@ -99,7 +103,7 @@ const search = async () => {
const onDownload = async (row: Backup.RecordInfo) => {
let params = {
source: row.source,
downloadAccountID: row.downloadAccountID,
fileDir: row.fileDir,
fileName: row.fileName,
};
frontend/src/views/cronjob/operate/index.vue
+32 -19
View File
@@ -245,11 +245,16 @@
<el-select
multiple
class="selectClass"
v-model="dialogData.rowData!.backupAccountList"
v-model="dialogData.rowData!.sourceAccounts"
@change="changeAccount"
>
<div v-for="item in backupOptions" :key="item.label">
<el-option :value="item.value" :label="item.label" />
<div v-for="item in backupOptions" :key="item.id">
<el-option
v-if="item.type !== $t('setting.LOCAL')"
:value="item.id"
:label="item.type + ' - ' + item.name"
/>
<el-option v-else :value="item.id" :label="item.type" />
</div>
</el-select>
<span class="input-help">
@@ -271,10 +276,15 @@
>
<el-input v-model="dialogData.rowData!.secret" />
</el-form-item>
<el-form-item :label="$t('cronjob.default_download_path')" prop="defaultDownload">
<el-select class="selectClass" v-model="dialogData.rowData!.defaultDownload">
<div v-for="item in accountOptions" :key="item.label">
<el-option :value="item.value" :label="item.label" />
<el-form-item :label="$t('cronjob.default_download_path')" prop="downloadAccountID">
<el-select class="selectClass" v-model="dialogData.rowData!.downloadAccountID">
<div v-for="item in accountOptions" :key="item.id">
<el-option
v-if="item.type !== $t('setting.LOCAL')"
:value="item.id"
:label="item.type + ' - ' + item.name"
/>
<el-option v-else :value="item.id" :label="item.type" />
</div>
</el-select>
</el-form-item>
@@ -360,10 +370,13 @@ const acceptParams = (params: DialogProps): void => {
if (dialogData.value.title === 'create') {
changeType();
dialogData.value.rowData.dbType = 'mysql';
dialogData.value.rowData.defaultDownload = 'LOCAL';
dialogData.value.rowData.downloadAccountID = 1;
}
if (dialogData.value.rowData.backupAccounts) {
dialogData.value.rowData.backupAccountList = dialogData.value.rowData.backupAccounts.split(',');
if (dialogData.value.rowData.sourceAccountIDs) {
let itemIDs = dialogData.value.rowData.sourceAccountIDs.split(',');
for (const item of itemIDs) {
dialogData.value.rowData.sourceAccounts.push(Number(item));
}
}
dialogData.value.rowData!.command = dialogData.value.rowData!.command || 'sh';
dialogData.value.rowData!.isCustom =
@@ -591,14 +604,14 @@ const handleSpecDelete = (index: number) => {
const loadBackups = async () => {
const res = await getBackupList();
backupOptions.value = [];
if (!dialogData.value.rowData!.backupAccountList) {
dialogData.value.rowData!.backupAccountList = ['LOCAL'];
if (!dialogData.value.rowData!.sourceAccounts) {
dialogData.value.rowData!.sourceAccounts = [1];
}
for (const item of res.data) {
if (item.id === 0) {
continue;
}
backupOptions.value.push({ label: i18n.global.t('setting.' + item.type), value: item.type });
backupOptions.value.push({ id: item.id, type: i18n.global.t('setting.' + item.type), name: item.name });
}
changeAccount();
};
@@ -608,21 +621,21 @@ const changeAccount = async () => {
let isInAccounts = false;
for (const item of backupOptions.value) {
let exist = false;
for (const ac of dialogData.value.rowData.backupAccountList) {
if (item.value == ac) {
for (const ac of dialogData.value.rowData.sourceAccounts) {
if (item.id == ac) {
exist = true;
break;
}
}
if (exist) {
if (item.value === dialogData.value.rowData.defaultDownload) {
if (item.value === dialogData.value.rowData.downloadAccountID) {
isInAccounts = true;
}
accountOptions.value.push(item);
}
}
if (!isInAccounts) {
dialogData.value.rowData.defaultDownload = '';
dialogData.value.rowData.downloadAccountID = undefined;
}
};
@@ -632,7 +645,7 @@ const loadAppInstalls = async () => {
};
const loadWebsites = async () => {
const res = await GetWebsiteOptions({});
const res = await GetWebsiteOptions();
websiteOptions.value = res.data || [];
};
@@ -674,7 +687,7 @@ const onSubmit = async (formEl: FormInstance | undefined) => {
}
specs.push(itemSpec);
}
dialogData.value.rowData.backupAccounts = dialogData.value.rowData.backupAccountList.join(',');
dialogData.value.rowData.sourceAccountIDs = dialogData.value.rowData.sourceAccounts.join(',');
dialogData.value.rowData.spec = specs.join(',');
if (!formEl) return;
formEl.validate(async (valid) => {