Skip to content

Commit

Permalink
feat: 统一备份文件大小获取接口
Browse files Browse the repository at this point in the history
  • Loading branch information
@ssongliu
ssongliu committed Jan 14, 2025
1 parent e86090c commit 37456d9
Show file tree
Hide file tree
Showing 22 changed files with 254 additions and 179 deletions.
23 changes: 23 additions & 0 deletions agent/app/api/v2/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,29 @@ func (b *BaseApi) GetLocalDir(c *gin.Context) {
helper.SuccessWithData(c, dir)
}

// @Tags Backup Account
// @Summary Page backup records
// @Description 获取备份记录列表分页
// @Accept json
// @Param request body dto.SearchForSize true "request"
// @Success 200 {object} dto.RecordFileSize
// @Security ApiKeyAuth
// @Router /backups/record/size [post]
func (b *BaseApi) LoadBackupRecordSize(c *gin.Context) {
var req dto.SearchForSize
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}

list, err := backupRecordService.LoadRecordSize(req)
if err != nil {
helper.InternalServer(c, err)
return
}

helper.SuccessWithData(c, list)
}

// @Tags Backup Account
// @Summary Page backup records
// @Description 获取备份记录列表分页
Expand Down
22 changes: 0 additions & 22 deletions agent/app/api/v2/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,28 +134,6 @@ func (b *BaseApi) SearchSnapshot(c *gin.Context) {
})
}

// @Tags System Setting
// @Summary Load system snapshot size
// @Description 获取系统快照文件大小
// @Accept json
// @Param request body dto.SearchWithPage true "request"
// @Success 200
// @Security ApiKeyAuth
// @Router /settings/snapshot/size [post]
func (b *BaseApi) LoadSnapshotSize(c *gin.Context) {
var req dto.SearchWithPage
if err := helper.CheckBindAndValidate(&req, c); err != nil {
return
}

accounts, err := snapshotService.LoadSize(req)
if err != nil {
helper.InternalServer(c, err)
return
}
helper.SuccessWithData(c, accounts)
}

// @Tags System Setting
// @Summary Recover system backup
// @Description 从系统快照恢复
Expand Down
15 changes: 14 additions & 1 deletion agent/app/dto/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,24 @@ type BackupRecords struct {
DownloadAccountID uint `json:"downloadAccountID"`
FileDir string `json:"fileDir"`
FileName string `json:"fileName"`
Size int64 `json:"size"`
}

type DownloadRecord struct {
DownloadAccountID uint `json:"downloadAccountID" validate:"required"`
FileDir string `json:"fileDir" validate:"required"`
FileName string `json:"fileName" validate:"required"`
}

type SearchForSize struct {
PageInfo
Type string `json:"type" validate:"required"`
Name string `json:"name"`
DetailName string `json:"detailName"`
Info string `json:"info"`
CronjobID uint `json:"cronjobID"`
}
type RecordFileSize struct {
ID uint `json:"id"`
Name string `json:"name"`
Size int64 `json:"size"`
}
22 changes: 11 additions & 11 deletions agent/app/dto/cronjob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,17 +106,17 @@ type CronjobInfo struct {
ContainerName string `json:"containerName"`
User string `json:"user"`

AppID string `json:"appID"`
Website string `json:"website"`
ExclusionRules string `json:"exclusionRules"`
DBType string `json:"dbType"`
DBName string `json:"dbName"`
URL string `json:"url"`
IsDir bool `json:"isDir"`
SourceDir string `json:"sourceDir"`
SourceAccountIDs string `json:"sourceAccountIDs"`
DownloadAccountID uint `json:"downloadAccountID"`
RetainCopies int `json:"retainCopies"`
AppID string `json:"appID"`
Website string `json:"website"`
ExclusionRules string `json:"exclusionRules"`
DBType string `json:"dbType"`
DBName string `json:"dbName"`
URL string `json:"url"`
IsDir bool `json:"isDir"`
SourceDir string `json:"sourceDir"`
SourceAccounts []string `json:"sourceAccounts"`
DownloadAccount string `json:"downloadAccount"`
RetainCopies int `json:"retainCopies"`

LastRecordStatus string `json:"lastRecordStatus"`
LastRecordTime string `json:"lastRecordTime"`
Expand Down
12 changes: 2 additions & 10 deletions agent/app/dto/snapshot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ type SnapshotInfo struct {
ID uint `json:"id"`
Name string `json:"name"`
Description string `json:"description" validate:"max=256"`
From string `json:"from"`
DefaultDownload string `json:"defaultDownload"`
SourceAccounts []string `json:"sourceAccounts"`
DownloadAccount string `json:"downloadAccount"`
Status string `json:"status"`
Message string `json:"message"`
CreatedAt time.Time `json:"createdAt"`
Expand All @@ -108,11 +108,3 @@ type SnapshotInfo struct {
RollbackMessage string `json:"rollbackMessage"`
LastRollbackedAt string `json:"lastRollbackedAt"`
}

type SnapshotFile struct {
ID uint `json:"id"`
Name string `json:"name"`
From string `json:"from"`
DefaultDownload string `json:"defaultDownload"`
Size int64 `json:"size"`
}
40 changes: 35 additions & 5 deletions agent/app/service/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ func (u *BackupService) SearchWithPage(req dto.SearchPageWithType) (int64, inter
if err := copier.Copy(&item, &account); err != nil {
global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
}
if item.Type != constant.Sftp && item.Type != constant.Local {
item.BackupPath = path.Join("/", strings.TrimPrefix(item.BackupPath, "/"))
}
if !item.RememberAuth {
item.AccessKey = ""
item.Credential = ""
Expand Down Expand Up @@ -124,6 +127,9 @@ func (u *BackupService) Create(req dto.BackupOperate) error {
if req.Type == constant.Local {
return buserr.New(constant.ErrBackupLocalCreate)
}
if req.Type != constant.Sftp && req.BackupPath != "/" {
req.BackupPath = strings.TrimPrefix(req.BackupPath, "/")
}
backup, _ := backupRepo.Get(repo.WithByName(req.Name))
if backup.ID != 0 {
return constant.ErrRecordExist
Expand Down Expand Up @@ -215,6 +221,9 @@ func (u *BackupService) Update(req dto.BackupOperate) error {
if backup.ID == 0 {
return constant.ErrRecordNotFound
}
if req.Type != constant.Sftp && req.Type != constant.Local && req.BackupPath != "/" {
req.BackupPath = strings.TrimPrefix(req.BackupPath, "/")
}
var newBackup model.BackupAccount
if err := copier.Copy(&newBackup, &req); err != nil {
return errors.WithMessage(constant.ErrStructTransform, err.Error())
Expand Down Expand Up @@ -428,14 +437,10 @@ func NewBackupClientMap(ids []string) (map[string]backupClientHelper, error) {
if err != nil {
return nil, err
}
pathItem := item.BackupPath
if item.Type != constant.Sftp && item.Type != constant.Local && pathItem != "/" {
pathItem = strings.TrimPrefix(item.BackupPath, "/")
}
clientMap[fmt.Sprintf("%v", item.ID)] = backupClientHelper{
client: backClient,
name: item.Name,
backupPath: pathItem,
backupPath: item.BackupPath,
accountType: item.Type,
id: item.ID,
}
Expand Down Expand Up @@ -502,3 +507,28 @@ func loadRefreshTokenByCode(backup *model.BackupAccount) error {
backup.Vars = string(itemVars)
return nil
}

func loadBackupNamesByID(accountIDs string, downloadID uint) ([]string, string, error) {
accountIDList := strings.Split(accountIDs, ",")
var ids []uint
for _, item := range accountIDList {
if len(item) != 0 {
itemID, _ := strconv.Atoi(item)
ids = append(ids, uint(itemID))
}
}
list, err := backupRepo.List(repo.WithByIDs(ids))
if err != nil {
return nil, "", err
}
var accounts []string
var downloadAccount string
for _, item := range list {
itemName := fmt.Sprintf("%s - %s", item.Type, item.Name)
accounts = append(accounts, itemName)
if item.ID == downloadID {
downloadAccount = itemName
}
}
return accounts, downloadAccount, nil
}
113 changes: 67 additions & 46 deletions agent/app/service/backup_record.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,14 @@ import (
"fmt"
"os"
"path"
"sort"
"strings"
"sync"

"github.com/1Panel-dev/1Panel/agent/app/dto"
"github.com/1Panel-dev/1Panel/agent/app/model"
"github.com/1Panel-dev/1Panel/agent/app/repo"
"github.com/1Panel-dev/1Panel/agent/constant"
"github.com/1Panel-dev/1Panel/agent/global"
"github.com/1Panel-dev/1Panel/agent/utils/cloud_storage"
"github.com/jinzhu/copier"
"github.com/pkg/errors"
)

type BackupRecordService struct{}
Expand All @@ -30,6 +26,7 @@ type IBackupRecordService interface {
ListAppRecords(name, detailName, fileName string) ([]model.BackupRecord, error)

ListFiles(req dto.OperateByID) []string
LoadRecordSize(req dto.SearchForSize) ([]dto.RecordFileSize, error)
}

func NewIBackupRecordService() IBackupRecordService {
Expand All @@ -47,15 +44,15 @@ func (u *BackupRecordService) SearchRecordsWithPage(search dto.RecordSearch) (in
if err != nil {
return 0, nil, err
}

if total == 0 {
return 0, nil, nil
var data []dto.BackupRecords
for _, account := range records {
var item dto.BackupRecords
if err := copier.Copy(&item, &account); err != nil {
global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
}
data = append(data, item)
}
datas, err := u.loadRecordSize(records)
sort.Slice(datas, func(i, j int) bool {
return datas[i].CreatedAt.After(datas[j].CreatedAt)
})
return total, datas, err
return total, data, err
}

func (u *BackupRecordService) SearchRecordsByCronjobWithPage(search dto.RecordSearchByCronjob) (int64, []dto.BackupRecords, error) {
Expand All @@ -67,15 +64,15 @@ func (u *BackupRecordService) SearchRecordsByCronjobWithPage(search dto.RecordSe
if err != nil {
return 0, nil, err
}

if total == 0 {
return 0, nil, nil
var data []dto.BackupRecords
for _, account := range records {
var item dto.BackupRecords
if err := copier.Copy(&item, &account); err != nil {
global.LOG.Errorf("copy backup account to dto backup info failed, err: %v", err)
}
data = append(data, item)
}
datas, err := u.loadRecordSize(records)
sort.Slice(datas, func(i, j int) bool {
return datas[i].CreatedAt.After(datas[j].CreatedAt)
})
return total, datas, err
return total, data, err
}

func (u *BackupRecordService) DownloadRecord(info dto.DownloadRecord) (string, error) {
Expand All @@ -94,7 +91,7 @@ func (u *BackupRecordService) DownloadRecord(info dto.DownloadRecord) (string, e
}
srcPath := fmt.Sprintf("%s/%s", info.FileDir, info.FileName)
if len(account.BackupPath) != 0 {
srcPath = path.Join(strings.TrimPrefix(account.BackupPath, "/"), srcPath)
srcPath = path.Join(account.BackupPath, srcPath)
}
if exist, _ := client.Exist(srcPath); exist {
isOK, err := client.Download(srcPath, targetPath)
Expand Down Expand Up @@ -181,36 +178,67 @@ func (u *BackupRecordService) ListFiles(req dto.OperateByID) []string {
return datas
}

func (u *BackupRecordService) loadRecordSize(records []model.BackupRecord) ([]dto.BackupRecords, error) {
type backupSizeHelper struct {
ID uint `json:"id"`
DownloadID uint `json:"downloadID"`
FilePath string `json:"filePath"`
Size uint `json:"size"`
}

func (u *BackupRecordService) LoadRecordSize(req dto.SearchForSize) ([]dto.RecordFileSize, error) {
var list []backupSizeHelper
switch req.Type {
case "snapshot":
_, records, err := snapshotRepo.Page(req.Page, req.PageSize, repo.WithByLikeName(req.Info))
if err != nil {
return nil, err
}
for _, item := range records {
list = append(list, backupSizeHelper{ID: item.ID, DownloadID: item.DownloadAccountID, FilePath: fmt.Sprintf("system_snapshot/%s.tar.gz", item.Name)})
}
case "cronjob":
_, records, err := backupRepo.PageRecord(req.Page, req.PageSize, backupRepo.WithByCronID(req.CronjobID))
if err != nil {
return nil, err
}
for _, item := range records {
list = append(list, backupSizeHelper{ID: item.ID, DownloadID: item.DownloadAccountID, FilePath: path.Join(item.FileDir, item.FileName)})
}
default:
_, records, err := backupRepo.PageRecord(
req.Page, req.PageSize,
repo.WithByName(req.Name),
repo.WithByType(req.Type),
repo.WithByDetailName(req.DetailName),
)
if err != nil {
return nil, err
}
for _, item := range records {
list = append(list, backupSizeHelper{ID: item.ID, DownloadID: item.DownloadAccountID, FilePath: path.Join(item.FileDir, item.FileName)})
}
}
recordMap := make(map[uint]struct{})
var recordIds []string
for _, record := range records {
if _, ok := recordMap[record.DownloadAccountID]; !ok {
recordMap[record.DownloadAccountID] = struct{}{}
recordIds = append(recordIds, fmt.Sprintf("%v", record.DownloadAccountID))
for _, record := range list {
if _, ok := recordMap[record.DownloadID]; !ok {
recordMap[record.DownloadID] = struct{}{}
recordIds = append(recordIds, fmt.Sprintf("%v", record.DownloadID))
}
}
clientMap, err := NewBackupClientMap(recordIds)
if err != nil {
return nil, err
}

var datas []dto.BackupRecords
var datas []dto.RecordFileSize
var wg sync.WaitGroup
for i := 0; i < len(records); i++ {
var item dto.BackupRecords
if err := copier.Copy(&item, &records[i]); err != nil {
return nil, errors.WithMessage(constant.ErrStructTransform, err.Error())
}

itemPath := path.Join(records[i].FileDir, records[i].FileName)
if val, ok := clientMap[fmt.Sprintf("%v", records[i].DownloadAccountID)]; ok {
item.AccountName = val.name
item.AccountType = val.accountType
item.DownloadAccountID = val.id
for i := 0; i < len(list); i++ {
item := dto.RecordFileSize{ID: list[i].ID}
if val, ok := clientMap[fmt.Sprintf("%v", list[i].DownloadID)]; ok {
wg.Add(1)
go func(index int) {
item.Size, _ = val.client.Size(path.Join(strings.TrimLeft(val.backupPath, "/"), itemPath))
item.Size, _ = val.client.Size(path.Join(val.backupPath, list[i].FilePath))
datas = append(datas, item)
wg.Done()
}(i)
Expand All @@ -221,10 +249,3 @@ func (u *BackupRecordService) loadRecordSize(records []model.BackupRecord) ([]dt
wg.Wait()
return datas, nil
}

type loadSizeHelper struct {
isOk bool
backupName string
backupPath string
client cloud_storage.CloudStorageClient
}
Loading

0 comments on commit 37456d9

Please sign in to comment.