Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: get file content with bytes will backfill cache if no cache exists #116

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

perf: get file content with bytes will backfill cache if no cache exists #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9ed56fd
perf: get file content with bytes will backfill cache if no cache fil…
fireyun Sep 3, 2024
42d1462
chore: adjust code
fireyun Sep 3, 2024
559a703
Merge branch 'master' of https://github.com/TencentBlueKing/bscp-go i…
fireyun Sep 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion client/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ func (c *ConfigItemFile) GetContent() ([]byte, error) {
return bytes, nil
}
}
bytes := make([]byte, c.FileMeta.ContentSpec.ByteSize)

bytes := make([]byte, c.FileMeta.ContentSpec.ByteSize)
if err := downloader.GetDownloader().Download(c.FileMeta.PbFileMeta(), c.FileMeta.RepositoryPath,
c.FileMeta.ContentSpec.ByteSize, downloader.DownloadToBytes, bytes, ""); err != nil {
logger.Error("download file failed", logger.ErrAttr(err))
Expand Down
68 changes: 37 additions & 31 deletions internal/cache/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"io/fs"
"math"
"os"
"path"
"path/filepath"
"sort"
"time"
Expand Down Expand Up @@ -131,21 +132,16 @@ func (c *Cache) checkFileCacheExists(ci *sfs.ConfigItemMetaV1) (bool, error) {
}

// GetFileContent return the config content bytes.
// get from cache first, if not exist, then get from remote repo and add it to cache
func (c *Cache) GetFileContent(ci *sfs.ConfigItemMetaV1) (bool, []byte) {
exists, err := c.checkFileCacheExists(ci)
if err != nil {
logger.Error("check config item cache exists failed",
slog.String("item", ci.ContentSpec.Signature), logger.ErrAttr(err))
if ok := c.ensureCache(ci); !ok {
return false, nil
}
if !exists {
return false, nil
}
filePath := filepath.Join(c.path, ci.ContentSpec.Signature)
bytes, err := os.ReadFile(filePath)

cacheFilePath := path.Join(c.path, ci.ContentSpec.Signature)
bytes, err := os.ReadFile(cacheFilePath)
if err != nil {
logger.Error("read config item cache file failed",
slog.String("file", filePath), logger.ErrAttr(err))
logger.Error("read config item cache file failed", slog.String("file", cacheFilePath), logger.ErrAttr(err))
return false, nil
}
return true, bytes
Expand All @@ -154,30 +150,13 @@ func (c *Cache) GetFileContent(ci *sfs.ConfigItemMetaV1) (bool, []byte) {
// CopyToFile copy the config content to the specified file.
// get from cache first, if not exist, then get from remote repo and add it to cache
func (c *Cache) CopyToFile(ci *sfs.ConfigItemMetaV1, filePath string) bool {
if ci.ContentSpec.ByteSize > uint64(MaxSingleFileCacheSizeRate*c.thrsholdGB*GByte) {
logger.Warn("config item size is too large, skip cache",
slog.String("item", filepath.Join(ci.ConfigItemSpec.Path, ci.ConfigItemSpec.Name)),
slog.Int64("size", int64(ci.ContentSpec.ByteSize)))
return false
}
exists, err := c.checkFileCacheExists(ci)
if err != nil {
logger.Error("check config item cache exists failed",
slog.String("item", ci.ContentSpec.Signature), logger.ErrAttr(err))
if ok := c.ensureCache(ci); !ok {
return false
}

cacheFilePath := filepath.Join(c.path, ci.ContentSpec.Signature)
if !exists {
// get from remote repo and add it to cache
if err = downloader.GetDownloader().Download(ci.PbFileMeta(), ci.RepositoryPath, ci.ContentSpec.ByteSize,
downloader.DownloadToFile, nil, cacheFilePath); err != nil {
logger.Error("download file failed", logger.ErrAttr(err))
return false
}
}

cacheFilePath := path.Join(c.path, ci.ContentSpec.Signature)
var src, dst *os.File
var err error
src, err = os.Open(cacheFilePath)
if err != nil {
logger.Error("open config item cache file failed", slog.String("file", cacheFilePath), logger.ErrAttr(err))
Expand All @@ -200,6 +179,33 @@ func (c *Cache) CopyToFile(ci *sfs.ConfigItemMetaV1, filePath string) bool {
return true
}

// ensureCache ensures cache exist, if not exist, then download file and add it to cache file
func (c *Cache) ensureCache(ci *sfs.ConfigItemMetaV1) bool {
if ci.ContentSpec.ByteSize > uint64(MaxSingleFileCacheSizeRate*c.thrsholdGB*GByte) {
logger.Warn("config item size is too large, skip copy to file",
slog.String("item", path.Join(ci.ConfigItemSpec.Path, ci.ConfigItemSpec.Name)),
slog.Int64("size", int64(ci.ContentSpec.ByteSize)))
return false
}
exists, err := c.checkFileCacheExists(ci)
if err != nil {
logger.Error("check config item cache exists failed",
slog.String("item", ci.ContentSpec.Signature), logger.ErrAttr(err))
return false
}

cacheFilePath := path.Join(c.path, ci.ContentSpec.Signature)
if !exists {
// get from remote repo and add it to cache
if err = downloader.GetDownloader().Download(ci.PbFileMeta(), ci.RepositoryPath, ci.ContentSpec.ByteSize,
downloader.DownloadToFile, nil, cacheFilePath); err != nil {
logger.Error("download file failed", logger.ErrAttr(err))
return false
}
}
return true
}

// AutoCleanupFileCache auto cleanup file cache
func AutoCleanupFileCache(cacheDir string, cleanupIntervalSeconds int64, thresholdGB, retentionRate float64) {
logger.Info("start auto cleanup file cache ",
Expand Down
Loading