feat(transcode): add cache pruning and config options

* Added config option to set size of transcode cache and cadence to enforce that sizing via ejection.

* Added cache eject to contrib/config.

* Added error return for CacheEject(). Changed to use WalkDir() instead of Walk().

* Lint fix.

* Added universal lock for cache eject.

* Removed accidentally committed binary.

README.md

@@ -71,6 +71,8 @@ password can then be changed from the web interface
 | `GONIC_MULTI_VALUE_GENRE`        | `-multi-value-genre`        | **optional** setting for multi-valued genre tags when scanning ([see more](#multi-valued-tags-v016))                                                                                                                                                                              |
 | `GONIC_MULTI_VALUE_ARTIST`       | `-multi-value-artist`       | **optional** setting for multi-valued artist tags when scanning ([see more](#multi-valued-tags-v016))                                                                                                                                                                             |
 | `GONIC_MULTI_VALUE_ALBUM_ARTIST` | `-multi-value-album-artist` | **optional** setting for multi-valued album artist tags when scanning ([see more](#multi-valued-tags-v016))                                                                                                                                                                       |
+| `GONIC_TRANSCODE_CACHE_SIZE` | `-transcode-cache-size` | **optional** size of the transcode cache in MB (0 = no limit) |
+| `GONIC_TRANSCODE_EJECT_INTERVAL` | `-transcode-eject-interval` | **optional** interval (in minutes) to eject transcode cache (0 = never) |
 | `GONIC_EXPVAR`                   | `-expvar`                   | **optional** enable the /debug/vars endpoint (exposes useful debugging attributes as well as database stats)                                                                                                                                                                      |
 
 ## multi valued tags (v0.16+)

cmd/gonic/gonic.go

@@ -93,6 +93,9 @@ func main() {
 
 	deprecatedConfGenreSplit := flag.String("genre-split", "", "(deprecated, see multi-value settings)")
 
+	confTranscodeCacheSize := flag.Int("transcode-cache-size", 0, "size of the transcode cache in MB (0 = no limit) (optional)")
+	confTranscodeEjectInterval := flag.Int("transcode-eject-interval", 0, "interval (in minutes) to eject transcode cache (0 = never) (optional)")
+
 	flag.Parse()
 	flagconf.ParseEnv()
 	flagconf.ParseConfig(*confConfigPath)
@@ -201,6 +204,7 @@ func main() {
 	transcoder := transcode.NewCachingTranscoder(
 		transcode.NewFFmpegTranscoder(),
 		cacheDirAudio,
+		*confTranscodeCacheSize,
 	)
 
 	lastfmClientKeySecretFunc := func() (string, string, error) {
@@ -404,6 +408,21 @@ func main() {
 		return nil
 	})
 
+	errgrp.Go(func() error {
+		if *confTranscodeEjectInterval == 0 || *confTranscodeCacheSize == 0 {
+			return nil
+		}
+
+		defer logJob("transcode cache eject")()
+
+		ctxTick(ctx, time.Duration(*confTranscodeEjectInterval)*time.Minute, func() {
+			if err := transcoder.CacheEject(); err != nil {
+				log.Printf("error ejecting transcode cache: %v", err)
+			}
+		})
+		return nil
+	})
+
 	errgrp.Go(func() error {
 		if *confScanIntervalMins == 0 {
 			return nil

contrib/config

@@ -49,6 +49,10 @@ playlists-path              <path to your m3u playlist dir>
 # regenerated.
 cache-path                  /var/cache/gonic
 
+# Option to eject least recently used items from transcode cache.
+#transcode-cache-size		5000 # in Mb (0 = no limit)
+#transcode-eject-interval	1440 # in minutes (0 = never eject)
+
 # Interval (in minutes) to check for new music. Default: don't scan
 #scan-interval               0
 #scan-at-start-enabled       false

transcode/transcode_test.go

@@ -113,7 +113,7 @@ func TestCachingParallelism(t *testing.T) {
 		callback:   func() { realTranscodeCount.Add(1) },
 	}
 
-	cacheTranscoder := transcode.NewCachingTranscoder(transcoder, t.TempDir())
+	cacheTranscoder := transcode.NewCachingTranscoder(transcoder, t.TempDir(), 1024)
 
 	var wg sync.WaitGroup
 	for i := 0; i < 5; i++ {

transcode/transcoder_caching.go

@@ -5,26 +5,34 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
+	"sort"
 	"sync"
+	"time"
 )
 
 const perm = 0o644
 
 type CachingTranscoder struct {
 	cachePath  string
 	transcoder Transcoder
+	limitMB    int
 	locks      keyedMutex
+	cleanLock  sync.RWMutex
 }
 
 var _ Transcoder = (*CachingTranscoder)(nil)
 
-func NewCachingTranscoder(t Transcoder, cachePath string) *CachingTranscoder {
-	return &CachingTranscoder{transcoder: t, cachePath: cachePath}
+func NewCachingTranscoder(t Transcoder, cachePath string, limitMB int) *CachingTranscoder {
+	return &CachingTranscoder{transcoder: t, cachePath: cachePath, limitMB: limitMB}
 }
 
 func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in string, out io.Writer) error {
+	t.cleanLock.RLock()
+	defer t.cleanLock.RUnlock()
+
 	// don't try cache partial transcodes
 	if profile.Seek() > 0 {
 		return t.transcoder.Transcode(ctx, profile, in, out)
@@ -52,6 +60,7 @@ func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in s
 
 	if i, err := cf.Stat(); err == nil && i.Size() > 0 {
 		_, _ = io.Copy(out, cf)
+		_ = os.Chtimes(path, time.Now(), time.Now()) // Touch for LRU cache purposes
 		return nil
 	}
 
@@ -64,6 +73,55 @@ func (t *CachingTranscoder) Transcode(ctx context.Context, profile Profile, in s
 	return nil
 }
 
+func (t *CachingTranscoder) CacheEject() error {
+	t.cleanLock.Lock()
+	defer t.cleanLock.Unlock()
+
+	// Delete LRU cache files that exceed size limit. Use last modified time.
+	type file struct {
+		path string
+		info os.FileInfo
+	}
+
+	var files []file
+	var total int64 = 0
+
+	err := filepath.WalkDir(t.cachePath, func(path string, de fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		if !de.IsDir() {
+			info, err := de.Info()
+			if err != nil {
+				return fmt.Errorf("walk cache path for eject: %w", err)
+			}
+			files = append(files, file{path, info})
+			total += info.Size()
+		}
+		return nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("walk cache path for eject: %w", err)
+	}
+
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].info.ModTime().Before(files[j].info.ModTime())
+	})
+
+	for total > int64(t.limitMB)*1024*1024 {
+		curFile := files[0]
+		files = files[1:]
+		total -= curFile.info.Size()
+		err = os.Remove(curFile.path)
+		if err != nil {
+			return fmt.Errorf("remove cache file: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func cacheKey(cmd string, args []string) string {
 	// the cache is invalid whenever transcode command (which includes the
 	// absolute filepath, bit rate args, replay gain args, etc.) changes