feat: vacuum unused packages (#3467)

* feat: vacuum unused packages

* fix: fix lint errors

* fix: fix lint errors

* fix: recreate a timestamp file instead of removing it

* fix: ignore a lint error

* ci: add integration tests

* fix: add vacuum command

* fix: add a newline

* feat: support initializing timestamp files

* ci: test vacuum --init

* fix: check if packages are installed

* fix: set 60 to vacuum-days by default

* fix: fix a lint error

* fix: move -vacuum-days to -days

* fix: fix the validation of command line arguments

* test: add tests

* refactor: fix a lint error

* fix: improve the help message

* fix(remove): remove timestamp files when removing packages

* refactor: remove an unused method

.github/workflows/wc-integration-test.yaml

@@ -203,6 +203,13 @@ jobs:
           git checkout -- .
         working-directory: tests/update3
 
+      - name: Test vacuum
+        env:
+          AQUA_VACUUM_DAYS: 1
+        run: aqua vacuum
+      - name: Test vacuum --init
+        run: aqua vacuum --init
+
       - run: aqua update-checksum -a
 
       - run: terraform --help

pkg/cli/runner.go

@@ -20,6 +20,7 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/cli/update"
 	"github.com/aquaproj/aqua/v2/pkg/cli/updateaqua"
 	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/cli/vacuum"
 	"github.com/aquaproj/aqua/v2/pkg/cli/version"
 	"github.com/aquaproj/aqua/v2/pkg/cli/which"
 	"github.com/urfave/cli/v2"
@@ -105,6 +106,7 @@ func Run(ctx context.Context, param *util.Param, args ...string) error { //nolin
 			upc.New,
 			remove.New,
 			update.New,
+			vacuum.New,
 		),
 	}
 

pkg/cli/vacuum/command.go

@@ -0,0 +1,104 @@
+package vacuum
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/aquaproj/aqua/v2/pkg/cli/profile"
+	"github.com/aquaproj/aqua/v2/pkg/cli/util"
+	"github.com/aquaproj/aqua/v2/pkg/config"
+	"github.com/aquaproj/aqua/v2/pkg/controller"
+	"github.com/urfave/cli/v2"
+)
+
+const description = `Remove unused installed packages.
+
+This command removes unused installed packages, which is useful to save storage and keep your machine clean.
+
+	$ aqua vacuum
+
+It removes installed packages which haven't been used for over the expiration days.
+The default expiration days is 60, but you can change it by the environment variable $AQUA_VACUUM_DAYS or the command line option "-days <expiration days>".
+
+e.g.
+
+	$ export AQUA_VACUUM_DAYS=90
+
+	$ aqua vacuum -d 30
+
+As of aqua v2.43.0, aqua records packages' last used date times.
+Date times are updated when packages are installed or executed.
+Packages installed by aqua v2.42.2 or older don't have records of last used date times, so aqua can't remove them.
+To solve the problem, "aqua vacuum --init" is available.
+
+	aqua vacuum --init
+
+"aqua vacuum --init" searches installed packages from aqua.yaml including $AQUA_GLOBAL_CONFIG and records the current date time as the last used date time of those packages if their last used date times aren't recorded.
+
+"aqua vacuum --init" can't record date times of install packages which are not found in aqua.yaml.
+If you want to record their date times, you need to remove them by "aqua rm" command and re-install them.
+`
+
+type command struct {
+	r *util.Param
+}
+
+func New(r *util.Param) *cli.Command {
+	i := &command{
+		r: r,
+	}
+	return &cli.Command{
+		Name:        "vacuum",
+		Usage:       "Remove unused installed packages",
+		Description: description,
+		Action:      i.action,
+		Flags: []cli.Flag{
+			&cli.BoolFlag{
+				Name:  "init",
+				Usage: "Create timestamp files.",
+			},
+			&cli.IntFlag{
+				Name:    "days",
+				Aliases: []string{"d"},
+				Usage:   "Expiration days",
+				EnvVars: []string{"AQUA_VACUUM_DAYS"},
+				Value:   60, //nolint:mnd
+			},
+		},
+	}
+}
+
+func (i *command) action(c *cli.Context) error {
+	profiler, err := profile.Start(c)
+	if err != nil {
+		return fmt.Errorf("start CPU Profile or tracing: %w", err)
+	}
+	defer profiler.Stop()
+
+	logE := i.r.LogE
+
+	param := &config.Param{}
+	if err := util.SetParam(c, logE, "vacuum", param, i.r.LDFlags); err != nil {
+		return fmt.Errorf("parse the command line arguments: %w", err)
+	}
+
+	if c.Bool("init") {
+		ctrl := controller.InitializeVacuumInitCommandController(c.Context, param, i.r.Runtime, &http.Client{})
+		if err := ctrl.Init(c.Context, logE, param); err != nil {
+			return err //nolint:wrapcheck
+		}
+		return nil
+	}
+
+	param.VacuumDays = c.Int("days")
+	if param.VacuumDays <= 0 {
+		return errors.New("vacuum days must be greater than 0")
+	}
+
+	ctrl := controller.InitializeVacuumCommandController(c.Context, param, i.r.Runtime)
+	if err := ctrl.Vacuum(logE, param); err != nil {
+		return err //nolint:wrapcheck
+	}
+	return nil
+}

pkg/config/package.go

@@ -31,7 +31,7 @@ func (p *Package) ExePath(rootDir string, file *registry.File, rt *runtime.Runti
 		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, p.Package.Version, "bin", file.Name), nil
 	}
 
-	pkgPath, err := p.PkgPath(rootDir, rt)
+	pkgPath, err := p.AbsPkgPath(rootDir, rt)
 	if err != nil {
 		return "", err
 	}
@@ -80,7 +80,7 @@ func (p *Package) RenderPath() (string, error) {
 	return p.RenderTemplateString(pkgInfo.GetPath(), &runtime.Runtime{})
 }
 
-func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) { //nolint:cyclop
+func (p *Package) PkgPath(rt *runtime.Runtime) (string, error) { //nolint:cyclop
 	pkgInfo := p.PackageInfo
 	pkg := p.Package
 	assetName, err := p.RenderAsset(rt)
@@ -89,23 +89,23 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) {
 	}
 	switch pkgInfo.Type {
 	case PkgInfoTypeGitHubArchive:
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version), nil
 	case PkgInfoTypeGoBuild:
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, "src"), nil
 	case PkgInfoTypeGoInstall:
 		p, err := p.RenderPath()
 		if err != nil {
 			return "", fmt.Errorf("render Go Module Path: %w", err)
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil
+		return filepath.Join("pkgs", pkgInfo.Type, p, pkg.Version, "bin"), nil
 	case PkgInfoTypeCargo:
 		registry := "crates.io"
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil
+		return filepath.Join("pkgs", pkgInfo.Type, registry, pkgInfo.Crate, strings.TrimPrefix(pkg.Version, "v")), nil
 	case PkgInfoTypeGitHubContent, PkgInfoTypeGitHubRelease:
 		if pkgInfo.RepoOwner == "aquaproj" && (pkgInfo.RepoName == "aqua" || pkgInfo.RepoName == "aqua-proxy") {
-			return filepath.Join(rootDir, "internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
+			return filepath.Join("internal", "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
+		return filepath.Join("pkgs", pkgInfo.Type, "github.com", pkgInfo.RepoOwner, pkgInfo.RepoName, pkg.Version, assetName), nil
 	case PkgInfoTypeHTTP:
 		uS, err := p.RenderURL(rt)
 		if err != nil {
@@ -115,11 +115,19 @@ func (p *Package) PkgPath(rootDir string, rt *runtime.Runtime) (string, error) {
 		if err != nil {
 			return "", fmt.Errorf("parse the URL: %w", err)
 		}
-		return filepath.Join(rootDir, "pkgs", pkgInfo.Type, u.Host, u.Path), nil
+		return filepath.Join("pkgs", pkgInfo.Type, u.Host, u.Path), nil
 	}
 	return "", nil
 }
 
+func (p *Package) AbsPkgPath(rootDir string, rt *runtime.Runtime) (string, error) {
+	pkgPath, err := p.PkgPath(rt)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(rootDir, pkgPath), nil
+}
+
 func (p *Package) RenderTemplateString(s string, rt *runtime.Runtime) (string, error) {
 	tpl, err := template.Compile(s)
 	if err != nil {
@@ -250,7 +258,6 @@ type RemoveMode struct {
 }
 
 type Param struct {
-	GlobalConfigFilePaths             []string
 	ConfigFilePath                    string
 	LogLevel                          string
 	File                              string
@@ -265,7 +272,11 @@ type Param struct {
 	OutTestData                       string
 	Limit                             int
 	MaxParallelism                    int
+	VacuumDays                        int
+	GlobalConfigFilePaths             []string
 	Args                              []string
+	PolicyConfigFilePaths             []string
+	Commands                          []string
 	Tags                              map[string]struct{}
 	ExcludedTags                      map[string]struct{}
 	DisableLazyInstall                bool
@@ -292,8 +303,6 @@ type Param struct {
 	GitHubArtifactAttestationDisabled bool
 	SLSADisabled                      bool
 	Installed                         bool
-	PolicyConfigFilePaths             []string
-	Commands                          []string
 }
 
 func appendExt(s, format string) string {

pkg/config/package_test.go

@@ -210,7 +210,7 @@ func TestPackageInfo_PkgPath(t *testing.T) { //nolint:funlen
 	for _, d := range data {
 		t.Run(d.title, func(t *testing.T) {
 			t.Parallel()
-			pkgPath, err := d.pkg.PkgPath(rootDir, rt)
+			pkgPath, err := d.pkg.AbsPkgPath(rootDir, rt)
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/controller/exec/controller.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"os"
 	"runtime"
+	"time"
 
 	"github.com/aquaproj/aqua/v2/pkg/config"
 	"github.com/aquaproj/aqua/v2/pkg/controller/which"
@@ -26,13 +27,18 @@ type Controller struct {
 	fs               afero.Fs
 	policyReader     PolicyReader
 	enabledXSysExec  bool
+	vacuum           Vacuum
+}
+
+type Vacuum interface {
+	Update(pkgPath string, timestamp time.Time) error
 }
 
 type Installer interface {
 	InstallPackage(ctx context.Context, logE *logrus.Entry, param *installpackage.ParamInstallPackage) error
 }
 
-func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader) *Controller {
+func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, osEnv osenv.OSEnv, fs afero.Fs, policyReader PolicyReader, vacuum Vacuum) *Controller {
 	return &Controller{
 		stdin:            os.Stdin,
 		stdout:           os.Stdout,
@@ -43,6 +49,7 @@ func New(pkgInstaller Installer, whichCtrl WhichController, executor Executor, o
 		enabledXSysExec:  getEnabledXSysExec(osEnv, runtime.GOOS),
 		fs:               fs,
 		policyReader:     policyReader,
+		vacuum:           vacuum,
 	}
 }
 

pkg/controller/exec/exec.go

@@ -13,12 +13,13 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/osexec"
 	"github.com/aquaproj/aqua/v2/pkg/osfile"
 	"github.com/aquaproj/aqua/v2/pkg/policy"
+	"github.com/aquaproj/aqua/v2/pkg/runtime"
 	"github.com/sirupsen/logrus"
 	"github.com/suzuki-shunsuke/go-error-with-exit-code/ecerror"
 	"github.com/suzuki-shunsuke/logrus-error/logerr"
 )
 
-func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) {
+func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config.Param, exeName string, args ...string) (gErr error) { //nolint:cyclop
 	logE = logE.WithField("exe_name", exeName)
 	defer func() {
 		if gErr != nil {
@@ -62,9 +63,25 @@ func (c *Controller) Exec(ctx context.Context, logE *logrus.Entry, param *config
 	if err := c.install(ctx, logE, findResult, policyCfgs, param); err != nil {
 		return err
 	}
+
+	if err := c.updateTimestamp(findResult.Package); err != nil {
+		logerr.WithError(logE, err).Warn("update the last used datetime")
+	}
+
 	return c.execCommandWithRetry(ctx, logE, findResult.ExePath, args...)
 }
 
+func (c *Controller) updateTimestamp(pkg *config.Package) error {
+	pkgPath, err := pkg.PkgPath(runtime.New())
+	if err != nil {
+		return fmt.Errorf("get a package path: %w", err)
+	}
+	if err := c.vacuum.Update(pkgPath, time.Now()); err != nil {
+		return fmt.Errorf("update the last used datetime: %w", err)
+	}
+	return nil
+}
+
 func (c *Controller) install(ctx context.Context, logE *logrus.Entry, findResult *which.FindResult, policies []*policy.Config, param *config.Param) error {
 	var checksums *checksum.Checksums
 	if findResult.Config.ChecksumEnabled(param.EnforceChecksum, param.Checksum) {

pkg/controller/exec/exec_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/slsa"
 	"github.com/aquaproj/aqua/v2/pkg/testutil"
 	"github.com/aquaproj/aqua/v2/pkg/unarchive"
+	"github.com/aquaproj/aqua/v2/pkg/vacuum"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/afero"
 	"github.com/suzuki-shunsuke/go-osenv/osenv"
@@ -152,9 +153,9 @@ packages:
 			whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker)
 			downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient))
 			executor := &osexec.Mock{}
-			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{})
+			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuum.NewMock(d.param.RootDir, nil, nil))
 			policyFinder := policy.NewConfigFinder(fs)
-			ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)))
+			ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, policy.NewReader(fs, policy.NewValidator(d.param, fs), policyFinder, policy.NewConfigReader(fs)), vacuum.NewMock(d.param.RootDir, nil, nil))
 			if err := ctrl.Exec(ctx, logE, d.param, d.exeName, d.args...); err != nil {
 				if d.isErr {
 					return
@@ -247,8 +248,9 @@ packages:
 			whichCtrl := which.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, ghDownloader, afero.NewOsFs(), d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), d.rt, osEnv, fs, linker)
 			downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient))
 			executor := &osexec.Mock{}
-			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{})
-			ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, &policy.MockReader{})
+			vacuumMock := vacuum.NewMock(d.param.RootDir, nil, nil)
+			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuumMock)
+			ctrl := execCtrl.New(pkgInstaller, whichCtrl, executor, osEnv, fs, &policy.MockReader{}, vacuumMock)
 			b.ResetTimer()
 			for range b.N {
 				func() {

pkg/controller/install/install_test.go

@@ -23,6 +23,7 @@ import (
 	"github.com/aquaproj/aqua/v2/pkg/slsa"
 	"github.com/aquaproj/aqua/v2/pkg/testutil"
 	"github.com/aquaproj/aqua/v2/pkg/unarchive"
+	"github.com/aquaproj/aqua/v2/pkg/vacuum"
 	"github.com/sirupsen/logrus"
 )
 
@@ -103,7 +104,8 @@ packages:
 			}
 			downloader := download.NewDownloader(nil, download.NewHTTPDownloader(http.DefaultClient))
 			executor := &osexec.Mock{}
-			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{})
+			vacuumMock := vacuum.NewMock(d.param.RootDir, nil, nil)
+			pkgInstaller := installpackage.New(d.param, downloader, d.rt, fs, linker, nil, &checksum.Calculator{}, unarchive.New(executor, fs), &cosign.MockVerifier{}, &slsa.MockVerifier{}, &minisign.MockVerifier{}, &ghattestation.MockVerifier{}, &installpackage.MockGoInstallInstaller{}, &installpackage.MockGoBuildInstaller{}, &installpackage.MockCargoPackageInstaller{}, vacuumMock)
 			policyFinder := policy.NewConfigFinder(fs)
 			policyReader := policy.NewReader(fs, &policy.MockValidator{}, policyFinder, policy.NewConfigReader(fs))
 			ctrl := install.New(d.param, finder.NewConfigFinder(fs), reader.New(fs, d.param), registry.New(d.param, registryDownloader, fs, d.rt, &cosign.MockVerifier{}, &slsa.MockVerifier{}), pkgInstaller, fs, d.rt, policyReader)