new: Add runner.autoCleanCache setting. (#1435)

* Add setting.

* Update docs.

* Fix lint.

crates/cli/src/commands/clean.rs

@@ -13,7 +13,9 @@ pub struct CleanArgs {
 pub async fn clean(args: ArgsRef<CleanArgs>, workspace: ResourceRef<Workspace>) {
     let done = create_progress_bar(format!("Cleaning stale cache older than {}", args.lifetime));
 
-    let (files_deleted, bytes_saved) = workspace.cache_engine.clean_stale_cache(&args.lifetime)?;
+    let (files_deleted, bytes_saved) = workspace
+        .cache_engine
+        .clean_stale_cache(&args.lifetime, true)?;
 
     done(
         format!("Deleted {files_deleted} files and saved {bytes_saved} bytes"),

crates/core/action-pipeline/src/subscribers/local_cache.rs

@@ -79,9 +79,11 @@ impl Subscriber for LocalCacheSubscriber {
 
             // After the run has finished, clean any stale archives.
             Event::PipelineFinished { .. } => {
-                workspace
-                    .cache_engine
-                    .clean_stale_cache(&workspace.config.runner.cache_lifetime)?;
+                if workspace.config.runner.auto_clean_cache {
+                    workspace
+                        .cache_engine
+                        .clean_stale_cache(&workspace.config.runner.cache_lifetime, false)?;
+                }
             }
             _ => {}
         }

nextgen/cache/src/cache_engine.rs

@@ -62,20 +62,33 @@ impl CacheEngine {
         self.cache(self.states_dir.join(path.as_ref()))
     }
 
-    pub fn clean_stale_cache(&self, lifetime: &str) -> miette::Result<(usize, u64)> {
+    pub fn clean_stale_cache(&self, lifetime: &str, all: bool) -> miette::Result<(usize, u64)> {
         let duration =
             parse_duration(lifetime).map_err(|e| miette::miette!("Invalid lifetime: {e}"))?;
 
         debug!(
-            "Cleaning up and deleting stale cache older than \"{}\"",
+            "Cleaning up and deleting stale cached artifacts older than \"{}\"",
             lifetime
         );
 
-        let stats = fs::remove_dir_stale_contents(&self.cache_dir, duration)?;
-        let deleted = stats.files_deleted;
-        let bytes = stats.bytes_saved;
+        let mut deleted = 0;
+        let mut bytes = 0;
 
-        debug!("Deleted {} files and saved {} bytes", deleted, bytes);
+        if all {
+            let stats = fs::remove_dir_stale_contents(&self.cache_dir, duration)?;
+            deleted += stats.files_deleted;
+            bytes += stats.bytes_saved;
+        } else {
+            let stats = fs::remove_dir_stale_contents(self.cache_dir.join("hashes"), duration)?;
+            deleted += stats.files_deleted;
+            bytes += stats.bytes_saved;
+
+            let stats = fs::remove_dir_stale_contents(self.cache_dir.join("outputs"), duration)?;
+            deleted += stats.files_deleted;
+            bytes += stats.bytes_saved;
+        }
+
+        debug!("Deleted {} artifacts and saved {} bytes", deleted, bytes);
 
         Ok((deleted, bytes))
     }

nextgen/config/src/workspace/runner_config.rs

@@ -8,6 +8,10 @@ pub struct RunnerConfig {
     /// cached and persisted.
     pub archivable_targets: Vec<Target>,
 
+    /// Automatically clean the cache after every task run.
+    #[setting(default = true)]
+    pub auto_clean_cache: bool,
+
     /// The lifetime in which task outputs will be cached.
     #[setting(default = "7 days")]
     pub cache_lifetime: String,

nextgen/process/src/command_inspector.rs

@@ -246,11 +246,7 @@ impl<'cmd> CommandInspector<'cmd> {
             workspace_root = Some(self.get_workspace_root());
         }
 
-        let working_dir_field = self
-            .command
-            .cwd
-            .as_deref()
-            .or_else(|| workspace_root.as_deref());
+        let working_dir_field = self.command.cwd.as_deref().or(workspace_root.as_deref());
 
         debug!(
             env_vars = ?env_vars_field,

packages/cli/CHANGELOG.md

@@ -17,6 +17,8 @@
 - Added an experimental `moon templates` command, that lists all available codegen templates.
 - Added a `--dependents` flag to `moon project-graph <id>` and `moon query projects`, to include
   downstream dependents of a focused/affected project.
+- Added a `runner.autoCleanCache` setting to `.moon/workspace.yml`, allowing the post-run clean
+  mechanism to be controlled.
 - Updated `moon ci` to automatically determine base/head revisions based on your current CI provider
   (when applicable).
 - Updated `moon generate`:

packages/types/src/tasks-config.ts

@@ -89,8 +89,8 @@ export interface TaskOptionsConfig {
 	 */
 	mergeOutputs: TaskMergeStrategy | null;
 	/**
-	 * Creates an exclusive lock on a virtual resource, preventing other tasks
-	 * using the same resource from running concurrently.
+	 * Creates an exclusive lock on a virtual resource, preventing other
+	 * tasks using the same resource from running concurrently.
 	 */
 	mutex: string | null;
 	/**
@@ -308,8 +308,8 @@ export interface PartialTaskOptionsConfig {
 	 */
 	mergeOutputs?: TaskMergeStrategy | null;
 	/**
-	 * Creates an exclusive lock on a virtual resource, preventing other tasks
-	 * using the same resource from running concurrently.
+	 * Creates an exclusive lock on a virtual resource, preventing other
+	 * tasks using the same resource from running concurrently.
 	 */
 	mutex?: string | null;
 	/**

packages/types/src/workspace-config.ts

@@ -148,6 +148,12 @@ export interface RunnerConfig {
 	 * cached and persisted.
 	 */
 	archivableTargets: string[];
+	/**
+	 * Automatically clean the cache after every task run.
+	 *
+	 * @default true
+	 */
+	autoCleanCache: boolean;
 	/**
 	 * The lifetime in which task outputs will be cached.
 	 *
@@ -386,6 +392,12 @@ export interface PartialRunnerConfig {
 	 * cached and persisted.
 	 */
 	archivableTargets?: string[] | null;
+	/**
+	 * Automatically clean the cache after every task run.
+	 *
+	 * @default true
+	 */
+	autoCleanCache?: boolean | null;
 	/**
 	 * The lifetime in which task outputs will be cached.
 	 *

website/blog/2024-04-17_moon-v1.24.mdx

@@ -55,6 +55,8 @@ $ moon query projects --affected --dependents
 View the [official release](https://github.com/moonrepo/moon/releases/tag/v1.24.0) for a full list
 of changes.
 
+- Added a `runner.autoCleanCache` setting to `.moon/workspace.yml`, allowing the post-run clean
+  mechanism to be controlled.
 - Updated `moon generate` with better argument to variable handling.
 - Updated action graph and project graph visualization to be more readable.
 - Updated root-level tasks to have no inputs by default, instead of `**/*`.

website/docs/config/project.mdx

@@ -981,14 +981,14 @@ The [strategy](../concepts/task-inheritance#merge-strategies) to use when mergin
 The [strategy](../concepts/task-inheritance#merge-strategies) to use when merging the
 [`outputs`](#outputs) list with an inherited task. Defaults to "append".
 
-#### `mutex`
+#### `mutex`<VersionLabel version="1.24.0" />
 
 <HeadingApiLink to="/api/types/interface/TaskOptionsConfig#mutex" />
 
 Creates an exclusive lock on a "virtual resource", preventing other tasks using the same "virtual
 resource" from running concurrently.
 
-If you have many tasks that require exclusive access to a resource that can't be tracked by Moon
+If you have many tasks that require exclusive access to a resource that can't be tracked by moon
 (like a database, an ignored file, a file that's not part of the project, or a remote resource) you
 can use the `mutex` option to prevent them from running at the same time.
 

website/docs/config/workspace.mdx

@@ -418,20 +418,7 @@ notifier:
 
 <HeadingApiLink to="/api/types/interface/WorkspaceConfig#runner" />
 
-Configures aspects of the action pipeline.
-
-### `cacheLifetime`
-
-<HeadingApiLink to="/api/types/interface/RunnerConfig#cacheLifetime" />
-
-The maximum lifetime of cached artifacts before they're marked as stale and automatically removed by
-the action pipeline. Defaults to "7 days". This field requires an integer and a timeframe unit that
-can be [parsed as a duration](https://docs.rs/humantime/2.1.0/humantime/fn.parse_duration.html).
-
-```yaml title=".moon/workspace.yml" {2}
-runner:
-  cacheLifetime: '24 hours'
-```
+Configures aspects of task running and the action pipeline.
 
 ### `archivableTargets`
 
@@ -451,6 +438,32 @@ runner:
 > This setting primarily exists for [remote caching](../guides/remote-cache) as it will create and
 > persist tar archives located in `.moon/cache/outputs`.
 
+### `autoCleanCache`<VersionLabel version="1.24.0" />
+
+<HeadingApiLink to="/api/types/interface/RunnerConfig#autoCleanCache" />
+
+Automatically cleans cached artifacts older than [`cacheLifetime`](#cachelifetime) from the cache
+directory (`.moon/cache`) after every run. This is useful for keeping the cache directory lean.
+Defaults to `true`.
+
+```yaml title=".moon/workspace.yml" {2}
+runner:
+  autoCleanCache: false
+```
+
+### `cacheLifetime`
+
+<HeadingApiLink to="/api/types/interface/RunnerConfig#cacheLifetime" />
+
+The maximum lifetime of cached artifacts before they're marked as stale and automatically removed by
+the action pipeline. Defaults to "7 days". This field requires an integer and a timeframe unit that
+can be [parsed as a duration](https://docs.rs/humantime/2.1.0/humantime/fn.parse_duration.html).
+
+```yaml title=".moon/workspace.yml" {2}
+runner:
+  cacheLifetime: '24 hours'
+```
+
 ### `inheritColorsForPipedTasks`
 
 <HeadingApiLink to="/api/types/interface/RunnerConfig#inheritColorsForPipedTasks" />

website/static/schemas/project.json

@@ -1108,8 +1108,7 @@
             {
               "type": "null"
             }
-          ],
-          "markdownDescription": "Creates an exclusive lock on a virtual resource, preventing other tasks using the same resource from running concurrently."
+          ]
         },
         "outputStyle": {
           "title": "outputStyle",

website/static/schemas/tasks.json

@@ -513,8 +513,7 @@
             {
               "type": "null"
             }
-          ],
-          "markdownDescription": "Creates an exclusive lock on a virtual resource, preventing other tasks using the same resource from running concurrently."
+          ]
         },
         "outputStyle": {
           "title": "outputStyle",

website/static/schemas/workspace.json

@@ -389,6 +389,12 @@
             "type": "string"
           }
         },
+        "autoCleanCache": {
+          "title": "autoCleanCache",
+          "description": "Automatically clean the cache after every task run.",
+          "default": true,
+          "type": "boolean"
+        },
         "cacheLifetime": {
           "title": "cacheLifetime",
           "description": "The lifetime in which task outputs will be cached.",