new: Support tokens in task environment variables. (#997)

* Expand env.

* Add tests.

* Update docs.

* Updated blog post.

nextgen/project-expander/src/tasks_expander.rs

@@ -5,7 +5,7 @@ use moon_common::{color, Id};
 use moon_config::InputPath;
 use moon_project::Project;
 use moon_task::{Target, TargetScope, Task};
-use rustc_hash::{FxHashMap, FxHashSet};
+use rustc_hash::FxHashSet;
 use starbase_utils::glob::GlobSet;
 use tracing::{trace, warn};
 
@@ -196,12 +196,13 @@ impl<'graph, 'query> TasksExpander<'graph, 'query> {
             "Expanding environment variables"
         );
 
-        // Substitute environment variables
-        let cloned_env = task.env.clone();
-        let mut env = FxHashMap::default();
+        // Expand tokens
+        let mut env = self.token.expand_env(task)?;
+        let cloned_env = env.clone();
 
-        for (key, val) in &task.env {
-            env.insert(key.to_owned(), substitute_env_var(val, &cloned_env));
+        // Substitute environment variables
+        for (_, value) in env.iter_mut() {
+            *value = substitute_env_var(value, &cloned_env);
         }
 
         // Load variables from an .env file

nextgen/project-expander/src/token_expander.rs

@@ -6,6 +6,7 @@ use moon_project::FileGroup;
 use moon_task::Task;
 use moon_time::{now_millis, now_timestamp};
 use pathdiff::diff_paths;
+use rustc_hash::FxHashMap;
 use tracing::warn;
 
 pub type ExpandedPaths = (Vec<WorkspaceRelativePathBuf>, Vec<WorkspaceRelativePathBuf>);
@@ -14,6 +15,7 @@ pub type ExpandedPaths = (Vec<WorkspaceRelativePathBuf>, Vec<WorkspaceRelativePa
 pub enum TokenScope {
     Command,
     Args,
+    Env,
     Inputs,
     Outputs,
 }
@@ -23,6 +25,7 @@ impl TokenScope {
         match self {
             TokenScope::Command => "commands",
             TokenScope::Args => "args",
+            TokenScope::Env => "env",
             TokenScope::Inputs => "inputs",
             TokenScope::Outputs => "outputs",
         }
@@ -127,6 +130,36 @@ impl<'graph, 'query> TokenExpander<'graph, 'query> {
         Ok(args)
     }
 
+    pub fn expand_env(&mut self, task: &Task) -> miette::Result<FxHashMap<String, String>> {
+        self.scope = TokenScope::Env;
+
+        let mut env = FxHashMap::default();
+
+        for (key, value) in &task.env {
+            if self.has_token_function(value) {
+                let (files, globs) = self.replace_function(task, value)?;
+
+                let mut list = vec![];
+                list.extend(files);
+                list.extend(globs);
+
+                env.insert(
+                    key.to_owned(),
+                    list.iter()
+                        .map(|i| i.as_str())
+                        .collect::<Vec<_>>()
+                        .join(","),
+                );
+            } else if self.has_token_variable(value) {
+                env.insert(key.to_owned(), self.replace_variables(task, value)?);
+            } else {
+                env.insert(key.to_owned(), value.to_owned());
+            }
+        }
+
+        Ok(env)
+    }
+
     pub fn expand_inputs(&mut self, task: &Task) -> miette::Result<ExpandedPaths> {
         self.scope = TokenScope::Inputs;
 
@@ -234,7 +267,12 @@ impl<'graph, 'query> TokenExpander<'graph, 'query> {
         let file_group = || -> miette::Result<&FileGroup> {
             self.check_scope(
                 token,
-                &[TokenScope::Args, TokenScope::Inputs, TokenScope::Outputs],
+                &[
+                    TokenScope::Args,
+                    TokenScope::Env,
+                    TokenScope::Inputs,
+                    TokenScope::Outputs,
+                ],
             )?;
 
             Ok(self.context.project.file_groups.get(arg).ok_or_else(|| {
@@ -359,6 +397,7 @@ impl<'graph, 'query> TokenExpander<'graph, 'query> {
             &[
                 TokenScope::Command,
                 TokenScope::Args,
+                TokenScope::Env,
                 TokenScope::Inputs,
                 TokenScope::Outputs,
             ],

nextgen/project-expander/tests/tasks_expander_test.rs

@@ -635,6 +635,52 @@ mod tasks_expander {
             );
         }
 
+        #[test]
+        fn replaces_tokens() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+
+            let mut task = create_task();
+            task.env.insert("KEY1".into(), "@globs(all)".into());
+            task.env.insert("KEY2".into(), "$project-$task".into());
+
+            let context = create_context(&project, sandbox.path());
+            TasksExpander::new(&context).expand_env(&mut task).unwrap();
+
+            assert_eq!(
+                task.env,
+                FxHashMap::from_iter([
+                    (
+                        "KEY1".into(),
+                        "project/source/*.md,project/source/**/*.json".into()
+                    ),
+                    ("KEY2".into(), "project-task".into()),
+                ])
+            );
+        }
+
+        #[test]
+        fn can_use_env_vars_and_token_vars() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+
+            let mut task = create_task();
+            task.env
+                .insert("KEY".into(), "$project-$FOO-$unknown".into());
+
+            env::set_var("FOO", "foo");
+
+            let context = create_context(&project, sandbox.path());
+            TasksExpander::new(&context).expand_env(&mut task).unwrap();
+
+            env::remove_var("FOO");
+
+            assert_eq!(
+                task.env,
+                FxHashMap::from_iter([("KEY".into(), "project-foo-$unknown".into()),])
+            );
+        }
+
         #[test]
         fn loads_from_env_file() {
             let sandbox = create_sandbox("env-file");

nextgen/project-expander/tests/token_expander_test.rs

@@ -3,6 +3,7 @@ mod utils;
 use moon_common::path::WorkspaceRelativePathBuf;
 use moon_config::{InputPath, LanguageType, OutputPath, ProjectType};
 use moon_project_expander::TokenExpander;
+use rustc_hash::FxHashMap;
 use starbase_sandbox::{create_empty_sandbox, create_sandbox, predicates::prelude::*};
 use utils::{create_context, create_project, create_task};
 
@@ -272,6 +273,186 @@ mod token_expander {
         }
     }
 
+    mod env {
+        use super::*;
+
+        #[test]
+        fn passes_through() {
+            let sandbox = create_empty_sandbox();
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("KEY".into(), "value".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([("KEY".into(), "value".into())])
+            );
+        }
+
+        #[test]
+        fn replaces_one_var() {
+            let sandbox = create_empty_sandbox();
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("VAR".into(), "$project-prod".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([("VAR".into(), "project-prod".into())])
+            );
+        }
+
+        #[test]
+        fn replaces_two_vars() {
+            let sandbox = create_empty_sandbox();
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env
+                .insert("VARS".into(), "$project-debug-$task".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([("VARS".into(), "project-debug-task".into())])
+            );
+        }
+
+        #[test]
+        fn supports_group_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("GROUP".into(), "@group(all)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([("GROUP".into(), "project/source/config.yml,project/source/dir/subdir,project/source/*.md,project/source/**/*.json".into())])
+            );
+        }
+
+        #[test]
+        fn supports_dirs_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("DIRS".into(), "@dirs(dirs)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([(
+                    "DIRS".into(),
+                    "project/source/dir/subdir,project/source/other".into()
+                )])
+            );
+        }
+
+        #[test]
+        fn supports_files_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("FILES".into(), "@files(all)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([(
+                    "FILES".into(),
+                    "project/source/config.yml,project/source/dir/subdir/nested.json,project/source/docs.md,project/source/other/file.json".into()
+                )])
+            );
+        }
+
+        #[test]
+        fn supports_globs_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("GLOBS".into(), "@globs(all)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([(
+                    "GLOBS".into(),
+                    "project/source/*.md,project/source/**/*.json".into()
+                )])
+            );
+        }
+
+        #[test]
+        fn supports_root_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("ROOT".into(), "@root(all)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            assert_eq!(
+                expander.expand_env(&task).unwrap(),
+                FxHashMap::from_iter([("ROOT".into(), "project/source/dir/subdir".into())])
+            );
+        }
+
+        #[test]
+        #[should_panic(expected = "Token @in(0) cannot be used within task env.")]
+        fn errors_for_in_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("IN".into(), "@in(0)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            expander.expand_env(&task).unwrap();
+        }
+
+        #[test]
+        #[should_panic(expected = "Token @out(0) cannot be used within task env.")]
+        fn errors_for_out_func() {
+            let sandbox = create_sandbox("file-group");
+            let project = create_project(sandbox.path());
+            let mut task = create_task();
+
+            task.env.insert("OUT".into(), "@out(0)".into());
+
+            let context = create_context(&project, sandbox.path());
+            let mut expander = TokenExpander::new(&context);
+
+            expander.expand_env(&task).unwrap();
+        }
+    }
+
     mod inputs {
         use super::*;
 

packages/cli/CHANGELOG.md

@@ -6,6 +6,7 @@
 
 - Added an `extends` field to task configurations. This allows tasks to extend and inherit settings
   from sibling tasks.
+- Updated task `env` values to support token functions and variables.
 
 ## 1.11.1
 

website/blog/2023-01-30_v0.23.mdx

@@ -77,7 +77,7 @@ tasks:
 The biggest change to task inheritance is that tasks can now be scoped by a project's
 [`language`](/docs/config/project#language) or [`type`](/docs/config/project#type) using the new
 `.moon/tasks/<language>.yml` or `.moon/tasks/<language>-<type>.yml` configuration files! Jump to the
-[official documentation on task inheritance](/docs/concepts/task#inheritance) for more information
+[official documentation on task inheritance](/docs/concepts/task-inheritance) for more information
 on how scoping works, the lookup order of files, and much more.
 
 As a demonstration, you can scope tasks to Node.js projects with `.moon/tasks/node.yml`, Rust

website/blog/2023-04-17_moon-v1.2.mdx

@@ -69,7 +69,7 @@ tasks:
 
 Each of these Astro applications will now inherit all 5 tasks and the file group automatically! This
 helps to greatly reduce maintenance overhead and help enforce consistency across projects. Jump to
-the official [task inheritance docs](/docs/concepts/task#inheritance) for more information on tag
+the official [task inheritance docs](/docs/concepts/task-inheritance) for more information on tag
 based inheritance.
 
 ## Other changes

website/blog/2023-06-12_moon-v1.8.mdx

@@ -116,7 +116,7 @@ configurations for popular programming languages, frameworks, libraries, and mor
 repository is kind of empty, but we're hoping to grow it over time, so feel free to contribute!
 
 If you're curious how this works in practice, we'll use our Rust configuration as an example. The
-entire system is based around [tag inheritance](/docs/concepts/task#inheritance), where a project
+entire system is based around [tag inheritance](/docs/concepts/task-inheritance), where a project
 can inherit tasks from a remote source, and then extend or override them as needed. For example,
 create the tag-based config: