Initial proof of concept for workload_groups

This change add a new directive: `workload_groups`

It is intended to give the user a way to group workloads together, and
unlocks a key CUJ where inherited applications want to apply variables
defined in parent classes

lib/ramble/ramble/language/application_language.py

@@ -86,6 +86,34 @@ def _execute_workload(app):
     return _execute_workload
 
 
+@application_directive('workload_groups')
+def workload_group(name, workloads=[], mode=None, **kwargs):
+    """Adds a workload group to this application
+
+    Defines a new workload group that can be used within the context of its
+    application.
+
+    Args:
+        name: The name of the group
+        workloads: A list of workloads to be grouped
+    """
+
+    def _execute_workload(app):
+        # TODO where is it best to validate the workloads
+        if mode == 'append':  # TODO: dry magic enum string
+            app.workload_groups[name] += workloads
+        else:
+            app.workload_groups[name] = workloads
+
+        # Apply any existing variables in the group to the workload
+        for workload in workloads:
+            if name in app.workload_group_vars:
+                for var in app.workload_group_vars[name]:
+                    app.workloads[workload].add_variable(var)
+
+    return _execute_workload
+
+
 @application_directive('executables')
 def executable(name, template, **kwargs):
     """Adds an executable to this application
@@ -153,8 +181,9 @@ def _execute_input_file(app):
 
 
 @application_directive(dicts=())
+@application_directive('workload_group_vars')
 def workload_variable(name, default, description, values=None, workload=None,
-                      workloads=None, expandable=True, **kwargs):
+                      workloads=None, workload_group=None, expandable=True, **kwargs):
     """Define a new variable to be used in experiments
 
     Defines a new variable that can be defined within the
@@ -165,20 +194,45 @@ def workload_variable(name, default, description, values=None, workload=None,
     """
 
     def _execute_workload_variable(app):
-        all_workloads = ramble.language.language_helpers.require_definition(workload,
-                                                                            workloads,
-                                                                            app.workloads,
-                                                                            'workload',
-                                                                            'workloads',
-                                                                            'workload_variable')
-
-        for wl_name in all_workloads:
-            app.workloads[wl_name].add_variable(
-                ramble.workload.WorkloadVariable(
-                    name, default=default, description=description,
-                    values=values, expandable=expandable
+        if (workload is not None or workloads is not None) and workload_group is None:
+            all_workloads =  \
+                ramble.language.language_helpers.require_definition(workload,
+                                                                    workloads,
+                                                                    app.workloads,
+                                                                    'workload',
+                                                                    'workloads',
+                                                                    'workload_variable')
+
+            for wl_name in all_workloads:
+                app.workloads[wl_name].add_variable(
+                    ramble.workload.WorkloadVariable(
+                        name, default=default, description=description,
+                        values=values, expandable=expandable
+                    )
                 )
-            )
+        elif workload is None and workloads is None and workload_group is not None:
+            # Find workload group, and iterate it
+            workload_group_list = app.workload_groups[workload_group]
+
+            # TODO: better place to init this?
+            if workload_group not in app.workload_group_vars:
+                app.workload_group_vars[workload_group] = []
+
+            # Track which vars we add to, to allow us to re-apply during inheritance
+
+            # TODO: dry this add with above?
+            workload_var = ramble.workload.WorkloadVariable(
+                name, default=default, description=description,
+                values=values, expandable=expandable)
+
+            app.workload_group_vars[workload_group].append(workload_var)
+
+            for wl_name in workload_group_list:
+                # Apply the variable
+                app.workloads[wl_name].add_variable(workload_var)
+        else:  # workload/workloads is not None and workload_group is not None:
+            raise DirectiveError('A workload variable may set either' +
+                                 '`workloads` or `workload_group` but not both')
 
     return _execute_workload_variable
 

lib/ramble/ramble/test/application_tests.py

@@ -24,6 +24,7 @@
 def test_app_features(mutable_mock_apps_repo, app):
     app_inst = mutable_mock_apps_repo.get(app)
     assert hasattr(app_inst, 'workloads')
+    assert hasattr(app_inst, 'workload_groups')
     assert hasattr(app_inst, 'executables')
     assert hasattr(app_inst, 'figures_of_merit')
     assert hasattr(app_inst, 'inputs')
@@ -533,3 +534,36 @@ def test_derive_variables_for_template_path(mutable_mock_apps_repo):
     test_answer = "/workspace/experiments/bar/test_wl2/baz/execute_experiment"
     executable_application_instance._derive_variables_for_template_path(ws1)
     assert executable_application_instance.variables['execute_experiment'] == test_answer
+
+
+def test_workload_groups(mutable_mock_apps_repo):
+    workload_group_inst = mutable_mock_apps_repo.get('workload-groups')
+
+    assert 'test_wl' in workload_group_inst.workloads
+
+    assert 'empty' in workload_group_inst.workload_groups
+    assert 'test_wlg' in workload_group_inst.workload_groups
+
+    my_var = workload_group_inst.workloads['test_wl'].find_variable('test_var')
+    assert my_var is not None
+    assert my_var.default == '2.0'
+    assert my_var.description == 'Test work load vars and groups'
+
+
+def test_workload_groups_inherited(mutable_mock_apps_repo):
+    wlgi_inst = mutable_mock_apps_repo.get('workload-groups-inherited')
+
+    assert 'test_wl' in wlgi_inst.workloads
+    assert 'test_wl3' in wlgi_inst.workloads
+
+    # check we inherit groups we don't touch
+    assert 'empty' in wlgi_inst.workload_groups
+    assert 'test_wlg' in wlgi_inst.workload_groups
+
+    assert 'test_wl' in wlgi_inst.workload_groups['test_wlg']
+
+    # Ensure a new workload can obtain the parent level vars via groups
+    my_var = wlgi_inst.workloads['test_wl3'].find_variable('test_var')
+    assert my_var is not None
+    assert my_var.default == '2.0'
+    assert my_var.description == 'Test work load vars and groups'

var/ramble/repos/builtin.mock/applications/workload-groups-inherited/application.py

@@ -0,0 +1,22 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+from ramble.appkit import *
+
+from ramble.app.builtin.mock.workload_groups import WorkloadGroups
+
+
+class WorkloadGroupsInherited(WorkloadGroups):
+    name = "workload-groups-inherited"
+
+    workload('test_wl3', executable='baz')
+
+    # Test populated group applies existing vars to new workload
+    workload_group('test_wlg',
+                   workloads=['test_wl3'],
+                   mode='append')

var/ramble/repos/builtin.mock/applications/workload-groups/application.py

@@ -0,0 +1,32 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+from ramble.appkit import *
+
+
+class WorkloadGroups(ExecutableApplication):
+    name = "workload-groups"
+
+    executable('foo', 'echo "bar"', use_mpi=False)
+    executable('bar', 'echo "baz"', use_mpi=False)
+
+    workload('test_wl', executable='foo')
+    workload('test_wl2', executable='bar')
+
+    # Test empty group
+    workload_group('empty',
+                   workloads=[])
+
+    # Test populated group
+    workload_group('test_wlg',
+                   workloads=['test_wl', 'test_wl2'])
+
+    # Test workload_variable that uses a group
+    workload_variable('test_var', default='2.0',
+                      description='Test workload vars and groups',
+                      workload_group='test_wlg')