Merge branch 'develop' into workload_groups

lib/ramble/ramble/application.py

@@ -45,6 +45,7 @@
 import ramble.util.directives
 import ramble.util.stats
 import ramble.util.graph
+import ramble.util.class_attributes
 from ramble.util.logger import logger
 
 from ramble.workspace import namespace
@@ -89,6 +90,8 @@ class ApplicationBase(object, metaclass=ApplicationMeta):
     def __init__(self, file_path):
         super().__init__()
 
+        ramble.util.class_attributes.convert_class_attributes(self)
+
         self.keywords = ramble.keywords.keywords
 
         self._vars_are_expanded = False

lib/ramble/ramble/cmd/repo.py

@@ -40,8 +40,12 @@ def setup_parser(subparser):
     create_parser.add_argument(
         'directory', help="directory to create the repo in")
     create_parser.add_argument(
-        'namespace', help="namespace to identify objects "
-        "in the repository. defaults to the directory name", nargs='?')
+        'namespace',
+        metavar='new_namespace',
+        help="namespace to identify objects "
+        "in the repository. defaults to the directory name",
+        nargs='?',
+    )
     create_parser.add_argument(
         '-d', '--subdirectory',
         action='store',

lib/ramble/ramble/cmd/workspace.py

@@ -302,7 +302,7 @@ def _workspace_create(name_or_path, dir=False,
 def workspace_remove_setup_parser(subparser):
     """remove an existing workspace"""
     subparser.add_argument(
-        'rm_wrkspc', metavar='wrkspc', nargs='+',
+        'rm_wrkspc', metavar='workspace', nargs='+',
         help='workspace(s) to remove')
     arguments.add_common_arguments(subparser, ['yes_to_all'])
 

lib/ramble/ramble/expander.py

@@ -767,7 +767,10 @@ def _eval_comp_in(self, node):
                                             f'"{var_name} in {namespace}"')
                     self.__raise_syntax_error(node)
                 return val
-        elif isinstance(node.left, ast.Constant):
+        # ast.Str was deprecated. short-circuit the test for it to avoid issues with newer python.
+        # TODO: Remove `or` logic after 3.6 & 3.7 series python are unsupported
+        elif isinstance(node.left, ast.Constant) or \
+                (hasattr(ast, 'Str') and isinstance(node.left, ast.Str)):
             lhs_value = self.eval_math(node.left)
 
             found = False

lib/ramble/ramble/language/language_base.py

@@ -95,7 +95,8 @@ def __init__(cls, name, bases, attr_dict):
 
             directive_attrs = {
                 '_directive_functions': {},
-                '_directive_classes': {}
+                '_directive_classes': {},
+                '_directive_names': DirectiveMeta._directive_names.copy()
             }
 
             for attr in directive_attrs.keys():

lib/ramble/ramble/modifier.py

@@ -20,6 +20,7 @@
 from ramble.error import RambleError
 import ramble.util.colors as rucolor
 import ramble.util.directives
+import ramble.util.class_attributes
 from ramble.util.logger import logger
 
 
@@ -41,6 +42,8 @@ class ModifierBase(object, metaclass=ModifierMeta):
     def __init__(self, file_path):
         super().__init__()
 
+        ramble.util.class_attributes.convert_class_attributes(self)
+
         self._file_path = file_path
         self._on_executables = ['*']
         self.expander = None

lib/ramble/ramble/spack_runner.py

@@ -519,7 +519,9 @@ def _env_file_dict(self):
         env_file[spack_namespace]['concretizer']['unify'] = True
 
         env_file[spack_namespace]['specs'] = syaml.syaml_list()
-        env_file[spack_namespace]['specs'].extend(self.env_contents)
+        # Ensure the specs content are consistently sorted.
+        # Otherwise the hash checking may artificially miss due to ordering.
+        env_file[spack_namespace]['specs'].extend(sorted(self.env_contents))
 
         env_file[spack_namespace]['include'] = self.includes
 

lib/ramble/ramble/test/modifier_language.py

@@ -549,3 +549,16 @@ def test_modifier_variable_directive(mod_class, func_type):
         assert test_def['name'] in mod_inst.modifier_variables[mode]
         assert test_def['description'] == mod_inst.modifier_variables[mode][var_name].description
         assert test_def['default'] == mod_inst.modifier_variables[mode][var_name].default
+
+
+@pytest.mark.parametrize('func_type', func_types)
+@pytest.mark.parametrize('mod_class', mod_types)
+def test_modifier_class_attributes(mod_class, func_type):
+    test_class = generate_mod_class(mod_class)
+    mod_inst = test_class('/not/a/path')
+    mod_copy = mod_inst.copy()
+
+    mod_copy.mode('added_mode', description='Mode added to test attributes')
+
+    assert 'added_mode' in mod_copy.modes
+    assert 'added_mode' not in mod_inst.modes

lib/ramble/ramble/test/spack_runner.py

@@ -79,6 +79,7 @@ def test_env_concretize_skips_already_concretized_envs(tmpdir, capsys):
         sr.create_env(env_path)
         sr.activate()
         sr.add_spec('zlib')
+        sr.add_spec('intel-oneapi-mpi')
 
         # Generate an initial env file
         sr.generate_env_file()

lib/ramble/ramble/util/class_attributes.py

@@ -0,0 +1,24 @@
+# Copyright 2022-2024 The Ramble Authors
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# https://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+def convert_class_attributes(obj):
+    """Convert class attributes defined from directives to instance attributes
+    Class attributes that are valid for conversion are stored in the _directive_names
+    attribute.
+
+    Args:
+        obj (Object): Input object instance to convert attributes in
+    """
+
+    if hasattr(obj, '_directive_names'):
+        dir_set = dir(obj)
+        var_set = vars(obj)
+        for attr in obj._directive_names:
+            if attr in dir_set and attr not in var_set:
+                inst_val = getattr(obj, attr).copy()
+                setattr(obj, attr, inst_val)

share/ramble/cloud-build/Dockerfile-centos7 → share/ramble/cloud-build/Dockerfile-yum

@@ -1,10 +1,16 @@
-FROM centos:7 as builder
+ARG BASE_IMG=centos
+ARG BASE_VER=7
+FROM ${BASE_IMG}:${BASE_VER} as builder
 
-RUN yum install -yq git python3 python3-pip wget mercurial which svn curl gcc && rm -rf /var/lib/apt/lists/*
+ARG SPACK_REF=releases/latest
+ARG CONDA_VER=4.10.3
+RUN yum install -yq git python3 python3-pip wget mercurial which svn curl gcc tar bzip2 && rm -rf /var/lib/apt/lists/*
 RUN cd /opt && \
-    git clone https://github.com/spack/spack -b v0.19.2 && \
+    git clone https://github.com/spack/spack && \
+    cd spack && \
+    git checkout $SPACK_REF && \
     . /opt/spack/share/spack/setup-env.sh && \
-    spack install miniconda3 && \
+    spack install miniconda3@${CONDA_VER} && \
     spack clean -a
 RUN echo -e "export PATH=$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:${PATH}\n. /opt/spack/share/spack/setup-env.sh" > /etc/profile.d/ramble.sh
 RUN cd /opt &&  \
@@ -14,7 +20,7 @@ RUN cd /opt &&  \
     conda install -qy pip && \
     python -m pip install -r /opt/requirements.txt
 
-FROM centos:7
+FROM ${BASE_IMG}:${BASE_VER}
 
 COPY --from=builder / /
 

share/ramble/cloud-build/ramble-image-builder.yaml

@@ -13,13 +13,27 @@ steps:
     args:
     - 'build'
     - '-t'
-    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest'
+    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest'
     - '--cache-from'
-    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest'
+    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest'
     - '-f'
-    - 'share/ramble/cloud-build/Dockerfile-centos7'
+    - 'share/ramble/cloud-build/Dockerfile-${_PKG_MANAGER}'
+    - '--build-arg'
+    - 'BASE_IMG=${_BASE_IMG}'
+    - '--build-arg'
+    - 'BASE_VER=${_BASE_VER}'
+    - '--build-arg'
+    - 'SPACK_REF=${_SPACK_REF}'
+    - '--build-arg'
+    - 'CONDA_VER=${_CONDA_VER}'
     - '.'
-images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7']
+substitutions:
+  _SPACK_REF: v0.21.2
+  _CONDA_VER: 22.11.1
+  _BASE_IMG: centos
+  _BASE_VER: '7'
+  _PKG_MANAGER: 'yum'
+images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}']
 timeout: 1500s
 options:
   machineType: N1_HIGHCPU_8

share/ramble/cloud-build/ramble-pr-software-conflicts.yaml

@@ -13,7 +13,7 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
     args:
       - '-c'
       - |
@@ -26,6 +26,9 @@ steps:
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        echo "Spack version is $(spack --version)"
+        echo "Python version is $(python3 --version)"
+
         ramble software-definitions -s
 
         ramble software-definitions -c -e
@@ -43,7 +46,11 @@ steps:
         exit $$error
     id: ramble-style-tests
     entrypoint: /bin/bash
-
+substitutions:
+  _SPACK_REF: v0.21.2
+  _CONDA_VER: 22.11.1
+  _BASE_IMG: centos
+  _BASE_VER: '7'
 timeout: 600s
 options:
   machineType: N1_HIGHCPU_8

share/ramble/cloud-build/ramble-pr-style.yaml

@@ -13,7 +13,7 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
     args:
       - '-c'
       - |
@@ -26,6 +26,9 @@ steps:
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        echo "Spack version is $(spack --version)"
+        echo "Python version is $(python3 --version)"
+
         ramble flake8 -U
         # $$ characters are required for cloud-build:
         # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values
@@ -87,7 +90,11 @@ steps:
         exit $$error
     id: ramble-style-tests
     entrypoint: /bin/bash
-
+substitutions:
+  _SPACK_REF: v0.21.2
+  _CONDA_VER: 22.11.1
+  _BASE_IMG: centos
+  _BASE_VER: '7'
 timeout: 600s
 options:
   machineType: N1_HIGHCPU_8

share/ramble/cloud-build/ramble-pr-unit-tests.yaml

@@ -13,7 +13,7 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-centos7:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
     args:
       - '-c'
       - |
@@ -26,6 +26,9 @@ steps:
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        echo "Spack version is $(spack --version)"
+        echo "Python version is $(python3 --version)"
+
         COVERAGE=true LONG=true /workspace/share/ramble/qa/run-unit-tests
         # $$ characters are required for cloud-build:
         # https://cloud.google.com/build/docs/configuring-builds/substitute-variable-values
@@ -64,7 +67,11 @@ steps:
         exit $$error
     id: ramble-unit-tests
     entrypoint: /bin/bash
-
+substitutions:
+  _SPACK_REF: v0.21.2
+  _CONDA_VER: 22.11.1
+  _BASE_IMG: centos
+  _BASE_VER: '7'
 timeout: 900s
 options:
   machineType: N1_HIGHCPU_8

share/ramble/ramble-completion.bash

@@ -586,7 +586,7 @@ _ramble_repo_create() {
     then
         RAMBLE_COMPREPLY="-h --help -d --subdirectory -t --type"
     else
-        _repos
+        RAMBLE_COMREPLY=""
     fi
 }
 
@@ -728,7 +728,7 @@ _ramble_workspace_remove() {
     then
         RAMBLE_COMPREPLY="-h --help -y --yes-to-all"
     else
-        RAMBLE_COMREPLY=""
+        _workspaces
     fi
 }
 
@@ -737,6 +737,6 @@ _ramble_workspace_rm() {
     then
         RAMBLE_COMPREPLY="-h --help -y --yes-to-all"
     else
-        RAMBLE_COMREPLY=""
+        _workspaces
     fi
 }