Merge branch 'develop' of https://github.com/easybuilders/easybuild-e…

…asyblocks into develop

easybuild/easyblocks/a/aocc.py

@@ -31,21 +31,38 @@
 @author: Sebastian Achilles (Forschungszentrum Juelich GmbH)
 """
 
+import glob
 import os
+import re
 import stat
+import tempfile
 
 from easybuild.tools import LooseVersion
 
 from easybuild.easyblocks.generic.packedbinary import PackedBinary
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import adjust_permissions, move_file, write_file
-from easybuild.tools.systemtools import get_shared_lib_ext
+from easybuild.tools.modules import get_software_root, get_software_version
+from easybuild.tools.run import run_cmd
+from easybuild.tools.systemtools import get_shared_lib_ext, get_cpu_architecture
 
 # Wrapper script definition
 WRAPPER_TEMPLATE = """#!/bin/sh
 
-%(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@"
+# Patch argv[0] to the actual compiler so that the correct driver is used internally
+(exec -a "%(actual_compiler_name)s" %(compiler_name)s --gcc-toolchain=$EBROOTGCCCORE "$@")
+"""
+
+AOCC_MINIMAL_CPP_EXAMPLE = """
+#include <iostream>
+
+int main(){ std::cout << "It works!" << std::endl; }
+"""
+
+AOCC_MINIMAL_FORTRAN_EXAMPLE = """
+program main
+end program main
 """
 
 
@@ -66,6 +83,7 @@ def __init__(self, *args, **kwargs):
         super(EB_AOCC, self).__init__(*args, **kwargs)
 
         self.clangversion = self.cfg['clangversion']
+        self.gcc_prefix = None
 
     def _aocc_guess_clang_version(self):
         map_aocc_to_clang_ver = {
@@ -76,6 +94,7 @@ def _aocc_guess_clang_version(self):
             '4.0.0': '14.0.6',
             '4.1.0': '16.0.3',
             '4.2.0': '16.0.3',
+            '5.0.0': '17.0.6',
         }
 
         if self.version in map_aocc_to_clang_ver:
@@ -89,43 +108,26 @@ def _aocc_guess_clang_version(self):
             ]
             raise EasyBuildError('\n'.join(error_lines))
 
-    def install_step(self):
-        # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option,
-        # or via 'accept_eula = True' in easyconfig file
-        self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf')
-
-        # AOCC is based on Clang. Try to guess the clangversion from the AOCC version
-        # if clangversion is not specified in the easyconfig
-        if self.clangversion is None:
-            self.clangversion = self._aocc_guess_clang_version()
+    def _create_compiler_wrappers(self, compilers_to_wrap):
+        if not compilers_to_wrap:
+            return
 
-        super(EB_AOCC, self).install_step()
-
-    def post_install_step(self):
-        """Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain"""
-
-        orig_compiler_tmpl = '%s.orig'
+        orig_compiler_tmpl = '%s/%s.orig'
 
         def create_wrapper(wrapper_comp):
             """Create for a particular compiler, with a particular name"""
             wrapper_f = os.path.join(self.installdir, 'bin', wrapper_comp)
-            write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl % wrapper_comp})
+            write_file(wrapper_f, WRAPPER_TEMPLATE % {'compiler_name': orig_compiler_tmpl %
+                                                      (os.path.join(self.installdir, 'bin'), wrapper_comp),
+                                                      'actual_compiler_name': wrapper_comp})
             perms = stat.S_IXUSR | stat.S_IRUSR | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH
             adjust_permissions(wrapper_f, perms)
 
-        compilers_to_wrap = [
-            'clang',
-            'clang++',
-            'clang-%s' % LooseVersion(self.clangversion).version[0],
-            'clang-cpp',
-            'flang',
-        ]
-
         # Rename original compilers and prepare wrappers to pick up GCCcore as GCC toolchain for the compilers
         for comp in compilers_to_wrap:
             actual_compiler = os.path.join(self.installdir, 'bin', comp)
             if os.path.isfile(actual_compiler):
-                move_file(actual_compiler, orig_compiler_tmpl % actual_compiler)
+                move_file(actual_compiler, orig_compiler_tmpl % (os.path.join(self.installdir, 'bin'), comp))
             else:
                 err_str = "Tried to move '%s' to '%s', but it does not exist!"
                 raise EasyBuildError(err_str, actual_compiler, '%s.orig' % actual_compiler)
@@ -137,6 +139,109 @@ def create_wrapper(wrapper_comp):
                 err_str = "Creating wrapper for '%s' not possible, since original compiler was not renamed!"
                 raise EasyBuildError(err_str, actual_compiler)
 
+    def _create_compiler_config_files(self, compilers_to_add_config_file):
+        if not compilers_to_add_config_file:
+            return
+
+        # For each of the compiler suites, add a .cfg file which points to the correct GCCcore as the GCC toolchain.
+        # We need the GCC prefix for this.
+        self._set_gcc_prefix()
+        bin_dir = os.path.join(self.installdir, 'bin')
+        prefix_str = '--gcc-install-dir=%s' % self.gcc_prefix
+        for comp in compilers_to_add_config_file:
+            with open(os.path.join(bin_dir, '%s.cfg' % comp), 'w') as f:
+                f.write(prefix_str)
+
+    def _sanity_check_gcc_prefix(self):
+        """Check if the GCC prefix is correct."""
+        compilers_to_check = [
+            'clang',
+            'clang++',
+            'clang-%s' % LooseVersion(self.clangversion).version[0],
+            'clang-cpp',
+            'flang',
+        ]
+
+        # Set prefix if not done during installation.
+        self._set_gcc_prefix()
+        rgx = re.compile('Selected GCC installation: (.*)')
+        for comp in compilers_to_check:
+            cmd = "%s -v" % os.path.join(self.installdir, 'bin', comp)
+            out, _ = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False)
+            mch = rgx.search(out)
+            if mch is None:
+                self.log.debug(out)
+                raise EasyBuildError("Failed to extract GCC installation path from output of `%s`", cmd)
+            gcc_prefix = mch.group(1)
+            if gcc_prefix != self.gcc_prefix:
+                raise EasyBuildError(
+                    "GCC installation path `%s` does not match expected path `%s`", gcc_prefix, self.gcc_prefix
+                    )
+
+    def _set_gcc_prefix(self):
+        """Set the GCC prefix for the build."""
+        if not self.gcc_prefix:
+            arch = get_cpu_architecture()
+            gcc_root = get_software_root('GCCcore')
+            gcc_version = get_software_version('GCCcore')
+            # If that doesn't work, try with GCC
+            if gcc_root is None:
+                gcc_root = get_software_root('GCC')
+                gcc_version = get_software_version('GCC')
+            # If that doesn't work either, print error and exit
+            if gcc_root is None:
+                raise EasyBuildError("Can't find GCC or GCCcore to use")
+
+            pattern = os.path.join(gcc_root, 'lib', 'gcc', '%s-*' % arch, '%s' % gcc_version)
+            matches = glob.glob(pattern)
+            if not matches:
+                raise EasyBuildError("Can't find GCC version %s for architecture %s in %s", gcc_version, arch, pattern)
+            self.gcc_prefix = os.path.abspath(matches[0])
+            self.log.debug("Using %s as the gcc install location", self.gcc_prefix)
+
+    def install_step(self):
+        # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option,
+        # or via 'accept_eula = True' in easyconfig file
+        self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf')
+
+        # AOCC is based on Clang. Try to guess the clangversion from the AOCC version
+        # if clangversion is not specified in the easyconfig
+        if self.clangversion is None:
+            self.clangversion = self._aocc_guess_clang_version()
+
+        super(EB_AOCC, self).install_step()
+
+    def post_install_step(self):
+        """
+        For AOCC <5.0.0:
+        Create wrappers for the compilers to make sure compilers picks up GCCcore as GCC toolchain.
+        For AOCC >= 5.0.0:
+        Create [compiler-name].cfg files to point the compiler to the correct GCCcore as GCC toolchain.
+        For compilers not supporting this option, wrap the compilers using the old method.
+        """
+        compilers_to_wrap = []
+        compilers_to_add_config_files = []
+
+        if LooseVersion(self.version) < LooseVersion("5.0.0"):
+            compilers_to_wrap += [
+                'clang',
+                'clang++',
+                'clang-%s' % LooseVersion(self.clangversion).version[0],
+                'clang-cpp',
+                'flang',
+            ]
+        else:
+            compilers_to_add_config_files += [
+                'clang',
+                'clang++',
+                'clang-cpp'
+            ]
+            compilers_to_wrap += [
+                'flang'
+            ]
+
+        self._create_compiler_config_files(compilers_to_add_config_files)
+        self._create_compiler_wrappers(compilers_to_wrap)
         super(EB_AOCC, self).post_install_step()
 
     def sanity_check_step(self):
@@ -167,6 +272,19 @@ def sanity_check_step(self):
             "flang --help",
             "llvm-config --cxxflags",
         ]
+
+        self._sanity_check_gcc_prefix()
+
+        # Check if clang++ can actually compile programs. This can fail if the wrong driver is picked up by LLVM.
+        tmpdir = tempfile.mkdtemp()
+        write_file(os.path.join(tmpdir, 'minimal.cpp'), AOCC_MINIMAL_CPP_EXAMPLE)
+        minimal_cpp_compiler_cmd = "cd %s && clang++ minimal.cpp -o minimal_cpp" % tmpdir
+        custom_commands.append(minimal_cpp_compiler_cmd)
+        # Check if flang can actually compile programs. This can fail if the wrong driver is picked up by LLVM.
+        write_file(os.path.join(tmpdir, 'minimal.f90'), AOCC_MINIMAL_FORTRAN_EXAMPLE)
+        minimal_f90_compiler_cmd = "cd %s && flang minimal.f90 -o minimal_f90" % tmpdir
+        custom_commands.append(minimal_f90_compiler_cmd)
+
         super(EB_AOCC, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
 
     def make_module_extra(self):

easybuild/easyblocks/c/cudnn.py

@@ -68,3 +68,13 @@ def __init__(self, *args, **kwargs):
         self.cfg['keepsymlinks'] = True
         self.cfg.template_values['cudnnarch'] = cudnnarch
         self.cfg.generate_template_values()
+
+    def fetch_step(self, *args, **kwargs):
+        """Check for EULA acceptance prior to getting sources."""
+        # EULA for cuDNN must be accepted via --accept-eula-for EasyBuild configuration option,
+        # or via 'accept_eula = True' in easyconfig file
+        self.check_accepted_eula(
+            name='cuDNN',
+            more_info='https://docs.nvidia.com/deeplearning/cudnn/latest/reference/eula.html'
+        )
+        return super(EB_cuDNN, self).fetch_step(*args, **kwargs)

easybuild/easyblocks/generic/cargo.py

@@ -293,7 +293,12 @@ def generate_crate_list(sourcedir):
     import toml
 
     cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml'))
-    cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock'))
+
+    try:
+        cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock'))
+    except FileNotFoundError as err:
+        print("\nNo Cargo.lock file found. Generate one with 'cargo generate-lockfile'\n")
+        raise err
 
     try:
         app_name = cargo_toml['package']['name']

easybuild/easyblocks/generic/cmakemake.py

@@ -87,6 +87,52 @@ def setup_cmake_env(tc):
     setvar("CMAKE_LIBRARY_PATH", library_paths)
 
 
+def setup_cmake_env_python_hints(cmake_version=None):
+    """Set environment variables as hints for CMake to prefer the Python module, if loaded.
+    Useful when there is no way to specify arguments for CMake directly,
+    e.g. when CMake is called from within another build system.
+    Otherwise get_cmake_python_config_[str/dict] should be used instead.
+    """
+    if cmake_version is None:
+        cmake_version = det_cmake_version()
+    if LooseVersion(cmake_version) < '3.12':
+        raise EasyBuildError("Setting Python hints for CMake requires CMake version 3.12 or newer")
+    python_root = get_software_root('Python')
+    if python_root:
+        python_version = LooseVersion(get_software_version('Python'))
+        setvar('Python_ROOT_DIR', python_root)
+        if python_version >= "3":
+            setvar('Python3_ROOT_DIR', python_root)
+        else:
+            setvar('Python2_ROOT_DIR', python_root)
+
+
+def get_cmake_python_config_dict():
+    """Get a dictionary with CMake configuration options for finding Python if loaded as a module."""
+    options = {}
+    python_root = get_software_root('Python')
+    if python_root:
+        python_version = LooseVersion(get_software_version('Python'))
+        python_exe = os.path.join(python_root, 'bin', 'python')
+        # This is required for (deprecated) `find_package(PythonInterp ...)`
+        options['PYTHON_EXECUTABLE'] = python_exe
+        # Ensure that both `find_package(Python) and find_package(Python2/3)` work as intended
+        options['Python_EXECUTABLE'] = python_exe
+        if python_version >= "3":
+            options['Python3_EXECUTABLE'] = python_exe
+        else:
+            options['Python2_EXECUTABLE'] = python_exe
+    return options
+
+
+def get_cmake_python_config_str():
+    """Get CMake configuration arguments for finding Python if loaded as a module.
+    This string is intended to be passed to the invocation of `cmake`.
+    """
+    options = get_cmake_python_config_dict()
+    return ' '.join('-D%s=%s' % (key, value) for key, value in options.items())
+
+
 class CMakeMake(ConfigureMake):
     """Support for configuring build with CMake instead of traditional configure script"""
 
@@ -273,17 +319,15 @@ def configure_step(self, srcdir=None, builddir=None):
             # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5
             options['CMAKE_SKIP_RPATH'] = 'ON'
 
-        # make sure that newer CMAKE picks python based on location, not just the newest python
-        # Avoids issues like e.g. https://github.com/EESSI/software-layer/pull/370#issuecomment-1785594932
-        if LooseVersion(self.cmake_version) >= '3.15':
-            options['CMAKE_POLICY_DEFAULT_CMP0094'] = 'NEW'
-
         # show what CMake is doing by default
         options['CMAKE_VERBOSE_MAKEFILE'] = 'ON'
 
         # disable CMake user package repository
         options['CMAKE_FIND_USE_PACKAGE_REGISTRY'] = 'OFF'
 
+        # ensure CMake uses EB python, not system or virtualenv python
+        options.update(get_cmake_python_config_dict())
+
         if not self.cfg.get('allow_system_boost', False):
             boost_root = get_software_root('Boost')
             if boost_root:
@@ -302,6 +346,8 @@ def configure_step(self, srcdir=None, builddir=None):
                 options['BOOST_ROOT'] = boost_root
                 options['Boost_NO_SYSTEM_PATHS'] = 'ON'
 
+        self.cmake_options = options
+
         if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD:
             self.prepend_config_opts(options)
             command = ' '.join([