From 2077aa809a815783d77afd3c2b773ff6ffe6544a Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 27 Oct 2022 16:26:25 +0200 Subject: [PATCH 01/87] use gfortran for -g77 in NVHPC --- easybuild/easyblocks/n/nvhpc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index eabb8f0600..b0216c5700 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -164,7 +164,7 @@ def install_step(self): line = re.sub(r"^PATH=/", r"#PATH=/", line) sys.stdout.write(line) - cmd = "%s -x %s -g77 /" % (makelocalrc_filename, compilers_subdir) + cmd = "%s -x %s -g77 gfortran" % (makelocalrc_filename, compilers_subdir) run_cmd(cmd, log_all=True, simple=True) # If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so From be5c7b421807c1b1fb1a246162e6ee9d5e257588 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 6 Jan 2023 09:47:22 +0100 Subject: [PATCH 02/87] fix TensorFlow easyblock for new versions of Bazel & TensorFlow --- easybuild/easyblocks/t/tensorflow.py | 45 ++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index e8912c2b2c..16f8079c33 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2023 Ghent University +# Copyright 2017-2022 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -35,14 +35,13 @@ import re import stat import tempfile -from distutils.version import LooseVersion import easybuild.tools.environment as env import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools import run +from easybuild.tools import run, LooseVersion from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option, IGNORE from easybuild.tools.filetools import adjust_permissions, apply_regex_substitutions, copy_file, mkdir, resolve_path @@ -760,6 +759,11 @@ def build_step(self): self.bazel_opts = [ '--output_user_root=%s' % self.output_user_root_dir, ] + # Environment variables and values needed for Bazel actions. + action_env = {} + # A value of None is interpreted as using the invoking environments value + INHERIT = None # For better readability + jvm_max_memory = self.cfg['jvm_max_memory'] if jvm_max_memory: jvm_startup_memory = min(512, int(jvm_max_memory)) @@ -801,21 +805,17 @@ def build_step(self): # Make TF find our modules. LD_LIBRARY_PATH gets automatically added by configure.py cpaths, libpaths = self.system_libs_info[1:] if cpaths: - self.target_opts.append("--action_env=CPATH='%s'" % ':'.join(cpaths)) + action_env['CPATH'] = ':'.join(cpaths) if libpaths: - self.target_opts.append("--action_env=LIBRARY_PATH='%s'" % ':'.join(libpaths)) - self.target_opts.append('--action_env=PYTHONPATH') + action_env['LIBRARY_PATH'] = ':'.join(libpaths) + action_env['PYTHONPATH'] = INHERIT # Also export $EBPYTHONPREFIXES to handle the multi-deps python setup # See https://github.com/easybuilders/easybuild-easyblocks/pull/1664 if 'EBPYTHONPREFIXES' in os.environ: - self.target_opts.append('--action_env=EBPYTHONPREFIXES') + action_env['EBPYTHONPREFIXES'] = INHERIT # Ignore user environment for Python - self.target_opts.append('--action_env=PYTHONNOUSERSITE=1') - - # Use the same configuration (i.e. environment) for compiling and using host tools - # This means that our action_envs are always passed - self.target_opts.append('--distinct_host_configuration=false') + action_env['PYTHONNOUSERSITE'] = '1' # TF 2 (final) sets this in configure if LooseVersion(self.version) < LooseVersion('2.0'): @@ -851,6 +851,26 @@ def build_step(self): # and download it if needed self.target_opts.append('--config=mkl') + bazel_version = LooseVersion(get_software_version('Bazel')) + if bazel_version.version is None: + raise EasyBuildError('Bazel version not found. Is it a dependency?') + + # Use the same configuration (i.e. environment) for compiling and using host tools + # This means that our action_envs are (almost) always passed + # Fully removed in Bazel 6.0 and limitted effect after at least 3.7 (see --host_action_env) + if bazel_version < '6.0.0': + self.target_opts.append('--distinct_host_configuration=false') + + for key, value in action_env.items(): + option = key + if value is not None: + option += "='%s'" % value + + self.target_opts.append('--action_env=' + option) + if bazel_version >= '3.7.0': + # See https://github.com/bazelbuild/bazel/commit/a463d9095386b22c121d20957222dbb44caef7d4 + self.target_opts.append('--host_action_env=' + option) + # Compose final command cmd = ( [self.cfg['prebuildopts']] @@ -1017,6 +1037,7 @@ def test_step(self): self.log.warning('Test %s failed with output\n%s', test_name, read_file(log_path, log_error=False)) if failed_tests: + failed_tests = sorted(set(failed_tests)) # Make unique to not count retries fail_msg = 'At least %s %s tests failed:\n%s' % ( len(failed_tests), device, ', '.join(failed_tests)) self.report_test_failure(fail_msg) From fdc069abcd9e51c8cf6e404dbc3694815fee93cb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 8 Jan 2023 15:41:30 +0100 Subject: [PATCH 03/87] make NAMD easyblock aware of (pre)testopts --- easybuild/easyblocks/n/namd.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/namd.py b/easybuild/easyblocks/n/namd.py index 4c84f9bcfd..a0f4c82b95 100644 --- a/easybuild/easyblocks/n/namd.py +++ b/easybuild/easyblocks/n/namd.py @@ -201,10 +201,12 @@ def test_step(self): ppn = '' if self.toolchain.options.get('openmp', False): ppn = '+ppn 2' - cmd = "%(namd)s %(ppn)s %(testdir)s" % { + cmd = "%(pretestopts)s %(namd)s %(ppn)s %(testopts)s %(testdir)s" % { 'namd': namdcmd, 'ppn': ppn, + 'pretestopts': self.cfg['pretestopts'], 'testdir': os.path.join(self.cfg['start_dir'], self.namd_arch, 'src', 'alanin'), + 'testopts': self.cfg['testopts'], } out, ec = run_cmd(cmd, simple=False) if ec == 0: From 652df1dcc52e45cc92f02ef7a13ef2eab235cf2a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 10 Jan 2023 18:45:09 +0100 Subject: [PATCH 04/87] bump version to 4.7.1dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 94dac2bea4..9619ef8946 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.7.0') +VERSION = LooseVersion('4.7.1.dev0') UNKNOWN = 'UNKNOWN' From 720d392d97013ee3324698f1da10389e1d11641b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Jan 2023 16:14:36 +0100 Subject: [PATCH 05/87] Increase Bazel start/connect timeout --- easybuild/easyblocks/t/tensorflow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 16f8079c33..719025a0a2 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -1,5 +1,5 @@ ## -# Copyright 2017-2022 Ghent University +# Copyright 2017-2023 Ghent University # # This file is part of EasyBuild, # originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), @@ -445,7 +445,7 @@ def configure_step(self): # and will hang forever building the TensorFlow package. # So limit to something high but still reasonable while allowing ECs to overwrite it if self.cfg['maxparallel'] is None: - self.cfg['parallel'] = min(self.cfg['parallel'], 64) + self.cfg['parallel'] = min(self.cfg['parallel'], 96) binutils_root = get_software_root('binutils') if not binutils_root: @@ -758,6 +758,7 @@ def build_step(self): # Options passed to the bazel command self.bazel_opts = [ '--output_user_root=%s' % self.output_user_root_dir, + '--local_startup_timeout_secs=600', # 5min for bazel to start ] # Environment variables and values needed for Bazel actions. action_env = {} From 3dddcaa15466605bd8b7a72aa75d3a5f1b27d470 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 12 Jan 2023 16:25:57 +0100 Subject: [PATCH 06/87] Make the maxparallel limitation depend on the Bazel version --- easybuild/easyblocks/t/tensorflow.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 719025a0a2..fbaeab509c 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -210,6 +210,14 @@ def is_version_ok(version_range): return result +def get_bazel_version(): + """Get the Bazel version as a LooseVersion. Error if not found""" + version = get_software_version('Bazel') + if version is None: + raise EasyBuildError('Bazel version not found. Is it a dependency?') + return LooseVersion(version) + + class EB_TensorFlow(PythonPackage): """Support for building/installing TensorFlow.""" @@ -445,7 +453,9 @@ def configure_step(self): # and will hang forever building the TensorFlow package. # So limit to something high but still reasonable while allowing ECs to overwrite it if self.cfg['maxparallel'] is None: - self.cfg['parallel'] = min(self.cfg['parallel'], 96) + # Seemingly Bazel around 3.x got better, so double the max there + bazel_max = 64 if get_bazel_version() < '3.0.0' else 128 + self.cfg['parallel'] = min(self.cfg['parallel'], bazel_max) binutils_root = get_software_root('binutils') if not binutils_root: @@ -852,9 +862,7 @@ def build_step(self): # and download it if needed self.target_opts.append('--config=mkl') - bazel_version = LooseVersion(get_software_version('Bazel')) - if bazel_version.version is None: - raise EasyBuildError('Bazel version not found. Is it a dependency?') + bazel_version = get_bazel_version() # Use the same configuration (i.e. environment) for compiling and using host tools # This means that our action_envs are (almost) always passed From 8c88c719df95614896958bf5e93b9d7800a1ad8a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 13 Jan 2023 18:03:31 +0100 Subject: [PATCH 07/87] Fix cmdline for Bazel < 4.0 --- easybuild/easyblocks/t/tensorflow.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index fbaeab509c..47c8a48f71 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -758,6 +758,8 @@ def patch_crosstool_files(self): def build_step(self): """Custom build procedure for TensorFlow.""" + bazel_version = get_bazel_version() + # pre-create target installation directory mkdir(os.path.join(self.installdir, self.pylibdir), parents=True) @@ -768,8 +770,10 @@ def build_step(self): # Options passed to the bazel command self.bazel_opts = [ '--output_user_root=%s' % self.output_user_root_dir, - '--local_startup_timeout_secs=600', # 5min for bazel to start ] + if bazel_version >= '4.0.0': + self.bazel_opts.append('--local_startup_timeout_secs=600') # 5min for bazel to start + # Environment variables and values needed for Bazel actions. action_env = {} # A value of None is interpreted as using the invoking environments value @@ -862,8 +866,6 @@ def build_step(self): # and download it if needed self.target_opts.append('--config=mkl') - bazel_version = get_bazel_version() - # Use the same configuration (i.e. environment) for compiling and using host tools # This means that our action_envs are (almost) always passed # Fully removed in Bazel 6.0 and limitted effect after at least 3.7 (see --host_action_env) From 4520a34ba9fc42023dc94e8cb7d3006f9f8c92a9 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Tue, 17 Jan 2023 10:26:11 +0000 Subject: [PATCH 08/87] update MesonNinja easyblock for Meson >=0.64.0 --- easybuild/easyblocks/generic/mesonninja.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index fab1a79296..601dbe5210 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -28,12 +28,16 @@ @author: Kenneth Hoste (Ghent University) """ +from distutils.version import LooseVersion from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, create_unused_dir, which +from easybuild.tools.modules import get_software_version from easybuild.tools.run import run_cmd +DEFAULT_CONFIGURE_CMD = 'meson' + class MesonNinja(EasyBlock): """ @@ -45,6 +49,7 @@ def extra_options(extra_vars=None): """Define extra easyconfig parameters specific to MesonNinja.""" extra_vars = EasyBlock.extra_options(extra_vars) extra_vars.update({ + 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], }) return extra_vars @@ -73,8 +78,16 @@ def configure_step(self, cmd_prefix=''): if no_Dlibdir and no_libdir: self.cfg.update('configopts', '-Dlibdir=lib') - cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { + configure_cmd = self.cfg.get('configure_cmd') or DEFAULT_CONFIGURE_CMD + # Meson >= 0.64.0 has a deprecatation warning for running `meson [options]` + # instead of `meson setup [options]` + if (LooseVersion(get_software_version('Meson')) >= LooseVersion('0.64.0') + and configure_cmd == DEFAULT_CONFIGURE_CMD): + configure_cmd += ' setup' + + cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { 'configopts': self.cfg['configopts'], + 'configure_cmd': configure_cmd, 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], 'sourcedir': self.start_dir, From fec5e3b8c3c9d73c186dcc49e56a3adfdbfc6c47 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Tue, 17 Jan 2023 10:31:46 +0000 Subject: [PATCH 09/87] linting --- easybuild/easyblocks/generic/mesonninja.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index 601dbe5210..c8ad0af51b 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -81,9 +81,9 @@ def configure_step(self, cmd_prefix=''): configure_cmd = self.cfg.get('configure_cmd') or DEFAULT_CONFIGURE_CMD # Meson >= 0.64.0 has a deprecatation warning for running `meson [options]` # instead of `meson setup [options]` - if (LooseVersion(get_software_version('Meson')) >= LooseVersion('0.64.0') - and configure_cmd == DEFAULT_CONFIGURE_CMD): - configure_cmd += ' setup' + if (LooseVersion(get_software_version('Meson')) >= LooseVersion('0.64.0') and + configure_cmd == DEFAULT_CONFIGURE_CMD): + configure_cmd += ' setup' cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { 'configopts': self.cfg['configopts'], From 182704176f49880be9f0e686dd4636e9e652fb82 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Tue, 17 Jan 2023 15:48:16 +0000 Subject: [PATCH 10/87] update scipy easyblock for scipy >= 1.9.0 to use meson/ninja --- easybuild/easyblocks/s/scipy.py | 79 +++++++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index c4ebd92ed9..79acdb95fe 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -30,33 +30,96 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Jasper Grimm (University of York) """ +import os from distutils.version import LooseVersion +import easybuild.tools.environment as env +import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage +from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.easyblocks.generic.pythonpackage import det_pylibdir -import easybuild.tools.toolchain as toolchain +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.modules import get_software_root +from easybuild.tools.run import run_cmd -class EB_scipy(FortranPythonPackage): +class EB_scipy(FortranPythonPackage, MesonNinja): """Support for installing the scipy Python package as part of a Python installation.""" def __init__(self, *args, **kwargs): """Set scipy-specific test command.""" super(EB_scipy, self).__init__(*args, **kwargs) + self.use_meson = LooseVersion(self.version) >= LooseVersion('1.9') + self.pylibdir = None self.testinstall = True self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" - super(EB_scipy, self).configure_step() + self.pylibdir = det_pylibdir() + + # scipy >= 1.9.0 uses Meson/Ninja + if self.use_meson: + # configure BLAS/LAPACK library to use with Meson for scipy >= 1.9.0 + lapack_lib = self.toolchain.lapack_family() + if lapack_lib == toolchain.FLEXIBLAS: + blas_lapack = 'flexiblas' + elif lapack_lib == toolchain.INTELMKL: + blas_lapack = 'mkl' + elif lapack_lib == toolchain.OPENBLAS: + blas_lapack = 'openblas' + else: + raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib) + + configopts = '-Dblas=%(blas_lapack)s -Dlapack=%(blas_lapack)s' % {'blas_lapack': blas_lapack} + self.cfg.update('configopts', configopts) + + pythonpath = os.getenv('PYTHONPATH') + env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath])) + + path = os.getenv('PATH') + env.setvar('PATH', os.pathsep.join([os.path.join(self.installdir, 'bin'), path])) + + MesonNinja.configure_step(self) + else: + super(EB_scipy, self).configure_step() + + if LooseVersion(self.version) >= LooseVersion('0.13'): + # in recent scipy versions, additional compilation is done in the install step, + # which requires unsetting $LDFLAGS + if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]: # @UndefinedVariable + self.cfg.update('preinstallopts', "unset LDFLAGS && ") + + def build_step(self): + """Custom build step for scipy: use ninja for scipy >= 1.9.0""" + if self.use_meson: + MesonNinja.build_step(self) + else: + super(EB_scipy, self).build_step() + + def test_step(self): + """Custom test step for scipy: skip if scipy >= 1.9.0""" + if self.use_meson: + # need to run `ninja install` before we can run the test + pass + else: + super(EB_scipy, self).build_step() - if LooseVersion(self.version) >= LooseVersion('0.13'): - # in recent scipy versions, additional compilation is done in the install step, - # which requires unsetting $LDFLAGS - if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]: # @UndefinedVariable - self.cfg.update('preinstallopts', "unset LDFLAGS && ") + def install_step(self): + """Custom install step for scipy: use ninja for scipy >= 1.9.0""" + if self.use_meson: + self.cfg['installopts'] = "" + MesonNinja.install_step(self) + + # also run the test + python_root = get_software_root('Python') + python_bin = os.path.join(python_root, 'bin', 'python') + run_cmd(self.testcmd % {'python': python_bin}, log_all=True, simple=True) + else: + super(EB_scipy, self).install_step() def sanity_check_step(self, *args, **kwargs): """Custom sanity check for scipy.""" From 57d52d990dca8f653e662739f1c4adac9a365670 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Tue, 17 Jan 2023 15:54:38 +0000 Subject: [PATCH 11/87] linting --- easybuild/easyblocks/s/scipy.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 79acdb95fe..826ed358d1 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs): def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" self.pylibdir = det_pylibdir() - + # scipy >= 1.9.0 uses Meson/Ninja if self.use_meson: # configure BLAS/LAPACK library to use with Meson for scipy >= 1.9.0 @@ -72,14 +72,14 @@ def configure_step(self): elif lapack_lib == toolchain.OPENBLAS: blas_lapack = 'openblas' else: - raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib) - + raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib) + configopts = '-Dblas=%(blas_lapack)s -Dlapack=%(blas_lapack)s' % {'blas_lapack': blas_lapack} self.cfg.update('configopts', configopts) - + pythonpath = os.getenv('PYTHONPATH') env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath])) - + path = os.getenv('PATH') env.setvar('PATH', os.pathsep.join([os.path.join(self.installdir, 'bin'), path])) @@ -113,8 +113,8 @@ def install_step(self): if self.use_meson: self.cfg['installopts'] = "" MesonNinja.install_step(self) - - # also run the test + + # also run the test python_root = get_software_root('Python') python_bin = os.path.join(python_root, 'bin', 'python') run_cmd(self.testcmd % {'python': python_bin}, log_all=True, simple=True) From cf9f565d89ee60642c0515665349be1aff9b339c Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Tue, 17 Jan 2023 20:37:03 +0100 Subject: [PATCH 12/87] do not use -g77 option when installing NVHPC 22.9+ --- easybuild/easyblocks/n/nvhpc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index 761fb1e468..b9b27cb009 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -164,7 +164,10 @@ def install_step(self): line = re.sub(r"^PATH=/", r"#PATH=/", line) sys.stdout.write(line) - cmd = "%s -x %s -g77 gfortran" % (makelocalrc_filename, compilers_subdir) + if LooseVersion(self.version) >= LooseVersion('22.9'): + cmd = "%s -x %s" % (makelocalrc_filename, compilers_subdir) + else: + cmd = "%s -x %s -g77 /" % (makelocalrc_filename, compilers_subdir) run_cmd(cmd, log_all=True, simple=True) # If an OS libnuma is NOT found, makelocalrc creates symbolic links to libpgnuma.so From 7f9bf4fb8544c3693d17db2dd197e6d44889bd1f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 18 Jan 2023 09:52:43 +0100 Subject: [PATCH 13/87] only give read permissions in GitHub Actions workflows --- .github/workflows/linting.yml | 3 +++ .github/workflows/unit_tests.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index d7482ec3f7..bfa19cc1a8 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -1,6 +1,9 @@ name: Static Analysis on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index c6b163cac3..b5164b143b 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -1,6 +1,9 @@ name: easyblocks unit tests on: [push, pull_request] +permissions: + contents: read # to fetch code (actions/checkout) + concurrency: group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}} cancel-in-progress: true From 795e8b66356edf44fea52f0e7f1baa642dcb6f4b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 19 Jan 2023 09:46:44 +0100 Subject: [PATCH 14/87] Improve readability --- easybuild/easyblocks/t/tensorflow.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 47c8a48f71..229cc72c1d 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -771,8 +771,9 @@ def build_step(self): self.bazel_opts = [ '--output_user_root=%s' % self.output_user_root_dir, ] + # Increase time to wait for bazel to start, available since 4.0+ if bazel_version >= '4.0.0': - self.bazel_opts.append('--local_startup_timeout_secs=600') # 5min for bazel to start + self.bazel_opts.append('--local_startup_timeout_secs=300') # 5min # Environment variables and values needed for Bazel actions. action_env = {} @@ -868,17 +869,19 @@ def build_step(self): # Use the same configuration (i.e. environment) for compiling and using host tools # This means that our action_envs are (almost) always passed - # Fully removed in Bazel 6.0 and limitted effect after at least 3.7 (see --host_action_env) + # Fully removed in Bazel 6.0 and limited effect after at least 3.7 (see --host_action_env) if bazel_version < '6.0.0': self.target_opts.append('--distinct_host_configuration=false') - for key, value in action_env.items(): + for key, value in sorted(action_env.items()): option = key if value is not None: option += "='%s'" % value self.target_opts.append('--action_env=' + option) if bazel_version >= '3.7.0': + # Since Bazel 3.7 action_env only applies to the "target" environment, not the "host" environment + # As we are not cross-compiling we need both be the same -> Duplicate the setting to host_action_env # See https://github.com/bazelbuild/bazel/commit/a463d9095386b22c121d20957222dbb44caef7d4 self.target_opts.append('--host_action_env=' + option) From 5f93ffc9fd61243d2c0068e0f08bbe6059f135c8 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 19 Jan 2023 09:52:39 +0100 Subject: [PATCH 15/87] Improve bazel error msg --- easybuild/easyblocks/t/tensorflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 229cc72c1d..d63a4bc3f6 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -214,7 +214,7 @@ def get_bazel_version(): """Get the Bazel version as a LooseVersion. Error if not found""" version = get_software_version('Bazel') if version is None: - raise EasyBuildError('Bazel version not found. Is it a dependency?') + raise EasyBuildError('Failed to determine Bazel version - is it listed as a (build) dependency?') return LooseVersion(version) From 1fec03336db8b93285c9ba2acf823e7b9c3d5cfa Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Thu, 19 Jan 2023 15:21:36 +0000 Subject: [PATCH 16/87] improve scipy testing --- easybuild/easyblocks/s/scipy.py | 84 ++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 826ed358d1..ecfa42b8ff 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -33,6 +33,7 @@ @author: Jasper Grimm (University of York) """ import os +import tempfile from distutils.version import LooseVersion import easybuild.tools.environment as env @@ -41,6 +42,7 @@ from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.easyblocks.generic.pythonpackage import det_pylibdir from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import change_dir, mkdir, remove_dir from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd @@ -53,13 +55,16 @@ def __init__(self, *args, **kwargs): super(EB_scipy, self).__init__(*args, **kwargs) self.use_meson = LooseVersion(self.version) >= LooseVersion('1.9') - self.pylibdir = None self.testinstall = True - self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" + self.testcmd = " && ".join([ + "cd ..", + "touch %(srcdir)s/.coveragerc", + "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", + ]) def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" - self.pylibdir = det_pylibdir() + pylibdir = det_pylibdir() # scipy >= 1.9.0 uses Meson/Ninja if self.use_meson: @@ -77,21 +82,23 @@ def configure_step(self): configopts = '-Dblas=%(blas_lapack)s -Dlapack=%(blas_lapack)s' % {'blas_lapack': blas_lapack} self.cfg.update('configopts', configopts) + # need to have already installed extensions in PATH, PYTHONPATH for configure/build/install steps pythonpath = os.getenv('PYTHONPATH') - env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath])) + env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, pylibdir), pythonpath])) path = os.getenv('PATH') env.setvar('PATH', os.pathsep.join([os.path.join(self.installdir, 'bin'), path])) MesonNinja.configure_step(self) + else: super(EB_scipy, self).configure_step() - if LooseVersion(self.version) >= LooseVersion('0.13'): - # in recent scipy versions, additional compilation is done in the install step, - # which requires unsetting $LDFLAGS - if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]: # @UndefinedVariable - self.cfg.update('preinstallopts', "unset LDFLAGS && ") + if LooseVersion(self.version) >= LooseVersion('0.13'): + # in recent scipy versions, additional compilation is done in the install step, + # which requires unsetting $LDFLAGS + if self.toolchain.comp_family() in [toolchain.GCC, toolchain.CLANGGCC]: # @UndefinedVariable + self.cfg.update('preinstallopts', "unset LDFLAGS && ") def build_step(self): """Custom build step for scipy: use ninja for scipy >= 1.9.0""" @@ -101,23 +108,66 @@ def build_step(self): super(EB_scipy, self).build_step() def test_step(self): - """Custom test step for scipy: skip if scipy >= 1.9.0""" + """Run available scipy unit tests. Adapted from numpy easyblock""" + if self.use_meson: - # need to run `ninja install` before we can run the test - pass + + # temporarily install scipy so we can run the test suite + tmpdir = tempfile.mkdtemp() + cwd = os.getcwd() + + cmd = " && ".join([ + # reconfigure meson to use temp dir + 'meson configure --prefix=%s' % tmpdir, + 'ninja -j %s install' % self.cfg['parallel'], + ]) + run_cmd(cmd, log_all=True, simple=True, verbose=False) + + abs_pylibdirs = [os.path.join(tmpdir, pylibdir) for pylibdir in self.all_pylibdirs] + for pylibdir in abs_pylibdirs: + mkdir(pylibdir, parents=True) + python_root = get_software_root('Python') + python_bin = os.path.join(python_root, 'bin', 'python') + + self.cfg['pretestopts'] = " && ".join([ + # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets + # see http://projects.scipy.org/numpy/ticket/182 + "unset LDFLAGS", + "export PYTHONPATH=%s:$PYTHONPATH" % os.pathsep.join(abs_pylibdirs), + "", + ]) + self.cfg['runtest'] = self.testcmd % { + 'python': python_bin, + 'srcdir': self.cfg['start_dir'], + 'parallel': self.cfg['parallel'], + } + + MesonNinja.test_step(self) + + # reconfigure meson to use real install dir + os.chdir(cwd) + cmd = 'meson configure --prefix=%s' % self.installdir + run_cmd(cmd, log_all=True, simple=True, verbose=False) + + try: + remove_dir(tmpdir) + except OSError as err: + raise EasyBuildError("Failed to remove directory %s: %s", tmpdir, err) + else: - super(EB_scipy, self).build_step() + self.testcmd = self.testcmd % { + 'srcdir': self.cfg['start_dir'], + 'parallel': self.cfg['parallel'] + } + super(EB_scipy, self).test_step() def install_step(self): """Custom install step for scipy: use ninja for scipy >= 1.9.0""" if self.use_meson: + # ignore installopts inherited from FortranPythonPackage self.cfg['installopts'] = "" MesonNinja.install_step(self) - # also run the test - python_root = get_software_root('Python') - python_bin = os.path.join(python_root, 'bin', 'python') - run_cmd(self.testcmd % {'python': python_bin}, log_all=True, simple=True) else: super(EB_scipy, self).install_step() From 4e8e47ffb95107c711c6bde33405c4160f5dc510 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Thu, 19 Jan 2023 15:22:44 +0000 Subject: [PATCH 17/87] use change_dir --- easybuild/easyblocks/s/scipy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index ecfa42b8ff..06efba71d3 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -145,7 +145,7 @@ def test_step(self): MesonNinja.test_step(self) # reconfigure meson to use real install dir - os.chdir(cwd) + change_dir(cwd) cmd = 'meson configure --prefix=%s' % self.installdir run_cmd(cmd, log_all=True, simple=True, verbose=False) From b71ca1bff3ae520b3dacda217f3883e114f0451e Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Thu, 19 Jan 2023 18:11:06 +0000 Subject: [PATCH 18/87] fix tests for non-meson builds --- easybuild/easyblocks/s/scipy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 06efba71d3..de8a30f847 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -156,8 +156,9 @@ def test_step(self): else: self.testcmd = self.testcmd % { + 'python': '%(python)s', 'srcdir': self.cfg['start_dir'], - 'parallel': self.cfg['parallel'] + 'parallel': self.cfg['parallel'], } super(EB_scipy, self).test_step() From 34ba9b8be384c1249c2c0ef60f7e6f328f4776cd Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 20 Jan 2023 11:02:58 +0100 Subject: [PATCH 19/87] check that sanity_check_module_loaded attribute exists before querying it in PythonPackage easyblock (fixes #2864) --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 449f699792..d89e495122 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -793,7 +793,7 @@ def sanity_check_step(self, *args, **kwargs): # load module early ourselves rather than letting parent sanity_check_step method do so, # since custom actions taken below require that environment is set up properly already # (especially when using --sanity-check-only) - if not self.sanity_check_module_loaded: + if hasattr(self, 'sanity_check_module_loaded') and not self.sanity_check_module_loaded: extension = self.is_extension or kwargs.get('extension', False) extra_modules = kwargs.get('extra_modules', None) self.fake_mod_data = self.sanity_check_load_module(extension=extension, extra_modules=extra_modules) From b18e343b17597ce673c58c0dac1e3374ead18577 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Fri, 20 Jan 2023 11:10:36 +0000 Subject: [PATCH 20/87] add extra variable to enable/disable running of slow scipy tests --- easybuild/easyblocks/s/scipy.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index de8a30f847..895a19cab3 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -41,6 +41,7 @@ from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.easyblocks.generic.pythonpackage import det_pylibdir +from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import change_dir, mkdir, remove_dir from easybuild.tools.modules import get_software_root @@ -50,6 +51,15 @@ class EB_scipy(FortranPythonPackage, MesonNinja): """Support for installing the scipy Python package as part of a Python installation.""" + @staticmethod + def extra_options(): + """Easyconfig parameters specific to scipy.""" + extra_vars = ({ + 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], + }) + + return FortranPythonPackage.extra_options(extra_vars=extra_vars) + def __init__(self, *args, **kwargs): """Set scipy-specific test command.""" super(EB_scipy, self).__init__(*args, **kwargs) @@ -61,6 +71,8 @@ def __init__(self, *args, **kwargs): "touch %(srcdir)s/.coveragerc", "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", ]) + if self.cfg['enable_slow_tests'] is True: + self.testcmd += " -m full " def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" From 0060fa23574695a804b42e5af64c048d7c0f0139 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Fri, 20 Jan 2023 11:44:25 +0000 Subject: [PATCH 21/87] add option to ignore scipy test results --- easybuild/easyblocks/s/scipy.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 895a19cab3..d42c6d92e2 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -56,6 +56,7 @@ def extra_options(): """Easyconfig parameters specific to scipy.""" extra_vars = ({ 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], + 'ignore_test_result': [False, "Run scipy test suite, but ignore result (only log)", CUSTOM], }) return FortranPythonPackage.extra_options(extra_vars=extra_vars) @@ -66,13 +67,17 @@ def __init__(self, *args, **kwargs): self.use_meson = LooseVersion(self.version) >= LooseVersion('1.9') self.testinstall = True - self.testcmd = " && ".join([ - "cd ..", - "touch %(srcdir)s/.coveragerc", - "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", - ]) - if self.cfg['enable_slow_tests'] is True: - self.testcmd += " -m full " + if self.cfg['ignore_test_result']: + # running tests this way exits with 0 regardless + self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" + else: + self.testcmd = " && ".join([ + "cd ..", + "touch %(srcdir)s/.coveragerc", + "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", + ]) + if self.cfg['enable_slow_tests'] is True: + self.testcmd += " -m full " def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" From 40deb9f83814fe5e68e67847592ea0c82fe69475 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Fri, 20 Jan 2023 11:56:39 +0000 Subject: [PATCH 22/87] ignore test results by default to maintain current behaviour for older easyconfigs --- easybuild/easyblocks/s/scipy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index d42c6d92e2..13e87d0937 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -56,7 +56,8 @@ def extra_options(): """Easyconfig parameters specific to scipy.""" extra_vars = ({ 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], - 'ignore_test_result': [False, "Run scipy test suite, but ignore result (only log)", CUSTOM], + # ignore tests by default to maintain behaviour in older easyconfigs + 'ignore_test_result': [True, "Run scipy test suite, but ignore result (only log)", CUSTOM], }) return FortranPythonPackage.extra_options(extra_vars=extra_vars) From f2808b01201ec407f63790b6516cc1bfcd739661 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Mon, 23 Jan 2023 15:50:46 +0000 Subject: [PATCH 23/87] make changes suggested in review --- easybuild/easyblocks/s/scipy.py | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 13e87d0937..22a7e1cc40 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -40,15 +40,15 @@ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.fortranpythonpackage import FortranPythonPackage from easybuild.easyblocks.generic.mesonninja import MesonNinja -from easybuild.easyblocks.generic.pythonpackage import det_pylibdir +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, mkdir, remove_dir +from easybuild.tools.filetools import change_dir, copy_dir, mkdir from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd -class EB_scipy(FortranPythonPackage, MesonNinja): +class EB_scipy(FortranPythonPackage, PythonPackage, MesonNinja): """Support for installing the scipy Python package as part of a Python installation.""" @staticmethod @@ -57,19 +57,31 @@ def extra_options(): extra_vars = ({ 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], # ignore tests by default to maintain behaviour in older easyconfigs - 'ignore_test_result': [True, "Run scipy test suite, but ignore result (only log)", CUSTOM], + 'ignore_test_result': [None, "Run scipy test suite, but ignore result (only log)", CUSTOM], }) - return FortranPythonPackage.extra_options(extra_vars=extra_vars) + return PythonPackage.extra_options(extra_vars=extra_vars) def __init__(self, *args, **kwargs): """Set scipy-specific test command.""" - super(EB_scipy, self).__init__(*args, **kwargs) - - self.use_meson = LooseVersion(self.version) >= LooseVersion('1.9') + # calling PythonPackage __init__ also lets MesonNinja work in an extension + PythonPackage.__init__(self, *args, **kwargs) self.testinstall = True + + if LooseVersion(self.version) >= LooseVersion('1.9'): + self.use_meson = True + # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 + if self.cfg['ignore_test_result'] is None: + self.cfg['ignore_test_result'] = True + # ignore installopts inherited from FortranPythonPackage + self.cfg['installopts'] = "" + else: + self.use_meson = False + if self.cfg['ignore_test_result']: - # running tests this way exits with 0 regardless + # maintains compatibility with easyconfigs predating scipy 1.9. Runs tests (serially) in + # a way that exits with code 0 regardless of test results, see: + # https://github.com/easybuilders/easybuild-easyblocks/issues/2237 self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" else: self.testcmd = " && ".join([ @@ -77,12 +89,11 @@ def __init__(self, *args, **kwargs): "touch %(srcdir)s/.coveragerc", "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", ]) - if self.cfg['enable_slow_tests'] is True: + if self.cfg['enable_slow_tests']: self.testcmd += " -m full " def configure_step(self): """Custom configure step for scipy: set extra installation options when needed.""" - pylibdir = det_pylibdir() # scipy >= 1.9.0 uses Meson/Ninja if self.use_meson: @@ -102,6 +113,7 @@ def configure_step(self): # need to have already installed extensions in PATH, PYTHONPATH for configure/build/install steps pythonpath = os.getenv('PYTHONPATH') + pylibdir = det_pylibdir() env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, pylibdir), pythonpath])) path = os.getenv('PATH') @@ -110,7 +122,7 @@ def configure_step(self): MesonNinja.configure_step(self) else: - super(EB_scipy, self).configure_step() + FortranPythonPackage.configure_step(self) if LooseVersion(self.version) >= LooseVersion('0.13'): # in recent scipy versions, additional compilation is done in the install step, @@ -123,72 +135,60 @@ def build_step(self): if self.use_meson: MesonNinja.build_step(self) else: - super(EB_scipy, self).build_step() + FortranPythonPackage.build_step(self) def test_step(self): """Run available scipy unit tests. Adapted from numpy easyblock""" if self.use_meson: - # temporarily install scipy so we can run the test suite tmpdir = tempfile.mkdtemp() cwd = os.getcwd() - cmd = " && ".join([ - # reconfigure meson to use temp dir - 'meson configure --prefix=%s' % tmpdir, - 'ninja -j %s install' % self.cfg['parallel'], - ]) - run_cmd(cmd, log_all=True, simple=True, verbose=False) + tmp_builddir = os.path.join(tmpdir, 'build') + tmp_installdir = os.path.join(tmpdir, 'install') - abs_pylibdirs = [os.path.join(tmpdir, pylibdir) for pylibdir in self.all_pylibdirs] - for pylibdir in abs_pylibdirs: - mkdir(pylibdir, parents=True) - python_root = get_software_root('Python') - python_bin = os.path.join(python_root, 'bin', 'python') + # create a copy of the builddir + copy_dir(cwd, tmp_builddir) + change_dir(tmp_builddir) + + # reconfigure (to update prefix), and install to tmpdir + MesonNinja.configure_step(self, cmd_prefix=tmp_installdir) + MesonNinja.install_step(self) + + tmp_pylibdir = [os.path.join(tmp_installdir, det_pylibdir())] + self.prepare_python() self.cfg['pretestopts'] = " && ".join([ # LDFLAGS should not be set when testing numpy/scipy, because it overwrites whatever numpy/scipy sets # see http://projects.scipy.org/numpy/ticket/182 "unset LDFLAGS", - "export PYTHONPATH=%s:$PYTHONPATH" % os.pathsep.join(abs_pylibdirs), + "export PYTHONPATH=%s:$PYTHONPATH" % tmp_pylibdir, "", ]) self.cfg['runtest'] = self.testcmd % { - 'python': python_bin, + 'python': self.python_cmd, 'srcdir': self.cfg['start_dir'], 'parallel': self.cfg['parallel'], } MesonNinja.test_step(self) - # reconfigure meson to use real install dir - change_dir(cwd) - cmd = 'meson configure --prefix=%s' % self.installdir - run_cmd(cmd, log_all=True, simple=True, verbose=False) - - try: - remove_dir(tmpdir) - except OSError as err: - raise EasyBuildError("Failed to remove directory %s: %s", tmpdir, err) - else: self.testcmd = self.testcmd % { 'python': '%(python)s', 'srcdir': self.cfg['start_dir'], 'parallel': self.cfg['parallel'], } - super(EB_scipy, self).test_step() + FortranPythonPackage.test_step(self) def install_step(self): """Custom install step for scipy: use ninja for scipy >= 1.9.0""" if self.use_meson: - # ignore installopts inherited from FortranPythonPackage - self.cfg['installopts'] = "" MesonNinja.install_step(self) else: - super(EB_scipy, self).install_step() + FortranPythonPackage.install_step(self) def sanity_check_step(self, *args, **kwargs): """Custom sanity check for scipy.""" @@ -200,4 +200,4 @@ def sanity_check_step(self, *args, **kwargs): 'dirs': [det_pylibdir()], } - return super(EB_scipy, self).sanity_check_step(custom_paths=custom_paths) + return PythonPackage.sanity_check_step(self, custom_paths=custom_paths) From 8d44a255b7f44b5019fbc62547d44867b8322f99 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Mon, 23 Jan 2023 15:53:07 +0000 Subject: [PATCH 24/87] appease the hound --- easybuild/easyblocks/s/scipy.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 22a7e1cc40..1df30766b9 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -43,9 +43,7 @@ from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, copy_dir, mkdir -from easybuild.tools.modules import get_software_root -from easybuild.tools.run import run_cmd +from easybuild.tools.filetools import change_dir, copy_dir class EB_scipy(FortranPythonPackage, PythonPackage, MesonNinja): @@ -77,7 +75,7 @@ def __init__(self, *args, **kwargs): self.cfg['installopts'] = "" else: self.use_meson = False - + if self.cfg['ignore_test_result']: # maintains compatibility with easyconfigs predating scipy 1.9. Runs tests (serially) in # a way that exits with code 0 regardless of test results, see: From b1fc372617ed6b20dbce563a67f0220fbd4e3990 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Jan 2023 15:27:51 +0100 Subject: [PATCH 25/87] use start dir of extension to install R packages --- easybuild/easyblocks/generic/rpackage.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 572dab6935..190e416db0 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -134,15 +134,10 @@ def make_cmdline_cmd(self, prefix=None): else: prefix = '' - if self.start_dir: - loc = os.path.join(self.ext_dir or os.path.sep, self.start_dir) - else: - loc = self.ext_dir or self.ext_src - cmd = ' '.join([ self.cfg['preinstallopts'], "R CMD INSTALL", - loc, + self.start_dir, confargs, confvars, prefix, From e357a408feb3b78f4e9a506f826dab2a20629545 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 26 Jan 2023 15:03:05 +0000 Subject: [PATCH 26/87] modified the logic to find the PyQt5 sipdir in more places --- easybuild/easyblocks/q/qscintilla.py | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index a712222c48..c4dd6bc3ef 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -118,16 +118,23 @@ def install_step(self): pyshortver = '.'.join(get_software_version('Python').split('.')[:2]) sip_incdir = find_glob_pattern(os.path.join(self.pyqt_root, 'include', 'python%s*' % pyshortver), False) - # in case PyQt5's sip was installed in directories that are specific to each version of python - # as could happen with multi_deps - pyqt_sipdir = find_glob_pattern(os.path.join(self.pyqt_root, 'share', 'python%s*' % pyshortver, - 'site-packages', 'sip', self.pyqt_pkg_name), False) - # fall back to a single sipdir + # depending on PyQt5 versions and how it was installed, the sip directory could be in various places + # test them and figure out the first one that matches + pyqt_sip_subidr = [os.path.join('share', 'python%s*' % pyshortver, 'site-packages', 'sip', + self.pyqt_pkg_name), + os.path.join('share', 'sip', self.pyqt_pkg_name), + os.path.join('share', 'sip'), + os.path.join('lib', 'python%s*' % pyshortver, 'site-packages', self.pyqt_pkg_name, + 'bindings') + ] + pyqt_sipdir_options = [os.path.join(self.pyqt_root, subdir) for subdir in pyqt_sip_subidr] + for pyqt_sipdir_option in pyqt_sipdir_options: + pyqt_sipdir = find_glob_pattern(pyqt_sipdir_option, False) + if pyqt_sipdir: + break + if not pyqt_sipdir: - if LooseVersion(get_software_version(self.pyqt_pkg_name)) >= LooseVersion('5.15'): - pyqt_sipdir = os.path.join(self.pyqt_root, 'share', 'sip') - else: - pyqt_sipdir = os.path.join(self.pyqt_root, 'share', 'sip', self.pyqt_pkg_name) + raise EasyBuildError("Failed to find PyQt5 sip directory") cfgopts = [ '--destdir %s' % os.path.join(self.installdir, pylibdir), From a90d61708a214cd7c72e2767c729c787619a2927 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 26 Jan 2023 15:07:11 +0000 Subject: [PATCH 27/87] appeasing hound --- easybuild/easyblocks/q/qscintilla.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index c4dd6bc3ef..ac7a2bbeeb 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -126,7 +126,7 @@ def install_step(self): os.path.join('share', 'sip'), os.path.join('lib', 'python%s*' % pyshortver, 'site-packages', self.pyqt_pkg_name, 'bindings') - ] + ] pyqt_sipdir_options = [os.path.join(self.pyqt_root, subdir) for subdir in pyqt_sip_subidr] for pyqt_sipdir_option in pyqt_sipdir_options: pyqt_sipdir = find_glob_pattern(pyqt_sipdir_option, False) From 3e491c0b28a8692d422d45685885ee4c74b0b91f Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 26 Jan 2023 15:14:26 +0000 Subject: [PATCH 28/87] fixed typos --- easybuild/easyblocks/q/qscintilla.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/q/qscintilla.py b/easybuild/easyblocks/q/qscintilla.py index ac7a2bbeeb..3a64d6c607 100644 --- a/easybuild/easyblocks/q/qscintilla.py +++ b/easybuild/easyblocks/q/qscintilla.py @@ -120,14 +120,14 @@ def install_step(self): sip_incdir = find_glob_pattern(os.path.join(self.pyqt_root, 'include', 'python%s*' % pyshortver), False) # depending on PyQt5 versions and how it was installed, the sip directory could be in various places # test them and figure out the first one that matches - pyqt_sip_subidr = [os.path.join('share', 'python%s*' % pyshortver, 'site-packages', 'sip', + pyqt_sip_subdir = [os.path.join('share', 'python%s*' % pyshortver, 'site-packages', 'sip', self.pyqt_pkg_name), os.path.join('share', 'sip', self.pyqt_pkg_name), os.path.join('share', 'sip'), os.path.join('lib', 'python%s*' % pyshortver, 'site-packages', self.pyqt_pkg_name, 'bindings') ] - pyqt_sipdir_options = [os.path.join(self.pyqt_root, subdir) for subdir in pyqt_sip_subidr] + pyqt_sipdir_options = [os.path.join(self.pyqt_root, subdir) for subdir in pyqt_sip_subdir] for pyqt_sipdir_option in pyqt_sipdir_options: pyqt_sipdir = find_glob_pattern(pyqt_sipdir_option, False) if pyqt_sipdir: From 8f5534a80e5b07f16e44af8e47d29de1354af845 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Thu, 26 Jan 2023 16:31:06 +0100 Subject: [PATCH 29/87] fix JULIA_DEPOT_PATH in installation of multiple JuliaPackage extensions --- easybuild/easyblocks/generic/juliapackage.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 239ed5f5d3..2490bc2823 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -96,11 +96,12 @@ def install_step(self): """Install Julia package with Pkg""" # prepend installation directory to Julia DEPOT_PATH + # extensions in a bundle can share their DEPOT_PATH # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH - depot_path = os.getenv('JULIA_DEPOT_PATH') - if depot_path is not None: - depot_path = ':'.join([self.installdir, depot_path]) - env.setvar('JULIA_DEPOT_PATH', depot_path) + depot_path = os.getenv('JULIA_DEPOT_PATH', '') + if self.installdir not in depot_path: + depot_path = ':'.join([depot for depot in (self.installdir, depot_path) if depot]) + env.setvar('JULIA_DEPOT_PATH', depot_path) # command sequence for Julia.Pkg julia_pkg_cmd = ['using Pkg'] From e30d17fde03fd13a40157dfb0a9808452a5ae89a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 29 Jan 2023 12:12:20 +0100 Subject: [PATCH 30/87] fix website/docs links in README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index b651c614a6..fd11b4fe04 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,7 @@ .. image:: https://github.com/easybuilders/easybuild-easyblocks/workflows/easyblocks%20unit%20tests/badge.svg?branch=develop -`EasyBuild `_ is a software build +`EasyBuild `_ is a software build and installation framework that allows you to manage (scientific) software on High Performance Computing (HPC) systems in an efficient way. @@ -13,7 +13,7 @@ EasyBuild. Easyblocks are Python modules that implement the install procedure fo (group of) software package(s). Together with the EasyBuild framework, they allow to easily build and install supported software packages. -The EasyBuild documentation is available at http://easybuild.readthedocs.org/. +The EasyBuild documentation is available at http://docs.easybuild.io/. The easybuild-easyblocks source code is hosted on GitHub, along with an issue tracker for bug reports and feature requests, see From 2d3de771df47fa0cb63c05cb29c875c7b6abe092 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 30 Jan 2023 14:25:34 +0100 Subject: [PATCH 31/87] Add `testinstall` EC option to PythonPackage --- easybuild/easyblocks/generic/pythonpackage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index d89e495122..00ea6183a0 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -250,6 +250,7 @@ def extra_options(extra_vars=None): 'sanity_pip_check': [False, "Run 'python -m pip check' to ensure all required Python packages are " "installed and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default + 'testinstall': [False, "Install into temporary directory prior to running the tests.", CUSTOM], 'unpack_sources': [None, "Unpack sources prior to build/install. Defaults to 'True' except for whl files", CUSTOM], # A version of 0.0.0 is usually an error on installation unless the package does really not provide a @@ -283,7 +284,7 @@ def __init__(self, *args, **kwargs): self.sitecfgfn = 'site.cfg' self.sitecfglibdir = None self.sitecfgincdir = None - self.testinstall = False + self.testinstall = self.cfg['testinstall'] self.testcmd = None self.unpack_options = self.cfg['unpack_options'] From 6f10887b1634025fc91ace2c67b5666f5e813119 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Jan 2023 14:57:29 +0100 Subject: [PATCH 32/87] fix checking of CUDA/ROCR-Runtime dependencies for Clang to determine default build targets (fixes #2871) --- easybuild/easyblocks/c/clang.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 4f1344187a..e9f195b275 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -177,18 +177,19 @@ def __init__(self, *args, **kwargs): build_targets = self.cfg['build_targets'] # define build_targets if not set if build_targets is None: + deps = [dep['name'].lower() for dep in self.cfg.dependencies()] arch = get_cpu_architecture() try: default_targets = DEFAULT_TARGETS_MAP[arch][:] # If CUDA is included as a dep, add NVPTX as a target - if get_software_root("CUDA"): - default_targets += ["NVPTX"] + if 'cuda' in deps: + default_targets += ['NVPTX'] # For AMDGPU support we need ROCR-Runtime and # ROCT-Thunk-Interface, however, since ROCT is a dependency of # ROCR we only check for the ROCR-Runtime here # https://openmp.llvm.org/SupportAndFAQ.html#q-how-to-build-an-openmp-amdgpu-offload-capable-compiler - if get_software_root("ROCR-Runtime"): - default_targets += ["AMDGPU"] + if 'rocr-runtime' in deps: + default_targets += ['AMDGPU'] self.cfg['build_targets'] = build_targets = default_targets self.log.debug("Using %s as default build targets for CPU/GPU architecture %s.", default_targets, arch) except KeyError: From 4f30aab3822002794251fdc94e9b69ff8afed9ed Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Jan 2023 16:08:22 +0100 Subject: [PATCH 33/87] take into account that CUDA may be an indirect dependency via the toolchain Co-authored-by: ocaisa --- easybuild/easyblocks/c/clang.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index e9f195b275..172811566d 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -182,7 +182,9 @@ def __init__(self, *args, **kwargs): try: default_targets = DEFAULT_TARGETS_MAP[arch][:] # If CUDA is included as a dep, add NVPTX as a target - if 'cuda' in deps: + # There are (old) toolchains with CUDA as part of the toolchain + cuda_toolchain = hasattr(self.toolchain, 'COMPILER_CUDA_FAMILY') + if 'cuda' in deps or cuda_toolchain: default_targets += ['NVPTX'] # For AMDGPU support we need ROCR-Runtime and # ROCT-Thunk-Interface, however, since ROCT is a dependency of From e98c8ea693907979ef23c5d0bcb83addd744e990 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 2 Feb 2023 13:42:31 +0100 Subject: [PATCH 34/87] Show template values of exts_default_options in PythonBundle --- easybuild/easyblocks/generic/pythonbundle.py | 22 ++++++++------------ 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonbundle.py b/easybuild/easyblocks/generic/pythonbundle.py index 8e38e0cfa5..292cc275aa 100644 --- a/easybuild/easyblocks/generic/pythonbundle.py +++ b/easybuild/easyblocks/generic/pythonbundle.py @@ -62,21 +62,17 @@ def __init__(self, *args, **kwargs): self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES # need to disable templating to ensure that actual value for exts_default_options is updated... - prev_enable_templating = self.cfg.enable_templating - self.cfg.enable_templating = False + with self.cfg.disable_templating(): + # set default options for extensions according to relevant top-level easyconfig parameters + pypkg_keys = PythonPackage.extra_options().keys() + for key in pypkg_keys: + if key not in self.cfg['exts_default_options']: + self.cfg['exts_default_options'][key] = self.cfg[key] - # set default options for extensions according to relevant top-level easyconfig parameters - pypkg_keys = PythonPackage.extra_options().keys() - for key in pypkg_keys: - if key not in self.cfg['exts_default_options']: - self.cfg['exts_default_options'][key] = self.cfg[key] + self.cfg['exts_default_options']['download_dep_fail'] = True + self.log.info("Detection of downloaded extension dependencies is enabled") - self.cfg['exts_default_options']['download_dep_fail'] = True - self.log.info("Detection of downloaded extension dependencies is enabled") - - self.cfg.enable_templating = prev_enable_templating - - self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) + self.log.info("exts_default_options: %s", self.cfg['exts_default_options']) self.pylibdir = None self.all_pylibdirs = [] From 46c81b79c51c8ba32cdcf3ab54b95049f1a70d3a Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 3 Feb 2023 15:03:23 +0100 Subject: [PATCH 35/87] Fix missing initialization of CMakeMake in CMakePythonPackage --- easybuild/easyblocks/generic/cmakepythonpackage.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakepythonpackage.py b/easybuild/easyblocks/generic/cmakepythonpackage.py index f70fd83340..fac9aeced4 100644 --- a/easybuild/easyblocks/generic/cmakepythonpackage.py +++ b/easybuild/easyblocks/generic/cmakepythonpackage.py @@ -53,11 +53,6 @@ def extra_options(extra_vars=None): extra_vars['runtest'][0] = None return extra_vars - def __init__(self, *args, **kwargs): - """Initialize with PythonPackage.""" - PythonPackage.__init__(self, *args, **kwargs) - self._lib_ext = None # From CMakeMake.__init__ - def configure_step(self, *args, **kwargs): """Main configuration using cmake""" From ebd73924ec5f68b04dbbe8ea7fb9dc69c2877fa7 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 6 Feb 2023 12:35:34 +0100 Subject: [PATCH 36/87] Fix error when failing PIP version check during PythonPackage sanity check. The arguments must be passed individually not as a tuple, else you get: > File "/easybuild-easyblocks/easybuild/easyblocks/generic/pythonpackage.py", line 921, in sanity_check_step > pip_version)) > File "/easybuild-framework/easybuild/tools/build_log.py", line 81, in __init__ > msg = msg % args > TypeError: not enough arguments for format string --- easybuild/easyblocks/generic/pythonpackage.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 00ea6183a0..3e5e5e3a02 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -917,8 +917,9 @@ def sanity_check_step(self, *args, **kwargs): if pip_check_errors: raise EasyBuildError('\n'.join(pip_check_errors)) else: - raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", (pip_check_command, - pip_version)) + raise EasyBuildError("pip >= 9.0.0 is required for running '%s', found %s", + pip_check_command, + pip_version) else: raise EasyBuildError("Failed to determine pip version!") From 844a3a3e95767b03324a889d9f001fe6c2ff4d9c Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Tue, 7 Feb 2023 10:38:23 +0100 Subject: [PATCH 37/87] add debug flag to gopackage easyblock --- easybuild/easyblocks/generic/gopackage.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index d6f00f51a0..b94845b8cd 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -34,6 +34,7 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd @@ -126,6 +127,9 @@ def build_step(self): def install_step(self): """Install Go package to a custom path""" + if build_option('debug'): + self.cfg['installopts'] = '-x ' + self.cfg['installopts'] + # actually install Go package cmd = ' '.join([ self.cfg['preinstallopts'], From 0194c09906ac8fa7565bb4acf0381bd2eb0ec638 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 8 Feb 2023 20:55:50 +0000 Subject: [PATCH 38/87] handle templating correctly when playing with configopts --- easybuild/easyblocks/generic/cmakemake.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index f747702fc9..d9ee0d53af 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -141,12 +141,18 @@ def build_type(self): def prepend_config_opts(self, config_opts): """Prepends configure options (-Dkey=value) to configopts ignoring those already set""" + # need to disable template resolution or it will remain the same for all runs + self.cfg.enable_templating = False + cfg_configopts = self.cfg['configopts'] # All options are of the form '-D=' new_opts = ' '.join('-D%s=%s' % (key, value) for key, value in config_opts.items() if '-D%s=' % key not in cfg_configopts) self.cfg['configopts'] = ' '.join([new_opts, cfg_configopts]) + #re-enable template resolution + self.cfg.enable_templating = True + def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" From 950582fb37a684cf5e20908d8d14d9b491ad2a6e Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Wed, 8 Feb 2023 20:58:28 +0000 Subject: [PATCH 39/87] appeasing hound --- easybuild/easyblocks/generic/cmakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index d9ee0d53af..8bf841f313 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -150,7 +150,7 @@ def prepend_config_opts(self, config_opts): if '-D%s=' % key not in cfg_configopts) self.cfg['configopts'] = ' '.join([new_opts, cfg_configopts]) - #re-enable template resolution + # re-enable template resolution self.cfg.enable_templating = True def configure_step(self, srcdir=None, builddir=None): From 84c56102051048f339568fd736afb69afa136582 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 9 Feb 2023 08:41:22 +0100 Subject: [PATCH 40/87] void pytorch test_step crashing if runtest is not a command --- easybuild/easyblocks/p/pytorch.py | 264 +++++++++++++++--------------- 1 file changed, 135 insertions(+), 129 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 9a7637fed1..53dca5176b 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -248,137 +248,143 @@ def _set_cache_dir(self): def test_step(self): """Run unit tests""" - self._set_cache_dir() - # Pretend to be on FB CI which disables some tests, especially those which download stuff - env.setvar('SANDCASTLE', '1') - # Skip this test(s) which is very flaky - env.setvar('SKIP_TEST_BOTTLENECK', '1') - # Parse excluded_tests and flatten into space separated string - excluded_tests = [] - for arch, tests in self.cfg['excluded_tests'].items(): - if not arch or arch == get_cpu_architecture(): - excluded_tests.extend(tests) - # -x should not be used if there are no excluded tests - if excluded_tests: - excluded_tests = ['-x'] + excluded_tests - self.cfg.template_values.update({ - 'python': self.python_cmd, - 'excluded_tests': ' '.join(excluded_tests) - }) - tests_out, tests_ec = super(EB_PyTorch, self).test_step(return_output_ec=True) - - def get_count_for_pattern(regex, text): - """Match the regexp containing a single group and return the integer value of the matched group. - Return zero if no or more than 1 match was found and warn for the latter case - """ - match = re.findall(regex, text) - if len(match) == 1: - return int(match[0]) - elif len(match) > 1: - # Shouldn't happen, but means something went wrong with the regular expressions. - # Throw warning, as the build might be fine, no need to error on this. - warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" - warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." - print_warning(warn_msg) - return 0 - - # Create clear summary report - failure_report = "" - failure_cnt = 0 - error_cnt = 0 - failed_test_suites = [] - - # Grep for patterns like: - # Ran 219 tests in 67.325s - # - # FAILED (errors=10, skipped=190, expected failures=6) - # test_fx failed! - regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" - r"FAILED \((?P.*)\)$\n" - r"(?:^(?:(?!failed!).)*$\n)*" - r"(?P.*) failed!(?: Received signal: \w+)?\s*$") - - for m in re.finditer(regex, tests_out, re.M): - # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' - failure_summary = m.group('failure_summary') - total, test_suite = m.group('test_cnt', 'failed_test_suite_name') - failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( - test_suite=test_suite, total=total, failure_summary=failure_summary + # Must not run this unless runtest is actually set to a command, + # otherwise PythonPackage.test_step doesn't return the expected "out, ec" values which makes the code crash + if isinstance(self.cfg['runtest'], string_type): + cmd = self.cfg['runtest'] + if self.cfg['runtest'] and cmd is not None: + self._set_cache_dir() + # Pretend to be on FB CI which disables some tests, especially those which download stuff + env.setvar('SANDCASTLE', '1') + # Skip this test(s) which is very flaky + env.setvar('SKIP_TEST_BOTTLENECK', '1') + # Parse excluded_tests and flatten into space separated string + excluded_tests = [] + for arch, tests in self.cfg['excluded_tests'].items(): + if not arch or arch == get_cpu_architecture(): + excluded_tests.extend(tests) + # -x should not be used if there are no excluded tests + if excluded_tests: + excluded_tests = ['-x'] + excluded_tests + self.cfg.template_values.update({ + 'python': self.python_cmd, + 'excluded_tests': ' '.join(excluded_tests) + }) + + tests_out, tests_ec = super(EB_PyTorch, self).test_step(return_output_ec=True) + + def get_count_for_pattern(regex, text): + """Match the regexp containing a single group and return the integer value of the matched group. + Return zero if no or more than 1 match was found and warn for the latter case + """ + match = re.findall(regex, text) + if len(match) == 1: + return int(match[0]) + elif len(match) > 1: + # Shouldn't happen, but means something went wrong with the regular expressions. + # Throw warning, as the build might be fine, no need to error on this. + warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" + warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." + print_warning(warn_msg) + return 0 + + # Create clear summary report + failure_report = "" + failure_cnt = 0 + error_cnt = 0 + failed_test_suites = [] + + # Grep for patterns like: + # Ran 219 tests in 67.325s + # + # FAILED (errors=10, skipped=190, expected failures=6) + # test_fx failed! + regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" + r"FAILED \((?P.*)\)$\n" + r"(?:^(?:(?!failed!).)*$\n)*" + r"(?P.*) failed!(?: Received signal: \w+)?\s*$") + + for m in re.finditer(regex, tests_out, re.M): + # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' + failure_summary = m.group('failure_summary') + total, test_suite = m.group('test_cnt', 'failed_test_suite_name') + failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( + test_suite=test_suite, total=total, failure_summary=failure_summary + ) + failure_cnt += get_count_for_pattern(r"(?.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + + for m in re.finditer(regex, tests_out, re.M): + # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' + failure_summary = m.group('failure_summary') + test_suite = m.group('failed_test_suite_name') + failure_report += "{test_suite} ({failure_summary})\n".format( + test_suite=test_suite, failure_summary=failure_summary + ) + failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary) + error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary) + failed_test_suites.append(test_suite) + + # Make the names unique and sorted + failed_test_suites = sorted(set(failed_test_suites)) + # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) + # Also unique and sorted to be able to compare the lists below + all_failed_test_suites = sorted(set( + re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) + )) + # If we missed any test suites prepend a list of all failed test suites + if failed_test_suites != all_failed_test_suites: + failure_report_save = failure_report + failure_report = 'Failed tests (suites/files):\n' + failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) + if failure_report_save: + failure_report += '\n' + failure_report_save + + # Calculate total number of unsuccesful and total tests + failed_test_cnt = failure_cnt + error_cnt + test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) + + if failed_test_cnt > 0: + max_failed_tests = self.cfg['max_failed_tests'] + + failure_or_failures = 'failure' if failure_cnt == 1 else 'failures' + error_or_errors = 'error' if error_cnt == 1 else 'errors' + msg = "%d test %s, %d test %s (out of %d):\n" % ( + failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt ) - failure_cnt += get_count_for_pattern(r"(?.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" - - for m in re.finditer(regex, tests_out, re.M): - # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' - failure_summary = m.group('failure_summary') - test_suite = m.group('failed_test_suite_name') - failure_report += "{test_suite} ({failure_summary})\n".format( - test_suite=test_suite, failure_summary=failure_summary - ) - failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary) - error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary) - failed_test_suites.append(test_suite) - - # Make the names unique and sorted - failed_test_suites = sorted(set(failed_test_suites)) - # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) - # Also unique and sorted to be able to compare the lists below - all_failed_test_suites = sorted(set( - re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) - )) - # If we missed any test suites prepend a list of all failed test suites - if failed_test_suites != all_failed_test_suites: - failure_report_save = failure_report - failure_report = 'Failed tests (suites/files):\n' - failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) - if failure_report_save: - failure_report += '\n' + failure_report_save - - # Calculate total number of unsuccesful and total tests - failed_test_cnt = failure_cnt + error_cnt - test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) - - if failed_test_cnt > 0: - max_failed_tests = self.cfg['max_failed_tests'] - - failure_or_failures = 'failure' if failure_cnt == 1 else 'failures' - error_or_errors = 'error' if error_cnt == 1 else 'errors' - msg = "%d test %s, %d test %s (out of %d):\n" % ( - failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt - ) - msg += failure_report - - # If no tests are supposed to fail or some failed for which we were not able to count errors fail now - if max_failed_tests == 0 or failed_test_suites != all_failed_test_suites: - raise EasyBuildError(msg) - else: - msg += '\n\n' + ' '.join([ - "The PyTorch test suite is known to include some flaky tests,", - "which may fail depending on the specifics of the system or the context in which they are run.", - "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests, - "We recommend to double check that the failing tests listed above ", - "are known to be flaky, or do not affect your intended usage of PyTorch.", - "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", - ]) - # Print to console, the user should really be aware that we are accepting failing tests here... - print_warning(msg) - - # Also log this warning in the file log - self.log.warning(msg) - - if failed_test_cnt > max_failed_tests: - raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", - failed_test_cnt, max_failed_tests) - elif failure_report: - raise EasyBuildError("Test command had non-zero exit code (%s)!\n%s", tests_ec, failure_report) - elif tests_ec: - raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) + msg += failure_report + + # If no tests are supposed to fail or some failed for which we were not able to count errors fail now + if max_failed_tests == 0 or failed_test_suites != all_failed_test_suites: + raise EasyBuildError(msg) + else: + msg += '\n\n' + ' '.join([ + "The PyTorch test suite is known to include some flaky tests,", + "which may fail depending on the specifics of the system or the context in which they are run.", + "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests, + "We recommend to double check that the failing tests listed above ", + "are known to be flaky, or do not affect your intended usage of PyTorch.", + "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", + ]) + # Print to console, the user should really be aware that we are accepting failing tests here... + print_warning(msg) + + # Also log this warning in the file log + self.log.warning(msg) + + if failed_test_cnt > max_failed_tests: + raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", + failed_test_cnt, max_failed_tests) + elif failure_report: + raise EasyBuildError("Test command had non-zero exit code (%s)!\n%s", tests_ec, failure_report) + elif tests_ec: + raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) def test_cases_step(self): self._set_cache_dir() From 0a8ebd285f546cc5164c51b00dc993c95700ba80 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 9 Feb 2023 13:10:27 +0100 Subject: [PATCH 41/87] test return from super(EB_PyTorch, self).test_step and handle the result instead of trying to avoid the problem explicitly. --- easybuild/easyblocks/p/pytorch.py | 272 +++++++++++++++--------------- 1 file changed, 137 insertions(+), 135 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 53dca5176b..f4fd1e80f5 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -248,143 +248,145 @@ def _set_cache_dir(self): def test_step(self): """Run unit tests""" + self._set_cache_dir() + # Pretend to be on FB CI which disables some tests, especially those which download stuff + env.setvar('SANDCASTLE', '1') + # Skip this test(s) which is very flaky + env.setvar('SKIP_TEST_BOTTLENECK', '1') + # Parse excluded_tests and flatten into space separated string + excluded_tests = [] + for arch, tests in self.cfg['excluded_tests'].items(): + if not arch or arch == get_cpu_architecture(): + excluded_tests.extend(tests) + # -x should not be used if there are no excluded tests + if excluded_tests: + excluded_tests = ['-x'] + excluded_tests + self.cfg.template_values.update({ + 'python': self.python_cmd, + 'excluded_tests': ' '.join(excluded_tests) + }) - # Must not run this unless runtest is actually set to a command, - # otherwise PythonPackage.test_step doesn't return the expected "out, ec" values which makes the code crash - if isinstance(self.cfg['runtest'], string_type): - cmd = self.cfg['runtest'] - if self.cfg['runtest'] and cmd is not None: - self._set_cache_dir() - # Pretend to be on FB CI which disables some tests, especially those which download stuff - env.setvar('SANDCASTLE', '1') - # Skip this test(s) which is very flaky - env.setvar('SKIP_TEST_BOTTLENECK', '1') - # Parse excluded_tests and flatten into space separated string - excluded_tests = [] - for arch, tests in self.cfg['excluded_tests'].items(): - if not arch or arch == get_cpu_architecture(): - excluded_tests.extend(tests) - # -x should not be used if there are no excluded tests - if excluded_tests: - excluded_tests = ['-x'] + excluded_tests - self.cfg.template_values.update({ - 'python': self.python_cmd, - 'excluded_tests': ' '.join(excluded_tests) - }) - - tests_out, tests_ec = super(EB_PyTorch, self).test_step(return_output_ec=True) - - def get_count_for_pattern(regex, text): - """Match the regexp containing a single group and return the integer value of the matched group. - Return zero if no or more than 1 match was found and warn for the latter case - """ - match = re.findall(regex, text) - if len(match) == 1: - return int(match[0]) - elif len(match) > 1: - # Shouldn't happen, but means something went wrong with the regular expressions. - # Throw warning, as the build might be fine, no need to error on this. - warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" - warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." - print_warning(warn_msg) - return 0 - - # Create clear summary report - failure_report = "" - failure_cnt = 0 - error_cnt = 0 - failed_test_suites = [] - - # Grep for patterns like: - # Ran 219 tests in 67.325s - # - # FAILED (errors=10, skipped=190, expected failures=6) - # test_fx failed! - regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" - r"FAILED \((?P.*)\)$\n" - r"(?:^(?:(?!failed!).)*$\n)*" - r"(?P.*) failed!(?: Received signal: \w+)?\s*$") - - for m in re.finditer(regex, tests_out, re.M): - # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' - failure_summary = m.group('failure_summary') - total, test_suite = m.group('test_cnt', 'failed_test_suite_name') - failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( - test_suite=test_suite, total=total, failure_summary=failure_summary - ) - failure_cnt += get_count_for_pattern(r"(?.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" - - for m in re.finditer(regex, tests_out, re.M): - # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' - failure_summary = m.group('failure_summary') - test_suite = m.group('failed_test_suite_name') - failure_report += "{test_suite} ({failure_summary})\n".format( - test_suite=test_suite, failure_summary=failure_summary - ) - failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary) - error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary) - failed_test_suites.append(test_suite) - - # Make the names unique and sorted - failed_test_suites = sorted(set(failed_test_suites)) - # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) - # Also unique and sorted to be able to compare the lists below - all_failed_test_suites = sorted(set( - re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) - )) - # If we missed any test suites prepend a list of all failed test suites - if failed_test_suites != all_failed_test_suites: - failure_report_save = failure_report - failure_report = 'Failed tests (suites/files):\n' - failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) - if failure_report_save: - failure_report += '\n' + failure_report_save - - # Calculate total number of unsuccesful and total tests - failed_test_cnt = failure_cnt + error_cnt - test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) - - if failed_test_cnt > 0: - max_failed_tests = self.cfg['max_failed_tests'] - - failure_or_failures = 'failure' if failure_cnt == 1 else 'failures' - error_or_errors = 'error' if error_cnt == 1 else 'errors' - msg = "%d test %s, %d test %s (out of %d):\n" % ( - failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt + test_result = super(EB_PyTorch, self).test_step(return_output_ec=True) + if test_result is None: + if self.cfg['runtest'] or self.cfg['runtest'] is None: + msg = "runtest must be set to a command to run." + else: + msg = "Do not set runtest to False, use --skip-test-step instead." + raise EasyBuildError(msg) + + tests_out, tests_ec = test_result + + def get_count_for_pattern(regex, text): + """Match the regexp containing a single group and return the integer value of the matched group. + Return zero if no or more than 1 match was found and warn for the latter case + """ + match = re.findall(regex, text) + if len(match) == 1: + return int(match[0]) + elif len(match) > 1: + # Shouldn't happen, but means something went wrong with the regular expressions. + # Throw warning, as the build might be fine, no need to error on this. + warn_msg = "Error in counting the number of test failures in the output of the PyTorch test suite.\n" + warn_msg += "Please check the EasyBuild log to verify the number of failures (if any) was acceptable." + print_warning(warn_msg) + return 0 + + # Create clear summary report + failure_report = "" + failure_cnt = 0 + error_cnt = 0 + failed_test_suites = [] + + # Grep for patterns like: + # Ran 219 tests in 67.325s + # + # FAILED (errors=10, skipped=190, expected failures=6) + # test_fx failed! + regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" + r"FAILED \((?P.*)\)$\n" + r"(?:^(?:(?!failed!).)*$\n)*" + r"(?P.*) failed!(?: Received signal: \w+)?\s*$") + + for m in re.finditer(regex, tests_out, re.M): + # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' + failure_summary = m.group('failure_summary') + total, test_suite = m.group('test_cnt', 'failed_test_suite_name') + failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( + test_suite=test_suite, total=total, failure_summary=failure_summary ) - msg += failure_report - - # If no tests are supposed to fail or some failed for which we were not able to count errors fail now - if max_failed_tests == 0 or failed_test_suites != all_failed_test_suites: - raise EasyBuildError(msg) - else: - msg += '\n\n' + ' '.join([ - "The PyTorch test suite is known to include some flaky tests,", - "which may fail depending on the specifics of the system or the context in which they are run.", - "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests, - "We recommend to double check that the failing tests listed above ", - "are known to be flaky, or do not affect your intended usage of PyTorch.", - "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", - ]) - # Print to console, the user should really be aware that we are accepting failing tests here... - print_warning(msg) - - # Also log this warning in the file log - self.log.warning(msg) - - if failed_test_cnt > max_failed_tests: - raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", - failed_test_cnt, max_failed_tests) - elif failure_report: - raise EasyBuildError("Test command had non-zero exit code (%s)!\n%s", tests_ec, failure_report) - elif tests_ec: - raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) + failure_cnt += get_count_for_pattern(r"(?.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + + for m in re.finditer(regex, tests_out, re.M): + # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' + failure_summary = m.group('failure_summary') + test_suite = m.group('failed_test_suite_name') + failure_report += "{test_suite} ({failure_summary})\n".format( + test_suite=test_suite, failure_summary=failure_summary + ) + failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary) + error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary) + failed_test_suites.append(test_suite) + + # Make the names unique and sorted + failed_test_suites = sorted(set(failed_test_suites)) + # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) + # Also unique and sorted to be able to compare the lists below + all_failed_test_suites = sorted(set( + re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) + )) + # If we missed any test suites prepend a list of all failed test suites + if failed_test_suites != all_failed_test_suites: + failure_report_save = failure_report + failure_report = 'Failed tests (suites/files):\n' + failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) + if failure_report_save: + failure_report += '\n' + failure_report_save + + # Calculate total number of unsuccesful and total tests + failed_test_cnt = failure_cnt + error_cnt + test_cnt = sum(int(hit) for hit in re.findall(r"^Ran (?P[0-9]+) tests in", tests_out, re.M)) + + if failed_test_cnt > 0: + max_failed_tests = self.cfg['max_failed_tests'] + + failure_or_failures = 'failure' if failure_cnt == 1 else 'failures' + error_or_errors = 'error' if error_cnt == 1 else 'errors' + msg = "%d test %s, %d test %s (out of %d):\n" % ( + failure_cnt, failure_or_failures, error_cnt, error_or_errors, test_cnt + ) + msg += failure_report + + # If no tests are supposed to fail or some failed for which we were not able to count errors fail now + if max_failed_tests == 0 or failed_test_suites != all_failed_test_suites: + raise EasyBuildError(msg) + else: + msg += '\n\n' + ' '.join([ + "The PyTorch test suite is known to include some flaky tests,", + "which may fail depending on the specifics of the system or the context in which they are run.", + "For this PyTorch installation, EasyBuild allows up to %d tests to fail." % max_failed_tests, + "We recommend to double check that the failing tests listed above ", + "are known to be flaky, or do not affect your intended usage of PyTorch.", + "In case of doubt, reach out to the EasyBuild community (via GitHub, Slack, or mailing list).", + ]) + # Print to console, the user should really be aware that we are accepting failing tests here... + print_warning(msg) + + # Also log this warning in the file log + self.log.warning(msg) + + if failed_test_cnt > max_failed_tests: + raise EasyBuildError("Too many failed tests (%d), maximum allowed is %d", + failed_test_cnt, max_failed_tests) + elif failure_report: + raise EasyBuildError("Test command had non-zero exit code (%s)!\n%s", tests_ec, failure_report) + elif tests_ec: + raise EasyBuildError("Test command had non-zero exit code (%s), but no failed tests found?!", tests_ec) def test_cases_step(self): self._set_cache_dir() From 1714017478e60e2fa003f5ebba71ae9128fd1446 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 9 Feb 2023 14:06:28 +0000 Subject: [PATCH 42/87] use with self.cfg.disable_templating(): --- easybuild/easyblocks/generic/cmakemake.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 8bf841f313..c16c101bbe 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -143,16 +143,14 @@ def prepend_config_opts(self, config_opts): """Prepends configure options (-Dkey=value) to configopts ignoring those already set""" # need to disable template resolution or it will remain the same for all runs self.cfg.enable_templating = False + with self.cfg.disable_templating(): + cfg_configopts = self.cfg['configopts'] - cfg_configopts = self.cfg['configopts'] # All options are of the form '-D=' new_opts = ' '.join('-D%s=%s' % (key, value) for key, value in config_opts.items() if '-D%s=' % key not in cfg_configopts) self.cfg['configopts'] = ' '.join([new_opts, cfg_configopts]) - # re-enable template resolution - self.cfg.enable_templating = True - def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" From 7622313e4e7764ded6162056522c5cd07660e93e Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Thu, 9 Feb 2023 14:20:48 +0000 Subject: [PATCH 43/87] removed forgotten enable_templating = False line --- easybuild/easyblocks/generic/cmakemake.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index c16c101bbe..48cdae593a 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -142,7 +142,6 @@ def build_type(self): def prepend_config_opts(self, config_opts): """Prepends configure options (-Dkey=value) to configopts ignoring those already set""" # need to disable template resolution or it will remain the same for all runs - self.cfg.enable_templating = False with self.cfg.disable_templating(): cfg_configopts = self.cfg['configopts'] From 8fff3f64e743be21af2131b408253854bc9fe4fb Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 9 Feb 2023 15:35:36 +0100 Subject: [PATCH 44/87] reverse test of runtest and clarify error message. --- easybuild/easyblocks/p/pytorch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index f4fd1e80f5..252bc09741 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -268,10 +268,10 @@ def test_step(self): test_result = super(EB_PyTorch, self).test_step(return_output_ec=True) if test_result is None: - if self.cfg['runtest'] or self.cfg['runtest'] is None: - msg = "runtest must be set to a command to run." + if self.cfg['runtest'] is False: + msg = "Do not set 'runtest' to False, use --skip-test-step instead." else: - msg = "Do not set runtest to False, use --skip-test-step instead." + msg = "Tests did not run. Make sure 'runtest' is set to a command." raise EasyBuildError(msg) tests_out, tests_ec = test_result From 1d7fa6225b207ab19898fa61ac93405e131fad8d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 9 Feb 2023 16:18:37 +0100 Subject: [PATCH 45/87] Fix to check for existance of config option in FlexiBLAS Only checking for `key` might match where it should not. Use the full `"-Dkey="` as the search string. --- easybuild/easyblocks/f/flexiblas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/f/flexiblas.py b/easybuild/easyblocks/f/flexiblas.py index 2a63c1650e..ab6fabfc50 100644 --- a/easybuild/easyblocks/f/flexiblas.py +++ b/easybuild/easyblocks/f/flexiblas.py @@ -138,7 +138,7 @@ def configure_step(self): # to allow easyconfig to override specifies settings for key, value in sorted(configopts.items()): opt = '-D%s=' % key - if key not in self.cfg['configopts']: + if opt not in self.cfg['configopts']: self.cfg.update('configopts', opt + "'%s'" % value) # specify compiler commands with absolute paths, to ensure that RPATH wrapper scripts are used From 6e30d90d2ba539f3be24eda20f906270ce82b50a Mon Sep 17 00:00:00 2001 From: Randy Pittman Date: Tue, 14 Feb 2023 14:38:17 -0800 Subject: [PATCH 46/87] Use older `ncgen -H` for older netCDF Fixes #2888. --- easybuild/easyblocks/n/netcdf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/n/netcdf.py b/easybuild/easyblocks/n/netcdf.py index 9ac62ff72c..45017748bb 100644 --- a/easybuild/easyblocks/n/netcdf.py +++ b/easybuild/easyblocks/n/netcdf.py @@ -143,7 +143,7 @@ def sanity_check_step(self): custom_commands = [ "nc-config --help", - "ncgen -h", + "ncgen -h" if LooseVersion(self.version) > LooseVersion("4.6.1") else "ncgen -H", ] super(EB_netCDF, self).sanity_check_step(custom_commands=custom_commands, custom_paths=custom_paths) From 566249a3f9ccfd9de8213efe48ecc8f17163fe5d Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Wed, 15 Feb 2023 17:05:55 +0100 Subject: [PATCH 47/87] always use -x option --- easybuild/easyblocks/generic/gopackage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index b94845b8cd..2dcc34c5d7 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -127,14 +127,12 @@ def build_step(self): def install_step(self): """Install Go package to a custom path""" - if build_option('debug'): - self.cfg['installopts'] = '-x ' + self.cfg['installopts'] - # actually install Go package cmd = ' '.join([ self.cfg['preinstallopts'], 'go', 'install', + '-x', self.cfg['installopts'], ]) run_cmd(cmd, log_all=True, log_ok=True, simple=True) From 574190c5714a7fd7e6c1d44fa4e14df0e8c3eb26 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Wed, 15 Feb 2023 17:06:54 +0100 Subject: [PATCH 48/87] remove stale update --- easybuild/easyblocks/generic/gopackage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index 2dcc34c5d7..f60ef94daf 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -34,7 +34,6 @@ from easybuild.framework.easyblock import EasyBlock from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import build_option from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.run import run_cmd From a2323ecc5419ac1e639fe56b90154c37b78f2a7d Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Wed, 15 Feb 2023 17:42:48 +0100 Subject: [PATCH 49/87] add comment --- easybuild/easyblocks/generic/gopackage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index f60ef94daf..aa51ea05cb 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -131,6 +131,8 @@ def install_step(self): self.cfg['preinstallopts'], 'go', 'install', + # print commands as they are executed, + # such as downloads and installs of package deps as listed in the go.mod file '-x', self.cfg['installopts'], ]) From 8f1c579a3cf21843411c3b205df85379b4aa6acd Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Wed, 15 Feb 2023 17:45:12 +0100 Subject: [PATCH 50/87] tweak comment --- easybuild/easyblocks/generic/gopackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py index aa51ea05cb..c06f724437 100644 --- a/easybuild/easyblocks/generic/gopackage.py +++ b/easybuild/easyblocks/generic/gopackage.py @@ -132,7 +132,7 @@ def install_step(self): 'go', 'install', # print commands as they are executed, - # such as downloads and installs of package deps as listed in the go.mod file + # including downloading and installing of package deps as listed in the go.mod file '-x', self.cfg['installopts'], ]) From 00efdf1823d74d84ce6c05b5d61f6ccd134f7b0a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 8 Feb 2023 15:54:59 +0100 Subject: [PATCH 51/87] add deprecation notice to RPackage extensions with relative paths in start_dir --- easybuild/easyblocks/generic/rpackage.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index 190e416db0..897408503c 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -134,10 +134,19 @@ def make_cmdline_cmd(self, prefix=None): else: prefix = '' + loc = self.start_dir + if loc is None: + loc = self.ext_dir or self.ext_src + elif not os.path.isabs(loc): + # TODO: deprecated behaviour in framework 4.7.1, remove after 5.0 + loc = os.path.join(self.ext_dir or os.path.sep, loc) + deprecation_msg = "Found relative path in start_dir, please upgrade to easybuild-framework>=4.7.1" + self.log.deprecated(deprecation_msg, '5.0') + cmd = ' '.join([ self.cfg['preinstallopts'], "R CMD INSTALL", - self.start_dir, + loc, confargs, confvars, prefix, From 016118af4c54d9b604d91132e65a6020232443f7 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Thu, 16 Feb 2023 11:17:42 +0000 Subject: [PATCH 52/87] apply suggestions from review --- easybuild/easyblocks/s/scipy.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 1df30766b9..9473a2f145 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -54,8 +54,9 @@ def extra_options(): """Easyconfig parameters specific to scipy.""" extra_vars = ({ 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], - # ignore tests by default to maintain behaviour in older easyconfigs - 'ignore_test_result': [None, "Run scipy test suite, but ignore result (only log)", CUSTOM], + # ignore test failures by default in scipy < 1.9 to maintain behaviour in older easyconfigs + 'ignore_test_result': [None, "Run scipy test suite, but ignore test failures (True/False/None). Default " + "(None) implies True for scipy < 1.9, and False for scipy ?= 1.9", CUSTOM], }) return PythonPackage.extra_options(extra_vars=extra_vars) @@ -68,17 +69,27 @@ def __init__(self, *args, **kwargs): if LooseVersion(self.version) >= LooseVersion('1.9'): self.use_meson = True + # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 if self.cfg['ignore_test_result'] is None: - self.cfg['ignore_test_result'] = True - # ignore installopts inherited from FortranPythonPackage - self.cfg['installopts'] = "" + self.cfg['ignore_test_result'] = False + + # strip inherited PythonPackage installopts + installopts = self.cfg['installopts'] + pythonpackage_installopts = ['--no-deps', '--ignore-installed', '--no-index', '--egg', + '--zip-ok', '--no-index'] + self.log.info("Stripping inherited PythonPackage installopts %s from installopts %s", + pythonpackage_installopts, installopts) + for i in pythonpackage_installopts: + installopts.replace(i, '') + self.cfg['installopts'] = installopts + else: self.use_meson = False if self.cfg['ignore_test_result']: - # maintains compatibility with easyconfigs predating scipy 1.9. Runs tests (serially) in - # a way that exits with code 0 regardless of test results, see: + # can be used to maintain compatibility with easyconfigs predating scipy 1.9. Runs tests + # (serially) in a way that exits with code 0 regardless of test results, see: # https://github.com/easybuilders/easybuild-easyblocks/issues/2237 self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" else: @@ -106,8 +117,8 @@ def configure_step(self): else: raise EasyBuildError("Unknown BLAS/LAPACK library used: %s", lapack_lib) - configopts = '-Dblas=%(blas_lapack)s -Dlapack=%(blas_lapack)s' % {'blas_lapack': blas_lapack} - self.cfg.update('configopts', configopts) + for opt in ('blas', 'lapack'): + self.cfg.update('configopts', '-D%(opt)s=%(blas_lapack)s' % {'opt': opt, 'blas_lapack': blas_lapack}) # need to have already installed extensions in PATH, PYTHONPATH for configure/build/install steps pythonpath = os.getenv('PYTHONPATH') @@ -120,6 +131,7 @@ def configure_step(self): MesonNinja.configure_step(self) else: + # scipy < 1.9.0 uses install procedure using setup.py FortranPythonPackage.configure_step(self) if LooseVersion(self.version) >= LooseVersion('0.13'): @@ -184,7 +196,6 @@ def install_step(self): """Custom install step for scipy: use ninja for scipy >= 1.9.0""" if self.use_meson: MesonNinja.install_step(self) - else: FortranPythonPackage.install_step(self) From a69b47f42eef6eddb57c2a403580fc7f89e12b57 Mon Sep 17 00:00:00 2001 From: Jasper <65227842+jfgrimm@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:38:32 +0000 Subject: [PATCH 53/87] Update easybuild/easyblocks/s/scipy.py Co-authored-by: Simon Branford <4967+branfosj@users.noreply.github.com> --- easybuild/easyblocks/s/scipy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 9473a2f145..77504eed40 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -81,7 +81,7 @@ def __init__(self, *args, **kwargs): self.log.info("Stripping inherited PythonPackage installopts %s from installopts %s", pythonpackage_installopts, installopts) for i in pythonpackage_installopts: - installopts.replace(i, '') + installopts = installopts.replace(i, '') self.cfg['installopts'] = installopts else: From f2b4131c0821019b47d96568fc604ee0bcf20de0 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 21 Feb 2023 13:47:20 +0100 Subject: [PATCH 54/87] split depot path in JuliaPackage with os.pathsep --- easybuild/easyblocks/generic/juliapackage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 2490bc2823..ff8bc74298 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -99,8 +99,10 @@ def install_step(self): # extensions in a bundle can share their DEPOT_PATH # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH depot_path = os.getenv('JULIA_DEPOT_PATH', '') + if depot_path: + depot_path = depot_path.split(os.pathsep) if self.installdir not in depot_path: - depot_path = ':'.join([depot for depot in (self.installdir, depot_path) if depot]) + depot_path = os.pathsep.join([depot for depot in (self.installdir, depot_path) if depot]) env.setvar('JULIA_DEPOT_PATH', depot_path) # command sequence for Julia.Pkg From b360347928de191ec945dab106dbb8d5e98337d9 Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 21 Feb 2023 15:44:30 +0000 Subject: [PATCH 55/87] allow disabling pybind11 tests --- easybuild/easyblocks/p/pybind11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pybind11.py b/easybuild/easyblocks/p/pybind11.py index 09d711b28c..1174d23a13 100644 --- a/easybuild/easyblocks/p/pybind11.py +++ b/easybuild/easyblocks/p/pybind11.py @@ -70,8 +70,9 @@ def configure_step(self): def test_step(self): """Run pybind11 tests""" - # always run tests - self.cfg['runtest'] = 'check' + # always run tests unless explicitly disabled + if self.cfg['runtest'] is not False: + self.cfg['runtest'] = 'check' super(EB_pybind11, self).test_step() def install_step(self): From c99e441abb542ab67f74fb4075e5ce695eb0903d Mon Sep 17 00:00:00 2001 From: Maxime Boissonneault Date: Tue, 21 Feb 2023 11:16:37 -0500 Subject: [PATCH 56/87] Update easybuild/easyblocks/p/pybind11.py Co-authored-by: Jasper <65227842+jfgrimm@users.noreply.github.com> --- easybuild/easyblocks/p/pybind11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/pybind11.py b/easybuild/easyblocks/p/pybind11.py index 1174d23a13..d964ffb9ac 100644 --- a/easybuild/easyblocks/p/pybind11.py +++ b/easybuild/easyblocks/p/pybind11.py @@ -70,7 +70,7 @@ def configure_step(self): def test_step(self): """Run pybind11 tests""" - # always run tests unless explicitly disabled + # run tests unless explicitly disabled if self.cfg['runtest'] is not False: self.cfg['runtest'] = 'check' super(EB_pybind11, self).test_step() From d6e4e25d5d5a10bceb87885fb16d421b7e4f4502 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francisco=20Jos=C3=A9=20Gonz=C3=A1lez=20Carrillo?= Date: Wed, 22 Feb 2023 12:00:38 +0100 Subject: [PATCH 57/87] Adding support for postinstallcmds in easybuildmeta --- easybuild/easyblocks/e/easybuildmeta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index baac89baa0..f0ff1f340a 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -157,6 +157,7 @@ def install_step(self): def post_install_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" + super(EB_EasyBuildMeta, self).post_install_step() setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): setuptools_pth_txt = read_file(setuptools_pth) From 4c11ffaf6bfe91309596bf22cf0f2198d3bde4c7 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 23 Feb 2023 09:40:16 +0100 Subject: [PATCH 58/87] add blank line in post_install_step method of EasyBuildMeta easyblock to improve readability --- easybuild/easyblocks/e/easybuildmeta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/e/easybuildmeta.py b/easybuild/easyblocks/e/easybuildmeta.py index f0ff1f340a..f4b52ff921 100644 --- a/easybuild/easyblocks/e/easybuildmeta.py +++ b/easybuild/easyblocks/e/easybuildmeta.py @@ -158,6 +158,7 @@ def post_install_step(self): """Remove setuptools.pth file that hard includes a system-wide (site-packages) path, if it is there.""" super(EB_EasyBuildMeta, self).post_install_step() + setuptools_pth = os.path.join(self.installdir, self.pylibdir, 'setuptools.pth') if os.path.exists(setuptools_pth): setuptools_pth_txt = read_file(setuptools_pth) From 91cee1036d35f229f132c94e361456b65598f581 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 13:30:33 +0100 Subject: [PATCH 59/87] Added GPU support to ELPA EasyBlock. If CUDA is specified in the EasyConfig, automatically enable nvidia GPU support --- easybuild/easyblocks/e/elpa.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 8ac4856207..116bd2abea 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -39,7 +39,7 @@ from easybuild.tools.systemtools import get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.tools.utilities import nub - +from easybuild.tools.modules import get_software_root ELPA_CPU_FEATURE_FLAGS = ['avx', 'avx2', 'avx512f', 'vsx', 'sse4_2'] @@ -176,6 +176,28 @@ def run_all_steps(self, *args, **kwargs): return super(EB_ELPA, self).run_all_steps(*args, **kwargs) + def configure_step(self): + """Configure step for ELPA""" + + # Add nvidia GPU support if requested + cuda_root = get_software_root('CUDA') + self.log.info("Got cuda root: %s", cuda_root) + if cuda_root: + self.cfg.update('configopts', '--enable-nvidia-gpu') + self.cfg.update('configopts', '--with-cuda-path=%s' % cuda_root) + self.cfg.update('configopts', '--with-cuda-sdk-path=%s' % cuda_root) + + cuda_cc = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities'] + if not cuda_cc: + raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' + 'cuda_compute_capabilities easyconfig parameter or via ' + '--cuda-compute-capabilities') + cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + + super(EB_ELPA, self).configure_step() + def patch_step(self, *args, **kwargs): """Patch manual_cpp script to avoid using hardcoded /usr/bin/python.""" super(EB_ELPA, self).patch_step(*args, **kwargs) From ca7775227ef921e480599428ef29ad8002c6480b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 13:46:58 +0100 Subject: [PATCH 60/87] Currently, ELPA only supports passing one architecture to --with-NVIDIA-GPU-compute-capability=VALUE --- easybuild/easyblocks/e/elpa.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 116bd2abea..189c99068c 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -192,6 +192,12 @@ def configure_step(self): raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' 'cuda_compute_capabilities easyconfig parameter or via ' '--cuda-compute-capabilities') + + # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture + if len(cuda_cc) > 1: + raise EasyBuildError('ELPA currently only supports specifying one architecture when ' + 'building. You specified cuda-compute-capabilities: %s', cuda_cc) + cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) From 4865037803d4b98837845f07c94a94f678995f1b Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 17:35:27 +0100 Subject: [PATCH 61/87] Make sure to set CPP, as newer ELPA configures require it --- easybuild/easyblocks/e/elpa.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 189c99068c..cdbce5f23c 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -31,6 +31,7 @@ """ import os +import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError @@ -202,6 +203,17 @@ def configure_step(self): self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + # From v2022.05.001 onwards, the config complains if CPP is not set + env_dict = env.read_environment({'cxx': 'CXX', 'cpp': 'CPP'}) + if 'cxx' in env_dict: + if 'cpp' in env_dict and env_dict['cxx'] != env_dict['cpp']: + self.log.warning("Overwriting value of CPP (%s) with the value for CXX (%s)", + env_dict['cpp'], env_dict['cxx']) + env.setvar('CPP', env_dict['cxx']) + else: + raise EasyBuildError('ELPA requires CPP to be set. EasyBuild tried setting it based on the value of CXX, ' + 'but could not retreive a value for CXX') + super(EB_ELPA, self).configure_step() def patch_step(self, *args, **kwargs): From 6642b40e99c847f3e30c3ba02e38e31fca2ca457 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Fri, 24 Feb 2023 18:55:57 +0100 Subject: [PATCH 62/87] Set CPP env var to cpp. This is valid for GCC, but we might make to make this neater and read it from the compiler definition from easybuild-framework. Still discussing that on EB slack --- easybuild/easyblocks/e/elpa.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index cdbce5f23c..51742b1d0d 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -204,15 +204,9 @@ def configure_step(self): self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) # From v2022.05.001 onwards, the config complains if CPP is not set - env_dict = env.read_environment({'cxx': 'CXX', 'cpp': 'CPP'}) - if 'cxx' in env_dict: - if 'cpp' in env_dict and env_dict['cxx'] != env_dict['cpp']: - self.log.warning("Overwriting value of CPP (%s) with the value for CXX (%s)", - env_dict['cpp'], env_dict['cxx']) - env.setvar('CPP', env_dict['cxx']) - else: - raise EasyBuildError('ELPA requires CPP to be set. EasyBuild tried setting it based on the value of CXX, ' - 'but could not retreive a value for CXX') + # Need to make this neater so that it is either set by easybuild-framework, OR query the toolchain for something like COMPILER_CPP + # Discussing that right now on EB Slack... + env.setvar('CPP', 'cpp') super(EB_ELPA, self).configure_step() From a11a1a4ddc2e5730df59bb008e7dff9684a804d0 Mon Sep 17 00:00:00 2001 From: Sven Hansen Date: Fri, 24 Feb 2023 23:19:38 +0100 Subject: [PATCH 63/87] update Maple easyblock for v2022 --- easybuild/easyblocks/m/maple.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/m/maple.py b/easybuild/easyblocks/m/maple.py index 14621e8515..bf3b75f2aa 100644 --- a/easybuild/easyblocks/m/maple.py +++ b/easybuild/easyblocks/m/maple.py @@ -30,6 +30,7 @@ @author: Kenneth Hoste (Ghent University) @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) +@author: Sven Hansen (RWTH Aachen University) """ import glob import os @@ -61,6 +62,8 @@ def install_step(self): "Do you accept this license? [y/n]:": 'y', 'ENTER AN ABSOLUTE PATH, OR PRESS TO ACCEPT THE DEFAULT :': self.installdir, 'IS THIS CORRECT? (Y/N):': 'Y', + 'Language Selection\n\nPlease select the installation language\n[1] English - English\n[2] Japanese - \n' + 'Please choose an option [1] : ': '1', 'Do you wish to have a shortcut installed on your desktop? ->1- Yes 2- No ENTER THE NUMBER ' + 'FOR YOUR CHOICE, OR PRESS TO ACCEPT THE DEFAULT::': '2', "Do you wish to have a shortcut installed on your desktop? [Y/n]:": 'n', @@ -83,6 +86,7 @@ def install_step(self): r"\[1\] Single Server.*\n.*\nPlease choose an option \[.\] :": '1', r"Port number \[[0-9]+\]:": '', r"Enable periodic checking for Maple .* updates after installation \[Y/n\]:": 'n', + r'Pre-Installation Summary[\s\S]*': '', } no_qa = [ @@ -91,10 +95,10 @@ def install_step(self): 'Launching installer...', "Configuring the installer for this system's environment...", 'Unpacking the JRE...', - r'\[[-|]*', + r'\[[-|#|]*', ] - run_cmd_qa(cmd, qa, std_qa=std_qa, no_qa=no_qa, log_all=True, simple=True) + run_cmd_qa(cmd, qa, std_qa=std_qa, no_qa=no_qa, log_all=True, simple=True, maxhits=150) upgrade_installers = glob.glob(os.path.join(self.builddir, 'Maple*Upgrade*')) if upgrade_installers: From 1b639923f34f011f5d087b914a293855425e3913 Mon Sep 17 00:00:00 2001 From: Sven Hansen Date: Fri, 24 Feb 2023 23:20:40 +0100 Subject: [PATCH 64/87] allow custom license server ports in Maple easyblock --- easybuild/easyblocks/m/maple.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/m/maple.py b/easybuild/easyblocks/m/maple.py index bf3b75f2aa..65868877ff 100644 --- a/easybuild/easyblocks/m/maple.py +++ b/easybuild/easyblocks/m/maple.py @@ -72,7 +72,7 @@ def install_step(self): 'PRESS TO EXIT THE INSTALLER:': '', 'License server (DEFAULT: ):': self.cfg['license_server'], "License server []:": self.cfg['license_server'], - 'Port number (optional) (DEFAULT: ):': '', + 'Port number (optional) (DEFAULT: ):': self.cfg['license_server_port'] or '', '->1- Configure toolbox for Matlab 2- Do not configure at this time ENTER THE NUMBER FOR YOUR CHOICE, ' + 'OR PRESS TO ACCEPT THE DEFAULT::': '2', "MATLAB Configuration [y/N]:": 'n', @@ -84,7 +84,7 @@ def install_step(self): r"Choose Install Folder \[.*\]:": self.installdir, r"\[2\] Network License.*\nPlease choose an option \[.\] :": '2', r"\[1\] Single Server.*\n.*\nPlease choose an option \[.\] :": '1', - r"Port number \[[0-9]+\]:": '', + r"Port number \[[0-9]+\]:": self.cfg['license_server_port'] or '', r"Enable periodic checking for Maple .* updates after installation \[Y/n\]:": 'n', r'Pre-Installation Summary[\s\S]*': '', } From 41b424c47091f1a6686ace61700a73d87ced8f66 Mon Sep 17 00:00:00 2001 From: Jasper Grimm Date: Sat, 25 Feb 2023 09:00:24 +0000 Subject: [PATCH 65/87] fix default ignore_test_result logic --- easybuild/easyblocks/s/scipy.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 77504eed40..2ba91030dc 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -56,7 +56,7 @@ def extra_options(): 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], # ignore test failures by default in scipy < 1.9 to maintain behaviour in older easyconfigs 'ignore_test_result': [None, "Run scipy test suite, but ignore test failures (True/False/None). Default " - "(None) implies True for scipy < 1.9, and False for scipy ?= 1.9", CUSTOM], + "(None) implies True for scipy < 1.9, and False for scipy >= 1.9", CUSTOM], }) return PythonPackage.extra_options(extra_vars=extra_vars) @@ -71,9 +71,6 @@ def __init__(self, *args, **kwargs): self.use_meson = True # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 - if self.cfg['ignore_test_result'] is None: - self.cfg['ignore_test_result'] = False - # strip inherited PythonPackage installopts installopts = self.cfg['installopts'] pythonpackage_installopts = ['--no-deps', '--ignore-installed', '--no-index', '--egg', @@ -87,6 +84,13 @@ def __init__(self, *args, **kwargs): else: self.use_meson = False + if self.cfg['ignore_test_result'] is None: + # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 + if self.use_meson: + self.cfg['ignore_test_result'] = False + else: + self.cfg['ignore_test_result'] = True + if self.cfg['ignore_test_result']: # can be used to maintain compatibility with easyconfigs predating scipy 1.9. Runs tests # (serially) in a way that exits with code 0 regardless of test results, see: From f53a4c67e8f56155d7dc4ac5ca2a995e0ee8a028 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Feb 2023 10:10:39 +0100 Subject: [PATCH 66/87] simplify logic to set default value for ignore_test_result in scipy easyblock --- easybuild/easyblocks/s/scipy.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 2ba91030dc..1c21a01fc2 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -54,7 +54,6 @@ def extra_options(): """Easyconfig parameters specific to scipy.""" extra_vars = ({ 'enable_slow_tests': [False, "Run scipy test suite, including tests marked as slow", CUSTOM], - # ignore test failures by default in scipy < 1.9 to maintain behaviour in older easyconfigs 'ignore_test_result': [None, "Run scipy test suite, but ignore test failures (True/False/None). Default " "(None) implies True for scipy < 1.9, and False for scipy >= 1.9", CUSTOM], }) @@ -85,16 +84,15 @@ def __init__(self, *args, **kwargs): self.use_meson = False if self.cfg['ignore_test_result'] is None: - # enforce scipy test suite results if not explicitly disabled for scipy >= 1.9 - if self.use_meson: - self.cfg['ignore_test_result'] = False - else: - self.cfg['ignore_test_result'] = True + # automatically ignore scipy test suite results for scipy < 1.9, as we did in older easyconfigs + self.cfg['ignore_test_result'] = LooseVersion(self.version) < '1.9' + self.log.info("ignore_test_result not specified, so automatically set to %s for scipy %s", + self.cfg['ignore_test_result'], self.version) if self.cfg['ignore_test_result']: - # can be used to maintain compatibility with easyconfigs predating scipy 1.9. Runs tests - # (serially) in a way that exits with code 0 regardless of test results, see: - # https://github.com/easybuilders/easybuild-easyblocks/issues/2237 + # used to maintain compatibility with easyconfigs predating scipy 1.9; + # runs tests (serially) in a way that exits with code 0 regardless of test results, + # see https://github.com/easybuilders/easybuild-easyblocks/issues/2237 self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" else: self.testcmd = " && ".join([ From 82472980646e5117c0783618e6319d4f3f4f0d52 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Feb 2023 19:15:13 +0100 Subject: [PATCH 67/87] relax glob pattern to find Mathematica install script --- easybuild/easyblocks/m/mathematica.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/m/mathematica.py b/easybuild/easyblocks/m/mathematica.py index 13a3080ff4..9f1fb78827 100644 --- a/easybuild/easyblocks/m/mathematica.py +++ b/easybuild/easyblocks/m/mathematica.py @@ -68,7 +68,7 @@ def install_step(self): install_script_glob = '%s_%s_LINUX*.sh' % (self.name, self.version) # Starting at V13, Mathematica have renamed their install file... if LooseVersion(self.version) >= LooseVersion("13"): - install_script_glob = '%s_%s_BNDL_LINUX*.sh' % (self.name, self.version) + install_script_glob = '%s_%s_*LINUX*.sh' % (self.name, self.version) matches = glob.glob(install_script_glob) if len(matches) == 1: From b587ae82fd739f7831fbeb7c1e90b413bc56ee50 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Feb 2023 19:22:25 +0100 Subject: [PATCH 68/87] fix linking numexpr with Intel MKL's VML library for imkl >= 2021.x --- easybuild/easyblocks/n/numexpr.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index b5d23e73d0..81ba8d8f0a 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -55,12 +55,16 @@ def __init__(self, *args, **kwargs): self.imkl_root = None + def prepare_step(self, *args, **kwargs): + """Custom prepare step for numexpr: check whether we're building on top of Intel MKL.""" + super(EB_numexpr, self).prepare_step(*args, **kwargs) + + self.imkl_root = get_software_root('imkl') + def configure_step(self): """Custom configuration procedure for numexpr.""" super(EB_numexpr, self).configure_step() - self.imkl_root = get_software_root('imkl') - # if Intel MKL is available, set up site.cfg such that the right VML library is used; # this makes a *big* difference in terms of performance; # see also https://github.com/pydata/numexpr/blob/master/site.cfg.example @@ -122,8 +126,15 @@ def sanity_check_step(self): custom_commands = [] + # imkl_root may still be None, for example when running with --sanity-check-only + if self.imkl_root is None: + self.imkl_root = get_software_root('imkl') + # if Intel MKL is available, make sure VML is used if self.imkl_root: - custom_commands.append("python -c 'import numexpr; assert(numexpr.use_vml)'") + custom_commands.extend([ + "python -c 'import numexpr; assert(numexpr.use_vml)'", + """python -c "import numexpr; numexpr.set_vml_accuracy_mode('low')" """, + ]) return super(EB_numexpr, self).sanity_check_step(custom_commands=custom_commands) From b372286d0b1074cdd269e251e4694a0bc1f20970 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Feb 2023 23:00:22 +0100 Subject: [PATCH 69/87] use libmkl_rt to link numexpr with imkl >= 2021.x --- easybuild/easyblocks/n/numexpr.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 81ba8d8f0a..deafddb122 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -28,7 +28,6 @@ import os from distutils.version import LooseVersion -import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file @@ -55,16 +54,12 @@ def __init__(self, *args, **kwargs): self.imkl_root = None - def prepare_step(self, *args, **kwargs): - """Custom prepare step for numexpr: check whether we're building on top of Intel MKL.""" - super(EB_numexpr, self).prepare_step(*args, **kwargs) - - self.imkl_root = get_software_root('imkl') - def configure_step(self): """Custom configuration procedure for numexpr.""" super(EB_numexpr, self).configure_step() + self.imkl_root = get_software_root('imkl') + # if Intel MKL is available, set up site.cfg such that the right VML library is used; # this makes a *big* difference in terms of performance; # see also https://github.com/pydata/numexpr/blob/master/site.cfg.example @@ -84,20 +79,12 @@ def configure_step(self): mkl_ver = get_software_version('imkl') - comp_fam = self.toolchain.comp_family() - self.log.info("Using toolchain with compiler family %s", comp_fam) - if LooseVersion(mkl_ver) >= LooseVersion('2021'): mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'latest', 'lib', 'intel64'), ] mkl_include_dirs = os.path.join(self.imkl_root, 'mkl', 'latest', 'include') - if comp_fam == toolchain.INTELCOMP: - mkl_libs = ['mkl_intel_lp64', 'mkl_intel_thread', 'mkl_core', 'iomp5'] - elif comp_fam == toolchain.GCC: - mkl_libs = ['mkl_intel_lp64', 'mkl_gnu_thread', 'mkl_core', 'gomp'] - else: - raise EasyBuildError("Unknown compiler family, don't know how to link MKL libraries: %s", comp_fam) + mkl_libs = ['mkl_rt'] else: mkl_lib_dirs = [ os.path.join(self.imkl_root, 'mkl', 'lib', 'intel64'), From 138b6629be6916e2ab6e3c4f13c3a140e29f268a Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 25 Feb 2023 23:03:33 +0100 Subject: [PATCH 70/87] remove unused import for EasyBuildError in numexpr easyblock --- easybuild/easyblocks/n/numexpr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index deafddb122..40e01f0cb8 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -29,7 +29,6 @@ from distutils.version import LooseVersion from easybuild.easyblocks.generic.pythonpackage import PythonPackage -from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import write_file from easybuild.tools.modules import get_software_root, get_software_version from easybuild.tools.systemtools import get_cpu_features From 88e4b3aab8c60afe8757641c8fb1e015ff290d3e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sun, 26 Feb 2023 19:43:44 +0100 Subject: [PATCH 71/87] only do a more extensive check for VML support for numexpr >= 2.7.3 --- easybuild/easyblocks/n/numexpr.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/n/numexpr.py b/easybuild/easyblocks/n/numexpr.py index 40e01f0cb8..1c62b4502c 100644 --- a/easybuild/easyblocks/n/numexpr.py +++ b/easybuild/easyblocks/n/numexpr.py @@ -118,9 +118,10 @@ def sanity_check_step(self): # if Intel MKL is available, make sure VML is used if self.imkl_root: - custom_commands.extend([ - "python -c 'import numexpr; assert(numexpr.use_vml)'", - """python -c "import numexpr; numexpr.set_vml_accuracy_mode('low')" """, - ]) + custom_commands.append("python -c 'import numexpr; assert(numexpr.use_vml)'") + + # for sufficiently recent versions of numexpr, also do a more extensive check for VML support + if LooseVersion(self.version) >= LooseVersion('2.7.3'): + custom_commands.append("""python -c "import numexpr; numexpr.set_vml_accuracy_mode('low')" """) return super(EB_numexpr, self).sanity_check_step(custom_commands=custom_commands) From c687f9095d5c0616cd33a963fd7244fb5db4b907 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 12:02:44 +0100 Subject: [PATCH 72/87] Fixed typo, added option to enable dedicated sm80 kernel, set CPP (C preprocessor environment variable) in a way that can easily be extended or modified per toolchain family --- easybuild/easyblocks/e/elpa.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 51742b1d0d..255abe4ee2 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -30,10 +30,13 @@ @author: Kenneth Hoste (Ghent University) """ import os +from distutils.version import LooseVersion import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import ConfigureMake from easybuild.framework.easyconfig import CUSTOM +from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC +from easybuild.toolchains.compiler.inteliccifort import TC_CONSTANT_INTELCOMP from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions @@ -201,12 +204,24 @@ def configure_step(self): cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capabilitie: %s", cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') # From v2022.05.001 onwards, the config complains if CPP is not set - # Need to make this neater so that it is either set by easybuild-framework, OR query the toolchain for something like COMPILER_CPP - # Discussing that right now on EB Slack... - env.setvar('CPP', 'cpp') + # C preprocessor to use for given comp_fam + cpp_dict = { + TC_CONSTANT_GCC: 'cpp', + TC_CONSTANT_INTELCOMP: 'cpp', + } + comp_fam = self.toolchain.comp_family() + if comp_fam in cpp_dict: + env.setvar('CPP', cpp_dict[comp_fam]) + else: + raise EasyBuildError('ELPA EasyBlock does not know which C preprocessor to use for the ' + 'current compiler family (%s). Please add the correct preprocessor ' + 'for this compiler family to cpp_dict in the ELPA EasyBlock', comp_fam) super(EB_ELPA, self).configure_step() From 515e57df7b1184597e3545f58d13f9755bb488fb Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:39:43 +0100 Subject: [PATCH 73/87] Clarify comment --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 255abe4ee2..a69d657345 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -209,7 +209,7 @@ def configure_step(self): if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - # From v2022.05.001 onwards, the config complains if CPP is not set + # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam cpp_dict = { TC_CONSTANT_GCC: 'cpp', From 1f86084c67c75c6caacfaa772ba7a8ef06417c21 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:56:37 +0100 Subject: [PATCH 74/87] Pull logic inside the if statement that checks if only a single cuda compute capability has been specified. Also, now check if the cuda_cc is larger or equal to 8.0, since you probably also want to build the 8.0 optimized kernel if you have 8.5 capability in your system --- easybuild/easyblocks/e/elpa.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index a69d657345..5bff3c695b 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -198,17 +198,18 @@ def configure_step(self): '--cuda-compute-capabilities') # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture - if len(cuda_cc) > 1: + if len(cuda_cc) == 1: + cuda_cc = cuda_cc[0] + cuda_cc_string = cuda_cc.replace('.', '') + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') + else: raise EasyBuildError('ELPA currently only supports specifying one architecture when ' 'building. You specified cuda-compute-capabilities: %s', cuda_cc) - cuda_cc_string = ','.join(['sm_%s' % x.replace('.', '') for x in cuda_cc]) - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) - # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards - if '8.0' in cuda_cc and LooseVersion(self.version) >= LooseVersion('2021.11.001'): - self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam cpp_dict = { From c3e7701f583c285d905b6879ef4f4c824d46849a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Mon, 27 Feb 2023 15:57:59 +0100 Subject: [PATCH 75/87] Make sure sm_ gets prepended in the argument for --with-NVIDIA-GPU-compute-capability --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 5bff3c695b..39ea64b41e 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -201,7 +201,7 @@ def configure_step(self): if len(cuda_cc) == 1: cuda_cc = cuda_cc[0] cuda_cc_string = cuda_cc.replace('.', '') - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=%s' % cuda_cc_string) + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): From 8b154aba5c1e61db84f432000386907dcd18d811 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Feb 2023 11:55:00 +0100 Subject: [PATCH 76/87] Reordered if-else statement to reduce indentation. Clarified error message that is raised --- easybuild/easyblocks/e/elpa.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 39ea64b41e..143952cbf7 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -198,17 +198,16 @@ def configure_step(self): '--cuda-compute-capabilities') # ELPA's --with-NVIDIA-GPU-compute-capability only accepts a single architecture - if len(cuda_cc) == 1: - cuda_cc = cuda_cc[0] - cuda_cc_string = cuda_cc.replace('.', '') - self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) - self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) - # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards - if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): - self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') - else: - raise EasyBuildError('ELPA currently only supports specifying one architecture when ' + if len(cuda_cc) != 1: + raise EasyBuildError('ELPA currently only supports specifying one CUDA architecture when ' 'building. You specified cuda-compute-capabilities: %s', cuda_cc) + cuda_cc = cuda_cc[0] + cuda_cc_string = cuda_cc.replace('.', '') + self.cfg.update('configopts', '--with-NVIDIA-GPU-compute-capability=sm_%s' % cuda_cc_string) + self.log.info("Enabling nvidia GPU support for compute capability: %s", cuda_cc_string) + # There is a dedicated kernel for sm80, but only from version 2021.11.001 onwards + if float(cuda_cc) >= 8.0 and LooseVersion(self.version) >= LooseVersion('2021.11.001'): + self.cfg.update('configopts', '--enable-nvidia-sm80-gpu') # From v2022.05.001 onwards, the config complains if CPP is not set, resulting in non-zero exit of configure # C preprocessor to use for given comp_fam From 40bc11790fef31a4f8d0499c5c2e35dc3308e8c1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 28 Feb 2023 13:38:07 +0100 Subject: [PATCH 77/87] Moved up from .. import to maintain alphabetical order --- easybuild/easyblocks/e/elpa.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/e/elpa.py b/easybuild/easyblocks/e/elpa.py index 143952cbf7..41b81e15e8 100644 --- a/easybuild/easyblocks/e/elpa.py +++ b/easybuild/easyblocks/e/elpa.py @@ -40,10 +40,10 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions +from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_features, get_shared_lib_ext from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC from easybuild.tools.utilities import nub -from easybuild.tools.modules import get_software_root ELPA_CPU_FEATURE_FLAGS = ['avx', 'avx2', 'avx512f', 'vsx', 'sse4_2'] From 2207b51b4be816425ccf7aa9243988a03eba5b79 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 1 Mar 2023 11:43:50 +0100 Subject: [PATCH 78/87] always handle depot paths in a list in JuliaPackage Co-authored-by: Sam Moors --- easybuild/easyblocks/generic/juliapackage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index ff8bc74298..2e32047665 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -98,11 +98,11 @@ def install_step(self): # prepend installation directory to Julia DEPOT_PATH # extensions in a bundle can share their DEPOT_PATH # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH - depot_path = os.getenv('JULIA_DEPOT_PATH', '') + depot_path = os.getenv('JULIA_DEPOT_PATH', []) if depot_path: depot_path = depot_path.split(os.pathsep) if self.installdir not in depot_path: - depot_path = os.pathsep.join([depot for depot in (self.installdir, depot_path) if depot]) + depot_path = os.pathsep.join([depot for depot in [self.installdir] + depot_path if depot]) env.setvar('JULIA_DEPOT_PATH', depot_path) # command sequence for Julia.Pkg From 54eff72c48218d1589ea956e81a9fb6b173a4fcd Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Wed, 1 Mar 2023 13:51:16 +0000 Subject: [PATCH 79/87] Allow disabling of building tests and forward deps, and support version 13 for Trilinos --- easybuild/easyblocks/t/trilinos.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index 4fa0ab9148..3af9614b0d 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -54,6 +54,8 @@ def extra_options(): extra_vars = { 'shared_libs': [None, "Deprecated. Use build_shared_libs", CUSTOM], 'openmp': [True, "Enable OpenMP support", CUSTOM], + 'forward_deps': [True, "Enable all forward dependencies", CUSTOM], + 'build_tests': [True, "Enable building tests/examples", CUSTOM], 'all_exts': [True, "Enable all Trilinos packages", CUSTOM], 'skip_exts': [[], "List of Trilinos packages to skip", CUSTOM], 'verbose': [False, "Configure for verbose output", CUSTOM], @@ -106,9 +108,11 @@ def configure_step(self): if self.toolchain.options.get('usempi', None): self.cfg.update('configopts', "-DTPL_ENABLE_MPI:BOOL=ON") - # enable full testing - self.cfg.update('configopts', "-DTrilinos_ENABLE_TESTS:BOOL=ON") - self.cfg.update('configopts', "-DTrilinos_ENABLE_ALL_FORWARD_DEP_PACKAGES:BOOL=ON") + if self.cfg['build_tests']: + # enable full testing + self.cfg.update('configopts', "-DTrilinos_ENABLE_TESTS:BOOL=ON") + if self.cfg['forward_deps']: + self.cfg.update('configopts', "-DTrilinos_ENABLE_ALL_FORWARD_DEP_PACKAGES:BOOL=ON") lib_re = re.compile("^lib(.*).a$") @@ -285,6 +289,11 @@ def sanity_check_step(self): libs.remove('Galeri') libs.extend(['galeri-epetra', 'galeri-xpetra']) + if LooseVersion(self.version) >= LooseVersion('13'): + libs.remove('GlobiPack') + libs.remove('Mesquite') + libs.remove('MOOCHO') + # Get the library extension if self.cfg['build_shared_libs']: lib_ext = get_shared_lib_ext() From f9bfd0d0140d250d0c5ae5f98157b26c93ea5865 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Wed, 1 Mar 2023 14:03:24 +0000 Subject: [PATCH 80/87] library changes from ComputeCanada --- easybuild/easyblocks/t/trilinos.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/t/trilinos.py b/easybuild/easyblocks/t/trilinos.py index 3af9614b0d..aa4b911473 100644 --- a/easybuild/easyblocks/t/trilinos.py +++ b/easybuild/easyblocks/t/trilinos.py @@ -289,11 +289,15 @@ def sanity_check_step(self): libs.remove('Galeri') libs.extend(['galeri-epetra', 'galeri-xpetra']) - if LooseVersion(self.version) >= LooseVersion('13'): - libs.remove('GlobiPack') + # Mesquite and MOOCHO packages gone in 12.18: + if LooseVersion(self.version) >= LooseVersion('12.18'): libs.remove('Mesquite') libs.remove('MOOCHO') + # GlobiPack package gone in 13.0: + if LooseVersion(self.version) >= LooseVersion('13.0'): + libs.remove('GlobiPack') + # Get the library extension if self.cfg['build_shared_libs']: lib_ext = get_shared_lib_ext() From de55fd4f28aaf79b3eec21c6c433e592c79c8219 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 9 Mar 2023 09:52:56 +0100 Subject: [PATCH 81/87] update scipy easyblock to copy PKG-INFO to scipy-.egg-info when installing with Meson/Ninja + enhance add sanity check command using 'pip list' --- easybuild/easyblocks/s/scipy.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 1c21a01fc2..4c8d7e0398 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -42,8 +42,8 @@ from easybuild.easyblocks.generic.mesonninja import MesonNinja from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, copy_dir +from easybuild.tools.build_log import EasyBuildError, print_warning +from easybuild.tools.filetools import change_dir, copy_dir, copy_file class EB_scipy(FortranPythonPackage, PythonPackage, MesonNinja): @@ -198,6 +198,18 @@ def install_step(self): """Custom install step for scipy: use ninja for scipy >= 1.9.0""" if self.use_meson: MesonNinja.install_step(self) + + # copy PKG-INFO file included in scipy source tarball to scipy-.egg-info in installation, + # so pip is aware of the scipy installation (required for 'pip list', 'pip check', etc.); + # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2901 + pkg_info = 'PKG-INFO' + target_egg_info = os.path.join(self.installdir, self.pylibdir, 'scipy-%s.egg-info' % self.version) + if os.path.isfile(pkg_info): + copy_file(pkg_info, target_egg_info) + else: + cwd = os.getcwd() + print_warning("%s not found in %s, so can't use it to create %s!", pkg_info, cwd, target_egg_info, + log=self.log) else: FortranPythonPackage.install_step(self) @@ -211,4 +223,9 @@ def sanity_check_step(self, *args, **kwargs): 'dirs': [det_pylibdir()], } - return PythonPackage.sanity_check_step(self, custom_paths=custom_paths) + # make sure that scipy is included in output of 'pip list', + # so that 'pip check' passes if scipy is a required dependency for another Python package; + # use case-insensitive match, since name is sometimes reported as 'SciPy' + custom_commands = [r"pip list | grep -i '^scipy[ \t]*%s[ \t]*$'" % self.version] + + return PythonPackage.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) From bd9e6938e17a56baf61873bea2553c4175e87988 Mon Sep 17 00:00:00 2001 From: Simon Branford <4967+branfosj@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:30:27 +0000 Subject: [PATCH 82/87] correct directory to find PKG-INFO and improve sanity check test Co-authored-by: Jasper <65227842+jfgrimm@users.noreply.github.com> --- easybuild/easyblocks/s/scipy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 4c8d7e0398..823116e1e8 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -202,7 +202,7 @@ def install_step(self): # copy PKG-INFO file included in scipy source tarball to scipy-.egg-info in installation, # so pip is aware of the scipy installation (required for 'pip list', 'pip check', etc.); # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2901 - pkg_info = 'PKG-INFO' + pkg_info = os.path.join(self.cfg['start_dir'], 'PKG-INFO') target_egg_info = os.path.join(self.installdir, self.pylibdir, 'scipy-%s.egg-info' % self.version) if os.path.isfile(pkg_info): copy_file(pkg_info, target_egg_info) @@ -226,6 +226,6 @@ def sanity_check_step(self, *args, **kwargs): # make sure that scipy is included in output of 'pip list', # so that 'pip check' passes if scipy is a required dependency for another Python package; # use case-insensitive match, since name is sometimes reported as 'SciPy' - custom_commands = [r"pip list | grep -i '^scipy[ \t]*%s[ \t]*$'" % self.version] + custom_commands = [r"pip list | grep -iE '^scipy\s+%s\s*$'" % self.version.replace('.', r'\.')] return PythonPackage.sanity_check_step(self, custom_paths=custom_paths, custom_commands=custom_commands) From 42d5b98cb433d718af73e326eab70c7ca3843dc5 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Tue, 14 Mar 2023 08:33:06 +0100 Subject: [PATCH 83/87] Create non-versioned symlink for python-config As discussed here https://easybuild.slack.com/archives/C34UA1HT7/p1678728109206609?thread_ts=1678722749.081689&cid=C34UA1HT7 --- easybuild/easyblocks/p/python.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 465d069a67..f839da581b 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -444,10 +444,13 @@ def install_step(self): super(EB_Python, self).install_step() - # Create non-versioned, relative symlinks for python and pip + # Create non-versioned, relative symlinks for python, python-config and pip python_binary_path = os.path.join(self.installdir, 'bin', 'python') if not os.path.isfile(python_binary_path): symlink('python' + self.pyshortver, python_binary_path, use_abspath_source=False) + python_config_binary_path = os.path.join(self.installdir, 'bin', 'python-config') + if not os.path.isfile(python_config_binary_path): + symlink('python' + self.pyshortver + '-config', python_config_binary_path, use_abspath_source=False) if self.install_pip: pip_binary_path = os.path.join(self.installdir, 'bin', 'pip') if not os.path.isfile(pip_binary_path): From f5649005218c00a2382f7350dec25a5faa7b170d Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Tue, 14 Mar 2023 11:12:37 +0100 Subject: [PATCH 84/87] Add extra sanity checks --- easybuild/easyblocks/p/python.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index f839da581b..09a954d9f0 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -537,7 +537,13 @@ def sanity_check_step(self): pyver = 'python' + self.pyshortver custom_paths = { - 'files': [os.path.join('bin', pyver), os.path.join('lib', 'lib' + pyver + abiflags + '.' + shlib_ext)], + 'files': [ + os.path.join('bin', pyver), + os.path.join('bin', 'python'), + os.path.join('bin', pyver + '-config'), + os.path.join('bin', 'python-config'), + os.path.join('lib', 'lib' + pyver + abiflags + '.' + shlib_ext), + ], 'dirs': [os.path.join('include', pyver + abiflags), os.path.join('lib', pyver, 'lib-dynload')], } @@ -549,6 +555,7 @@ def sanity_check_step(self): "python -c 'import _ctypes'", # make sure that foreign function interface (libffi) works "python -c 'import _ssl'", # make sure SSL support is enabled one way or another "python -c 'import readline'", # make sure readline support was built correctly + "python-config --help", # make sure that symlink was created correctly ] if self.install_pip: From 7b7ad98305fee35ae1a65892e06055219a1cd6b1 Mon Sep 17 00:00:00 2001 From: Viktor Rehnberg <35767167+VRehnberg@users.noreply.github.com> Date: Wed, 15 Mar 2023 08:30:34 +0100 Subject: [PATCH 85/87] Move python-config --help sanity check --- easybuild/easyblocks/p/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/p/python.py b/easybuild/easyblocks/p/python.py index 09a954d9f0..975935cdfb 100644 --- a/easybuild/easyblocks/p/python.py +++ b/easybuild/easyblocks/p/python.py @@ -552,10 +552,10 @@ def sanity_check_step(self): custom_commands = [ "python --version", + "python-config --help", # make sure that symlink was created correctly "python -c 'import _ctypes'", # make sure that foreign function interface (libffi) works "python -c 'import _ssl'", # make sure SSL support is enabled one way or another "python -c 'import readline'", # make sure readline support was built correctly - "python-config --help", # make sure that symlink was created correctly ] if self.install_pip: From 474f4566c10d93e522e8c5b412480ff093d43fd1 Mon Sep 17 00:00:00 2001 From: Sebastian Achilles Date: Thu, 16 Mar 2023 22:08:37 +0100 Subject: [PATCH 86/87] prepare release notes for EasyBuild v4.7.1 + bump version to 4.7.1 --- RELEASE_NOTES | 40 ++++++++++++++++++++++++++++++++ easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index e71ea4acad..c3829c8e86 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -5,6 +5,46 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 39 generic easyblocks. +v4.7.1 (March 20th 2023) +------------------------ + +update/bugfix release + +- minor enhancements and updates, including: + - make NAMD easyblock aware of (pre)testopts (#2856) + - update MesonNinja easyblock for Meson >=0.64.0 (#2861) + - update scipy easyblock for scipy >= 1.9.0 to use meson/ninja (#2862) + - modify logic in QScintilla easyblock to find the PyQt5 sipdir in more places (#2868) + - add `testinstall` custom easyconfig parameter to PythonPackage easyblock (#2872) + - use -x option for "go install" in GoPackage generic easyblock (#2878) + - allow disabling pybind11 tests with '`runtest = False`' (#2892) + - call parent post_install_step in EasyBuildMeta easyblock (so postinstallcmds are taken into account) (#2893) + - update and enhance Maple easyblock for recent versions (#2895) + - relax glob pattern to find Mathematica install script (#2896) + - implement CUDA support in the ELPA EasyBlock & fix CPP configure issue on newer ELPA versions (#2898) + - update Trilinos easyblock to allow disabling of building tests and forward deps + support Trilinos v13.x (#2900) + - enhance Python easyblock to create non-versioned symlink for python-config + check for bin/python and bin/python-config in sanity check (#2904) +- various bug fixes, including: + - do not use -g77 option when installing NVHPC 22.9+ (#2819) + - check that sanity_check_module_loaded attribute exists before querying it in PythonPackage easyblock (#2865) + - fix $JULIA_DEPOT_PATH in installation of multiple JuliaPackage extensions (#2869) + - fix website/docs links in README (#2870) + - fix checking of CUDA/ROCR-Runtime dependencies for Clang to determine default build targets (#2873) + - show template values of exts_default_options in PythonBundle (#2874) + - Fix missing initialization of CMakeMake in CMakePythonPackage (#2876) + - fix error when failing pip version check during PythonPackage sanity check (#2877) + - handle templating correctly in CMakeMake when playing with configopts (#2882) + - avoid crash in test step of PyTorch easyblock if runtest is not a command (#2883) + - fix check configure option in FlexiBLAS easyblock (#2886) + - Use older `ncgen -H` for older netCDF (#2889) + - relax glob pattern to find Mathematica install script (#2896) + - fix linking numexpr with Intel MKL's VML library for imkl >= 2021.x (#2897) + - update scipy easyblock to copy PKG-INFO to scipy-.egg-info when installing with Meson/Ninja + add sanity check command using 'pip list' (#2903) +- other changes: + - only give read permissions in GitHub Actions workflows (#2863) + - use start dir of extension to install R packages (#2867) + - add deprecation notice to RPackage extensions with relative paths in start_dir (#2879) + v4.7.0 (January 9th 2023) ------------------------- diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 9619ef8946..d634d41d22 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.7.1.dev0') +VERSION = LooseVersion('4.7.1') UNKNOWN = 'UNKNOWN' From 9080685cce03807216781dac575a537275feaecc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Mar 2023 08:44:52 +0100 Subject: [PATCH 87/87] minor tweaks to release notes for EasyBuild v4.7.1 --- RELEASE_NOTES | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index c3829c8e86..96346e430c 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -11,13 +11,14 @@ v4.7.1 (March 20th 2023) update/bugfix release - minor enhancements and updates, including: + - fix TensorFlow easyblock for new versions of Bazel & TensorFlow (#2854) - make NAMD easyblock aware of (pre)testopts (#2856) - update MesonNinja easyblock for Meson >=0.64.0 (#2861) - - update scipy easyblock for scipy >= 1.9.0 to use meson/ninja (#2862) + - update scipy easyblock for scipy >= 1.9.0 to use meson/ninja (#2862, #2903) - modify logic in QScintilla easyblock to find the PyQt5 sipdir in more places (#2868) - add `testinstall` custom easyconfig parameter to PythonPackage easyblock (#2872) - - use -x option for "go install" in GoPackage generic easyblock (#2878) - - allow disabling pybind11 tests with '`runtest = False`' (#2892) + - use -x option for "go install" in GoPackage generic easyblock, to print commands as they are executed (#2878) + - allow disabling pybind11 tests with `runtest = False` (#2892) - call parent post_install_step in EasyBuildMeta easyblock (so postinstallcmds are taken into account) (#2893) - update and enhance Maple easyblock for recent versions (#2895) - relax glob pattern to find Mathematica install script (#2896) @@ -28,21 +29,19 @@ update/bugfix release - do not use -g77 option when installing NVHPC 22.9+ (#2819) - check that sanity_check_module_loaded attribute exists before querying it in PythonPackage easyblock (#2865) - fix $JULIA_DEPOT_PATH in installation of multiple JuliaPackage extensions (#2869) - - fix website/docs links in README (#2870) - fix checking of CUDA/ROCR-Runtime dependencies for Clang to determine default build targets (#2873) - show template values of exts_default_options in PythonBundle (#2874) - - Fix missing initialization of CMakeMake in CMakePythonPackage (#2876) + - fix missing initialization of CMakeMake in CMakePythonPackage (#2876) - fix error when failing pip version check during PythonPackage sanity check (#2877) - handle templating correctly in CMakeMake when playing with configopts (#2882) - avoid crash in test step of PyTorch easyblock if runtest is not a command (#2883) - fix check configure option in FlexiBLAS easyblock (#2886) - - Use older `ncgen -H` for older netCDF (#2889) - - relax glob pattern to find Mathematica install script (#2896) + - use older `ncgen -H` for older netCDF (#2889) - fix linking numexpr with Intel MKL's VML library for imkl >= 2021.x (#2897) - - update scipy easyblock to copy PKG-INFO to scipy-.egg-info when installing with Meson/Ninja + add sanity check command using 'pip list' (#2903) - other changes: - only give read permissions in GitHub Actions workflows (#2863) - use start dir of extension to install R packages (#2867) + - fix website/docs links in README (#2870) - add deprecation notice to RPackage extensions with relative paths in start_dir (#2879)