From f6ee972f2a8d1c38fd23fc93fe45415793a85513 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 10 Jan 2024 17:43:10 +0100 Subject: [PATCH 1/5] {2023.06}[2022b] SciPy-bundle v2023.02 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..fdacd95c55 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - SciPy-bundle-2023.02-gfbf-2022b.eb From b55efbf4d6d066dd292240e3863c8028c6336098 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 10 Jan 2024 18:41:52 +0100 Subject: [PATCH 2/5] add pre_single_extension_scipy hook to replace -mcpu=native with -march=armv8.4-a when building scipy 1.10.1 on aarch64/neoverse_v1 --- eb_hooks.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 73eac9286d..883d79c6e8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -384,16 +384,6 @@ def pre_single_extension_hook(ext, *args, **kwargs): PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) -def pre_single_extension_testthat(ext, *args, **kwargs): - """ - Pre-extension hook for testthat R package, to fix build on top of recent glibc. - """ - if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): - # use constant value instead of SIGSTKSZ for stack size, - # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 - ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " - - def pre_single_extension_isoband(ext, *args, **kwargs): """ Pre-extension hook for isoband R package, to fix build on top of recent glibc. @@ -404,6 +394,33 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " +def pre_single_extension_scipy(ext, *args, **kwargs): + """ + Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for + aarch64/neoverse_v1 CPU target. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1: + cflags = os.getenv('CFLAGS') + if '-mcpu=native' in cflags: + cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a') + ext.cfg.update('configopts', ' '.join([ + "-Dc_args='%s'" % cflags, + "-Dcpp_args='%s'" % cflags, + "-Dfortran_args='%s'" % cflags, + ])) + + +def pre_single_extension_testthat(ext, *args, **kwargs): + """ + Pre-extension hook for testthat R package, to fix build on top of recent glibc. + """ + if ext.name == 'testthat' and LooseVersion(ext.version) < LooseVersion('3.1.0'): + # use constant value instead of SIGSTKSZ for stack size, + # cfr. https://github.com/r-lib/testthat/issues/1373 + https://github.com/r-lib/testthat/pull/1403 + ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " + + def post_sanitycheck_hook(self, *args, **kwargs): """Main post-sanity-check hook: trigger custom functions based on software name.""" if self.name in POST_SANITYCHECK_HOOKS: @@ -531,6 +548,7 @@ def inject_gpu_property(ec): PRE_SINGLE_EXTENSION_HOOKS = { 'isoband': pre_single_extension_isoband, + 'scipy': pre_single_extension_scipy, 'testthat': pre_single_extension_testthat, } From da6dc24615a7892dbea809bae173db588f2140a5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 12 Jan 2024 18:34:08 +0100 Subject: [PATCH 3/5] build numpy with -march=armv8.4-a instead of -mcpu=native (instead of scipy) --- eb_hooks.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 883d79c6e8..9d84a9e9f8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -394,21 +394,19 @@ def pre_single_extension_isoband(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' src/testthat/vendor/catch.h && " -def pre_single_extension_scipy(ext, *args, **kwargs): +def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for scipy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if ext.name == 'scipy' and ext.version == '1.10.1' and cpu_target == CPU_TARGET_NEOVERSE_V1: - cflags = os.getenv('CFLAGS') - if '-mcpu=native' in cflags: - cflags = cflags.replace('-mcpu=native', '-march=armv8.4-a') - ext.cfg.update('configopts', ' '.join([ - "-Dc_args='%s'" % cflags, - "-Dcpp_args='%s'" % cflags, - "-Dfortran_args='%s'" % cflags, - ])) + if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + # unsure which of these actually matter for numpy, so changing all of them + for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'): + value = os.getenv(envvar) + if '-mcpu=native' in value: + value = value.replace('-mcpu=native', '-march=armv8.4-a') + env.setvar(envvar, value) def pre_single_extension_testthat(ext, *args, **kwargs): @@ -548,7 +546,7 @@ def inject_gpu_property(ec): PRE_SINGLE_EXTENSION_HOOKS = { 'isoband': pre_single_extension_isoband, - 'scipy': pre_single_extension_scipy, + 'numpy': pre_single_extension_numpy, 'testthat': pre_single_extension_testthat, } From 9943e357127f3ecea98517b1c78c09929710626d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Sat, 13 Jan 2024 18:23:00 +0100 Subject: [PATCH 4/5] update optarch build option to make sure that -march=armv8.4-a is used instead of -mcpu=native for numpy, reset it after installing numpy, and allow 2 failing scipy tests in SciPy-bundle v2023.02 --- eb_hooks.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9d84a9e9f8..5e6ffef823 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -367,23 +367,30 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): FAILED optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 - A... FAILED optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 = 2 failed, 30554 passed, 2064 skipped, 10992 deselected, 76 xfailed, 7 xpassed, 40 warnings in 380.27s (0:06:20) = - In versions 2023.07 and 2023.11, 2 failing tests in scipy 1.11.1 and 1.11.4: + In versions 2023.02, 2023.07, and 2023.11, 2 failing tests in scipy (versions 1.10.1, 1.11.1, 1.11.4): FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 = 2 failed, 54409 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 892.04s (0:14:52) = In previous versions we were not as strict yet on the numpy/SciPy tests """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if self.name == 'SciPy-bundle' and self.version in ['2021.10', '2023.07', '2023.11'] and cpu_target == CPU_TARGET_NEOVERSE_V1: + scipy_bundle_versions = ('2021.10', '2023.02', '2023.07', '2023.11') + if self.name == 'SciPy-bundle' and self.version in scipy_bundle_versions and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_single_extension_hook(ext, *args, **kwargs): - """Main pre-configure hook: trigger custom functions based on software name.""" + """Main pre-extension: trigger custom functions based on software name.""" if ext.name in PRE_SINGLE_EXTENSION_HOOKS: PRE_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) +def post_single_extension_hook(ext, *args, **kwargs): + """Main post-extension hook: trigger custom functions based on software name.""" + if ext.name in POST_SINGLE_EXTENSION_HOOKS: + POST_SINGLE_EXTENSION_HOOKS[ext.name](ext, *args, **kwargs) + + def pre_single_extension_isoband(ext, *args, **kwargs): """ Pre-extension hook for isoband R package, to fix build on top of recent glibc. @@ -396,17 +403,25 @@ def pre_single_extension_isoband(ext, *args, **kwargs): def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x when buidling for - aarch64/neoverse_v1 CPU target. + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x + when building for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + # note: this hook is called before build environment is set up (by calling toolchain.prepare()), + # so environment variables like $CFLAGS are not defined yet # unsure which of these actually matter for numpy, so changing all of them - for envvar in ('CFLAGS', 'CXXFLAGS', 'F90FLAGS', 'FFLAGS'): - value = os.getenv(envvar) - if '-mcpu=native' in value: - value = value.replace('-mcpu=native', '-march=armv8.4-a') - env.setvar(envvar, value) + ext.orig_optarch = build_option('optarch') + update_build_option('optarch', 'march=armv8.4-a') + + +def post_single_extension_numpy(ext, *args, **kwargs): + """ + Post-extension hook for numpy, to reset 'optarch' build option. + """ + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if ext.name == 'numpy' and ext.version == '1.24.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: + update_build_option('optarch', ext.orig_optarch) def pre_single_extension_testthat(ext, *args, **kwargs): @@ -550,6 +565,10 @@ def inject_gpu_property(ec): 'testthat': pre_single_extension_testthat, } +POST_SINGLE_EXTENSION_HOOKS = { + 'numpy': post_single_extension_numpy, +} + POST_SANITYCHECK_HOOKS = { 'CUDA': post_sanitycheck_cuda, } From 937efa6d8520e5af5f8292a21cbd0d3a9c5a8344 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 15 Jan 2024 18:06:00 +0100 Subject: [PATCH 5/5] fix docstring in pre_single_extension_numpy hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 5e6ffef823..7c52e24ff1 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -403,7 +403,7 @@ def pre_single_extension_isoband(ext, *args, **kwargs): def pre_single_extension_numpy(ext, *args, **kwargs): """ - Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for scipy 1.10.x + Pre-extension hook for numpy, to change -march=native to -march=armv8.4-a for numpy 1.24.2 when building for aarch64/neoverse_v1 CPU target. """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')