From f95918f635d1c4701469e5d485cf71d03f5f2e86 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 5 Jun 2024 00:38:55 +0200 Subject: [PATCH] add changes from librosa PR --- EESSI-install-software.sh | 5 ++ eb_hooks.py | 63 +++++++++++++++++ install_scripts.sh | 6 ++ scripts/extra/install_custom_ctypes.sh | 96 ++++++++++++++++++++++++++ 4 files changed, 170 insertions(+) create mode 100755 scripts/extra/install_custom_ctypes.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index ca6fed71d5..7d6cad7add 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -256,6 +256,11 @@ if command_exists "nvidia-smi"; then ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh fi +# Install extra software that is needed (e.g., for providing a custom ctypes +# library when needed) +echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)" +${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then diff --git a/eb_hooks.py b/eb_hooks.py index e4a957acdc..14c7c30d91 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -5,6 +5,7 @@ import easybuild.tools.environment as env from easybuild.easyblocks.generic.configuremake import obtain_config_guess +from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option @@ -349,6 +350,32 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): + """ + Add exts_filter to soundfile extension in exts_list + """ + if ec.name == 'librosa' and ec.version in ('0.10.1',): + ec_dict = ec.asdict() + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) + custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) + custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes") + ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path + exts_list_new = [] + for item in ec_dict['exts_list']: + if item[0] == 'soundfile': + ext_dict = item[2] + ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0], + EXTS_FILTER_PYTHON_PACKAGES[1]) + exts_list_new.append((item[0], item[1], ext_dict)) + else: + exts_list_new.append(item) + ec['exts_list'] = exts_list_new + print_msg("New exts_list: '%s'", ec['exts_list']) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both @@ -852,6 +879,37 @@ def inject_gpu_property(ec): return ec +def pre_module_hook(self, *args, **kwargs): + """Main pre-module-check hook: trigger custom functions based on software name.""" + if self.name in PRE_MODULE_HOOKS: + PRE_MODULE_HOOKS[self.name](self, *args, **kwargs) + + +def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): + """ + Add EBPYTHONPREFIXES to modluafooter + """ + if self.name == 'librosa' and self.version == '0.10.1': + eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') + custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) + custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) + custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes') + key = 'modluafooter' + values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] + print_msg("Adding '%s' to modluafooter", values[0]) + if not key in self.cfg: + self.cfg[key] = '\n'.join(values) + else: + new_value = self.cfg[key] + for value in values: + if not value in new_value: + new_value = '\n'.join([new_value, value]) + self.cfg[key] = new_value + print_msg("Full modluafooter is '%s'", self.cfg[key]) + else: + raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!") + + PARSE_HOOKS = { 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, @@ -859,6 +917,7 @@ def inject_gpu_property(ec): 'GPAW': parse_hook_gpaw_harcoded_path, 'ImageMagick': parse_hook_imagemagick_add_dependency, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'librosa': parse_hook_librosa_custom_ctypes, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths, 'pybind11': parse_hook_pybind11_replace_catch2, @@ -909,3 +968,7 @@ def inject_gpu_property(ec): 'cuDNN': post_sanitycheck_cudnn, 'cuTENSOR': post_sanitycheck_cutensor, } + +PRE_MODULE_HOOKS = { + 'librosa': pre_module_hook_librosa_augment_modluafooter, +} diff --git a/install_scripts.sh b/install_scripts.sh index 07643a39e6..000ad99444 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -116,6 +116,12 @@ nvidia_files=( ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" +# Copy files for the scripts/extra directory +extra_files=( + install_custom_ctypes.sh +) +copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}" + # Copy over EasyBuild hooks file used for installations hook_files=( eb_hooks.py diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh new file mode 100755 index 0000000000..6392da00e5 --- /dev/null +++ b/scripts/extra/install_custom_ctypes.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +# This script can be used to install custom ctypes under +# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes +# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes +# before loading a module that needs the custom ctypes implementation +# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of +# https://github.com/ComputeCanada/custom_ctypes + +# The `host_injections` directory is a variant symlink that by default points to +# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see +# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the +# installation to be successful, this directory needs to be writeable by the user +# executing this script. + +# some logging +echo ">>> Running ${BASH_SOURCE}" + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) +source "${TOPDIR}"/../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation" +} + +# Initialize variables +TEMP_DIR= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and +# `software` with `extra` +# also append `/custom_ctypes` +NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} +NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" + +# check if custom_ctypes has already been installed +if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then + echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" + exit 0 +fi + +# download custom_ctypes to temp directory +wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir} + +# make sure target directory exists +mkdir -p ${NESSI_SITE_INSTALL} + +# unpack custom_ctypes to target directory +tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL} + +# clean up tmpdir +rm -rf "${tmpdir}"