Skip to content

Commit

Permalink
add changes from librosa PR
Browse files Browse the repository at this point in the history
  • Loading branch information
truib committed Jun 4, 2024
1 parent bfdf0ab commit f95918f
Show file tree
Hide file tree
Showing 4 changed files with 170 additions and 0 deletions.
5 changes: 5 additions & 0 deletions EESSI-install-software.sh
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,11 @@ if command_exists "nvidia-smi"; then
${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
fi

# Install extra software that is needed (e.g., for providing a custom ctypes
# library when needed)
echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)"
${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp

# use PR patch file to determine in which easystack files stuff was added
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing')
if [ -z "${changed_easystacks}" ]; then
Expand Down
63 changes: 63 additions & 0 deletions eb_hooks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import easybuild.tools.environment as env
from easybuild.easyblocks.generic.configuremake import obtain_config_guess
from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
from easybuild.tools.build_log import EasyBuildError, print_msg
from easybuild.tools.config import build_option, update_build_option
Expand Down Expand Up @@ -349,6 +350,32 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs):
raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")


def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs):
"""
Add exts_filter to soundfile extension in exts_list
"""
if ec.name == 'librosa' and ec.version in ('0.10.1',):
ec_dict = ec.asdict()
eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes")
ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path
exts_list_new = []
for item in ec_dict['exts_list']:
if item[0] == 'soundfile':
ext_dict = item[2]
ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0],
EXTS_FILTER_PYTHON_PACKAGES[1])
exts_list_new.append((item[0], item[1], ext_dict))
else:
exts_list_new.append(item)
ec['exts_list'] = exts_list_new
print_msg("New exts_list: '%s'", ec['exts_list'])
else:
raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!")


def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs):
"""
Solve issues with compiling or running the tests on both
Expand Down Expand Up @@ -852,13 +879,45 @@ def inject_gpu_property(ec):
return ec


def pre_module_hook(self, *args, **kwargs):
"""Main pre-module-check hook: trigger custom functions based on software name."""
if self.name in PRE_MODULE_HOOKS:
PRE_MODULE_HOOKS[self.name](self, *args, **kwargs)


def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs):
"""
Add EBPYTHONPREFIXES to modluafooter
"""
if self.name == 'librosa' and self.version == '0.10.1':
eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes')
key = 'modluafooter'
values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)]
print_msg("Adding '%s' to modluafooter", values[0])
if not key in self.cfg:
self.cfg[key] = '\n'.join(values)
else:
new_value = self.cfg[key]
for value in values:
if not value in new_value:
new_value = '\n'.join([new_value, value])
self.cfg[key] = new_value
print_msg("Full modluafooter is '%s'", self.cfg[key])
else:
raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!")


PARSE_HOOKS = {
'casacore': parse_hook_casacore_disable_vectorize,
'CGAL': parse_hook_cgal_toolchainopts_precise,
'fontconfig': parse_hook_fontconfig_add_fonts,
'GPAW': parse_hook_gpaw_harcoded_path,
'ImageMagick': parse_hook_imagemagick_add_dependency,
'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64,
'librosa': parse_hook_librosa_custom_ctypes,
'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths,
'pybind11': parse_hook_pybind11_replace_catch2,
Expand Down Expand Up @@ -909,3 +968,7 @@ def inject_gpu_property(ec):
'cuDNN': post_sanitycheck_cudnn,
'cuTENSOR': post_sanitycheck_cutensor,
}

PRE_MODULE_HOOKS = {
'librosa': pre_module_hook_librosa_augment_modluafooter,
}
6 changes: 6 additions & 0 deletions install_scripts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,12 @@ nvidia_files=(
)
copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}"

# Copy files for the scripts/extra directory
extra_files=(
install_custom_ctypes.sh
)
copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}"

# Copy over EasyBuild hooks file used for installations
hook_files=(
eb_hooks.py
Expand Down
96 changes: 96 additions & 0 deletions scripts/extra/install_custom_ctypes.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
#!/usr/bin/env bash

# This script can be used to install custom ctypes under
# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes
# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes
# before loading a module that needs the custom ctypes implementation
# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of
# https://github.com/ComputeCanada/custom_ctypes

# The `host_injections` directory is a variant symlink that by default points to
# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
# installation to be successful, this directory needs to be writeable by the user
# executing this script.

# some logging
echo ">>> Running ${BASH_SOURCE}"

# Initialise our bash functions
TOPDIR=$(dirname $(realpath ${BASH_SOURCE}))
source "${TOPDIR}"/../utils.sh

# Function to display help message
show_help() {
echo "Usage: $0 [OPTIONS]"
echo "Options:"
echo " --help Display this help message"
echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary"
echo " storage during the installation"
}

# Initialize variables
TEMP_DIR=

# Parse command-line options
while [[ $# -gt 0 ]]; do
case "$1" in
--help)
show_help
exit 0
;;
-t|--temp-dir)
if [ -n "$2" ]; then
TEMP_DIR="$2"
shift 2
else
echo "Error: Argument required for $1"
show_help
exit 1
fi
;;
*)
show_help
fatal_error "Error: Unknown option: $1"
;;
esac
done

# Make sure NESSI is initialised
check_eessi_initialised

# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and
# `software` with `extra`
# also append `/custom_ctypes`
NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections}
NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes

# we need a directory we can use for temporary storage
if [[ -z "${TEMP_DIR}" ]]; then
tmpdir=$(mktemp -d)
else
mkdir -p ${TEMP_DIR}
tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX)
if [[ ! -d "$tmpdir" ]] ; then
fatal_error "Could not create directory ${tmpdir}"
fi
fi
echo "Created temporary directory '${tmpdir}'"

# check if custom_ctypes has already been installed
if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then
echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'"
exit 0
fi

# download custom_ctypes to temp directory
wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir}

# make sure target directory exists
mkdir -p ${NESSI_SITE_INSTALL}

# unpack custom_ctypes to target directory
tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL}

# clean up tmpdir
rm -rf "${tmpdir}"

0 comments on commit f95918f

Please sign in to comment.