From 0aa6defa423a49d9c9aa52b2785f10abbc91368d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 8 Jul 2024 09:33:06 +0000 Subject: [PATCH 1/3] support for multiple CVMFS repositories in eessi_container.sh --- .../workflows/test_eessi_container_script.yml | 14 +- bot/build.sh | 2 +- bot/test.sh | 2 +- create_tarball.sh | 3 +- eessi_container.sh | 431 ++++++++++++------ 5 files changed, 302 insertions(+), 150 deletions(-) mode change 100755 => 100644 eessi_container.sh diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 347ac12bf0..c1d79fc3b5 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -45,9 +45,10 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} - - # test use of --list-repos with custom repos.cfg + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} + + # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then outfile=out_listrepos.txt outfile2=out_listrepos_2.txt @@ -57,11 +58,12 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} - + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} + export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} - grep "[EESSI/2023.02]" ${outfile2} + grep "EESSI/20AB.CD" ${outfile2} # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then diff --git a/bot/build.sh b/bot/build.sh index de73faef0b..222d8d8ab5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -138,7 +138,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/bot/test.sh b/bot/test.sh index d48fd74734..d3cd9a27ab 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -160,7 +160,7 @@ echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE} # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/create_tarball.sh b/create_tarball.sh index 0a7669f73f..2dee665060 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,7 +26,8 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +cvmfs_repo_name=${cvmfs_repo#/cvmfs/} +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then diff --git a/eessi_container.sh b/eessi_container.sh old mode 100755 new mode 100644 index 6495ee5401..f6410fcb80 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,7 +70,8 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -a | --access {ro,rw} - sets access globally for all CVMFS repositories:" + echo " ro (read-only), rw (read & write) [default: ro]" echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." echo " To specify multiple bind paths, separate by comma." echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" @@ -93,9 +94,12 @@ display_help() { echo " fuse-overlayfs will merge all lower directories;" echo " the option can be used to make certain directories" echo " in the CVMFS repo writable [default: none]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI via" - echo " default container, see --container]" + echo " -r | --repository CFG - configuration file or identifier defining thie" + echo " repository to use; can be given multiple times;" + echo " CFG may include a suffix ',access={ro,rw}' to" + echo " overwrite the global access mode for this repository" + echo " [default: software.eessi.io via CVMFS config available" + echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -129,7 +133,7 @@ LIST_REPOS=0 LOWER_DIRS= MODE="shell" SETUP_NVIDIA=0 -REPOSITORY="EESSI" +REPOSITORIES=() RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -189,9 +193,9 @@ while [[ $# -gt 0 ]]; do -o|--lower-dirs) LOWER_DIRS="$2" shift 2 - ;; - -r|--repository) - REPOSITORY="$2" + ;; + -r|--repository) + REPOSITORIES+=("$2") shift 2 ;; -s|--save) @@ -234,22 +238,53 @@ done set -- "${POSITIONAL_ARGS[@]}" +# define a list of CVMFS repositories that are accessible via the +# CVMFS config repository which is always mounted +# TODO instead of hard-coding the 'extra' and 'default' repositories here one +# could have another script in the GitHub and/or CVMFS repository which +# provides this "configuration" +declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") +eessi_default_cvmfs_repo="software.eessi.io,access=${ACCESS}" + +# define a list of CVMFS repositories that are accessible via the +# configuration file provided via $EESSI_REPOS_CFG_FILE +declare -A cfg_cvmfs_repos=() +if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + cfg_cvmfs_repos[${repo_id}]=${EESSI_REPOS_CFG_FILE} + done <<< "${sections}" +fi + if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Listing available repositories with format 'name [source]':" - echo " EESSI [default]" - if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" - done <<< "${sections}" - fi + echo "Listing available repositories with format 'name [source[, 'default']]'." + echo "Note, without argument '--repository' the one labeled 'default' will be mounted." + for cvmfs_repo in "${!eessi_cvmfs_repos[@]}" + do + if [[ ${eessi_cvmfs_repos[${cvmfs_repo}]} == "default" ]] ; then + default_label=", default" + else + default_label="" + fi + echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" + done + for cfg_repo in "${!cfg_cvmfs_repos[@]}" + do + echo " ${cfg_repo} [${cfg_cvmfs_repos[$cfg_repo]}]" + done exit 0 fi +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported +# use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -273,11 +308,47 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then fi fi -# TODO (arg -r|--repository) check if repository is known +# TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then - fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" -fi +# iterate over entries in REPOSITORIES and check if they are known +for cvmfs_repo in "${REPOSITORIES[@]}" +do + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + else + cvmfs_repo_name="${cvmfs_repo}" + fi + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + fatal_error "The repository '${cvmfs_repo_name}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi +done + +# make sure each repository is only listed once +declare -A listed_repos=() +for cvmfs_repo in "${REPOSITORIES[@]}" +do + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode + [[ ${VERBOSE} -eq 1 ]] && echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos + # and obtain actual repo_name from config + cfg_repo_id='' + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi + if [[ -n "${listed_repos[${cvmfs_repo_name}]}" ]] ; then + via_cfg="" + if [[ -n "${cfg_repo_id}" ]] ; then + via_cfg=" (via repository ID '${cfg_repo_id}')" + fi + fatal_error "CVMFS repository '${cvmfs_repo_name}'${via_cfg} listed multiple times" + fi + listed_repos+=([${cvmfs_repo_name}]=true) +done # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -353,19 +424,30 @@ fi # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} -# |-overlay-upper -# |-overlay-work # |-home # |-repos_cfg -# |-opt-eessi (unless otherwise specificed for host_injections) +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} +# |-CVMFS_REPO_1 +# | |-repo_settings.sh (name, id, access, host_injections) +# | |-overlay-upper +# | |-overlay-work +# | |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_n +# |-repo_settings.sh (name, id, access, host_injections) +# |-overlay-upper +# |-overlay-work +# |-opt-eessi (unless otherwise specificed for host_injections) # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +# TODO make this specific to repository? +# TODO move this code to when we already know which repositories we want to access +# actually we should know this already here, but we should rather move this to +# where repository args are being processed # Set host_injections directory and ensure it is a writable directory (if user provided) if [ -z ${USER_HOST_INJECTIONS+x} ]; then # Not set, so use our default @@ -457,6 +539,7 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" + # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then @@ -502,91 +585,111 @@ fi # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI" ]]; then - # need to source defaults as late as possible (see other sourcing below) - source ${TOPDIR}/init/eessi_defaults - - # strip "/cvmfs/" from default setting - repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} -else - # TODO implement more flexible specification of repo cfgs - # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR - # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) - # - # for now, assuming repo-id is defined in config file pointed to - # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory - # (could also become part of the software layer to define multiple - # standard EESSI repositories) - cfg_load ${EESSI_REPOS_CFG_FILE} - - # copy repos.cfg to job directory --> makes it easier to inspect the job - cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - - # cfg file should include: repo_name, repo_version, config_bundle, - # map { local_filepath -> container_filepath } - # - # repo_name_domain is the domain part of the repo_name, e.g., - # eessi.io for software.eessi.io - # - # where config bundle includes the files (-> target location in container) - # - default.local -> /etc/cvmfs/default.local - # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... - # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf - # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), - # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... - # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} - # a directory that contains the public key to access the repository, key - # itself then doesn't need to be BIND mounted - # - ${repo_name_domain}/${repo_name}.pub - # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub - # the public key to access the repository, key itself is BIND mounted - # via directory ${repo_name_domain} - repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") - # derive domain part from repo_name (everything after first '.') - repo_name_domain=${repo_name#*.} - repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") - config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") - config_map=$(cfg_get_value ${REPOSITORY} "config_map") - - # convert config_map into associative array cfg_file_map - cfg_init_file_map "${config_map}" - [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - - # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - # if config_bundle is relative path (no '/' at start) prepend it with - # EESSI_REPOS_CFG_DIR - config_bundle_path= - if [[ ! "${config_bundle}" =~ ^/ ]]; then - config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} - else - config_bundle_path=${config_bundle} - fi +[[ ${VERBOSE} -eq 1 ]] && echo +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo +# iterate over repositories in array REPOSITORIES +for cvmfs_repo in "${REPOSITORIES[@]}" +do + [[ ${VERBOSE} -eq 1 ]] && echo "process CVMFS repo spec '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # if cvmfs_repo_name is in cfg_cvmfs_repos, it is a "repository ID" and was + # derived from information in EESSI_REPOS_CFG_FILE, namely the section + # names in that .ini-type file + # in the if-block below, we'll use cfg_repo_id to refer to that ID + # we need to process/provide the config from EESSI_REPOS_CFG_FILE, such + # that the necessary information for accessing a CVMFS repository is made + # available inside the container + if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + cfg_repo_id=${cvmfs_repo_name} + + # obtain CVMFS repository name from section for the given ID + cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + # derive domain part from (cfg_)repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + + # cfg_cvmfs_repos is populated through reading the file pointed to by + # EESSI_REPOS_CFG_FILE. We need to copy that file and data it needs + # into the job's working directory. + + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } + # + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${cfg_repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${cfg_repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + cfg_repo_version=$(cfg_get_value ${cfg_repo_id} "repo_version") + cfg_config_bundle=$(cfg_get_value ${cfg_repo_id} "config_bundle") + cfg_config_map=$(cfg_get_value ${cfg_repo_id} "config_map") + + # convert cfg_config_map into associative array cfg_file_map + cfg_init_file_map "${cfg_config_map}" + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map + + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define + # BIND mounts + # check if config_bundle exists, if so, unpack it into + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${cfg_config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${cfg_config_bundle} + else + config_bundle_path=${cfg_config_bundle} + fi - if [[ ! -r ${config_bundle_path} ]]; then - fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} - fi + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi - # only unpack config_bundle if we're not resuming from a previous run - if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg - fi + # only unpack cfg_config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg + fi - for src in "${!cfg_file_map[@]}" - do - target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" - done - export EESSI_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" - # need to source defaults as late as possible (after *_OVERRIDEs) - source ${TOPDIR}/init/eessi_defaults -fi + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + # if target is alreay BIND mounted, exit with an error + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} + fi + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + done + fi + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo +done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to -# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is @@ -604,52 +707,98 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then - export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + src=${EESSI_TMPDIR}/repos_cfg/default.local + target=/etc/cvmfs/default.local + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi + BIND_PATHS="${BIND_PATHS},${src}:${target}" fi # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() -# always mount cvmfs-config repo (to get access to software.eessi.io) +# always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) # Commented out intentionally EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") -if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS -fi - -if [[ "${ACCESS}" == "rw" ]]; then - mkdir -p ${EESSI_TMPDIR}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/overlay-work - - # set environment variables for fuse mounts in Singularity container - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" - - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - - EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - if [[ ! -z ${LOWER_DIRS} ]]; then - # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as - # separator while the lowerdir overlayfs option uses ':' - export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${LOWER_DIRS}:/cvmfs_ro/${repo_name}" - else - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - fi - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_WRITABLE_OVERLAY +# iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) +for cvmfs_repo in "${REPOSITORIES[@]}" +do + unset cfg_repo_id + [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # obtain cvmfs_repo_name from EESSI_REPOS_CFG_FILE if cvmfs_repo is in cfg_cvmfs_repos + if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") - export EESSI_FUSE_MOUNTS -fi + # always create a directory for the repository (e.g., to store settings, ...) + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} + + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) + if [[ ${cvmfs_repo_access} == "ro" ]] ; then + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + elif [[ ${cvmfs_repo_access} == "rw" ]] ; then + # use repo-specific overlay directories + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" + + # set environment variables for fuse mounts in Singularity container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" + + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + if [[ ! -z ${LOWER_DIRS} ]]; then + # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as + # separator while the lowerdir overlayfs option uses ':' + export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${LOWER_DIRS}:/cvmfs_ro/${repo_name}" + else + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + fi + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" + exit ${REPOSITORY_ERROR_EXITCODE} + fi + # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store + # (intention is that the file could be just sourced to obtain the settings) + # repo_name = ${cvmfs_repo_name} + # repo_id = ${cfg_repo_id} # empty if not an EESSI repo + # repo_access = ${cvmfs_repo_access} + # repo_host_injections = [ {"src_path":"target_path"}... ] # TODO + settings= + #[[ -n ${cfg_repo_id} ]] && settings="[${cvmfs_repo_name}]\n" || settings="[${cfg_repo_id}]\n" + settings="${settings}repo_name = ${cvmfs_repo_name}\n" + settings="${settings}repo_id = ${cfg_repo_id}\n" + settings="${settings}repo_access = ${cvmfs_repo_access}\n" + # TODO iterate over host_injections (first need means to define them (globally and/or per repository) + # settings="${settings}repo_host_injections = ${host_injections}\n" + echo -e "${settings}" > ${EESSI_TMPDIR}/${cvmfs_repo_name}/repo_settings.sh +done # 5. run container # final settings From b956a855e991ea07b50d753566dc6b2eb02d2bed Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 8 Jul 2024 09:38:29 +0000 Subject: [PATCH 2/3] support for multiple CVMFS repositories in eessi_container.sh --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index f6410fcb80..6ba9a0d9f4 100644 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -194,7 +194,7 @@ while [[ $# -gt 0 ]]; do LOWER_DIRS="$2" shift 2 ;; - -r|--repository) + -r|--repository) REPOSITORIES+=("$2") shift 2 ;; From 3f254a251517dc370390b6d3da052ca131cacb43 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 8 Jul 2024 09:49:07 +0000 Subject: [PATCH 3/3] support for multiple CVMFS repositories in eessi_container.sh --- eessi_container.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 eessi_container.sh diff --git a/eessi_container.sh b/eessi_container.sh old mode 100644 new mode 100755