From 110e8f840dd250e6edcc4b211315306f923ed3f5 Mon Sep 17 00:00:00 2001 From: dli-astro Date: Tue, 7 Mar 2023 08:19:05 -0700 Subject: [PATCH 1/7] Do not install pip packages on desi-7. --- conf/pip-pkgs.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/conf/pip-pkgs.sh b/conf/pip-pkgs.sh index d930e42..e79e64b 100644 --- a/conf/pip-pkgs.sh +++ b/conf/pip-pkgs.sh @@ -1,15 +1,17 @@ # Install pip packages. -echo Installing pip packages at $(date) +if [ "$HOSTNAME" != "desi-7" ]; then + echo Installing pip packages at $(date) -pip install --no-binary :all: hpsspy -pip install threadpoolctl + pip install --no-binary :all: hpsspy + pip install threadpoolctl -# see https://docs.nersc.gov/development/languages/python/parallel-python/ -pip install --force --no-cache-dir --no-binary=mpi4py mpi4py + # see https://docs.nersc.gov/development/languages/python/parallel-python/ + pip install --force --no-cache-dir --no-binary=mpi4py mpi4py -if [ $? != 0 ]; then - echo "ERROR installing pip packages; exiting" - exit 1 -fi + if [ $? != 0 ]; then + echo "ERROR installing pip packages; exiting" + exit 1 + fi -echo Current time $(date) Done installing conda packages + echo Current time $(date) Done installing conda packages +fi From 579f4758ffba78761d97b1335ebb920ad81b7f1d Mon Sep 17 00:00:00 2001 From: dli-astro Date: Tue, 7 Mar 2023 10:18:21 -0700 Subject: [PATCH 2/7] Don't install specex and QuasarNP at KPNO. --- scripts/bootstrap-desi.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/bootstrap-desi.sh b/scripts/bootstrap-desi.sh index cbca6bf..31423a1 100644 --- a/scripts/bootstrap-desi.sh +++ b/scripts/bootstrap-desi.sh @@ -13,7 +13,11 @@ pip install git+https://github.com/desihub/desiutil.git if [[ "${NERSC_HOST}" == "datatran" ]]; then pkgs="desiutil desitree desiBackup desidatamodel desitransfer desida" else - pkgs="desiutil desitree desispec specter gpu_specter desimodel desitarget specsim desisim fiberassign desisurvey surveysim redrock redrock-templates prospect desimeter simqso speclite specex QuasarNP" + if [[ "${HOSTNAME}" == "desi-7" ]]; then + pkgs="desiutil desitree desispec specter gpu_specter desimodel desitarget specsim desisim fiberassign desisurvey surveysim redrock redrock-templates prospect desimeter simqso speclite nightwatch" + else + pkgs="desiutil desitree desispec specter gpu_specter desimodel desitarget specsim desisim fiberassign desisurvey surveysim redrock redrock-templates prospect desimeter simqso speclite specex QuasarNP" + fi fi export DESI_SPX_MKL=true base=$(realpath $DESICONDA/..) From 64321228065149b8cb8c6c9aa0fc0d6540fb03a0 Mon Sep 17 00:00:00 2001 From: dli-astro Date: Mon, 20 Mar 2023 09:05:20 -0700 Subject: [PATCH 3/7] Settings specific to desi-7. --- conf/kpno-env.sh | 12 ++++++++++++ conf/kpno-pkgs.sh | 2 ++ 2 files changed, 14 insertions(+) create mode 100644 conf/kpno-env.sh create mode 100644 conf/kpno-pkgs.sh diff --git a/conf/kpno-env.sh b/conf/kpno-env.sh new file mode 100644 index 0000000..dd20ca3 --- /dev/null +++ b/conf/kpno-env.sh @@ -0,0 +1,12 @@ +# export MINICONDA=https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh +# miniforge solves fast and works well with conda-forge +export MINICONDA=https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh +export CONDAVERSION=2.0 +export GRP=datasystems +export CONDAPRGENV=gnu + +export CC="gcc" +export FC="gfortran" +export CFLAGS="-O3 -fPIC -pthread" +export FCFLAGS="-O3 -fPIC -pthread -fexceptions" +export NTMAKE=8 diff --git a/conf/kpno-pkgs.sh b/conf/kpno-pkgs.sh new file mode 100644 index 0000000..8962811 --- /dev/null +++ b/conf/kpno-pkgs.sh @@ -0,0 +1,2 @@ +source $CONFDIR/conda-pkgs.sh + From f8f1123af224eba5278f1368d0419a3133e6f273 Mon Sep 17 00:00:00 2001 From: dli-astro Date: Mon, 20 Mar 2023 09:05:37 -0700 Subject: [PATCH 4/7] Undo guards for desi-7 (offloaded to custom env). --- conf/pip-pkgs.sh | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/conf/pip-pkgs.sh b/conf/pip-pkgs.sh index e79e64b..d930e42 100644 --- a/conf/pip-pkgs.sh +++ b/conf/pip-pkgs.sh @@ -1,17 +1,15 @@ # Install pip packages. -if [ "$HOSTNAME" != "desi-7" ]; then - echo Installing pip packages at $(date) +echo Installing pip packages at $(date) - pip install --no-binary :all: hpsspy - pip install threadpoolctl +pip install --no-binary :all: hpsspy +pip install threadpoolctl - # see https://docs.nersc.gov/development/languages/python/parallel-python/ - pip install --force --no-cache-dir --no-binary=mpi4py mpi4py +# see https://docs.nersc.gov/development/languages/python/parallel-python/ +pip install --force --no-cache-dir --no-binary=mpi4py mpi4py - if [ $? != 0 ]; then - echo "ERROR installing pip packages; exiting" - exit 1 - fi - - echo Current time $(date) Done installing conda packages +if [ $? != 0 ]; then + echo "ERROR installing pip packages; exiting" + exit 1 fi + +echo Current time $(date) Done installing conda packages From 24cab6c6d585893726828688bcc4625d9b8cc218 Mon Sep 17 00:00:00 2001 From: dli-astro Date: Wed, 26 Jul 2023 07:07:58 -0700 Subject: [PATCH 5/7] Use the libmamba package solver. --- install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/install.sh b/install.sh index b8434e3..91dee7b 100755 --- a/install.sh +++ b/install.sh @@ -54,6 +54,11 @@ source $CONDADIR/bin/activate export PYVERSION=$(python -c "import sys; print(str(sys.version_info[0])+'.'+str(sys.version_info[1]))") echo Using Python version $PYVERSION +# Prior to installing packages, switch to the fast libmamba package solver. +echo Installing the libmamba package solver +conda install -n base conda-libmamba-solver +conda config --set solver libmamba + # Install packages source $INSTALLPKGS From 48d923e9a4ce278820fed4b074a312c0ce8dec49 Mon Sep 17 00:00:00 2001 From: dli-astro Date: Thu, 27 Jul 2023 17:17:35 -0700 Subject: [PATCH 6/7] Fix to ensure libmamba solver is recognized. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 91dee7b..6b71566 100755 --- a/install.sh +++ b/install.sh @@ -56,7 +56,8 @@ echo Using Python version $PYVERSION # Prior to installing packages, switch to the fast libmamba package solver. echo Installing the libmamba package solver -conda install -n base conda-libmamba-solver +conda config --set solver classic +conda install -n base conda-libmamba-solver -y conda config --set solver libmamba # Install packages From 962575847c4f7fe1a9f7f895d246c9b9236afdcf Mon Sep 17 00:00:00 2001 From: dli-astro Date: Thu, 27 Jul 2023 17:21:46 -0700 Subject: [PATCH 7/7] Option to install on desi-7 or desi-8. --- scripts/bootstrap-desi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bootstrap-desi.sh b/scripts/bootstrap-desi.sh index 15b911a..133a4fd 100644 --- a/scripts/bootstrap-desi.sh +++ b/scripts/bootstrap-desi.sh @@ -13,7 +13,7 @@ pip install git+https://github.com/desihub/desiutil.git if [[ "${NERSC_HOST}" == "datatran" ]]; then pkgs="desiutil desitree desiBackup desidatamodel desitransfer desida" else - if [[ "${HOSTNAME}" == "desi-7" ]]; then + if [ "${HOSTNAME}" == "desi-7" ] || [ "${HOSTNAME}" == "desi-8" ]; then pkgs="desiutil desitree desispec specter gpu_specter desimodel desitarget specsim desisim fiberassign desisurvey surveysim redrock redrock-templates prospect desimeter simqso speclite nightwatch" else pkgs="desiutil desitree desispec specter gpu_specter desimodel desitarget specsim desisim fiberassign desisurvey surveysim redrock redrock-templates prospect desimeter simqso speclite specex QuasarNP desisim-testdata desisurveyops"