Upgrade ascent build system and use [email protected] on CI platforms (#20)

* Boilerplate scripts to install modules on Ascent via submodule Spack

* Fix '--nnodes'-->'-nodes' on Ascent

* Improve Ascent env.sh

* [email protected] on Ascent

* Apply pre-commmit fixes

* Relax constraints on exago dependencies on Ascent and build ~python

* concretizer: reuse was causing several packages to be duplicated in the environment. Require clean concretizations on  Ascent.

* Minor module update on Ascent

* Add LAPACK_LIBRARIES to Ascent base script. CMAKE was picking up python's openblas otherwise.

* Error with unzip.

* Apply pre-commmit fixes

* Add working build on ascent.

* Add working gcc11.2.0 spack spec.

* Add Ascent Spack pipeline. [ascent-rebuild]

* Update gcc version to 11.2.0 in base.sh [skip-ci]

* Fix stages of Ascent pipeline [ascent-rebuild]

* Add working ascent spack build.

* Add hiop@develop force rebuild to PNNL CI [ascent-rebuild] [newell-rebuild] [deception-rebuild] [incline-rebuild].

* Update Ascent spack built tcl modules

* Only test ascent on tcl module update [ci-skip]

* Update base.sh to disable python on ascent [skip ci]

* Remove LAPACK_LIBRARIES spec [ascent-test]

* Update ascent.gitlab-ci.yml to fix needs/dependencies [ascent-test]

* Update deception spack built tcl modules - [deception-test]

* Try again with Python, but have Spack build it instead of using the external module [ascent-rebuild]

* Force python rebuild on ascent and use [email protected] on incline [ascent-rebuild] [newell-rebuild] [incline-rebuild]

* Pin [email protected] on all CI platforms [decetpion-rebuild] [ascent-rebuild] [newell-rebuild] [incline-rebuild]

* Fix false positive/negative in Ascent pipelines [deception-rebuild] [ascent-test]

* Update incline spack built tcl modules - [incline-test]

* Update newell spack built tcl modules - [newell-test]

* Fix HiOp spec on Ascent [ascent-rebuild].

* Update deception spack built tcl modules - [deception-test]

* Update CPU Spack build with issue for each failing build [ci skip]

* Update Ascent spack built tcl modules [ascent-test]

* Add 1.0.0 dep into CHANGELOG.

* Add ascent-skip to CI to get tests passing [ascent-test]

---------

Co-authored-by: nkoukpaizan <[email protected]>
Co-authored-by: Cameron Rutherford <[email protected]>
Co-authored-by: cameronrutherford <[email protected]>
Co-authored-by: spack-auto-module <[email protected]>

.github/workflows/spack_cpu_build.yaml

@@ -12,8 +12,8 @@ jobs:
     strategy:
       matrix:
         spack_spec:
-          # Minimal Build
-           # See #18 
+          # Minimal Build(s)
+          # See #39 - ~python~mpi causes issues
         # - exago@develop~mpi~ipopt~hiop~python~raja
           - exago@develop+mpi~ipopt+hiop~python~raja ^openmpi
           # See #18 - +hiop~mpi causes issues

.gitignore

@@ -12,3 +12,5 @@ node_modules
 .env
 package-lock.json
 config.py
+spack_*
+hiop/

.gitlab/ornl/ascent.gitlab-ci.yml

@@ -102,6 +102,7 @@
     - rm -rf "$WORKDIR"
 
 Ascent Build:
+  needs: []
   stage: build
   tags:
     - nobatch
@@ -112,27 +113,71 @@ Ascent Build:
 
 Ascent Test:
   stage: test
-  dependencies:
-    - 'Ascent Build'
+  needs: ['Ascent Build']
   tags:
     - batch
   variables:
     # Don't clone for test jobs
     GIT_STRATEGY: none
-    CTESTARGS: "--output-on-failure"
+    CTESTARGS: "--output-on-failure -LE ascent-skip "
   extends:
     - .ascent_rules
     - .ascent_test_script_template
     - .ascent_environment_template
 
+# -- Spack Module Rebuilding
+
+Ascent-Module-Init:
+  needs: []
+  extends: .ascent_environment_template
+  variables:
+    GIT_SUBMODULE_STRATEGY: recursive
+  rules:
+    - if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
+      when: always
+  stage: build
+  tags:
+    - nobatch
+  script:
+    - set -xv
+    - mkdir -p "$WORKDIR"
+    - cp -r . "$WORKDIR"
+    - cd "$WORKDIR"
+    - MY_CLUSTER=ascent ./buildsystem/spack/binary_mirror.sh
+    - res=$?
+    - exit $res
+
+Ascent-Module-Build:
+  needs: ["Ascent-Module-Init"]
+  rules:
+    - if: '$CI_COMMIT_TITLE =~ /\[ascent-rebuild\]/'
+      when: always
+  extends: .ascent_environment_template
+  stage: test
+  tags:
+    - batch
+  script:
+    - cd "$WORKDIR"
+    - git checkout -b ${CI_COMMIT_REF_NAME}
+    - ./buildsystem/spack/ascent/bsub.sh
+    - git add ./buildsystem/spack/ascent/modules/*
+    - git config user.name "spack-auto-module"
+    - git config user.email "[email protected]"
+    - git commit -m "Update Ascent spack built tcl modules [ascent-test]"
+    - git remote set-url origin https://gitlab-ci-token:${SPACK_GIT_TOKEN}@github.com/pnnl/ExaGO.git
+    - git pull --rebase origin ${CI_COMMIT_REF_NAME}
+    - git push origin ${CI_COMMIT_REF_NAME}
+  after_script:
+    - cd "$WORKDIR/.."
+    - rm -rf "$WORKDIR"
+
 # -- Reporting Ascent Status to PNNL GitHub
 .report-job:
-  when: always
   variables:
     MY_CLUSTER: Ascent
     GIT_STRATEGY: none
     STATUS_PROJECT: pnnl/ExaGO
-    STATUS_NAME: ppc64le/gcc@10.2/cuda@11.4.2/v100@70
+    STATUS_NAME: ppc64le/gcc@11.2/cuda@11.8/v100@70
   tags:
     - nobatch
   extends: .ascent_environment_template

CHANGELOG.md

@@ -8,9 +8,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [develop]
 
 ### General
+- Transition from GitLab to GitHub
+- ExaGO now depends on HiOp 1.0.0. Next release will require HiOp 1.0.1+
 
 ### Build system
 - Add CI on Incline with Spack module pipelines (AMD / HIP GPU machine at PNNL)
+- Add Spack module pipelines to Ascent
 
 ### PS
 

buildsystem/gcc-cuda/ascent/base.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export MY_CLUSTER=ascent
+
+module reset
+
+# Load system modules
+module load gcc/11.2.0
+module load spectrum-mpi/10.4.0.3-20210112
+
+export CC=$(which gcc)
+export CXX=$(which g++)
+export FC=$(which gfortran)
+
+export MY_PETSC_DIR=$PETSC_DIR
+export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_CTEST_LAUNCH_COMMAND='jsrun -g 1 -n 1'"
+export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_ENABLE_PYTHON=OFF"

buildsystem/gcc-cuda/ascentVariables.sh

@@ -1,125 +1,9 @@
-export MY_CLUSTER=ascent
+#!/bin/bash
 
-module purge
+export SRCDIR=${SRCDIR:-$PWD}
 
-module use -a /gpfs/wolf/proj-shared/csc359/src/spack/share/spack/modules/linux-rhel8-power9le
+# Shared system configuration
+source $SRCDIR/buildsystem/gcc-cuda/ascent/base.sh
 
-# Load spack modules
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-blt/0.4.1/gcc-10.2.0-ndtzigt
-# [email protected]%[email protected]+cuda~ipo~rocm~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel8-power9le
-module load exasgd-camp/0.2.2/cuda-11.4.2/gcc-10.2.0-2xxjut5
-# [email protected]%[email protected]~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel8-power9le
-module load exasgd-cmake/3.22.2/gcc-10.2.0-sj55k3d
-# [email protected]%[email protected]+blas arch=linux-rhel8-power9le
-module load exasgd-coinhsl/2015.06.23/gcc-10.2.0-zr37tzv
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-cub/1.16.0/gcc-10.2.0-zeukz4z
-# [email protected]%[email protected]~allow-unsupported-compilers~dev arch=linux-rhel8-power9le
-module load exasgd-cuda/11.4.2/gcc-10.2.0-yf564wn
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-diffutils/3.8/gcc-10.2.0-n2gttub
-# [email protected]_experimental%[email protected]+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_system=cmake build_type=Debug cuda_arch=70 dev_path=/gpfs/wolf/proj-shared/csc359/src/ginkgo arch=linux-rhel8-power9le
-module load exasgd-ginkgo/1.5.0.glu_experimental/cuda-11.4.2/gcc-10.2.0-h4v5snw
-# [email protected]%[email protected] libs=shared,static arch=linux-rhel8-power9le
-module load exasgd-gmp/6.2.1/gcc-10.2.0-6oxkdwz
-# gnuconfig@2021-08-14%[email protected] arch=linux-rhel8-power9le
-module load exasgd-gnuconfig/2021-08-14/gcc-10.2.0-j2pelq3
-# [email protected]%[email protected]~cxx~fortran~hl~ipo~java+mpi+shared~szip~threadsafe+tools api=default build_type=RelWithDebInfo arch=linux-rhel8-power9le
-module load exasgd-hdf5/1.12.2/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-yh642ea
-# hiop@develop%[email protected]+cuda+cusolver+deepchecking+ginkgo~ipo~jsrun+kron+mpi+raja~rocm~shared+sparse build_system=cmake build_type=RelWithDebInfo cuda_arch=70 dev_path=/gpfs/wolf/proj-shared/csc359/src/nicholson-hiop arch=linux-rhel8-power9le
-module load exasgd-hiop/develop/cuda-11.4.2/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-ucuclxs
-# [email protected]%[email protected]~complex~cuda~debug+fortran~gptune~int64~internal-superlu~mixedint+mpi~openmp~rocm+shared~superlu-dist~unified-memory arch=linux-rhel8-power9le
-module load exasgd-hypre/2.24.0/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-oybqs3s
-# [email protected]%[email protected]+coinhsl~debug~metis~mumps build_system=autotools arch=linux-rhel8-power9le
-module load exasgd-ipopt/3.12.10/gcc-10.2.0-i4gqiqg
-# [email protected]%[email protected] libs=shared,static arch=linux-rhel8-power9le
-module load exasgd-libiconv/1.16/gcc-10.2.0-cpn6euq
-# [email protected]%[email protected]+cuda+fortran~ipo~rocm+shared build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel8-power9le
-module load exasgd-magma/2.6.2/cuda-11.4.2/gcc-10.2.0-ffujy7o
-# [email protected]%[email protected]~gdb~int64~real64+shared build_type=Release patches=4991da9,b1225da arch=linux-rhel8-power9le
-module load exasgd-metis/5.1.0/gcc-10.2.0-kdt6x4i
-# [email protected]%[email protected] libs=shared,static arch=linux-rhel8-power9le
-module load exasgd-mpfr/4.1.0/gcc-10.2.0-ua5zak3
-# [email protected]%[email protected]~bignuma~consistent_fpcsr~ilp64+locking+pic+shared symbol_suffix=none threads=none arch=linux-rhel8-power9le
-module load exasgd-openblas/0.3.20/gcc-10.2.0-lkjyfws
-# [email protected]%[email protected]~gdb~int64~ipo+shared build_type=RelWithDebInfo patches=4f89253,50ed208,704b84f arch=linux-rhel8-power9le
-module load exasgd-parmetis/4.0.3/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-gqz32yk
-# [email protected]%[email protected]~X~batch~cgns~complex~cuda~debug+double~exodusii~fftw+fortran~giflib+hdf5~hpddm~hwloc+hypre~int64~jpeg~knl~kokkos~libpng~libyaml~memkind+metis~mkl-pardiso~mmg~moab~mpfr+mpi~mumps~openmp~p4est~parmmg~ptscotch~random123~rocm~saws~scalapack+shared~strumpack~suite-sparse+superlu-dist~tetgen~trilinos~valgrind build_system=generic clanguage=C arch=linux-rhel8-power9le
-module load exasgd-petsc/3.18.1/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-cwok3k4
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-pkgconf/1.8.0/gcc-10.2.0-nngfigd
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-attrs/21.4.0/gcc-10.2.0-av2tzhn
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-flit-core/3.6.0/gcc-10.2.0-wdoqgro
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-importlib-metadata/4.8.2/gcc-10.2.0-klg3xix
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-iniconfig/1.1.1/gcc-10.2.0-o6bmfak
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-mpi4py/3.1.2/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-q53tcvd
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-packaging/21.3/gcc-10.2.0-ej3e6zm
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-pip/21.3.1/gcc-10.2.0-awkoczv
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-pluggy/1.0.0/gcc-10.2.0-tlqdjwe
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-py/1.11.0/gcc-10.2.0-fwbtxbd
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-pyparsing/3.0.6/gcc-10.2.0-qxfb6l5
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-pytest/6.2.5/gcc-10.2.0-2vv5hdj
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-setuptools/59.4.0/gcc-10.2.0-elx3acw
-# [email protected]%[email protected]+toml arch=linux-rhel8-power9le
-module load exasgd-py-setuptools-scm/6.3.2/gcc-10.2.0-xdvaqbm
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-toml/0.10.2/gcc-10.2.0-6lhs4ix
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-tomli/1.2.2/gcc-10.2.0-4glf4eg
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-typing-extensions/4.1.1/gcc-10.2.0-qg6k6g2
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-wheel/0.37.0/gcc-10.2.0-tg3twdl
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-py-zipp/3.6.0/gcc-10.2.0-4vxciug
-# [email protected]%[email protected]+bz2+ctypes+dbm~debug+ensurepip+libxml2+lzma~nis~optimizations+pic+pyexpat+pythoncmd+readline+shared+sqlite3+ssl~tix~tkinter~ucs4+uuid+zlib patches=c129e34 arch=linux-rhel8-power9le
-module load exasgd-python/3.6.8/gcc-10.2.0-273robh
-# [email protected]%[email protected]+cuda~examples~exercises~ipo+openmp~rocm+shared~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel8-power9le
-module load exasgd-raja/0.14.0/cuda-11.4.2/gcc-10.2.0-hkkig4v
-# [email protected]%[email protected] arch=linux-rhel8-power9le
-module load exasgd-spectrum-mpi/10.4.0.3-20210112/gcc-10.2.0-c5uqeqr
-# [email protected]%[email protected]~cuda~graphblas~openmp+pic~tbb arch=linux-rhel8-power9le
-module load exasgd-suite-sparse/5.10.1/gcc-10.2.0-sq5isrj
-# [email protected]%[email protected]~cuda~int64~ipo~openmp~rocm+shared build_type=RelWithDebInfo patches=8da9e21 arch=linux-rhel8-power9le
-module load exasgd-superlu-dist/7.2.0/spectrum-mpi-10.4.0.3-20210112/gcc-10.2.0-7cwmhkg
-# [email protected]%[email protected]~c+cuda~device_alloc~deviceconst~examples~fortran~ipo~numa~openmp~rocm~shared build_type=RelWithDebInfo cuda_arch=70 tests=none arch=linux-rhel8-power9le
-module load exasgd-umpire/6.0.0/cuda-11.4.2/gcc-10.2.0-selkipw
-# [email protected]%[email protected]+optimize+pic+shared patches=0d38234 arch=linux-rhel8-power9le
-module load exasgd-zlib/1.2.12/gcc-10.2.0-lfmc24z
-
-# Load system modules
-module load cuda/11.4.2
-module load gcc/10.2.0
-module load spectrum-mpi/10.4.0.3-20210112
-module load cmake/3.22.2
-
-export MY_PETSC_DIR=$PETSC_DIR
-export EXTRA_CMAKE_ARGS="$EXTRA_CMAKE_ARGS -DEXAGO_CTEST_LAUNCH_COMMAND='jsrun -g 1 -n 1'"
-
-[ -f $PWD/nvblas.conf ] && rm $PWD/nvblas.conf
-cat > $PWD/nvblas.conf <<-EOD
-NVBLAS_LOGFILE  nvblas.log
-NVBLAS_CPU_BLAS_LIB $OPENBLAS_LIBRARY_DIR/libopenblas.so
-NVBLAS_GPU_LIST ALL
-NVBLAS_TILE_DIM 2048
-NVBLAS_AUTOPIN_MEM_ENABLED
-EOD
-export NVBLAS_CONFIG_FILE=$PWD/nvblas.conf
-echo "Generated $PWD/nvblas.conf"
-
-export CC=$(which gcc)
-export CXX=$(which g++)
-export FC=$(which gfortran)
+# Spack modules
+source $SRCDIR/buildsystem/spack/$MY_CLUSTER/modules/dependencies.sh