diff --git a/CMakeLists.txt b/CMakeLists.txt index 29cfbecb..e85e9ec0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ option(WITH_EXAMPLES "Build examples" OFF) option(WITH_MPI "Option to enable MPI" OFF) option(WITH_DB "Use Database for Caching Simulation Results" OFF) option(WITH_CALIPER "Use Caliper for Profiling" OFF) +option(WITH_ADIAK "Use Adiak for recording metadata" OFF) option(WITH_FAISS "Use C/C++ FAISS interface for HD Cache" OFF) option(WITH_TORCH "Use C/C++ Torch interface for Surrogate Model Inference" OFF) option(WITH_TORCH_DEBUG "Compute RMSE of Surrogate Model and Physics Module" OFF) @@ -88,6 +89,14 @@ if (WITH_CALIPER) list(APPEND AMS_APP_DEFINES "-D__ENABLE_CALIPER__") endif() +# ------------------------------------------------------------------------------ +if (WITH_ADIAK) + find_package(adiak REQUIRED) + list(APPEND AMS_APP_INCLUDES ${adiak_INCLUDE_DIR}) + list(APPEND AMS_APP_LIBRARIES adiak::adiak) + list(APPEND AMS_APP_DEFINES "-D__ENABLE_ADIAK__") +endif() + if (WITH_AMS_DEBUG) list(APPEND AMS_APP_DEFINES "-DLIBAMS_VERBOSE") endif() @@ -334,6 +343,13 @@ if (WITH_EXAMPLES) list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_CALIPER__") endif() + if (WITH_ADIAK) + list(APPEND AMS_EXAMPLE_INCLUDES ${adiak_INCLUDE_DIR}) + list(APPEND AMS_EXAMPLE_LIBRARIES adiak::adiak) + list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_ADIAK__") + endif() + + if (WITH_FAISS) list(APPEND AMS_EXAMPLE_DEFINES "-D__ENABLE_FAISS__") endif() diff --git a/examples/main.cpp b/examples/main.cpp index f1b645c7..26a2d882 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -5,6 +5,8 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ +#include + #include #include #include @@ -73,6 +75,29 @@ int main(int argc, char **argv) std::cout.setstate(std::ios::failbit); } + // add adiak init here + // ------------------------------------------------------------------------ + adiak::init(nullptr); + + // adiak::value("compiler", adiak::version("gcc@8.1.0")); + const adiak::path adiak_ams_compiler = std::string("@RAJAPERF_COMPILER@"); + adiak::uid(); + adiak::launchdate(); + adiak::launchday(); + adiak::executable(); + adiak::executablepath(); + adiak::workdir(); + adiak::libraries(); + adiak::cmdline(); + adiak::hostname(); + adiak::clustername(); + adiak::walltime(); + adiak::systime(); + adiak::cputime(); + adiak::jobsize(); + adiak::hostlist(); + adiak::numhosts(); + const char *device_name = "cpu"; const char *eos_name = "ideal_gas"; const char *model_path = ""; @@ -693,6 +718,11 @@ int main(int argc, char **argv) MPI_CALL(MPI_Barrier(MPI_COMM_WORLD)); } CALIPER(CALI_MARK_END("TimeStepLoop");); + +// --------------------------------------------------------------------------- +// adiak finalize + adiak::fini(); + MPI_CALL(MPI_Finalize()); return 0; } diff --git a/setup_env_with_pfa.sh b/setup_env_with_pfa.sh new file mode 100644 index 00000000..05d30b97 --- /dev/null +++ b/setup_env_with_pfa.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + + +## need to know the code root +#miniapp_code_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +#echo "miniapp_code_root = $miniapp_code_root" + + +## load relevant modules on lassen +module load clang/10.0.1-gcc-8.3.1 +module load cuda/11.1.0 +module load cmake/3.20.2 +module load spectrum-mpi/2019.06.24 + +SPACK_ENV_PATH="0.19" +host=$(hostname) +host=${host//[0-9]/} + +if [[ "$SYS_TYPE" == "blueos_3_ppc64le_ib_p9" ]]; then + echo "Loading mpi and cuda" + if [[ "${SPACK_ENV_PATH}" == "0.19" ]]; then + module load cuda/11.6.1 + elif [[ "${SPACK_ENV_PATH}" == "0.18" ]]; then + module load cuda/11.4.1 + fi + module load spectrum-mpi + CUDA_ARCH=70 +elif [[ "$SYS_TYPE" == "toss_3_x86_64_ib" ]]; then + module load mvapich2/2.3 + if [[ "${SPACK_ENV_PATH}" == "0.19" ]]; then + module load cuda/11.6.1 + elif [[ "${SPACK_ENV_PATH}" == "0.18" ]]; then + module load cuda/11.4.1 + fi + CUDA_ARCH=60 +fi + + +## activate spack +source /usr/workspace/AMS/ams-spack-environments/${SPACK_ENV_PATH}/spack/share/spack/setup-env.sh + + +## activate spack env (comes with the repo) +#spack env activate $miniapp_code_root/spack_env +spack env activate /usr/workspace/AMS/ams-spack-environments/${SPACK_ENV_PATH}/$host + +# export the paths (currently cmake needs these) +export AMS_MFEM_PATH=`spack location -i mfem` +export AMS_TORCH_PATH=`spack location -i py-torch` +export AMS_FAISS_PATH=`spack location -i faiss` +export AMS_UMPIRE_PATH=`spack location -i umpire` +export AMS_HIREDIS_PATH=`spack location -i hiredis` +export AMS_REDIS_PLUS_PLUS_PATH=`spack location -i redis-plus-plus` +export AMS_HDF5_PATH=`spack location -i hdf5` +export AMS_CUDA_ARCH=${CUDA_ARCH} +#export AMS_PFA_PATH=/usr/workspace/AMS/miniapp_resources/PerfFlowAspect/src/c/install-apr2023 + +echo "AMS_MFEM_PATH = $AMS_MFEM_PATH" +echo "AMS_TORCH_PATH = $AMS_TORCH_PATH" +echo "AMS_FAISS_PATH = $AMS_FAISS_PATH" +echo "AMS_UMPIRE_PATH = $AMS_UMPIRE_PATH" +echo "AMS_CUDA_ARCH = $AMS_CUDA_ARCH" +echo "AMS_HIREDIS_PATH = $AMS_HIREDIS_PATH" +echo "AMS_REDIS_PLUS_PLUS_PATH = $AMS_REDIS_PLUS_PLUS_PATH" +echo "AMS_HDF5_PATH = $AMS_HDF5_PATH" +#echo "AMS_PFA_PATH = $AMS_PFA_PATH" + +export AMS_TORCH_PATH=$(echo $AMS_TORCH_PATH/lib/python3.*/site-packages/torch/share/cmake/Torch) +echo "(for cmake) AMS_TORCH_PATH = $AMS_TORCH_PATH"