CI: NVHPC for CUDA (#2066)

.github/workflows/cuda.yml

@@ -72,6 +72,47 @@ jobs:
 
         cmake --build build -j 2
 
+  # Build libamrex and all tests with NVHPC (recent supported)
+  tests-nvhpc21-9-nvcc:
+    name: [email protected] NVCC/NVC++ C++17 Release [tests]
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - name: Dependencies
+      run: .github/workflows/dependencies/dependencies_nvhpc21-9.sh
+    - name: Build & Install
+      run: |
+        source /etc/profile.d/modules.sh
+        module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.9
+
+        which nvcc || echo "nvcc not in PATH!"
+        which nvc++ || echo "nvc++ not in PATH!"
+        which nvc || echo "nvc not in PATH!"
+        which nvfortran || echo "nvfortran not in PATH!"
+        nvcc --version
+        nvc++ --version
+        nvc --version
+        nvfortran --version
+        cmake --version
+
+        cmake -S . -B build                              \
+            -DCMAKE_VERBOSE_MAKEFILE=ON                  \
+            -DAMReX_ENABLE_TESTS=ON                      \
+            -DAMReX_PARTICLES=ON                         \
+            -DAMReX_FORTRAN=OFF                          \
+            -DAMReX_GPU_BACKEND=CUDA                     \
+            -DCMAKE_C_COMPILER=$(which nvc)              \
+            -DCMAKE_CXX_COMPILER=$(which nvc++)          \
+            -DCMAKE_CUDA_HOST_COMPILER=$(which nvc++)    \
+            -DCMAKE_Fortran_COMPILER=$(which nvfortran)  \
+            -DCMAKE_CUDA_STANDARD=17                     \
+            -DCMAKE_CXX_STANDARD=17                      \
+            -DAMReX_CUDA_ARCH=8.0                        \
+            -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \
+            -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON
+
+        cmake --build build -j 2
+
   # Build 3D libamrex cuda build with configure
   configure-3d-cuda:
     name: [email protected] [email protected] [configure 3D]

.github/workflows/dependencies/dependencies_nvhpc21-9.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+#
+# Copyright 2021 Axel Huebl
+#
+# License: BSD-3-Clause-LBNL
+
+set -eu -o pipefail
+
+sudo apt-get -qqq update
+sudo apt-get install -y \
+    build-essential     \
+    ca-certificates     \
+    cmake               \
+    environment-modules \
+    gnupg               \
+    pkg-config          \
+    wget
+
+wget -q https://developer.download.nvidia.com/hpc-sdk/21.9/nvhpc-21-9_21.9_amd64.deb \
+        https://developer.download.nvidia.com/hpc-sdk/21.9/nvhpc-2021_21.9_amd64.deb
+sudo apt-get update
+sudo apt-get install -y ./nvhpc-21-9_21.9_amd64.deb ./nvhpc-2021_21.9_amd64.deb
+rm -rf ./nvhpc-21-9_21.9_amd64.deb ./nvhpc-2021_21.9_amd64.deb
+
+# things should reside in /opt/nvidia/hpc_sdk now
+
+# activation via:
+#   source /etc/profile.d/modules.sh
+#   module load /opt/nvidia/hpc_sdk/modulefiles/nvhpc/21.9

Src/Base/AMReX_BLBackTrace.H

@@ -28,7 +28,7 @@ struct BLBackTrace
 
     static std::stack<std::pair<std::string, std::string> > bt_stack;
 // threadprivate here doesn't work with Cray and Intel
-#if defined(AMREX_USE_OMP) && !defined(_CRAYC) && !defined(__INTEL_COMPILER) && !defined(__PGI)
+#if defined(AMREX_USE_OMP) && !defined(_CRAYC) && !defined(__INTEL_COMPILER) && !defined(__PGI) && !defined(__NVCOMPILER)
 #pragma omp threadprivate(bt_stack)
 #endif
 };

Src/Base/AMReX_Extension.H

@@ -15,6 +15,8 @@
 #define AMREX_CXX_CRAY
 #elif defined(__PGI)
 #define AMREX_CXX_PGI
+#elif defined(__NVCOMPILER)
+#define AMREX_CXX_NVHPC
 #elif defined(__NEC__)
 #define AMREX_CXX_NEC
 #elif defined(__ibmxl__)
@@ -61,6 +63,9 @@
 #elif defined(__PGI)
 #define AMREX_PRAGMA_SIMD _Pragma("loop ivdep")
 
+#elif defined(__NVCOMPILER)
+#define AMREX_PRAGMA_SIMD _Pragma("loop ivdep")
+
 #elif defined(__NEC__)
 #define AMREX_PRAGMA_SIMD
 
@@ -94,6 +99,9 @@
 #elif defined(__PGI)
 #define AMREX_FORCE_INLINE inline
 
+#elif defined(__NVCOMPILER)
+#define AMREX_FORCE_INLINE inline
+
 #elif defined(__NEC__)
 #define AMREX_FORCE_INLINE inline
 

Tools/CMake/AMReXGenerateConfigHeader.cmake

@@ -41,6 +41,8 @@ function ( generate_config_header )
            set(COMPILER_ID_MACRO  __CRAYC)
        elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "PGI" )
            set(COMPILER_ID_MACRO  __PGI)
+       elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "NVHPC" )
+           set(COMPILER_ID_MACRO  __NVCOMPILER)
        elseif ( ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" )       OR
                 ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang" )  OR
                 ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "ROCMClang" )   OR