Cleaning up the CMake build process - adding compiler flags to enabl…

…e different build configuration types.
estrozi · Nov 12, 2022 · 9c1c7a1 · 9c1c7a1
1 parent a64c5bd
commit 9c1c7a1
Show file tree

Hide file tree

Showing 4 changed files with 78 additions and 75 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -73,6 +73,7 @@ endif()
 # ----------------------------------------------------------------SET CXX STANDARD--
 set(CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_STANDARD 14)
+set(CXX_EXTENSIONS OFF)
 
 # ------------------OPTIONS WHICH ARE NEEDED TO SET BUILD-TYPES (COMPILATION FLAGS)--
 # ----------------------------------------------------------CUDA-ARCH--OR--HIP-ARCH--
@@ -82,16 +83,16 @@ if(NOT (DEFINED CUDA_ARCH OR DEFINED HIP_ARCH))
     set(CUDARCH "-arch=sm_50")
   else (HIP)
     message(STATUS "Setting fallback HIP_ARCH=gfx906")
-    set(AMDGPU_TARGETS "gfx906" CACHE STRING "AMD GPU targets to compile for")
+    set(AMDGPU_TARGETS "gfx906" CACHE STRING "Semicolon delimited list of AMD GPU targets to compile for (e.g gf908;gfx90a)")
   endif()
 else(NOT (DEFINED CUDA_ARCH OR DEFINED HIP_ARCH))
 if (CUDA)
   message(STATUS "Using provided CUDA_ARCH=${CUDA_ARCH}")
   set(CUDARCH "-arch=sm_${CUDA_ARCH}")
 else (HIP)
   message(STATUS "Using provided HIP_ARCH=${HIP_ARCH}")
-  set(AMDGPU_TARGETS "${HIP_ARCH}" CACHE STRING "AMD GPU targets to compile for")
-endif()
+  set(AMDGPU_TARGETS "${HIP_ARCH}" CACHE STRING "Semicolon delimited list of AMD GPU targets to compile for (e.g gf908;gfx90a)")
+  endif()
 endif(NOT (DEFINED CUDA_ARCH OR DEFINED HIP_ARCH))
 
 # -------------------------------------------------------------------FURTHER OPTIONS--
@@ -130,7 +131,7 @@ else()
 endif()
 
 if(CUDA OR ALTCPU)
-    add_definitions(-DACC_CUDA=2 -DACC_CPU=1)
+    add_definitions(-DACC_HIP=3 -DACC_CUDA=2 -DACC_CPU=1)
 endif()
 
 # -----------------------------------------------DOUBLE PRECISION (GPU-CODE) OR NOT--
@@ -185,16 +186,18 @@ else(HIP)
         include($ENV{ROCM_PATH}/hip/cmake/FindHIP.cmake)
     elseif(DEFINED ENV{HIP_PATH})
         include($ENV{HIP_PATH}/cmake/FindHIP.cmake)
+    elseif(EXISTS "${CMAKE_SOURCE_DIR}/cmake/FindHIP.cmake")
+        include(${CMAKE_SOURCE_DIR}/cmake/FindHIP.cmake)
     else()
         include(/opt/rocm/hip/cmake/FindHIP.cmake)
     endif()
-    
+
     find_package(HIP REQUIRED)
 
     if(HIP_FOUND)
       message(STATUS "Using hip wrapper to compile....")
-      if( ${HIP_VERSION} VERSION_LESS "4.0")
-		      message(WARNING "ROCm version is less than 4.0, so relion will be compiled without GPU support.")
+      if( ${HIP_VERSION} VERSION_LESS "5.0")
+		      message(WARNING "ROCm version is less than 5.0, so relion will be compiled without GPU support.")
           set(HIP OFF)
       endif()
       if(HIP)
@@ -207,7 +210,7 @@ else(HIP)
           endforeach()
       endif()
     else(HIP_FOUND)
-        message(FATAL_ERROR "HIP enabled but unlable to locate packages...")
+        message(FATAL_ERROR "HIP enabled but unlable to locate packages. ROCm >= 5.0 is required to configure RELION with HIP using CMake.")
     endif(HIP_FOUND)
 endif()
 # ------------------------------------------------------------------ALLOCATOR CHOICE--

diff --git a/cmake/BuildTypes.cmake b/cmake/BuildTypes.cmake
@@ -1,53 +1,53 @@
 # Extra flags defined on each build type (this file is all optional to include)
 #
-# Because gcc is compliant with a float128 type, fftw has become as well. nvcc is NOT. 
+# Because gcc is compliant with a float128 type, fftw has become as well. nvcc is NOT.
 # So -D__INTEL_COMPILER just manages to avoid compiling float128-targets (see fftw3.h, for instance).
 # Add -G to allow cuda-gdb to break inside kernels.
 if(CUDA)
     set(EXTRA_NVCC_FLAGS "-D__INTEL_COMPILER --default-stream per-thread --std=c++11")
     set(RELION_NVCC_FLAGS "${CUDARCH} ${WARN_DBL} ${EXTRA_NVCC_FLAGS}" CACHE STRING "" FORCE)
 else (HIP)
-    set(EXTRA_HIPCC_FLAGS "-v -fPIC -fno-gpu-rdc --default-stream per-thread --std=c++14 ")
-    set(RELION_HIPCC_FLAGS "${HIPARCH} ${WARN_DBL} ${EXTRA_HIPCC_FLAGS}" CACHE STRING "" FORCE)
+    set(EXTRA_HIPCC_FLAGS "-fno-gpu-rdc -munsafe-fp-atomics -fgpu-default-stream=legacy")
+    set(RELION_HIPCC_FLAGS "${EXTRA_HIPCC_FLAGS}" CACHE STRING "Compiler flags for HIP" FORCE)
 endif()
 #message(STATUS "RELION_NVCC_FLAGS: ${RELION_NVCC_FLAGS}")
 #message(STATUS "RELION_HIPCC_FLAGS: ${RELION_HIPCC_FLAGS}")
-# -------------------------- 
-#        Debug BUILD 
-# -------------------------- 
+# --------------------------
+#        Debug BUILD
+# --------------------------
 # Additional useful nvcc-flags for debugging
 #
 #      -keep	            Keep all intermediate files that are generated during internal compilation steps.
-#     --resource-usage      how resource usage such as registers and memeory of the GPU code. This option implies 
-#                           --nvlink-options=--verbose when --relocatable-device-code=true is set. Otherwise, 
+#     --resource-usage      how resource usage such as registers and memeory of the GPU code. This option implies
+#                           --nvlink-options=--verbose when --relocatable-device-code=true is set. Otherwise,
 #                           it implies --ptxas-options=--verbose.
 
 # -- Compiler flags -------------------------------------------------
 set(RELION_FLAGS_DEBUG "-O0" CACHE STRING "")
 if(CUDA)
     set(RELION_NVCC_FLAGS_DEBUG "${RELION_NVCC_FLAGS}" CACHE STRING "")
 else(HIP)
-    set(RELION_HIPCC_FLAGS_DEBUG "${RELION_HIPCC_FLAGS} -ggdb" CACHE STRING "")
+    set(RELION_HIPCC_FLAGS_DEBUG "${RELION_HIPCC_FLAGS} -g -ggdb" CACHE STRING "Compiler flags for HIPCC with DEBUG configuration")
 endif()
 
 # -- Linker flags ---------------------------------------------------
 set(RELION_LINKER_FLAGS_DEBUG  " ")
 
 # -- Append compiler and linker flags -------------------------------
-set(CMAKE_CXX_FLAGS_DEBUG        "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_FLAGS_DEBUG}" CACHE STRING "")
-set(CMAKE_C_FLAGS_DEBUG          "${CMAKE_C_FLAGS_DEBUG} ${RELION_FLAGS_DEBUG}" CACHE STRING "")
-set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${RELION_LINKER_FLAGS_DEBUG}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_DEBUG        "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_FLAGS_DEBUG}")
+set(CMAKE_C_FLAGS_DEBUG          "${CMAKE_C_FLAGS_DEBUG} ${RELION_FLAGS_DEBUG}")
+set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${RELION_LINKER_FLAGS_DEBUG}")
 
-if(CUDA) 
-    set(CUDA_NVCC_FLAGS_DEBUG        "${RELION_NVCC_FLAGS_DEBUG}" CACHE STRING "")
+if(CUDA)
+    set(CUDA_NVCC_FLAGS_DEBUG    "${RELION_NVCC_FLAGS_DEBUG}" CACHE STRING "")
 else(HIP)
-    set(HIP_HIPCC_FLAGS_DEBUG        "${RELION_HIPCC_FLAGS_DEBUG}" CACHE STRING "")
+    set(CMAKE_CXX_FLAGS_DEBUG    "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_HIPCC_FLAGS_DEBUG}")
 endif()
 
 # -- Add preprocessor defintions ------------------------------------
 if(CUDA)
     set(RELION_DEFINITIONS_DEBUG "-DDEBUG_CUDA")
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_DEFINITIONS_DEBUG}")    
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_DEFINITIONS_DEBUG}")
 else(HIP)
     set(RELION_DEFINITIONS_DEBUG "-DDEBUG_HIP")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${RELION_DEFINITIONS_DEBUG}")
@@ -56,62 +56,64 @@ endif()
 #message(STATUS "Set the extra flags for Debug build type")
 #message(STATUS "RELION_NVCC_FLAGS_DEBUG : ${RELION_NVCC_FLAGS_DEBUG}")
 #message(STATUS "CUDA_NVCC_FLAGS_DEBUG : ${CUDA_NVCC_FLAGS_DEBUG}")
-#message(STATUS "CMAKE_CXX_FLAGS_DEBUG : ${CMAKE_CXX_FLAGS_DEBUG}")
+# message(STATUS "CMAKE_CXX_FLAGS_DEBUG : ${CMAKE_CXX_FLAGS_DEBUG}")
 #--------------------------------------------------------------------
 
 
 
 
-# -------------------------- 
-#        RELWITHDEBINFO BUILD 
-# -------------------------- 
-
+# --------------------------
+#        RELWITHDEBINFO BUILD
+# --------------------------
+set(RELION_FLAGS_RELWITHDEBINFO "-O2" CACHE STRING "")
 # -- Compiler flags -------------------------------------------------
 if(CUDA)
     set(RELION_NVCC_FLAGS_RELWITHDEBINFO "${RELION_NVCC_FLAGS}" CACHE STRING "")
 else(HIP)
-    set(RELION_HIPCC_FLAGS_RELWITHDEBINFO "${RELION_HIPCC_FLAGS} -ggdb" CACHE STRING "")
+    set(RELION_HIPCC_FLAGS_RELWITHDEBINFO "${RELION_HIPCC_FLAGS} -g -ggdb" CACHE STRING "Compiler flags for HIPCC with RELWITHDEBINFO configuration")
 endif()
 
 # -- Linker flags ---------------------------------------------------
 set(RELION_LINKER_FLAGS_RELWITHDEBINFO  " ")
 
 # -- Append compiler and linker flags -------------------------------
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO        "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${RELION_FLAGS_RELWITHDEBINFO}" CACHE STRING "")
-set(CMAKE_C_FLAGS_RELWITHDEBINFO          "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${RELION_FLAGS_RELWITHDEBINFO}" CACHE STRING "")
-set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} ${RELION_LINKER_FLAGS_RELWITHDEBINFO}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELWITHDEBINFO        "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${RELION_FLAGS_RELWITHDEBINFO}")
+set(CMAKE_C_FLAGS_RELWITHDEBINFO          "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${RELION_FLAGS_RELWITHDEBINFO}")
+set(CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} ${RELION_LINKER_FLAGS_RELWITHDEBINFO}")
 if(CUDA)
     set(CUDA_NVCC_FLAGS_RELWITHDEBINFO        "${RELION_NVCC_FLAGS_RELWITHDEBINFO}" CACHE STRING "")
     # -- Add preprocessor defintions ------------------------------------
     set(RELION_DEFINITIONS_RELWITHDEBINFO "-DDEBUG_CUDA")
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${RELION_DEFINITIONS_RELWITHDEBINFO}")
 else(HIP)
-    set(HIP_HIPCC_FLAGS_RELWITHDEBINFO        "${RELION_HIPCC_FLAGS_RELWITHDEBINFO}" CACHE STRING "")
+    set(CMAKE_CXX__FLAGS_RELWITHDEBINFO        "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${RELION_HIPCC_FLAGS_RELWITHDEBINFO}")
     # -- Add preprocessor defintions ------------------------------------
     set(RELION_DEFINITIONS_RELWITHDEBINFO "-DDEBUG_HIP")
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${RELION_DEFINITIONS_RELWITHDEBINFO}")
 endif()
 
 
 #message(STATUS "Set the extra flags for RELWITHDEBINFO build type")
 #message(STATUS "RELION_NVCC_FLAGS_RELWITHDEBINFO : ${RELION_NVCC_FLAGS_RELWITHDEBINFO}")
 #message(STATUS "CUDA_NVCC_FLAGS_RELWITHDEBINFO : ${CUDA_NVCC_FLAGS_RELWITHDEBINFO}")
-#message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO : ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+# message(STATUS "CMAKE_CXX_FLAGS_RELWITHDEBINFO : ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
 #--------------------------------------------------------------------
 
 
 
 
-# -------------------------- 
-#       Release BUILD 
-# -------------------------- 
+# --------------------------
+#       Release BUILD
+# --------------------------
 # Additional useful nvcc-flags for optimization
 #
 #     --use_fast_math
-#     --prec-div         This option controls single-precision floating-point division and reciprocals. 
-#                        --prec-div=true enables the IEEE round-to-nearest mode and --prec-div=false enables   
+#     --prec-div         This option controls single-precision floating-point division and reciprocals.
+#                        --prec-div=true enables the IEEE round-to-nearest mode and --prec-div=false enables
 #                        the fast approximation mode. --use_fast_math implies --prec-div=false.
 #     --prec-sqrt        -||- sqrt
-#     --fmad             This option enables (disables) the contraction of floating-point multiplies and 
-#                        adds/subtracts into floating-point multiply-add operations (FMAD, FFMA, or DFMA). 
+#     --fmad             This option enables (disables) the contraction of floating-point multiplies and
+#                        adds/subtracts into floating-point multiply-add operations (FMAD, FFMA, or DFMA).
 #                        --use_fast_math implies --fmad=true.
 #     --restrict         Programmer assertion that all kernel pointer parameters are restrict pointers.
 
@@ -120,42 +122,42 @@ set(RELION_FLAGS_RELEASE " " CACHE STRING "")
 if(CUDA)
     set(RELION_NVCC_FLAGS_RELEASE "${RELION_NVCC_FLAGS} --disable-warnings" CACHE STRING "")
 else(HIP)
-    set(RELION_HIPCC_FLAGS_RELEASE "${RELION_HIPCC_FLAGS} --disable-warnings" CACHE STRING "")
+    set(RELION_HIPCC_FLAGS_RELEASE "${RELION_HIPCC_FLAGS} -w" CACHE STRING "Compiler flags for HIPCC with RELEASE configuration")
 endif()
 # -- Linker flags ---------------------------------------------------
 set(RELION_LINKER_FLAGS_RELEASE  "")
 
 # -- Append compiler and linker flags -------------------------------
 #message(STATUS "CCF_RELEASE :       ${CMAKE_CXX_FLAGS_RELEASE}")
-set(CMAKE_CXX_FLAGS_RELEASE        "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_FLAGS_RELEASE}"        CACHE STRING "")
-set(CMAKE_C_FLAGS_RELEASE          "${CMAKE_C_FLAGS_RELEASE} ${RELION_FLAGS_RELEASE}"          CACHE STRING "")
-set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${RELION_LINKER_FLAGS_RELEASE}" CACHE STRING "")
+set(CMAKE_CXX_FLAGS_RELEASE        "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_FLAGS_RELEASE}")
+set(CMAKE_C_FLAGS_RELEASE          "${CMAKE_C_FLAGS_RELEASE} ${RELION_FLAGS_RELEASE}")
+set(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${RELION_LINKER_FLAGS_RELEASE}")
 if(CUDA)
-    set(CUDA_NVCC_FLAGS_RELEASE        "${RELION_NVCC_FLAGS_RELEASE}" CACHE STRING "")
+    set(CUDA_NVCC_FLAGS_RELEASE    "${RELION_NVCC_FLAGS_RELEASE}" CACHE STRING "")
 else(HIP)
-    set(HIP_HIPCC_FLAGS_RELEASE        "${RELION_HIPCC_FLAGS_RELEASE}" CACHE STRING "")
+    set(CMAKE_CXX_FLAGS_RELEASE    "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_HIPCC_FLAGS_RELEASE}")
 endif()
 # -- Add preprocessor defintions ------------------------------------
 set(RELION_DEFINITIONS_RELEASE "")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_DEFINITIONS_RELEASE}")
 
 #message(STATUS "RELION_FLAGS_PROFILING : ${RELION_FLAGS_PROFILING}")
-#message(STATUS "CMAKE_CXX_FLAGS_PROFILING : ${CMAKE_CXX_FLAGS_PROFILING}")
+# message(STATUS "CMAKE_CXX_FLAGS_RELEASE : ${CMAKE_CXX_FLAGS_RELEASE}")
 #--------------------------------------------------------------------
 
 
 if(CUDA)
-    # ---------------------------------- 
-    #       NVIDIA Profiling BUILD 
-    #        (Release for nvprof) 
+    # ----------------------------------
+    #       NVIDIA Profiling BUILD
+    #        (Release for nvprof)
     # ----------------------------------
     # ** NOTE: this will not have overall Release perf. **
 
     # Additional useful nvcc-flags for profiling
     #
     #   -pg		               gprof profiling output (needs linker flag)
-    #   --resource-usage       how resource usage such as registers and memeory of the GPU code. This option implies 
-    #                           --nvlink-options=--verbose when --relocatable-device-code=true is set. Otherwise, 
+    #   --resource-usage       how resource usage such as registers and memeory of the GPU code. This option implies
+    #                           --nvlink-options=--verbose when --relocatable-device-code=true is set. Otherwise,
     #                           it implies --ptxas-options=--verbose#
 
     # -- Compiler flags -------------------------------------------------
@@ -179,46 +181,45 @@ if(CUDA)
     #--------------------------------------------------------------------
 
 else(HIP)
-    # ---------------------------------- 
-    #       AMD Profiling BUILD 
-    #        (Release for rocprof) 
+    # ----------------------------------
+    #       AMD Profiling BUILD
+    #        (Release for rocprof)
     # ----------------------------------
     # ** NOTE: this will not have overall Release perf. **
 
 
     # -- Compiler flags -------------------------------------------------
     set(RELION_FLAGS_PROFILING "" CACHE STRING "")
-    set(RELION_HIPCC_FLAGS_PROFILING "${RELION_HIPCC_FLAGS} -lineinfo -ggdb" CACHE STRING "")
+    set(RELION_HIPCC_FLAGS_PROFILING "${RELION_HIPCC_FLAGS} -g -ggdb" CACHE STRING "Semicolon delimited flags")
     # -- Linker flags ---------------------------------------------------
     set(RELION_LINKER_FLAGS_PROFILING  "")
 
     # -- Append compiler and linker flags -------------------------------
-    set(CMAKE_CXX_FLAGS_PROFILING        "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_FLAGS_PROFILING}"               CACHE STRING "")
-    set(CMAKE_C_FLAGS_PROFILING          "${CMAKE_C_FLAGS_RELEASE} ${RELION_FALAGS_PROFILING}"                 CACHE STRING "")
-    set(CMAKE_EXE_LINKER_FLAGS_PROFILING "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${RELION_LINKER_FLAGS_PROFILING}" CACHE STRING "")
-    set(HIP_HIPCC_FLAGS_PROFILING        "${RELION_HIPCC_FLAGS_PROFILING}"                                     CACHE STRING "")
+    set(CMAKE_CXX_FLAGS_PROFILING        "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_FLAGS_PROFILING} ${RELION_HIPCC_FLAGS_RELEASE}")
+    set(CMAKE_C_FLAGS_PROFILING          "${CMAKE_C_FLAGS_RELEASE} ${RELION_FALAGS_PROFILING}")
+    set(CMAKE_EXE_LINKER_FLAGS_PROFILING "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${RELION_LINKER_FLAGS_PROFILING}")
 
     # -- Add preprocessor defintions ------------------------------------
     set(RELION_DEFINITIONS_PROFILING "-DHIP_PROFILING")
     set(CMAKE_CXX_FLAGS_PROFILING "${CMAKE_CXX_FLAGS_PROFILING} ${RELION_DEFINITIONS_PROFILING}")
 
     #message(STATUS "RELION_FLAGS_PROFILING : ${RELION_FLAGS_PROFILING}")
-    #message(STATUS "CMAKE_CXX_FLAGS_PROFILING : ${CMAKE_CXX_FLAGS_PROFILING}")
+    # message(STATUS "CMAKE_CXX_FLAGS_PROFILING : ${CMAKE_CXX_FLAGS_PROFILING}")
     #--------------------------------------------------------------------
 endif()
 
 
 
-# ---------------------------------- 
-#       Benchmarking BUILD 
-#  (Release with profiling output) 
+# ----------------------------------
+#       Benchmarking BUILD
+#  (Release with profiling output)
 # ----------------------------------
 # -- Compiler flags -------------------------------------------------
 set(RELION_FLAGS_BENCHMARKING "" CACHE STRING "")
 if(CUDA)
     set(RELION_NVCC_FLAGS_BENCHMARKING "${RELION_NVCC_FLAGS} " CACHE STRING "")
 else(HIP)
-    set(RELION_HIPCC_FLAGS_BENCHMARKING "${RELION_HIPCC_FLAGS} " CACHE STRING "")
+    set(RELION_HIPCC_FLAGS_BENCHMARKING "${RELION_HIPCC_FLAGS} " CACHE STRING "Semicolon delimited flags for HIPCC with Benchmarking configuration")
 endif()
     # -- Linker flags ---------------------------------------------------
 set(RELION_LINKER_FLAGS_BENCHMARKING  "")
@@ -228,9 +229,9 @@ set(CMAKE_CXX_FLAGS_BENCHMARKING        "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_FLA
 set(CMAKE_C_FLAGS_BENCHMARKING          "${CMAKE_C_FLAGS_RELEASE} ${RELION_FLAGS_BENCHMARKING}"                 CACHE STRING "" FORCE)
 set(CMAKE_EXE_LINKER_FLAGS_BENCHMARKING "${CMAKE_EXE_LINKER_FLAGS_RELEASE} ${RELION_LINKER_FLAGS_BENCHMARKING}" CACHE STRING "" FORCE)
 if(CUDA)
-    set(CUDA_NVCC_FLAGS_BENCHMARKING        "${RELION_NVCC_FLAGS_BENCHMARKING}"                                       CACHE STRING "" FORCE)
+    set(CUDA_NVCC_FLAGS_BENCHMARKING    "${RELION_NVCC_FLAGS_BENCHMARKING}" CACHE STRING "" FORCE)
 else(HIP)
-    set(HIP_HIPCC_FLAGS_BENCHMARKING        "${RELION_HIPCC_FLAGS_BENCHMARKING}"                                       CACHE STRING "" FORCE)
+    set(CMAKE_CXX_FLAGS_BENCHMARKING    "${CMAKE_CXX_FLAGS_RELEASE} ${RELION_HIPCC_FLAGS_BENCHMARKING}")
 endif()
 # -- Add preprocessor defintions ------------------------------------
 if(CUDA)

diff --git a/src/acc/hip/hip_kernels/hip_device_utils.h b/src/acc/hip/hip_kernels/hip_device_utils.h
@@ -22,6 +22,7 @@ __device__ inline void hip_atomic_add(float* address, float value)
 {
 	atomicAddNoRet(address,value);
 	// unsafeAtomicAdd(address,value);
+	// atomicAdd(address,value);
 }
 #endif
-Original file line number
+Diff line change
@@ Expand Up @@
     {
     	atomicAddNoRet(address,value);
     	// unsafeAtomicAdd(address,value);
+    	// atomicAdd(address,value);
     }
     #endif
@@ Expand Down @@