make.inc.basecamp.nv.acc

#==============================================================================
# Exciting-Plus make.inc file for wyp-BaseCamp with NVIDIA HPC SDK
# Last edited: Dec 14, 2020 (WYP)
#==============================================================================

MAKE = make
COMPILER = nv

# Compiler switch to turn on the preprocessor
CPP_OPTS = -Mpreprocess

# Add your custom preprocessor defines here
#CPP_OPTS += -D_DEBUG_bmegqblh_ # Output debugging info related to bmegqblh
#CPP_OPTS += -D_DEBUG_megqblh_ # Output debugging info related to genmegqblh
#CPP_OPTS += -D_DUMP_spinor_ud_ # Dump spinor_ud when spinpol = .TRUE.
#CPP_OPTS += -D_DUMP_megqblh_ # Dump matrix elements

# Common compile time options
F90_OPTS = $(CPP_OPTS)

# Add your common compile time options here
#F90_OPTS += -Minfo=ftn,loop,opt,mp,par,vect # Show optimizer log
F90_OPTS += -Minfo=accel # Show optimizer log only for OpenACC
F90_OPTS += -Mstack_arrays # Use stack instead of heap for all arrays

#==============================================================================
# OpenACC
#==============================================================================

#F90_OPTS += -gpu=managed:debug # Debug build, managed memory
F90_OPTS += -gpu=managed # Optimized build, managed memory

#F90_OPTS += -gpu=debug # Debug build. non-managed memory

F90_OPTS += -acc=gpu # run on GPU
#F90_OPTS += -acc=multicore # Debug build, run on CPU

#F90_OPTS += -acc=verystrict # debug build
F90_OPTS += -acc=autopar # optimized build

#==============================================================================
# MPI
#==============================================================================

F90 = mpifort
CC  = mpicc
CXX = mpic++
MPI_CPP_OPTS = -D_MPI_
CPP_OPTS += $(MPI_CPP_OPTS)

#==============================================================================
# OpenMP
#==============================================================================

# These are passed at compile time and link time
# Don't forget to `export OMP_STACKSIZE=2G` at runtime
OMP_OPTS = -mp
F90_OPTS += $(OMP_OPTS)

#==============================================================================
# Serial compiler (for utilities)
#==============================================================================

F90SERIAL = nvfortran
F90_OPTS_SERIAL = $(filter-out $(OMP_OPTS),$(filter-out $(MPI_CPP_OPTS),$(F90_OPTS)))

#==============================================================================
# Compiler and linker options
#==============================================================================

# Debugging
#CPP_OPTS += -DEBUG=1
#F90_OPTS += -g -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS)
#EXE_SFX = dbg-acc

# Debugging with extra checks
# (Don't forget to enable core dump generation using `ulimit -c unlimited`)
#CPP_OPTS += -DEBUG=2
#F90_OPTS += -g -O0 -Minform=warn -Mbounds -traceback -Ktrap=inv
#F90_LINK_OPTS = $(F90_OPTS)
#EXE_SFX = dbgchk-acc

# Optimized build with equivalent options to '-O2'
F90_OPTS += -gopt -O -Mlre -Mvect=simd -Mflushz -Mcache_align -Mnoinline -Minform=warn
F90_LINK_OPTS = $(F90_OPTS) -fpic
EXE_SFX = opt-acc

# Profiling
# Note: don't use '-Mpfi', it is incompatible with '-mp' and '-fpic'
#F90_OPTS += -Minstrument -Mprof=ccff -O -Mlre -Mflushz -Mcache_align -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = prof-acc

# Fully optimized build
#F90_OPTS += -fast -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fast-acc

# Profiling, fully optimized
#F90_OPTS += -Minstrument -Mprof=ccff -fast -Mnoinline -Minform=warn
#F90_LINK_OPTS = $(F90_OPTS) -fpic
#EXE_SFX = fastprof-acc

#==============================================================================
# BLAS and LAPACK
#==============================================================================

# Use PGI's bundled BLAS and LAPACK
#LAPACK_LIB = -lblas -llapack

# Use reference BLAS and LAPACK 3.9.0
LAPACK_LIB = -L${LAPACK_DIR}/lib -lblas -llapack

# Use OpenBLAS
#LAPACK_LIB = -L${OPENBLAS_DIR}/lib -lopenblas

# Use MAGMA
MAGMA_INC   = -I${MAGMA_DIR}/include
LAPACK_LIB += -L${MAGMA_DIR}/lib -lmagma
CPP_OPTS   += -D_MAGMA_ -DNGPUS=1
F90_OPTS   += $(MAGMA_INC)

#==============================================================================
# FFTW 3
#==============================================================================

# To disable, comment these three lines
FFT_INC  = -I${FFTW_DIR}/include
FFT_LIB  = -L${FFTW_DIR}/lib -lfftw3_omp -lfftw3 -lm
FFT_CPP_OPTS = -D_FFTW3_

CPP_OPTS += $(FFT_CPP_OPTS)
F90_OPTS += $(FFT_INC)

#==============================================================================
# HDF5
#==============================================================================

# To disable, comment these three lines
HDF5_INC = -I${HDF5_DIR}/include
HDF5_LIB = -L${HDF5_DIR}/lib -lhdf5_fortran -lhdf5_hl -lhdf5 -ldl
HDF5_CPP_OPTS = -D_HDF5_

CPP_OPTS += $(HDF5_CPP_OPTS)
F90_OPTS += $(HDF5_INC)

#==============================================================================
# Other libraries (retained here for historical purposes)
#==============================================================================

# ==- compile with libXC support ===
#CPP_OPTS := $(CPP_OPTS) -D_LIBXC_
#XC_LIB =

# ==- compile with NFFT support ===
#CPP_OPTS := $(CPP_OPTS) -D_NFFT_
#NFFT_INC = -I$(HOME)/local/include
#NFFT_LIB = $(HOME)/local/lib/libnfft3.a $(HOME)/local/lib/libfftw3.a

# === compile with Madness API ===
#CPP_OPTS := $(CPP_OPTS) -D_MAD_
#MADNESS_INC = -I$(HOME)/local/include
#MADNESS_LIB = -L$(HOME)/local/lib/ -lMADmra -lMADlinalg -lMADtensor -lMADmisc -lMADmuparser -lMADtinyxml -lMADworld -lmpichcxx -lstdc++

#==============================================================================
# List all libraries to link
#==============================================================================

LIBS = $(LAPACK_LIB) $(FFT_LIB) $(HDF5_LIB)