CMakeLists.txt

cmake_minimum_required(VERSION 3.10)
project(fidibench)

enable_language(Fortran)
enable_testing()

set(CMAKE_CXX_STANDARD 11)

option(OPENACC "Enable OpenACC support" OFF)
option(ACC_MULTICORE "Whether for OpenACC to compile for multicore" OFF)
option(ACC_KERNELS "Whether for OpenACC to use the kernels directives" OFF)
option(OPENMP "Enable OpenMP support" ON)
option(OPENMP_OFFLOAD "Enable OpenMP offloading to GPU support" OFF)
option(ROCM "Enable ROCm, offloading to MI100 using OpenMPI 4.5" OFF)
option(INTEL_ANALYZER "Add compiler options for Intel Analyzer" OFF)
option(GPROF "Add gprof compiler option for profiling" OFF)

set(OPT_FLAGS "" CACHE STRING "compiler optimization flags")

set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OPT_FLAGS}")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPT_FLAGS}")
set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OPT_FLAGS}")

find_package(OpenMP REQUIRED)

set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")
# CMake needs help with clang 
find_package(PythonExecutable)
find_package(JuliaExecutable)

# Use cmake -D NUM_CELLS=# and -D NUM_TIME_STEPS=# to change default values
set(NUM_CELLS "32" CACHE STRING "Number of cells along each direction when running tests")
set(NUM_TIME_STEPS "10" CACHE STRING "Number of time steps when running tests")
set(NUM_PROCS "4" CACHE STRING "Number of processes when running tests")
set(THREADS_PER_TEAM "32" CACHE STRING "Number of threads (vector length) per team (gang) when offloading to GPU")
set(TEAMS "64" CACHE STRING "Number of teams (gangs) when offloading to GPU")
add_definitions(-D_THREADS_PER_TEAM=${THREADS_PER_TEAM} -D_TEAMS=${TEAMS})

set(EXTRA_LIBS "" CACHE STRING "Extra libraries to link against")

set(ROCM_COMPILER_PATH "/opt/rocm/llvm/bin" CACHE STRING "Path to the ROCm compilers (only used if ROCM is set)")
if (${ROCM}) 
  list(APPEND CMAKE_PREFIX_INSTALL /opt/rocm/hip /opt/rocm)
  set(CMAKE_CXX_COMPILER "${ROCM_COMPILER_PATH}/clang++")
  set(CMAKE_C_COMPILER "${ROCM_COMPILER_PATH}/clang")
  set(CMAKE_Fortran_COMPILER "${ROCM_COMPILER_PATH}/flang")
  #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -target x86_64-pc-linux-gnu -fopenmp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx908")
  find_package(hip)
  message(STATUS "Will add the hip library to OpenMP 4.5 targets")
endif()


if (NOT DEFINED "${CMAKE_BUILD_TYPE}")
    set(CMAKE_BUILD_TYPE "RelWithDebInfo")
    message(STATUS "Build type is ${CMAKE_BUILD_TYPE}")
endif()

message(STATUS "System: ${CMAKE_SYSTEM_NAME}")
message(STATUS "Fortran compiler Id: ${CMAKE_Fortran_COMPILER_ID}")
message(STATUS "C compiler Id      : ${CMAKE_C_COMPILER_ID}")
message(STATUS "C++ compiler Id    : ${CMAKE_CXX_COMPILER_ID}")
if (${CMAKE_Fortran_COMPILER_ID} STREQUAL "Cray")
  set(MPI_Fortran_COMPILER "ftn")
endif()
if (${CMAKE_C_COMPILER_ID} STREQUAL "Cray")
  set(MPI_C_COMPILER "cc")
endif()
if (${CMAKE_CXX_COMPILER_ID} STREQUAL "Cray")
  set(MPI_CXX_COMPILER "CC")
endif()

message(STATUS "Looking for MPI")
find_package(MPI)
if (MPI_FOUND)
    set(CMAKE_CXX_COMPILER ${MPI_CXX_COMPILER})
    set(CMAKE_C_COMPILER ${MPI_C_COMPILER})
    set(CMAKE_Fortran_COMPILER ${MPI_Fortran_COMPILER})
endif()

if (INTEL_ANALYZER)
   message(STATUS "You selected INTEL_ANALYZER")
   message(STATUS "You may need to load the following modules:")
   message(STATUS "ml intel/2017a")
   message(STATUS "ml intel/2017a")
   message(STATUS "ml itac/2017.2.028")
   message(STATUS "source itacvars.sh impi5")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -tcollect $ENV{VT_ADD_LIBS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -tcollect $ENV{VT_ADD_LIBS}")
   set(CMAKE_Fortran_FLAGS 
      "${CMAKE_Fortran_FLAGS} -g -tcollect $ENV{VT_ADD_LIBS}")
endif()

if (GPROF)
   message(STATUS "Adding -pg compiler option for gprof")
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -pg")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -pg")
endif()

if (OPENMP_OFFLOAD)
  add_definitions(-DHAVE_OPENMP_OFFLOAD)
  if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^NVHPC|PGI")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Minfo=all -mp=gpu")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -Minfo=all -mp=gpu")
  elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^GNU")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -foffload=nvptx-none")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenmp -foffload=nvptx-none")
  elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^Clang")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -fopenmp-targets=nvptx64-nvidia-cuda")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenmp -fopenmp-targets=nvptx64-nvidia-cuda")
  elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^IntelLLVM")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -fopenmp-targets=nvptx64-nvidia-cuda")
    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenmp -fopenmp-targets=nvptx64-nvidia-cuda")
  endif()
endif ()

if ((NOT OPENACC) AND OPENMP_FOUND)
  add_definitions(-DHAVE_OPENMP)
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} ${OpenMP_Fortran_FLAGS}")
else ()
  # OpenACC is incompatible with OpenMP, either one or the other or none
  if (OPENACC)
    add_definitions(-DHAVE_OPENACC)
    if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^NVHPC|PGI")
      if (ACC_KERNELS)
        message(STATUS "Apply OpenACC kernels directives")
        add_definitions(-DHAVE_OPENACC_KERNELS)
      endif ()
	    if (ACC_MULTICORE)
        message(STATUS "Apply OpenACC directives for offloading to multicore CPU")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc=multicore -Minfo=acc")
	      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc=multicore -Minfo=acc")
	    else ()
        message(STATUS "Apply OpenACC directives for offloading to GPU")
        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -acc -gpu=mem:managed -Minfo=acc")
	      set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -acc -gpu=mem:managed -Minfo=acc")
	    endif ()
	    message(STATUS "Using PGI or Nvidia C++ compiler, added OpenACC compiler switches")
    elseif ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^GNU")
	    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenacc -fopt-info-optimized-omp")
	    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fopenacc -fopt-info-optimized-omp")
	    message(STATUS "Using GNU C++ compiler, added OpenACC compiler switches")
    endif ()
    message(STATUS "OpenACC enabled")
  endif () 
endif ()

# enable profiling and tracing with TAU
if (NOT "${TAU_MAKEFILE}" STREQUAL "")
  if ("${CMAKE_VERSION}" VERSION_LESS 2.8.12)
    get_filename_component(TAU_LIBRARY_DIR "${TAU_MAKEFILE}" PATH)
    get_filename_component(TAU_DIR "${TAU_LIBRARY_DIR}" PATH)
  else ()
    get_filename_component(TAU_LIBRARY_DIR "${TAU_MAKEFILE}" DIRECTORY)
    get_filename_component(TAU_DIR "${TAU_LIBRARY_DIR}" DIRECTORY)
  endif ()
  message("*******************************************************************")
  message("Building with TAU enabled, set the following prior to typing 'make'")
  message("export PATH=${TAU_DIR}/bin:$PATH")
  message("export TAU_MAKEFILE=${TAU_MAKEFILE}")
  set(ENV{PATH} "${TAU_DIR}/bin:$PATH")
  set(ENV{TAU_MAKEFILE} "${TAU_MAKEFILE}")
  message("********************************************************************")
  set(CMAKE_CXX_COMPILER "tau_cxx.sh")
endif()

message(STATUS "==============================================================")
message(STATUS "C       compiler: ${CMAKE_C_COMPILER}")
message(STATUS "           flags: ${CMAKE_C_FLAGS}")
message(STATUS "==============================================================")
message(STATUS "C++     compiler: ${CMAKE_CXX_COMPILER}")
message(STATUS "           flags: ${CMAKE_CXX_FLAGS}")
message(STATUS "==============================================================")
message(STATUS "Fortran compiler: ${CMAKE_Fortran_COMPILER}")
message(STATUS "           flags: ${CMAKE_Fortran_FLAGS}")
message(STATUS "==============================================================")

add_subdirectory(scripts)
add_subdirectory(cxx)
add_subdirectory(laplacian)
add_subdirectory(upwind)
add_subdirectory(matmult)