lammps/lib/kokkos/Makefile.kokkos

# Default settings common options.

#LAMMPS specific settings:
KOKKOS_PATH=../../lib/kokkos
CXXFLAGS=$(CCFLAGS)

# Options: Cuda,ROCm,OpenMP,Pthreads,Qthreads,Serial
KOKKOS_DEVICES ?= "OpenMP"
#KOKKOS_DEVICES ?= "Pthreads"
# Options: 
# Intel:    KNC,KNL,SNB,HSW,BDW,SKX
# NVIDIA:   Kepler,Kepler30,Kepler32,Kepler35,Kepler37,Maxwell,Maxwell50,Maxwell52,Maxwell53,Pascal60,Pascal61,Volta70,Volta72
# ARM:      ARMv80,ARMv81,ARMv8-ThunderX,ARMv8-TX2
# IBM:      BGQ,Power7,Power8,Power9
# AMD-GPUS: Kaveri,Carrizo,Fiji,Vega
# AMD-CPUS: AMDAVX,Ryzen,Epyc
KOKKOS_ARCH ?= ""
# Options: yes,no
KOKKOS_DEBUG ?= "no"
# Options: hwloc,librt,experimental_memkind
KOKKOS_USE_TPLS ?= ""
# Options: c++11,c++1z
KOKKOS_CXX_STANDARD ?= "c++11"
# Options: aggressive_vectorization,disable_profiling,disable_deprecated_code
KOKKOS_OPTIONS ?= ""

# Default settings specific options.
# Options: force_uvm,use_ldg,rdc,enable_lambda
KOKKOS_CUDA_OPTIONS ?= "enable_lambda"

# Return a 1 if a string contains a substring and 0 if not
# Note the search string should be without '"'
# Example: $(call kokkos_has_string,"hwloc,librt",hwloc)
#   Will return a 1
kokkos_has_string=$(if $(findstring $2,$1),1,0)

# Check for general settings.
KOKKOS_INTERNAL_ENABLE_DEBUG := $(call kokkos_has_string,$(KOKKOS_DEBUG),yes)
KOKKOS_INTERNAL_ENABLE_CXX11 := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++11)
KOKKOS_INTERNAL_ENABLE_CXX1Z := $(call kokkos_has_string,$(KOKKOS_CXX_STANDARD),c++1z)

# Check for external libraries.
KOKKOS_INTERNAL_USE_HWLOC := $(call kokkos_has_string,$(KOKKOS_USE_TPLS),hwloc)
KOKKOS_INTERNAL_USE_LIBRT := $(call kokkos_has_string,$(KOKKOS_USE_TPLS),librt)
KOKKOS_INTERNAL_USE_MEMKIND := $(call kokkos_has_string,$(KOKKOS_USE_TPLS),experimental_memkind)

# Check for advanced settings.
KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS := $(call kokkos_has_string,$(KOKKOS_OPTIONS),compiler_warnings)
KOKKOS_INTERNAL_OPT_RANGE_AGGRESSIVE_VECTORIZATION := $(call kokkos_has_string,$(KOKKOS_OPTIONS),aggressive_vectorization)
KOKKOS_INTERNAL_DISABLE_PROFILING := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_profiling)
KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_deprecated_code)
KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK := $(call kokkos_has_string,$(KOKKOS_OPTIONS),disable_dualview_modify_check)
KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT := $(call kokkos_has_string,$(KOKKOS_OPTIONS),enable_profile_load_print)
KOKKOS_INTERNAL_CUDA_USE_LDG := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),use_ldg)
KOKKOS_INTERNAL_CUDA_USE_UVM := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),force_uvm)
KOKKOS_INTERNAL_CUDA_USE_RELOC := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),rdc)
KOKKOS_INTERNAL_CUDA_USE_LAMBDA := $(call kokkos_has_string,$(KOKKOS_CUDA_OPTIONS),enable_lambda)


# Check for Kokkos Host Execution Spaces one of which must be on.
KOKKOS_INTERNAL_USE_OPENMP := $(call kokkos_has_string,$(subst OpenMPTarget,,$(KOKKOS_DEVICES)),OpenMP)
KOKKOS_INTERNAL_USE_PTHREADS := $(call kokkos_has_string,$(KOKKOS_DEVICES),Pthread)
KOKKOS_INTERNAL_USE_QTHREADS := $(call kokkos_has_string,$(KOKKOS_DEVICES),Qthreads)
KOKKOS_INTERNAL_USE_SERIAL := $(call kokkos_has_string,$(KOKKOS_DEVICES),Serial)

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 0)
  ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 0)
    ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 0)
      KOKKOS_INTERNAL_USE_SERIAL := 1
    endif
  endif
endif

# Check for other Execution Spaces.
KOKKOS_INTERNAL_USE_CUDA := $(call kokkos_has_string,$(KOKKOS_DEVICES),Cuda)
KOKKOS_INTERNAL_USE_ROCM := $(call kokkos_has_string,$(KOKKOS_DEVICES),ROCm)
KOKKOS_INTERNAL_USE_OPENMPTARGET := $(call kokkos_has_string,$(KOKKOS_DEVICES),OpenMPTarget)

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc)
  CUDA_PATH ?= $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=)
  KOKKOS_INTERNAL_COMPILER_NVCC_VERSION := $(shell nvcc --version 2>&1 | grep release | cut -d' ' -f5 | cut -d',' -f1 | tr -d .)
endif

# Check OS.
KOKKOS_OS                      := $(strip $(shell uname -s))
KOKKOS_INTERNAL_OS_CYGWIN      := $(call kokkos_has_string,$(KOKKOS_OS),CYGWIN)
KOKKOS_INTERNAL_OS_LINUX       := $(call kokkos_has_string,$(KOKKOS_OS),Linux)
KOKKOS_INTERNAL_OS_DARWIN      := $(call kokkos_has_string,$(KOKKOS_OS),Darwin)

# Check compiler.
KOKKOS_CXX_VERSION                   := $(strip $(shell $(CXX) --version       2>&1))
KOKKOS_INTERNAL_COMPILER_INTEL       := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),Intel Corporation)
KOKKOS_INTERNAL_COMPILER_PGI         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),PGI)
KOKKOS_INTERNAL_COMPILER_XL          := $(strip $(shell $(CXX) -qversion       2>&1 | grep XL                  | wc -l))
KOKKOS_INTERNAL_COMPILER_CRAY        := $(strip $(shell $(CXX) -craype-verbose 2>&1 | grep "CC-"               | wc -l))
KOKKOS_INTERNAL_COMPILER_NVCC        := $(strip $(shell export OMPI_CXX=$(OMPI_CXX); export MPICH_CXX=$(MPICH_CXX); $(CXX) --version 2>&1 | grep nvcc | wc -l))
KOKKOS_INTERNAL_COMPILER_CLANG       := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),clang)
KOKKOS_INTERNAL_COMPILER_APPLE_CLANG := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),apple-darwin)
KOKKOS_INTERNAL_COMPILER_HCC         := $(call kokkos_has_string,$(KOKKOS_CXX_VERSION),HCC)

ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 2)
  KOKKOS_INTERNAL_COMPILER_CLANG = 1
endif
ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 2)
  KOKKOS_INTERNAL_COMPILER_XL = 1
endif

# Apple Clang passes both clang and apple clang tests, so turn off clang.
ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
  KOKKOS_INTERNAL_COMPILER_CLANG = 0
endif
# AMD HCC passes both clang and hcc test so turn off clang
ifeq ($(KOKKOS_INTERNAL_COMPILER_HCC), 1)
  KOKKOS_INTENAL_COMPILER_CLANG = 0
endif

ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
  KOKKOS_INTERNAL_COMPILER_CLANG_VERSION := $(shell clang --version | grep version | cut -d ' ' -f3 | tr -d '.')

  ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
    ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_CLANG_VERSION) -lt 400; echo $$?),0)
      $(error Compiling Cuda code directly with Clang requires version 4.0.0 or higher)
    endif

    KOKKOS_INTERNAL_CUDA_USE_LAMBDA := 1
  endif
endif

# Set compiler warnings flags.
ifeq ($(KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
    # TODO check if PGI accepts GNU style warnings
    KOKKOS_INTERNAL_COMPILER_WARNINGS =
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized
    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
        KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized
      else
        ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1)
          KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wuninitialized
        else
          ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
            # TODO check if cray accepts GNU style warnings
            KOKKOS_INTERNAL_COMPILER_WARNINGS =
          else
            #gcc
            KOKKOS_INTERNAL_COMPILER_WARNINGS = -Wall -Wshadow -pedantic -Wsign-compare -Wtype-limits -Wignored-qualifiers -Wempty-body -Wclobbered -Wuninitialized
          endif
        endif
      endif
    endif
  endif
else
  KOKKOS_INTERNAL_COMPILER_WARNINGS =
endif

# Set OpenMP flags.
ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
  KOKKOS_INTERNAL_OPENMP_FLAG := -mp
else
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp=libomp
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_APPLE_CLANG), 1)
      KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp=libomp
    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1)
        KOKKOS_INTERNAL_OPENMP_FLAG := -qsmp=omp
      else
        ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
          # OpenMP is turned on by default in Cray compiler environment.
          KOKKOS_INTERNAL_OPENMP_FLAG :=
        else
          KOKKOS_INTERNAL_OPENMP_FLAG := -fopenmp
        endif
      endif
    endif
  endif
endif
ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1)
  KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_IBM_XL_OMP45_WORKAROUND -qsmp=omp -qoffload -qnoeh
else
  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    KOKKOS_INTERNAL_OPENMPTARGET_FLAG := -DKOKKOS_BUG_WORKAROUND_IBM_CLANG_OMP45_VIEW_INIT -fopenmp-implicit-declare-target -fopenmp-targets=nvptx64-nvidia-cuda -fopenmp -fopenmp=libomp
  endif
endif

# Set C++11 flags.
ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
  KOKKOS_INTERNAL_CXX11_FLAG := --c++11
else
  ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1)
     KOKKOS_INTERNAL_CXX11_FLAG := -std=c++11
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
      KOKKOS_INTERNAL_CXX11_FLAG := -hstd=c++11
    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_HCC), 1)
        KOKKOS_INTERNAL_CXX11_FLAG := 
      else
        KOKKOS_INTERNAL_CXX11_FLAG := --std=c++11
        KOKKOS_INTERNAL_CXX1Z_FLAG := --std=c++1z
      endif
    endif
  endif
endif

# Check for Kokkos Architecture settings.

# Intel based.
KOKKOS_INTERNAL_USE_ARCH_KNC := $(call kokkos_has_string,$(KOKKOS_ARCH),KNC)
KOKKOS_INTERNAL_USE_ARCH_WSM := $(call kokkos_has_string,$(KOKKOS_ARCH),WSM)
KOKKOS_INTERNAL_USE_ARCH_SNB := $(call kokkos_has_string,$(KOKKOS_ARCH),SNB)
KOKKOS_INTERNAL_USE_ARCH_HSW := $(call kokkos_has_string,$(KOKKOS_ARCH),HSW)
KOKKOS_INTERNAL_USE_ARCH_BDW := $(call kokkos_has_string,$(KOKKOS_ARCH),BDW)
KOKKOS_INTERNAL_USE_ARCH_SKX := $(call kokkos_has_string,$(KOKKOS_ARCH),SKX)
KOKKOS_INTERNAL_USE_ARCH_KNL := $(call kokkos_has_string,$(KOKKOS_ARCH),KNL)

# NVIDIA based.
NVCC_WRAPPER := $(KOKKOS_PATH)/bin/nvcc_wrapper
KOKKOS_INTERNAL_USE_ARCH_KEPLER30 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler30)
KOKKOS_INTERNAL_USE_ARCH_KEPLER32 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler32)
KOKKOS_INTERNAL_USE_ARCH_KEPLER35 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler35)
KOKKOS_INTERNAL_USE_ARCH_KEPLER37 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler37)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL50 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell50)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL52 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell52)
KOKKOS_INTERNAL_USE_ARCH_MAXWELL53 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell53)
KOKKOS_INTERNAL_USE_ARCH_PASCAL61 := $(call kokkos_has_string,$(KOKKOS_ARCH),Pascal61)
KOKKOS_INTERNAL_USE_ARCH_PASCAL60 := $(call kokkos_has_string,$(KOKKOS_ARCH),Pascal60)
KOKKOS_INTERNAL_USE_ARCH_VOLTA70 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volta70)
KOKKOS_INTERNAL_USE_ARCH_VOLTA72 := $(call kokkos_has_string,$(KOKKOS_ARCH),Volta72)
KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLER30)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER32)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER35)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER37)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL61)  \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL60)  \
					      + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA70) \
					      + $(KOKKOS_INTERNAL_USE_ARCH_VOLTA72) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52) \
                                              + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53))

#SEK: This seems like a bug to me
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0)
  KOKKOS_INTERNAL_USE_ARCH_MAXWELL50 := $(call kokkos_has_string,$(KOKKOS_ARCH),Maxwell)
  KOKKOS_INTERNAL_USE_ARCH_KEPLER35 := $(call kokkos_has_string,$(KOKKOS_ARCH),Kepler)
  KOKKOS_INTERNAL_USE_ARCH_NVIDIA := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KEPLER30)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER32)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER35)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_KEPLER37)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL61)  \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_PASCAL60)  \
						+ $(KOKKOS_INTERNAL_USE_ARCH_VOLTA70) \
						+ $(KOKKOS_INTERNAL_USE_ARCH_VOLTA72) \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50) \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52) \
                                                + $(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53))
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 1)
  ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_INTERNAL_NVCC_PATH := $(shell which nvcc)
      CUDA_PATH ?= $(KOKKOS_INTERNAL_NVCC_PATH:/bin/nvcc=)
      KOKKOS_INTERNAL_OPENMPTARGET_FLAG := $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG) --cuda-path=$(CUDA_PATH)
    endif
  endif
endif
# ARM based.
KOKKOS_INTERNAL_USE_ARCH_ARMV80 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv80)
KOKKOS_INTERNAL_USE_ARCH_ARMV81 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv81)
KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv8-ThunderX)
KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2 := $(call kokkos_has_string,$(KOKKOS_ARCH),ARMv8-TX2)
KOKKOS_INTERNAL_USE_ARCH_ARM := $(strip $(shell echo $(KOKKOS_INTERNAL_USE_ARCH_ARMV80)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV81)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX)+$(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2) | bc))

# IBM based.
KOKKOS_INTERNAL_USE_ARCH_BGQ := $(call kokkos_has_string,$(KOKKOS_ARCH),BGQ)
KOKKOS_INTERNAL_USE_ARCH_POWER7 := $(call kokkos_has_string,$(KOKKOS_ARCH),Power7)
KOKKOS_INTERNAL_USE_ARCH_POWER8 := $(call kokkos_has_string,$(KOKKOS_ARCH),Power8)
KOKKOS_INTERNAL_USE_ARCH_POWER9 := $(call kokkos_has_string,$(KOKKOS_ARCH),Power9)
KOKKOS_INTERNAL_USE_ARCH_IBM := $(strip $(shell echo $(KOKKOS_INTERNAL_USE_ARCH_BGQ)+$(KOKKOS_INTERNAL_USE_ARCH_POWER7)+$(KOKKOS_INTERNAL_USE_ARCH_POWER8)+$(KOKKOS_INTERNAL_USE_ARCH_POWER9) | bc))

# AMD based.
KOKKOS_INTERNAL_USE_ARCH_AMDAVX := $(call kokkos_has_string,$(KOKKOS_ARCH),AMDAVX)
KOKKOS_INTERNAL_USE_ARCH_RYZEN := $(call kokkos_has_string,$(KOKKOS_ARCH),Ryzen)
KOKKOS_INTERNAL_USE_ARCH_EPYC := $(call kokkos_has_string,$(KOKKOS_ARCH),Epyc)
KOKKOS_INTERNAL_USE_ARCH_KAVERI := $(call kokkos_has_string,$(KOKKOS_ARCH),Kaveri)
KOKKOS_INTERNAL_USE_ARCH_CARRIZO := $(call kokkos_has_string,$(KOKKOS_ARCH),Carrizo)
KOKKOS_INTERNAL_USE_ARCH_FIJI := $(call kokkos_has_string,$(KOKKOS_ARCH),Fiji)
KOKKOS_INTERNAL_USE_ARCH_VEGA := $(call kokkos_has_string,$(KOKKOS_ARCH),Vega)
KOKKOS_INTERNAL_USE_ARCH_GFX901 := $(call kokkos_has_string,$(KOKKOS_ARCH),gfx901)

# Any AVX?
KOKKOS_INTERNAL_USE_ARCH_SSE42      := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_WSM))
KOKKOS_INTERNAL_USE_ARCH_AVX        := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_SNB) + $(KOKKOS_INTERNAL_USE_ARCH_AMDAVX))
KOKKOS_INTERNAL_USE_ARCH_AVX2       := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_HSW) + $(KOKKOS_INTERNAL_USE_ARCH_BDW))
KOKKOS_INTERNAL_USE_ARCH_AVX512MIC  := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KNL))
KOKKOS_INTERNAL_USE_ARCH_AVX512XEON := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_SKX))

# Decide what ISA level we are able to support.
KOKKOS_INTERNAL_USE_ISA_X86_64    := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_WSM) + $(KOKKOS_INTERNAL_USE_ARCH_SNB) + $(KOKKOS_INTERNAL_USE_ARCH_HSW) + $(KOKKOS_INTERNAL_USE_ARCH_BDW) + $(KOKKOS_INTERNAL_USE_ARCH_KNL) + $(KOKKOS_INTERNAL_USE_ARCH_SKX))
KOKKOS_INTERNAL_USE_ISA_KNC       := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_KNC))
KOKKOS_INTERNAL_USE_ISA_POWERPCLE := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_POWER8) + $(KOKKOS_INTERNAL_USE_ARCH_POWER9))
KOKKOS_INTERNAL_USE_ISA_POWERPCBE := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_POWER7))

# Decide whether we can support transactional memory
KOKKOS_INTERNAL_USE_TM            := $(shell expr $(KOKKOS_INTERNAL_USE_ARCH_BDW) + $(KOKKOS_INTERNAL_USE_ARCH_SKX))

# Incompatible flags?
KOKKOS_INTERNAL_USE_ARCH_MULTIHOST := $(strip $(shell echo "$(KOKKOS_INTERNAL_USE_ARCH_SSE42)+$(KOKKOS_INTERNAL_USE_ARCH_AVX)+$(KOKKOS_INTERNAL_USE_ARCH_AVX2)+$(KOKKOS_INTERNAL_USE_ARCH_AVX512MIC)+$(KOKKOS_INTERNAL_USE_ARCH_AVX512XEON)+$(KOKKOS_INTERNAL_USE_ARCH_KNC)+$(KOKKOS_INTERNAL_USE_ARCH_IBM)+$(KOKKOS_INTERNAL_USE_ARCH_ARM)>1" | bc ))
KOKKOS_INTERNAL_USE_ARCH_MULTIGPU := $(strip $(shell echo "$(KOKKOS_INTERNAL_USE_ARCH_NVIDIA)>1" | bc))

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MULTIHOST), 1)
  $(error Defined Multiple Host architectures: KOKKOS_ARCH=$(KOKKOS_ARCH) )
endif
ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MULTIGPU), 1)
  $(error Defined Multiple GPU architectures: KOKKOS_ARCH=$(KOKKOS_ARCH) )
endif

# Generating the list of Flags.

KOKKOS_CPPFLAGS = -I./ -I$(KOKKOS_PATH)/core/src -I$(KOKKOS_PATH)/containers/src -I$(KOKKOS_PATH)/algorithms/src
KOKKOS_TPL_INCLUDE_DIRS =
KOKKOS_TPL_LIBRARY_DIRS =
KOKKOS_TPL_LIBRARY_NAMES =

KOKKOS_CXXFLAGS =
ifeq ($(KOKKOS_INTERNAL_ENABLE_COMPILER_WARNINGS), 1)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_COMPILER_WARNINGS)
endif

KOKKOS_LIBS = -ldl
KOKKOS_TPL_LIBRARY_NAMES += dl
KOKKOS_LDFLAGS = -L$(shell pwd)
KOKKOS_LINK_FLAGS = 
KOKKOS_SRC =
KOKKOS_HEADERS =

# Generating the KokkosCore_config.h file.

KOKKOS_INTERNAL_CONFIG_TMP=KokkosCore_config.tmp
KOKKOS_CONFIG_HEADER=KokkosCore_config.h
# Functions for generating config header file
kokkos_append_header = $(shell echo $1 >> $(KOKKOS_INTERNAL_CONFIG_TMP))

# Do not append first line
tmp := $(shell echo "/* ---------------------------------------------" > KokkosCore_config.tmp)
tmp := $(call kokkos_append_header,"Makefile constructed configuration:")
tmp := $(call kokkos_append_header,"$(shell date)")
tmp := $(call kokkos_append_header,"----------------------------------------------*/")

tmp := $(call kokkos_append_header,'\#if !defined(KOKKOS_MACROS_HPP) || defined(KOKKOS_CORE_CONFIG_H)')
tmp := $(call kokkos_append_header,'\#error "Do not include $(KOKKOS_CONFIG_HEADER) directly; include Kokkos_Macros.hpp instead."')
tmp := $(call kokkos_append_header,'\#else')
tmp := $(call kokkos_append_header,'\#define KOKKOS_CORE_CONFIG_H')
tmp := $(call kokkos_append_header,'\#endif')
	
tmp := $(call kokkos_append_header,"/* Execution Spaces */")

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_CUDA")
endif

ifeq ($(KOKKOS_INTERNAL_USE_ROCM), 1)
  tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_ROCM')
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  tmp := $(call kokkos_append_header,'\#define KOKKOS_ENABLE_OPENMPTARGET')
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
  tmp := $(call kokkos_append_header,'\#define KOKKOS_HAVE_OPENMP')
endif

ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_PTHREAD")
endif

ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_QTHREADS")
endif

ifeq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_SERIAL")
endif

ifeq ($(KOKKOS_INTERNAL_USE_TM), 1)
  tmp := $(call kokkos_append_header,"\#ifndef __CUDA_ARCH__")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_TM")
  tmp := $(call kokkos_append_header,"\#endif")
endif

ifeq ($(KOKKOS_INTERNAL_USE_ISA_X86_64), 1)
  tmp := $(call kokkos_append_header,"\#ifndef __CUDA_ARCH__")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_USE_ISA_X86_64")
  tmp := $(call kokkos_append_header,"\#endif")
endif

ifeq ($(KOKKOS_INTERNAL_USE_ISA_KNC), 1)
  tmp := $(call kokkos_append_header,"\#ifndef __CUDA_ARCH__")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_USE_ISA_KNC")
  tmp := $(call kokkos_append_header,"\#endif")
endif

ifeq ($(KOKKOS_INTERNAL_USE_ISA_POWERPCLE), 1)
  tmp := $(call kokkos_append_header,"\#ifndef __CUDA_ARCH__")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_USE_ISA_POWERPCLE")
  tmp := $(call kokkos_append_header,"\#endif")
endif

ifeq ($(KOKKOS_INTERNAL_USE_ISA_POWERPCBE), 1)
  tmp := $(call kokkos_append_header,"\#ifndef __CUDA_ARCH__")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_USE_ISA_POWERPCBE")
  tmp := $(call kokkos_append_header,"\#endif")
endif

tmp := $(call kokkos_append_header,"/* General Settings */")
ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX11), 1)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX11_FLAG)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_CXX11")
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_CXX1Z), 1)
  KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CXX1Z_FLAG)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_CXX11")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_CXX1Z")
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_DEBUG), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_CXXFLAGS += -lineinfo
  endif

  KOKKOS_CXXFLAGS += -g
  KOKKOS_LDFLAGS += -g -ldl
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEBUG_BOUNDS_CHECK")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_DEBUG")
  ifeq ($(KOKKOS_INTERNAL_DISABLE_DUALVIEW_MODIFY_CHECK), 0)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEBUG_DUALVIEW_MODIFY_CHECK")
  endif
endif

ifeq ($(KOKKOS_INTERNAL_ENABLE_PROFILING_LOAD_PRINT), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_PROFILING_LOAD_PRINT")
endif

ifeq ($(KOKKOS_INTERNAL_USE_HWLOC), 1)
  ifneq ($(HWLOC_PATH),)
    KOKKOS_CPPFLAGS += -I$(HWLOC_PATH)/include
    KOKKOS_LDFLAGS += -L$(HWLOC_PATH)/lib
    KOKKOS_TPL_INCLUDE_DIRS += $(HWLOC_PATH)/include
    KOKKOS_TPL_LIBRARY_DIRS += $(HWLOC_PATH)/lib
  endif
  KOKKOS_LIBS += -lhwloc
  KOKKOS_TPL_LIBRARY_NAMES += hwloc
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_HWLOC")
endif

ifeq ($(KOKKOS_INTERNAL_USE_LIBRT), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_USE_LIBRT")
  KOKKOS_LIBS += -lrt
  KOKKOS_TPL_LIBRARY_NAMES += rt
endif

ifeq ($(KOKKOS_INTERNAL_USE_MEMKIND), 1)
  ifneq ($(MEMKIND_PATH),)
    KOKKOS_CPPFLAGS += -I$(MEMKIND_PATH)/include
    KOKKOS_LDFLAGS += -L$(MEMKIND_PATH)/lib
    KOKKOS_TPL_INCLUDE_DIRS += $(MEMKIND_PATH)/include
    KOKKOS_TPL_LIBRARY_DIRS += $(MEMKIND_PATH)/lib
  endif
  KOKKOS_LIBS += -lmemkind -lnuma
  KOKKOS_TPL_LIBRARY_NAMES += memkind numa
  tmp := $(call kokkos_append_header,"\#define KOKKOS_HAVE_HBWSPACE")
endif

ifeq ($(KOKKOS_INTERNAL_DISABLE_PROFILING), 0)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_PROFILING")
endif

ifeq ($(KOKKOS_INTERNAL_DISABLE_DEPRECATED_CODE), 0)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ENABLE_DEPRECATED_CODE")
endif

tmp := $(call kokkos_append_header,"/* Optimization Settings */")

ifeq ($(KOKKOS_INTERNAL_OPT_RANGE_AGGRESSIVE_VECTORIZATION), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_OPT_RANGE_AGGRESSIVE_VECTORIZATION")
endif

tmp := $(call kokkos_append_header,"/* Cuda Settings */")

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_LDG), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_LDG_INTRINSIC")
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_LDG_INTRINSIC")
    endif
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_UVM), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_UVM")
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_RELOC), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_RELOCATABLE_DEVICE_CODE")
    KOKKOS_CXXFLAGS += --relocatable-device-code=true
    KOKKOS_LDFLAGS += --relocatable-device-code=true
  endif

  ifeq ($(KOKKOS_INTERNAL_CUDA_USE_LAMBDA), 1)
    ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
      ifeq ($(shell test $(KOKKOS_INTERNAL_COMPILER_NVCC_VERSION) -gt 70; echo $$?),0)
        tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_LAMBDA")
        KOKKOS_CXXFLAGS += -expt-extended-lambda
      else
        $(warning Warning: Cuda Lambda support was requested but NVCC version is too low. This requires NVCC for Cuda version 7.5 or higher. Disabling Lambda support now.)
      endif
    endif

    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_USE_LAMBDA")
    endif
  endif

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_CUDA_CLANG_WORKAROUND")
  endif
endif

# Add Architecture flags.

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV80), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV80")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
      KOKKOS_CXXFLAGS +=
      KOKKOS_LDFLAGS +=
    else
      KOKKOS_CXXFLAGS += -march=armv8-a
      KOKKOS_LDFLAGS += -march=armv8-a
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV81), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV81")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
      KOKKOS_CXXFLAGS +=
      KOKKOS_LDFLAGS +=
    else
      KOKKOS_CXXFLAGS += -march=armv8.1-a
      KOKKOS_LDFLAGS += -march=armv8.1-a
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV80")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV8_THUNDERX")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
      KOKKOS_CXXFLAGS +=
      KOKKOS_LDFLAGS +=
    else
      KOKKOS_CXXFLAGS += -march=armv8-a -mtune=thunderx
      KOKKOS_LDFLAGS += -march=armv8-a -mtune=thunderx
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_ARMV8_THUNDERX2), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV81")
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ARMV8_THUNDERX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)
    KOKKOS_CXXFLAGS +=
    KOKKOS_LDFLAGS +=
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
      KOKKOS_CXXFLAGS +=
      KOKKOS_LDFLAGS +=
    else
      KOKKOS_CXXFLAGS += -mtune=thunderx2t99 -mcpu=thunderx2t99
      KOKKOS_LDFLAGS += -mtune=thunderx2t99 -mcpu=thunderx2t99
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_SSE42), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_SSE42")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xSSE4.2
    KOKKOS_LDFLAGS  += -xSSE4.2
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
        KOKKOS_CXXFLAGS += -tp=nehalem
        KOKKOS_LDFLAGS  += -tp=nehalem
      else
        # Assume that this is a really a GNU compiler.
        KOKKOS_CXXFLAGS += -msse4.2
        KOKKOS_LDFLAGS  += -msse4.2
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AVX), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_AVX")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -mavx
    KOKKOS_LDFLAGS  += -mavx
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
        KOKKOS_CXXFLAGS += -tp=sandybridge
        KOKKOS_LDFLAGS  += -tp=sandybridge
      else
        # Assume that this is a really a GNU compiler.
        KOKKOS_CXXFLAGS += -mavx
        KOKKOS_LDFLAGS  += -mavx
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER7), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_POWER7")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)

  else
    # Assume that this is a really a GNU compiler or it could be XL on P8.
    KOKKOS_CXXFLAGS += -mcpu=power7 -mtune=power7
    KOKKOS_LDFLAGS  += -mcpu=power7 -mtune=power7
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER8), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_POWER8")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)

  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1) 
        KOKKOS_CXXFLAGS += -mcpu=power8 -mtune=power8
        KOKKOS_LDFLAGS  += -mcpu=power8 -mtune=power8
    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)

      else 
        # Assume that this is a really a GNU compiler on P8.
        KOKKOS_CXXFLAGS += -mcpu=power8 -mtune=power8
        KOKKOS_LDFLAGS  += -mcpu=power8 -mtune=power8
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_POWER9), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_POWER9")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)

  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_XL), 1) 
        KOKKOS_CXXFLAGS += -mcpu=power9 -mtune=power9
        KOKKOS_LDFLAGS  += -mcpu=power9 -mtune=power9
    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)

      else 
        # Assume that this is a really a GNU compiler on P9
        KOKKOS_CXXFLAGS += -mcpu=power9 -mtune=power9
        KOKKOS_LDFLAGS  += -mcpu=power9 -mtune=power9
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_HSW), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX2
    KOKKOS_LDFLAGS  += -xCORE-AVX2
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
        KOKKOS_CXXFLAGS += -tp=haswell
        KOKKOS_LDFLAGS  += -tp=haswell
      else
        # Assume that this is a really a GNU compiler.
        KOKKOS_CXXFLAGS += -march=core-avx2 -mtune=core-avx2
        KOKKOS_LDFLAGS  += -march=core-avx2 -mtune=core-avx2
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_BDW), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_AVX2")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX2
    KOKKOS_LDFLAGS  += -xCORE-AVX2
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)
        KOKKOS_CXXFLAGS += -tp=haswell
        KOKKOS_LDFLAGS  += -tp=haswell
      else
        # Assume that this is a really a GNU compiler.
        KOKKOS_CXXFLAGS += -march=core-avx2 -mtune=core-avx2 -mrtm
        KOKKOS_LDFLAGS  += -march=core-avx2 -mtune=core-avx2 -mrtm
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AVX512MIC), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_AVX512MIC")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xMIC-AVX512
    KOKKOS_LDFLAGS  += -xMIC-AVX512
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)

      else
        # Asssume that this is really a GNU compiler.
        KOKKOS_CXXFLAGS += -march=knl -mtune=knl
        KOKKOS_LDFLAGS  += -march=knl -mtune=knl
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_AVX512XEON), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_AVX512XEON")

  ifeq ($(KOKKOS_INTERNAL_COMPILER_INTEL), 1)
    KOKKOS_CXXFLAGS += -xCORE-AVX512
    KOKKOS_LDFLAGS  += -xCORE-AVX512
  else
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CRAY), 1)

    else
      ifeq ($(KOKKOS_INTERNAL_COMPILER_PGI), 1)

      else
        # Nothing here yet.
        KOKKOS_CXXFLAGS += -march=skylake-avx512 -mtune=skylake-avx512 -mrtm
        KOKKOS_LDFLAGS  += -march=skylake-avx512 -mtune=skylake-avx512 -mrtm
      endif
    endif
  endif
endif

ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KNC), 1)
  tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KNC")
  KOKKOS_CXXFLAGS += -mmic
  KOKKOS_LDFLAGS += -mmic
endif

# Figure out the architecture flag for Cuda.
ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG=-arch
  else ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
		KOKKOS_INTERNAL_CUDA_ARCH_FLAG=--cuda-gpu-arch
		KOKKOS_CXXFLAGS += -x cuda
  else
    $(error Makefile.kokkos: CUDA is enabled but the compiler is neither NVCC nor Clang)
  endif

  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER30), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER30")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_30
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER32), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER32")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_32
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER35), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER35")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_35
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KEPLER37), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KEPLER37")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_37
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL50), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL50")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_50
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL52), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL52")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_52
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_MAXWELL53), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_MAXWELL53")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_53
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL60), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_PASCAL")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_PASCAL60")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_60
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_PASCAL61), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_PASCAL")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_PASCAL61")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_61
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA70), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VOLTA")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VOLTA70")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_70
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VOLTA72), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VOLTA")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VOLTA72")
    KOKKOS_INTERNAL_CUDA_ARCH_FLAG := $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)=sm_72
  endif

  ifneq ($(KOKKOS_INTERNAL_USE_ARCH_NVIDIA), 0)
    KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)

    ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
      KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_CUDA_ARCH_FLAG)
    endif
  endif
endif

# Figure out the architecture flag for ROCm.
ifeq ($(KOKKOS_INTERNAL_USE_ROCM), 1)
  # Lets start with adding architecture defines
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_KAVERI), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 701")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_KAVERI")
    KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx701 
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_CARRIZO), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 801")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_CARRIZO")
    KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx801 
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_FIJI), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 803")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_FIJI")
    KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx803
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_VEGA), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 900")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_VEGA")
    KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx900 
  endif
  ifeq ($(KOKKOS_INTERNAL_USE_ARCH_GFX901), 1)
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_ROCM 901")
    tmp := $(call kokkos_append_header,"\#define KOKKOS_ARCH_GFX901")
    KOKKOS_INTERNAL_ROCM_ARCH_FLAG := --amdgpu-target=gfx901 
  endif
 
  
  KOKKOS_INTERNAL_HCC_PATH := $(shell which $(CXX))
  ROCM_HCC_PATH ?= $(KOKKOS_INTERNAL_HCC_PATH:/bin/clang++=)

  KOKKOS_CXXFLAGS += $(shell $(ROCM_HCC_PATH)/bin/hcc-config --cxxflags) 
  KOKKOS_LDFLAGS += $(shell $(ROCM_HCC_PATH)/bin/hcc-config --ldflags) -lhc_am -lm 
  KOKKOS_TPL_LIBRARY_NAMES += hc_am m
  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_ROCM_ARCH_FLAG)

  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/ROCm/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/ROCm/*.hpp)
endif

KOKKOS_INTERNAL_LS_CONFIG := $(shell ls KokkosCore_config.h 2>&1)

ifeq ($(KOKKOS_INTERNAL_LS_CONFIG), KokkosCore_config.h)
  KOKKOS_INTERNAL_NEW_CONFIG := $(strip $(shell diff KokkosCore_config.h KokkosCore_config.tmp | grep define | wc -l))
else
  KOKKOS_INTERNAL_NEW_CONFIG := 1
endif

ifneq ($(KOKKOS_INTERNAL_NEW_CONFIG), 0)
  tmp := $(shell cp KokkosCore_config.tmp KokkosCore_config.h)
endif

KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.hpp)
KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/algorithms/src/*.hpp)

KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/impl/*.cpp)
KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/containers/src/impl/*.cpp)

ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Cuda/*.hpp)
  ifneq ($(CUDA_PATH),)
    KOKKOS_CPPFLAGS += -I$(CUDA_PATH)/include
    KOKKOS_LDFLAGS += -L$(CUDA_PATH)/lib64
    KOKKOS_TPL_INCLUDE_DIRS += $(CUDA_PATH)/include
    KOKKOS_TPL_LIBRARY_DIRS += $(CUDA_PATH)/lib64
    ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
      KOKKOS_CXXFLAGS += --cuda-path=$(CUDA_PATH)
    endif
  endif
  KOKKOS_LIBS += -lcudart -lcuda
  KOKKOS_TPL_LIBRARY_NAMES += cudart cuda
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMPTARGET), 1)
  KOKKOS_SRC += $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTarget_Exec.cpp $(KOKKOS_PATH)/core/src/OpenMPTarget/Kokkos_OpenMPTargetSpace.cpp
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMPTarget/*.hpp)
  ifeq ($(KOKKOS_INTERNAL_USE_CUDA), 1)
    KOKKOS_CXXFLAGS += -Xcompiler $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
  else
    KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
  endif
  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENMPTARGET_FLAG)
endif

ifeq ($(KOKKOS_INTERNAL_USE_OPENMP), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/OpenMP/*.hpp)

  ifeq ($(KOKKOS_INTERNAL_COMPILER_NVCC), 1)
    KOKKOS_CXXFLAGS += -Xcompiler $(KOKKOS_INTERNAL_OPENMP_FLAG)
  else
    KOKKOS_CXXFLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
  endif

  KOKKOS_LDFLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
  KOKKOS_LINK_FLAGS += $(KOKKOS_INTERNAL_OPENMP_FLAG)
endif

ifeq ($(KOKKOS_INTERNAL_USE_PTHREADS), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Threads/*.hpp)
  KOKKOS_LIBS += -lpthread
  KOKKOS_TPL_LIBRARY_NAMES += pthread
endif

ifeq ($(KOKKOS_INTERNAL_USE_QTHREADS), 1)
  KOKKOS_SRC += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.cpp)
  KOKKOS_HEADERS += $(wildcard $(KOKKOS_PATH)/core/src/Qthreads/*.hpp)
  ifneq ($(QTHREADS_PATH),)
    KOKKOS_CPPFLAGS += -I$(QTHREADS_PATH)/include
    KOKKOS_LDFLAGS += -L$(QTHREADS_PATH)/lib
    KOKKOS_TPL_INCLUDE_DIRS += $(QTHREADS_PATH)/include
    KOKKOS_TPL_LIBRARY_DIRS += $(QTHREADS_PATH)/lib64
  endif
  KOKKOS_LIBS += -lqthread
  KOKKOS_TPL_LIBRARY_NAMES += qthread
endif

# Explicitly set the GCC Toolchain for Clang.
ifeq ($(KOKKOS_INTERNAL_COMPILER_CLANG), 1)
  KOKKOS_INTERNAL_GCC_PATH = $(shell which g++)
  KOKKOS_INTERNAL_GCC_TOOLCHAIN = $(KOKKOS_INTERNAL_GCC_PATH:/bin/g++=)
  KOKKOS_CXXFLAGS += --gcc-toolchain=$(KOKKOS_INTERNAL_GCC_TOOLCHAIN)
  KOKKOS_LDFLAGS += --gcc-toolchain=$(KOKKOS_INTERNAL_GCC_TOOLCHAIN)
endif

# Don't include Kokkos_HBWSpace.cpp if not using MEMKIND to avoid a link warning.
ifneq ($(KOKKOS_INTERNAL_USE_MEMKIND), 1)
  KOKKOS_SRC := $(filter-out $(KOKKOS_PATH)/core/src/impl/Kokkos_HBWSpace.cpp,$(KOKKOS_SRC))
endif

# Don't include Kokkos_Serial.cpp or Kokkos_Serial_Task.cpp if not using Serial
# device to avoid a link warning.
ifneq ($(KOKKOS_INTERNAL_USE_SERIAL), 1)
  KOKKOS_SRC := $(filter-out $(KOKKOS_PATH)/core/src/impl/Kokkos_Serial.cpp,$(KOKKOS_SRC))
  KOKKOS_SRC := $(filter-out $(KOKKOS_PATH)/core/src/impl/Kokkos_Serial_Task.cpp,$(KOKKOS_SRC))
endif

# With Cygwin functions such as fdopen and fileno are not defined
# when strict ansi is enabled. strict ansi gets enabled with --std=c++11
# though. So we hard undefine it here. Not sure if that has any bad side effects
# This is needed for gtest actually, not for Kokkos itself!
ifeq ($(KOKKOS_INTERNAL_OS_CYGWIN), 1)
  KOKKOS_CXXFLAGS += -U__STRICT_ANSI__
endif

# Set KokkosExtraLibs and add -lkokkos to link line
KOKKOS_EXTRA_LIBS := ${KOKKOS_LIBS}
KOKKOS_LIBS := -lkokkos ${KOKKOS_LIBS}

# Setting up dependencies.

KokkosCore_config.h:

KOKKOS_CPP_DEPENDS := KokkosCore_config.h $(KOKKOS_HEADERS)

KOKKOS_OBJ = $(KOKKOS_SRC:.cpp=.o)
KOKKOS_OBJ_LINK = $(notdir $(KOKKOS_OBJ))

include $(KOKKOS_PATH)/Makefile.targets

kokkos-clean:
	rm -f $(KOKKOS_OBJ_LINK) KokkosCore_config.h KokkosCore_config.tmp libkokkos.a

libkokkos.a: $(KOKKOS_OBJ_LINK) $(KOKKOS_SRC) $(KOKKOS_HEADERS)
	ar cr libkokkos.a $(KOKKOS_OBJ_LINK)
	ranlib libkokkos.a

KOKKOS_LINK_DEPENDS=libkokkos.a